From 40c187d10e2b0657598e6c07af38cbd7562670dc Mon Sep 17 00:00:00 2001 From: Timothy Blamires Date: Thu, 12 Mar 2026 12:49:09 -0600 Subject: [PATCH 1/2] Revert most of #10 while keeping c315cfa and 4a7f2bd --- .bazelci/cache_test.sh | 22 +- .bazelci/docker_unit_test.sh | 14 - .bazelci/format.sh | 14 - .bazelci/ingest_grpc_logs.py | 2 +- .bazelci/integration_test.sh | 14 - .bazelci/package_name_check.sh | 14 - .bazelci/presubmit.yml | 28 +- .bazelci/redis_unit_tests.sh | 14 - .bazelci/run_abseil_test.sh | 14 - .bazelci/run_checkstyle.sh | 14 - .../run_generative_cc_many_double_test.sh | 14 - .bazelci/run_generative_cc_many_test.sh | 14 - .bazelci/run_server_test.sh | 13 - .bazelci/static_analyze.sh | 14 - .bazelci/test_buildfarm_container.sh | 14 - .bazelci/unused_deps.sh | 14 - .bazelrc | 12 +- .bazelversion | 2 +- .../workflows/buildfarm-helm-chart-lint.yml | 2 +- .../buildfarm-helm-chart-publish.yml | 2 +- .../buildfarm-images-build-and-deploy.yml | 9 +- .../buildfarm-release-build-and-deploy.yml | 9 +- ...buildfarm-worker-base-build-and-deploy.yml | 14 +- .github/workflows/codeql.yml | 8 +- .github/workflows/license.yaml | 19 - .github/workflows/main.yml | 4 +- .github/workflows/scorecards.yml | 6 +- CODEOWNERS | 2 +- Dockerfile | 10 +- MODULE.bazel | 115 +- README.md | 2 +- _site/docs/configuration/configuration.md | 43 +- _site/docs/execution/execution_properties.md | 16 - _site/docs/tools/bf-cat.md | 4 - as-nobody-windows.c | 15 - as-nobody.c | 16 - ci/base-worker-image/jammy/Dockerfile | 2 +- ci/license-header.txt | 13 - config/BUILD | 12 + {examples => config}/backplane.yml | 0 config/config.minimal.yml | 3 + config/logging.properties | 4 + {examples => config}/queues.yml | 0 {examples => config}/server.yml | 0 {examples => config}/worker.yml | 4 + container/BUILD | 2 +- delay.sh | 14 - examples/BUILD | 6 - examples/config.minimal.yml | 14 +- examples/config.yml | 4 +- examples/development-redis-cluster.sh | 14 - examples/logging.properties | 4 +- extensions.bzl | 6 +- generate_coverage.sh | 14 - kubernetes/helm-charts/buildfarm/Chart.yaml | 4 +- .../buildfarm/templates/_helpers.tpl | 2 +- .../buildfarm/templates/configmap.yaml | 45 +- .../templates/exec-worker/deployment.yaml | 124 -- .../buildfarm/templates/exec-worker/hpa.yaml | 30 - .../buildfarm/templates/exec-worker/pdb.yaml | 19 - .../templates/exec-worker/service.yaml | 27 - .../templates/exec-worker/servicemonitor.yaml | 40 - .../templates/server/deployment.yaml | 16 +- .../{hpa.yaml => autoscaler.yaml} | 0 .../{statefulset.yaml => statefulsets.yaml} | 17 +- kubernetes/helm-charts/buildfarm/values.yaml | 169 +-- licenserc.toml | 17 - macos-wrapper.sh | 14 - maven_install.json | 1274 ++++++++++++++--- .../examples/src/main/java/BUILD | 2 +- .../examples/src/test/java/BUILD | 4 +- .../src/main/java/persistent/bazel/BUILD | 28 +- .../bazel/client/BasicWorkerKey.java | 107 -- .../bazel/client/CommonsWorkerPool.java | 15 +- .../bazel/client/PersistentWorker.java | 8 +- .../persistent/bazel/client/WorkerIndex.java | 55 - .../persistent/bazel/client/WorkerKey.java | 121 +- .../bazel/client/WorkerSupervisor.java | 23 +- .../src/main/java/persistent/common/BUILD | 27 +- persistentworkers/src/main/protobuf/BUILD | 4 +- .../src/test/java/persistent/bazel/BUILD | 10 +- .../src/test/java/persistent/common/BUILD | 7 +- .../src/test/java/persistent/testutil/BUILD | 10 +- .../java/persistent/testutil/WorkerUtils.java | 16 +- renovate.json | 9 +- run_server | 20 +- run_worker | 18 +- src/main/java/build/buildfarm/BUILD | 10 +- .../java/build/buildfarm/actioncache/BUILD | 16 +- src/main/java/build/buildfarm/backplane/BUILD | 9 +- .../build/buildfarm/backplane/Backplane.java | 5 - src/main/java/build/buildfarm/cas/BUILD | 31 +- .../cas/ContentAddressableStorage.java | 6 - .../java/build/buildfarm/cas/GrpcCAS.java | 13 - .../java/build/buildfarm/cas/MemoryCAS.java | 9 - .../build/buildfarm/cas/cfc/CASFileCache.java | 344 ++--- .../cas/cfc/CasFallbackDelegate.java | 6 +- .../buildfarm/cas/cfc/DirectoryEntryCFC.java | 5 + .../java/build/buildfarm/cas/cfc/LRUDB.java | 14 - .../buildfarm/cas/cfc/LegacyDirectoryCFC.java | 14 +- .../build/buildfarm/cas/cfc/TextLRUDB.java | 71 - src/main/java/build/buildfarm/common/BUILD | 82 +- .../java/build/buildfarm/common/Claim.java | 16 +- .../build/buildfarm/common/DigestUtil.java | 15 +- .../build/buildfarm/common/Dispenser.java | 90 ++ .../java/build/buildfarm/common/Queue.java | 3 - .../buildfarm/common/SystemProcessors.java | 14 +- .../java/build/buildfarm/common/blake3/BUILD | 8 +- .../java/build/buildfarm/common/config/BUILD | 26 +- .../buildfarm/common/config/Backplane.java | 1 - .../common/config/ExecutionPolicy.java | 1 - .../build/buildfarm/common/config/Worker.java | 2 +- .../java/build/buildfarm/common/grpc/BUILD | 22 +- .../buildfarm/common/grpc/NullObserver.java | 28 - .../java/build/buildfarm/common/redis/BUILD | 16 +- .../common/redis/BalancedRedisQueue.java | 81 +- .../buildfarm/common/redis/RedisMap.java | 15 - .../common/redis/RedisPriorityQueue.java | 33 - .../buildfarm/common/redis/RedisQueue.java | 17 - .../build/buildfarm/common/resources/BUILD | 42 +- src/main/java/build/buildfarm/common/s3/BUILD | 10 +- .../build/buildfarm/common/services/BUILD | 22 +- .../common/services/ByteStreamService.java | 12 - src/main/java/build/buildfarm/instance/BUILD | 18 +- .../build/buildfarm/instance/Instance.java | 8 +- .../build/buildfarm/instance/server/BUILD | 24 +- .../instance/server/NodeInstance.java | 17 +- .../java/build/buildfarm/instance/shard/BUILD | 32 +- .../instance/shard/ExecutionQueue.java | 98 +- .../instance/shard/JedisCasWorkerMap.java | 2 - .../instance/shard/JedisClusterFactory.java | 1 - .../shard/ReadOnlyAwareOutputStream.java | 80 -- .../instance/shard/ReadOnlyAwareWrite.java | 115 -- .../instance/shard/RedisShardBackplane.java | 116 +- .../instance/shard/ServerInstance.java | 264 +--- .../java/build/buildfarm/instance/stub/BUILD | 20 +- .../buildfarm/instance/stub/StubInstance.java | 44 +- .../metrics/AbstractMetricsPublisher.java | 4 +- src/main/java/build/buildfarm/metrics/BUILD | 10 +- .../java/build/buildfarm/metrics/log/BUILD | 6 +- .../build/buildfarm/metrics/prometheus/BUILD | 8 +- .../java/build/buildfarm/proxy/http/BUILD | 52 +- .../java/build/buildfarm/rpms/server/BUILD | 2 +- .../java/build/buildfarm/rpms/worker/BUILD | 2 +- src/main/java/build/buildfarm/server/BUILD | 24 +- .../buildfarm/server/BuildFarmServer.java | 2 - .../build/buildfarm/server/services/BUILD | 22 +- .../server/services/WorkerProfileService.java | 76 - src/main/java/build/buildfarm/tools/BUILD | 185 +-- src/main/java/build/buildfarm/tools/Cat.java | 124 +- .../buildfarm/tools/GracefulShutdown.java | 8 +- src/main/java/build/buildfarm/tools/Hist.java | 2 +- .../java/build/buildfarm/tools/Mount.java | 4 +- .../build/buildfarm/tools/WorkerProfile.java | 118 +- .../buildfarm/tools/WorkerProfilePrinter.java | 45 + src/main/java/build/buildfarm/worker/BUILD | 53 +- .../{filesystem => }/CFCExecFileSystem.java | 88 +- .../CFCLinkExecFileSystem.java | 54 +- .../worker/DequeueMatchEvaluator.java | 3 + .../{filesystem => }/ExecDirException.java | 4 +- .../{filesystem => }/ExecDirectoryStream.java | 10 +- .../{filesystem => }/ExecFileSystem.java | 11 +- .../worker/{filesystem => }/ExecTree.java | 4 +- .../{filesystem => }/ExecTreeWalker.java | 4 +- .../buildfarm/worker/ExecuteActionStage.java | 36 +- .../buildfarm/worker/ExecutionContext.java | 20 +- .../java/build/buildfarm/worker/Executor.java | 140 +- .../worker/{filesystem => }/FuseCAS.java | 2 +- .../buildfarm/worker/InputFetchStage.java | 15 - .../build/buildfarm/worker/InputFetcher.java | 37 +- .../build/buildfarm/worker/MatchStage.java | 29 +- .../{filesystem => }/OutputDirectory.java | 2 +- .../java/build/buildfarm/worker/Pipeline.java | 28 +- .../build/buildfarm/worker/PipelineStage.java | 12 +- .../buildfarm/worker/ReportResultStage.java | 3 - .../buildfarm/worker/ResultReporter.java | 3 +- .../worker/SuperscalarPipelineStage.java | 38 +- .../build/buildfarm/worker/WorkerContext.java | 8 +- .../java/build/buildfarm/worker/cgroup/BUILD | 12 +- .../worker/cgroup/CGroupVersionProvider.java | 1 + .../build/buildfarm/worker/cgroup/Cpu.java | 1 + .../build/buildfarm/worker/cgroup/Group.java | 1 + .../buildfarm/worker/executionwrappers/BUILD | 13 - .../ExecutionWrapperUtils.java | 112 -- .../buildfarm/worker/filesystem/BUILD.bazel | 21 - .../build/buildfarm/worker/persistent/BUILD | 35 +- .../worker/persistent/FileAccessUtils.java | 44 +- .../buildfarm/worker/persistent/Keymaker.java | 63 +- .../worker/persistent/PersistentExecutor.java | 76 +- .../PersistentWorkerAwareExecOwnerPool.java | 115 -- .../worker/persistent/ProtoCoordinator.java | 31 +- .../worker/persistent/WorkerInputs.java | 6 +- .../build/buildfarm/worker/resources/BUILD | 13 +- .../worker/resources/LocalResource.java | 24 - .../worker/resources/LocalResourceSet.java | 102 +- .../resources/LocalResourceSetUtils.java | 209 +-- .../java/build/buildfarm/worker/shard/BUILD | 32 +- .../worker/shard/FuseExecFileSystem.java | 15 +- .../worker/shard/LocalCasWriter.java | 2 +- .../worker/shard/ShardWorkerContext.java | 90 +- ...rol.java => ShutDownWorkerGracefully.java} | 24 +- .../build/buildfarm/worker/shard/Worker.java | 241 ++-- .../worker/shard/WorkerInstance.java | 5 - .../worker/shard/WorkerProfileService.java | 93 +- .../java/build/buildfarm/worker/util/BUILD | 25 +- .../protobuf/build/buildfarm/v1test/BUILD | 8 +- .../build/buildfarm/v1test/buildfarm.proto | 75 +- src/test/java/build/buildfarm/BUILD | 4 +- src/test/java/build/buildfarm/cas/BUILD | 22 +- .../buildfarm/cas/cfc/CASFileCacheTest.java | 100 +- src/test/java/build/buildfarm/common/BUILD | 24 +- .../buildfarm/common/DigestUtilTest.java | 2 +- .../java/build/buildfarm/common/config/BUILD | 10 +- .../java/build/buildfarm/common/grpc/BUILD | 30 +- src/test/java/build/buildfarm/common/io/BUILD | 24 +- .../java/build/buildfarm/common/redis/BUILD | 12 +- .../redis/BalancedRedisQueueMockTest.java | 8 +- .../common/redis/BalancedRedisQueueTest.java | 4 +- .../common/redis/RedisMapMockTest.java | 21 - .../build/buildfarm/common/resources/BUILD | 22 +- src/test/java/build/buildfarm/common/s3/BUILD | 8 +- .../build/buildfarm/common/services/BUILD | 18 +- src/test/java/build/buildfarm/examples/BUILD | 22 +- src/test/java/build/buildfarm/instance/BUILD | 10 +- .../buildfarm/instance/DummyInstanceBase.java | 11 +- .../build/buildfarm/instance/server/BUILD | 14 +- .../instance/server/NodeInstanceTest.java | 7 +- .../java/build/buildfarm/instance/shard/BUILD | 148 +- .../shard/RedisShardBackplaneTest.java | 40 +- .../java/build/buildfarm/instance/stub/BUILD | 20 +- src/test/java/build/buildfarm/metrics/BUILD | 20 +- .../java/build/buildfarm/proxy/http/BUILD | 16 +- src/test/java/build/buildfarm/server/BUILD | 28 +- .../build/buildfarm/server/services/BUILD | 10 +- src/test/java/build/buildfarm/worker/BUILD | 40 +- .../CFCExecFileSystemTest.java | 2 +- .../worker/DequeueMatchEvaluatorTest.java | 38 +- .../worker/ExecuteActionStageTest.java | 5 +- .../worker/{filesystem => }/FuseCASTest.java | 2 +- .../buildfarm/worker/InputFetchStageTest.java | 1 - .../buildfarm/worker/InputFetcherTest.java | 7 +- .../{filesystem => }/OutputDirectoryTest.java | 2 +- .../buildfarm/worker/StubWorkerContext.java | 14 +- .../worker/SuperscalarPipelineStageTest.java | 6 +- .../java/build/buildfarm/worker/cgroup/BUILD | 8 +- .../cgroup/CGroupVersionProviderTest.java | 4 + .../buildfarm/worker/cgroup/GroupTest.java | 1 + .../buildfarm/worker/cgroup/GroupV1Test.java | 1 + .../buildfarm/worker/cgroup/GroupV2Test.java | 1 + .../buildfarm/worker/filesystem/BUILD.bazel | 32 - .../build/buildfarm/worker/persistent/BUILD | 29 +- .../persistent/ProtoCoordinatorTest.java | 25 +- .../build/buildfarm/worker/resources/BUILD | 16 +- .../java/build/buildfarm/worker/shard/BUILD | 16 +- .../worker/shard/ShardWorkerContextTest.java | 49 +- .../java/build/buildfarm/worker/util/BUILD | 57 +- third_party/grpc-java-12207.patch | 66 - third_party/grpc-java-12222.patch | 66 - third_party/remote_apis_grpc.patch | 11 - 259 files changed, 3367 insertions(+), 5463 deletions(-) delete mode 100644 .github/workflows/license.yaml delete mode 100644 ci/license-header.txt rename {examples => config}/backplane.yml (100%) create mode 100644 config/config.minimal.yml create mode 100644 config/logging.properties rename {examples => config}/queues.yml (100%) rename {examples => config}/server.yml (100%) rename {examples => config}/worker.yml (54%) delete mode 100644 kubernetes/helm-charts/buildfarm/templates/exec-worker/deployment.yaml delete mode 100644 kubernetes/helm-charts/buildfarm/templates/exec-worker/hpa.yaml delete mode 100644 kubernetes/helm-charts/buildfarm/templates/exec-worker/pdb.yaml delete mode 100644 kubernetes/helm-charts/buildfarm/templates/exec-worker/service.yaml delete mode 100644 kubernetes/helm-charts/buildfarm/templates/exec-worker/servicemonitor.yaml rename kubernetes/helm-charts/buildfarm/templates/shard-worker/{hpa.yaml => autoscaler.yaml} (100%) rename kubernetes/helm-charts/buildfarm/templates/shard-worker/{statefulset.yaml => statefulsets.yaml} (91%) delete mode 100644 licenserc.toml delete mode 100644 persistentworkers/src/main/java/persistent/bazel/client/BasicWorkerKey.java delete mode 100644 persistentworkers/src/main/java/persistent/bazel/client/WorkerIndex.java delete mode 100644 src/main/java/build/buildfarm/cas/cfc/LRUDB.java delete mode 100644 src/main/java/build/buildfarm/cas/cfc/TextLRUDB.java create mode 100644 src/main/java/build/buildfarm/common/Dispenser.java delete mode 100644 src/main/java/build/buildfarm/common/grpc/NullObserver.java delete mode 100644 src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareOutputStream.java delete mode 100644 src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareWrite.java delete mode 100644 src/main/java/build/buildfarm/server/services/WorkerProfileService.java rename src/main/java/build/buildfarm/worker/{filesystem => }/CFCExecFileSystem.java (83%) rename src/main/java/build/buildfarm/worker/{filesystem => }/CFCLinkExecFileSystem.java (90%) rename src/main/java/build/buildfarm/worker/{filesystem => }/ExecDirException.java (98%) rename src/main/java/build/buildfarm/worker/{filesystem => }/ExecDirectoryStream.java (84%) rename src/main/java/build/buildfarm/worker/{filesystem => }/ExecFileSystem.java (91%) rename src/main/java/build/buildfarm/worker/{filesystem => }/ExecTree.java (95%) rename src/main/java/build/buildfarm/worker/{filesystem => }/ExecTreeWalker.java (97%) rename src/main/java/build/buildfarm/worker/{filesystem => }/FuseCAS.java (99%) rename src/main/java/build/buildfarm/worker/{filesystem => }/OutputDirectory.java (99%) delete mode 100644 src/main/java/build/buildfarm/worker/executionwrappers/BUILD delete mode 100644 src/main/java/build/buildfarm/worker/executionwrappers/ExecutionWrapperUtils.java delete mode 100644 src/main/java/build/buildfarm/worker/filesystem/BUILD.bazel delete mode 100644 src/main/java/build/buildfarm/worker/persistent/PersistentWorkerAwareExecOwnerPool.java delete mode 100644 src/main/java/build/buildfarm/worker/resources/LocalResource.java rename src/main/java/build/buildfarm/worker/shard/{WorkerControl.java => ShutDownWorkerGracefully.java} (66%) rename src/test/java/build/buildfarm/worker/{filesystem => }/CFCExecFileSystemTest.java (97%) rename src/test/java/build/buildfarm/worker/{filesystem => }/FuseCASTest.java (99%) rename src/test/java/build/buildfarm/worker/{filesystem => }/OutputDirectoryTest.java (99%) delete mode 100644 src/test/java/build/buildfarm/worker/filesystem/BUILD.bazel delete mode 100644 third_party/grpc-java-12207.patch delete mode 100644 third_party/grpc-java-12222.patch delete mode 100644 third_party/remote_apis_grpc.patch diff --git a/.bazelci/cache_test.sh b/.bazelci/cache_test.sh index 122be99e02..53f05964cf 100755 --- a/.bazelci/cache_test.sh +++ b/.bazelci/cache_test.sh @@ -1,25 +1,11 @@ #!/bin/bash -# Copyright 2022-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # This is a CI integation test for a typical eployment of buildfarm. # It ensures that the buildfarm services can startup and function as expected given PR changes. # All of the needed buildfarm services are initialized (redis, server, worker). # We ensure that the system can build a set of bazel targets. # Run redis container -docker run --rm -d --name buildfarm-redis --network host redis:7.2.4 --bind localhost +docker run -d --name buildfarm-redis --network host redis:7.2.4 --bind localhost # Build a container for buildfarm services cp `which bazel` bazel @@ -27,7 +13,6 @@ docker build -t buildfarm . #Start the servies and do a test build docker run \ - --rm \ -v /tmp:/tmp \ --network host \ --env CACHE_TEST=$CACHE_TEST \ @@ -37,8 +22,3 @@ docker run \ --env TEST_ARG2=$TEST_ARG2 \ --env SHA1_TOOLS_REMOTE=$SHA1_TOOLS_REMOTE \ buildfarm buildfarm/.bazelci/test_buildfarm_container.sh -status=$? - -docker stop buildfarm-redis - -exit $status diff --git a/.bazelci/docker_unit_test.sh b/.bazelci/docker_unit_test.sh index 5c36ef6929..13ed343f2d 100755 --- a/.bazelci/docker_unit_test.sh +++ b/.bazelci/docker_unit_test.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # This is to run buildfarm's unit tests from within a docker container # Build a container for unit tests and run them diff --git a/.bazelci/format.sh b/.bazelci/format.sh index 159fc43682..44d866386c 100755 --- a/.bazelci/format.sh +++ b/.bazelci/format.sh @@ -1,18 +1,4 @@ #!/usr/bin/env bash -# Copyright 2020-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Run from the root of repository. # This script will format all of the java source files. # Use the flag --check if you want the script to fail when formatting is not correct. diff --git a/.bazelci/ingest_grpc_logs.py b/.bazelci/ingest_grpc_logs.py index 1ae2cd2671..8ceb28988e 100755 --- a/.bazelci/ingest_grpc_logs.py +++ b/.bazelci/ingest_grpc_logs.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/python2 import json import time import sys diff --git a/.bazelci/integration_test.sh b/.bazelci/integration_test.sh index e93175e447..84ab3f2897 100755 --- a/.bazelci/integration_test.sh +++ b/.bazelci/integration_test.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # This is a CI integation test for a typical deployment of buildfarm. # It ensures that the buildfarm services can startup and function as expected given PR changes. # All of the needed buildfarm services are initialized (redis, server, worker). diff --git a/.bazelci/package_name_check.sh b/.bazelci/package_name_check.sh index ae923ca9bf..3912284cb9 100755 --- a/.bazelci/package_name_check.sh +++ b/.bazelci/package_name_check.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Run this from the root of repository. # The script will check that each java source file has the a package name that matches its file path. # This is done for more than just consistency. Bazel's code coverage seems to exclude files when the package name does not match. diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 77dff7077f..8e67019b39 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -4,27 +4,27 @@ tasks: # Linting jobs format_check: name: Code Format - platform: ubuntu2404 + platform: ubuntu2204 shell_commands: - ./.bazelci/format.sh --check package_layout: name: Package Layout - platform: ubuntu2404 + platform: ubuntu2204 shell_commands: - ./.bazelci/package_name_check.sh java_unused_deps_check: name: Java Unused Dependencies - platform: ubuntu2404 + platform: ubuntu2204 shell_commands: - ./.bazelci/unused_deps.sh --check static_analysis_check: name: Static Analysis - platform: ubuntu2404 + platform: ubuntu2204 shell_commands: - ./.bazelci/static_analyze.sh checkstyle: name: Check Style - platform: ubuntu2404 + platform: ubuntu2204 shell_commands: - ./.bazelci/run_checkstyle.sh code_coverage: @@ -42,16 +42,6 @@ tasks: - ./generate_coverage.sh # Typical unit tests that run across a variety of operating systems. - ubuntu2404: - name: Unit Tests - build_targets: - - '...' - build_flags: - - --build_tag_filters=-container - test_flags: - - --test_tag_filters=-integration,-redis - test_targets: - - '...' ubuntu2204: name: Unit Tests build_targets: @@ -96,7 +86,7 @@ tasks: - '...' rpm_builds: name: Rpm Builds - platform: rockylinux8 + platform: centos7_java11_devtoolset10 shell_commands: - bazel build //src/main/java/build/buildfarm/rpms/server:buildfarm-server-rpm //src/main/java/build/buildfarm/rpms/worker:buildfarm-worker-rpm @@ -170,9 +160,9 @@ tasks: - export CACHE_TEST=true - export RUN_TEST=./.bazelci/run_generative_cc_many_double_test.sh - export BUILDFARM_CONFIG="/buildfarm/examples/config.minimal.yml" - - export TEST_ARG1=--remote_grpc_log='./grpc.log' - - export TEST_ARG2=--remote_grpc_log='./grpc2.log' - - export SHA1_TOOLS_REMOTE=8abbff48ada653a42dfef977e6ba91d31e900fd5 + - export TEST_ARG1=--experimental_remote_grpc_log='./grpc.log' + - export TEST_ARG2=--experimental_remote_grpc_log='./grpc2.log' + - export SHA1_TOOLS_REMOTE=eeddeff416118c86856fd76c95733307256dc855 - export EXECUTION_STAGE_WIDTH=5 - ./.bazelci/cache_test.sh diff --git a/.bazelci/redis_unit_tests.sh b/.bazelci/redis_unit_tests.sh index 49c9ae7d90..d3298c34ff 100755 --- a/.bazelci/redis_unit_tests.sh +++ b/.bazelci/redis_unit_tests.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2022-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Typically our redis implementations are mocked. # However this runs unit tests that interact directly with redis. diff --git a/.bazelci/run_abseil_test.sh b/.bazelci/run_abseil_test.sh index 40d06612ec..e9662a9c07 100755 --- a/.bazelci/run_abseil_test.sh +++ b/.bazelci/run_abseil_test.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Test bazel targets with buildfarm cd src/test/abseil; ../../../bazel test --jobs=25 $1 --noenable_bzlmod --test_output=errors --incompatible_enable_cc_toolchain_resolution --verbose_failures --verbose_explanations --test_tag_filters=-benchmark --remote_executor=grpc://localhost:8980 @com_google_absl//... -- -@com_google_absl//absl/time/... diff --git a/.bazelci/run_checkstyle.sh b/.bazelci/run_checkstyle.sh index 3d9a30094a..8cda8b8a8c 100755 --- a/.bazelci/run_checkstyle.sh +++ b/.bazelci/run_checkstyle.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2023-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Run from the root of repository. # This script will perform static analysis on all of the java source files. diff --git a/.bazelci/run_generative_cc_many_double_test.sh b/.bazelci/run_generative_cc_many_double_test.sh index 99d989fc82..6aa228b577 100755 --- a/.bazelci/run_generative_cc_many_double_test.sh +++ b/.bazelci/run_generative_cc_many_double_test.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2022-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Build bazel targets with buildfarm cd src/test/many; MANY_CC_BINARIES=50 MANY_CC_LIBRARIES=2 MANY_CC_LIBRARY_SOURCES=1 ../../../bazel build :cc --remote_executor=grpc://localhost:8980 $1 diff --git a/.bazelci/run_generative_cc_many_test.sh b/.bazelci/run_generative_cc_many_test.sh index ff997d41ab..d724200d09 100755 --- a/.bazelci/run_generative_cc_many_test.sh +++ b/.bazelci/run_generative_cc_many_test.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - set -o xtrace # Build bazel targets with buildfarm diff --git a/.bazelci/run_server_test.sh b/.bazelci/run_server_test.sh index c876f61090..ffcb082591 100755 --- a/.bazelci/run_server_test.sh +++ b/.bazelci/run_server_test.sh @@ -1,17 +1,4 @@ #!/bin/bash -# Copyright 2022-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. # Start redis container docker run -d --rm --name buildfarm-redis --network host redis:7.2.4 --bind localhost diff --git a/.bazelci/static_analyze.sh b/.bazelci/static_analyze.sh index 35e487affe..dfdbc586a8 100755 --- a/.bazelci/static_analyze.sh +++ b/.bazelci/static_analyze.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Run from the root of repository. # This script will perform static analysis on all of the java source files. diff --git a/.bazelci/test_buildfarm_container.sh b/.bazelci/test_buildfarm_container.sh index aa581ae3d5..70a51efb7e 100755 --- a/.bazelci/test_buildfarm_container.sh +++ b/.bazelci/test_buildfarm_container.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - set -e set -o pipefail diff --git a/.bazelci/unused_deps.sh b/.bazelci/unused_deps.sh index 159474fa67..59059d1a39 100755 --- a/.bazelci/unused_deps.sh +++ b/.bazelci/unused_deps.sh @@ -1,18 +1,4 @@ #!/bin/bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Run from the root of repository. # This script will remove any unused java dependencies # Use the flag --check if you want the script to fail when the dependencies not correct. diff --git a/.bazelrc b/.bazelrc index 291eaf6b24..0cf0abd075 100644 --- a/.bazelrc +++ b/.bazelrc @@ -18,6 +18,10 @@ test --nojava_header_compilation test --enable_runfiles test --test_tag_filters=-redis,-integration +# Ensure buildfarm is compatible with future versions of bazel. +# https://buildkite.com/bazel/bazelisk-plus-incompatible-flags +common --incompatible_disallow_empty_glob + # Stop recompiling protoc. # https://fzakaria.com/2024/10/23/bazel-knowledge-mind-your-path.html build --incompatible_strict_action_env @@ -33,11 +37,3 @@ common --lockfile_mode=off build:linux --sandbox_add_mount_pair=/tmp build:macos --sandbox_add_mount_pair=/var/tmp build:windows --sandbox_add_mount_pair=C:\Temp - -# Protobuf will be dropping support for MSVC + Bazel in 34.0. -# To continue using it until then, use the flag --define=protobuf_allow_msvc=true. -# For feedback or discussion, see github.com/protocolbuffers/protobuf/issues/20085. -build:windows --define=protobuf_allow_msvc=true - -# Planned to be default in Bazel 9; see https://github.com/bazelbuild/bazel/issues/22080 -common --incompatible_disable_native_repo_rules diff --git a/.bazelversion b/.bazelversion index 6da4de57dc..2b0aa21219 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -8.4.1 +8.2.1 diff --git a/.github/workflows/buildfarm-helm-chart-lint.yml b/.github/workflows/buildfarm-helm-chart-lint.yml index e52d41f43b..6636d34039 100644 --- a/.github/workflows/buildfarm-helm-chart-lint.yml +++ b/.github/workflows/buildfarm-helm-chart-lint.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - id: helm-lint name: Lint Helm Chart run: |- diff --git a/.github/workflows/buildfarm-helm-chart-publish.yml b/.github/workflows/buildfarm-helm-chart-publish.yml index 3400548363..9f8c2d39c8 100644 --- a/.github/workflows/buildfarm-helm-chart-publish.yml +++ b/.github/workflows/buildfarm-helm-chart-publish.yml @@ -21,7 +21,7 @@ jobs: packages: write steps: - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - id: get-chart-ver name: Extracting Helm Chart Version from Tag run: | diff --git a/.github/workflows/buildfarm-images-build-and-deploy.yml b/.github/workflows/buildfarm-images-build-and-deploy.yml index 90a470231d..daf8ef020c 100644 --- a/.github/workflows/buildfarm-images-build-and-deploy.yml +++ b/.github/workflows/buildfarm-images-build-and-deploy.yml @@ -14,16 +14,13 @@ jobs: name: Build Buildfarm Images runs-on: ubuntu-latest steps: - - uses: bazel-contrib/setup-bazel@4fd964a13a440a8aeb0be47350db2fc640f19ca8 # 0.15.0 - with: - # Avoid downloading Bazel every time. - bazelisk-cache: true + - uses: bazelbuild/setup-bazelisk@b39c379c82683a5f25d34f0d062761f62693e0b2 # v3 - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Login to Bazelbuild Docker Hub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3 with: username: ${{ secrets.BAZELBUILD_DOCKERHUB_USERNAME }} password: ${{ secrets.BAZELBUILD_DOCKERHUB_TOKEN }} diff --git a/.github/workflows/buildfarm-release-build-and-deploy.yml b/.github/workflows/buildfarm-release-build-and-deploy.yml index 5703193103..715337ccce 100644 --- a/.github/workflows/buildfarm-release-build-and-deploy.yml +++ b/.github/workflows/buildfarm-release-build-and-deploy.yml @@ -13,16 +13,13 @@ jobs: name: Build Buildfarm Images runs-on: ubuntu-latest steps: - - uses: bazel-contrib/setup-bazel@4fd964a13a440a8aeb0be47350db2fc640f19ca8 # 0.15.0 - with: - # Avoid downloading Bazel every time. - bazelisk-cache: true + - uses: bazelbuild/setup-bazelisk@b39c379c82683a5f25d34f0d062761f62693e0b2 # v3 - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Login to Bazelbuild Docker Hub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3 with: username: ${{ secrets.BAZELBUILD_DOCKERHUB_USERNAME }} password: ${{ secrets.BAZELBUILD_DOCKERHUB_TOKEN }} diff --git a/.github/workflows/buildfarm-worker-base-build-and-deploy.yml b/.github/workflows/buildfarm-worker-base-build-and-deploy.yml index 082eb085f5..3d685d69ec 100644 --- a/.github/workflows/buildfarm-worker-base-build-and-deploy.yml +++ b/.github/workflows/buildfarm-worker-base-build-and-deploy.yml @@ -18,18 +18,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Set up QEMU - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3 # QEMU needed for the ARM variant. - name: Set up Docker Buildx - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3 # Docker Buildx needed for the ARM variant. - name: Login to Bazelbuild Docker Hub - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3 with: username: ${{ secrets.BAZELBUILD_DOCKERHUB_USERNAME }} password: ${{ secrets.BAZELBUILD_DOCKERHUB_TOKEN }} @@ -37,7 +37,7 @@ jobs: - name: Build Jammy Docker image # AKA Ubuntu 22 - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@84ad562665bb303b549fec655d1b64f9945f3f91 with: context: . file: ./ci/base-worker-image/jammy/Dockerfile @@ -46,7 +46,7 @@ jobs: - name: Build Lunar Docker image # AKA Ubuntu 23 - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@84ad562665bb303b549fec655d1b64f9945f3f91 with: context: . file: ./ci/base-worker-image/lunar/Dockerfile @@ -55,7 +55,7 @@ jobs: - name: Build Multi-arch Noble Docker image # AKA Ubuntu 24 - uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + uses: docker/build-push-action@84ad562665bb303b549fec655d1b64f9945f3f91 with: context: . platforms: linux/amd64,linux/arm64/v8 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ea62f5e043..925a0d206c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -23,16 +23,16 @@ jobs: language: [java-kotlin] steps: - - uses: bazel-contrib/setup-bazel@4fd964a13a440a8aeb0be47350db2fc640f19ca8 # 0.15.0 + - uses: bazel-contrib/setup-bazel@e8776f58fb6a6e9055cbaf1b38c52ccc5247e9c4 # 0.14.0 with: # Avoid downloading Bazel every time. bazelisk-cache: true - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Initialize CodeQL - uses: github/codeql-action/init@192325c86100d080feab897ff886c34abd4c83a3 # v3.30.3 + uses: github/codeql-action/init@1b549b9259bda1cb5ddde3b41741a82a2d15a841 # v3 with: languages: ${{ matrix.language }} build-mode: manual @@ -53,6 +53,6 @@ jobs: //src/main/java/build/buildfarm:buildfarm-shard-worker bazel shutdown - - uses: github/codeql-action/analyze@192325c86100d080feab897ff886c34abd4c83a3 # v3.30.3 + - uses: github/codeql-action/analyze@1b549b9259bda1cb5ddde3b41741a82a2d15a841 # v3 with: category: /language:${{ matrix.language }} diff --git a/.github/workflows/license.yaml b/.github/workflows/license.yaml deleted file mode 100644 index 50d889f12c..0000000000 --- a/.github/workflows/license.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: PR Tests - -on: - pull_request: - branches: [main] - types: [opened, synchronize, reopened] - -jobs: - license: - name: Check license headers - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - fetch-depth: 0 - - - name: Check License Header - run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye:v6.1.1 check diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4b0e7185b2..dc402e9bf1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,10 +14,10 @@ jobs: permissions: pages: write steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 # Use GitHub Actions' cache to cache dependencies on servers - - uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 with: path: vendor/bundle key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile.lock') }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 7e896254e9..3e33e42c6b 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -28,12 +28,12 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 + uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 with: results_file: results.sarif results_format: sarif @@ -63,6 +63,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@192325c86100d080feab897ff886c34abd4c83a3 # v3.30.3 + uses: github/codeql-action/upload-sarif@1b549b9259bda1cb5ddde3b41741a82a2d15a841 # v3.28.13 with: sarif_file: results.sarif diff --git a/CODEOWNERS b/CODEOWNERS index 26b1c716ee..12eb32b3af 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,6 +1,6 @@ # Helm chart -/kubernetes/helm-charts/buildfarm/ @jasonschroeder-sfdc +kubernetes/helm-charts/buildfarm @jasonschroeder-sfdc # Default owner * @werkt diff --git a/Dockerfile b/Dockerfile index ee68152dfa..039138d40c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,14 @@ # A minimal container for building and running buildfarm services. # We copy in the current state of the reposiory to test against PR changes. -#FROM ubuntu:18.04@sha256:152dc042452c496007f07ca9127571cb9c29697f42acbfad72324b2bb2e43c98 -FROM ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233 +FROM ubuntu:18.04@sha256:152dc042452c496007f07ca9127571cb9c29697f42acbfad72324b2bb2e43c98 # For network stability we allow apt-get to retry. # The "80" is required for config priority but its not specifically important. RUN echo 'APT::Acquire::Retries "5";' > /etc/apt/apt.conf.d/80retries -RUN apt update -RUN apt -y install wget git zip python3-pip python3-dateutil gcc default-jdk-headless g++ redis redis-server +RUN apt-get update +RUN apt-get -y install wget git zip python gcc openjdk-8-jdk g++ redis redis-server +RUN wget --tries=10 -O get-pip.py https://bootstrap.pypa.io/pip/2.7/get-pip.py +RUN python2 get-pip.py +RUN pip install python-dateutil==2.8.2 COPY . buildfarm diff --git a/MODULE.bazel b/MODULE.bazel index bda3cf29e9..16888de897 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -5,22 +5,21 @@ module( repo_name = "build_buildfarm", ) -bazel_dep(name = "grpc", version = "1.74.1", repo_name = "com_github_grpc_grpc") -bazel_dep(name = "rules_foreign_cc", version = "0.15.0") -bazel_dep(name = "aspect_bazel_lib", version = "2.21.1") -bazel_dep(name = "bazel_skylib", version = "1.8.1") -bazel_dep(name = "blake3", version = "1.8.2") -bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2") -bazel_dep(name = "gazelle", version = "0.45.0") -bazel_dep(name = "grpc-java", version = "1.72.0") +bazel_dep(name = "aspect_bazel_lib", version = "2.15.3") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "blake3", version = "1.5.4") +bazel_dep(name = "buildifier_prebuilt", version = "8.0.3") +bazel_dep(name = "gazelle", version = "0.43.0") +bazel_dep(name = "grpc-java", version = "1.69.0") bazel_dep(name = "googleapis", version = "0.0.0-20240819-fe8ba054a") bazel_dep(name = "grpc-proto", version = "0.0.0-20240627-ec30f58", repo_name = "io_grpc_grpc_proto") bazel_dep(name = "hermetic_cc_toolchain", version = "3.2.0") -bazel_dep(name = "platforms", version = "1.0.0") -bazel_dep(name = "protobuf", version = "29.5", repo_name = "com_google_protobuf") -bazel_dep(name = "rules_cc", version = "0.2.8") -bazel_dep(name = "rules_java", version = "8.15.2") -bazel_dep(name = "rules_jvm_external", version = "6.8") +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "protobuf", version = "29.0-rc3") +bazel_dep(name = "rules_cc", version = "0.1.1") +bazel_dep(name = "rules_go", version = "0.54.0") +bazel_dep(name = "rules_java", version = "8.11.0") +bazel_dep(name = "rules_jvm_external", version = "6.6") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.6") bazel_dep(name = "rules_pkg", version = "1.1.0") @@ -32,28 +31,14 @@ bazel_dep( dev_dependency = True, ) -bazel_dep(name = "bazel_remote_apis", version = "e94a7ece2a1e8da1dcf278a0baf2edfe7baafb94", repo_name = "remoteapis") +# TODO: remove this after https://github.com/bazelbuild/remote-apis/pull/293 is merged +bazel_dep(name = "bazel_remote_apis", version = "6777112ef7defa6705b1ebd2831d6c7efeb12ba2", repo_name = "remoteapis") archive_override( module_name = "bazel_remote_apis", - integrity = "sha256-ZDtGnl29H5spbghbCEbMF/wOKzq8td40CVnXuD2nvZo=", - patches = [ - "//third_party:remote_apis_grpc.patch", - ], - strip_prefix = "remote-apis-e94a7ece2a1e8da1dcf278a0baf2edfe7baafb94", + integrity = "sha256-PTtx9Vz9LjLvvUhv/n4QnksoD19BfqoFssrb3QvxcSE=", + strip_prefix = "remote-apis-6777112ef7defa6705b1ebd2831d6c7efeb12ba2", urls = [ - "https://github.com/bazelbuild/remote-apis/archive/e94a7ece2a1e8da1dcf278a0baf2edfe7baafb94.zip", - ], -) - -# Remove once the following PRs are available in a grpc-java release. -# https://github.com/grpc/grpc-java/pull/12207 -# https://github.com/grpc/grpc-java/pull/12222 -single_version_override( - module_name = "grpc-java", - patch_strip = 1, - patches = [ - "//third_party:grpc-java-12207.patch", - "//third_party:grpc-java-12222.patch", + "https://github.com/bazelbuild/remote-apis/archive/6777112ef7defa6705b1ebd2831d6c7efeb12ba2.zip", ], ) @@ -67,14 +52,13 @@ register_toolchains( maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") maven.install( - name = "buildfarm_maven", artifacts = [ # keep sorted "com.amazonaws:aws-java-sdk-core", # Version is specified via the BOM below "com.amazonaws:aws-java-sdk-s3", "com.amazonaws:aws-java-sdk-secretsmanager", - "com.fasterxml.jackson.core:jackson-databind:2.20.0", - "com.github.ben-manes.caffeine:caffeine:3.2.2", + "com.fasterxml.jackson.core:jackson-databind:2.18.3", + "com.github.ben-manes.caffeine:caffeine:3.2.0", "com.github.docker-java:docker-java", "com.github.docker-java:docker-java-api", "com.github.docker-java:docker-java-core", @@ -83,24 +67,24 @@ maven.install( "com.github.jnr:jnr-constants:0.10.4", "com.github.jnr:jnr-ffi:2.2.17", "com.github.jnr:jnr-posix:3.1.20", - "com.github.luben:zstd-jni:1.5.7-4", - "com.github.oshi:oshi-core:6.8.3", + "com.github.luben:zstd-jni:1.5.7-2", + "com.github.oshi:oshi-core:6.8.0", "com.github.pcj:google-options:1.0.0", "com.github.serceman:jnr-fuse:0.5.8", - "com.google.api.grpc:proto-google-common-protos:2.61.1", + "com.google.api.grpc:proto-google-common-protos:2.29.0", "com.google.auth:google-auth-library-credentials", # Version is specified via the BOM below "com.google.auth:google-auth-library-oauth2-http", # Version is specified via the BOM below "com.google.code.findbugs:jsr305:3.0.2", - "com.google.code.gson:gson:2.13.2", - "com.google.errorprone:error_prone_annotations:2.41.0", - "com.google.errorprone:error_prone_core:2.41.0", + "com.google.code.gson:gson:2.12.1", + "com.google.errorprone:error_prone_annotations:2.37.0", + "com.google.errorprone:error_prone_core:2.37.0", "com.google.guava:failureaccess:1.0.3", - "com.google.guava:guava:33.4.8-jre", - "com.google.j2objc:j2objc-annotations:3.1", - "com.google.protobuf:protobuf-java:4.32.1", - "com.google.protobuf:protobuf-java-util:4.32.1", + "com.google.guava:guava:33.4.6-jre", + "com.google.j2objc:j2objc-annotations:3.0.0", + "com.google.protobuf:protobuf-java:3.19.1", + "com.google.protobuf:protobuf-java-util:3.19.1", "com.owteam.engUtils:netrc:2.0.1", - "commons-io:commons-io:2.20.0", + "commons-io:commons-io:2.18.0", "io.grpc:grpc-api", # Version is specified via the BOM below "io.grpc:grpc-auth", "io.grpc:grpc-context", @@ -110,7 +94,7 @@ maven.install( "io.grpc:grpc-protobuf", "io.grpc:grpc-services", "io.grpc:grpc-stub", - "io.grpc:grpc-util", + "io.grpc:grpc-testing", "io.netty:netty-buffer", # Version is specified via the BOM below "io.netty:netty-codec", "io.netty:netty-codec-http", @@ -133,23 +117,23 @@ maven.install( "me.dinowernli:java-grpc-prometheus:0.6.0", "net.javacrumbs.future-converter:future-converter-java8-guava:1.2.0", "net.jcip:jcip-annotations:1.0", - "org.apache.commons:commons-compress:1.28.0", - "org.apache.commons:commons-lang3:3.18.0", + "org.apache.commons:commons-compress:1.27.1", + "org.apache.commons:commons-lang3:3.17.0", "org.apache.commons:commons-pool2:2.12.1", "org.apache.httpcomponents:httpclient:4.5.14", "org.apache.tomcat:annotations-api:6.0.53", "org.bouncycastle:bcprov-jdk15on:1.70", - "org.checkerframework:checker-qual:3.51.0", - "org.jetbrains:annotations:26.0.2-1", - "org.mockito:mockito-core:5.19.0", + "org.checkerframework:checker-qual:3.49.1", + "org.jetbrains:annotations:26.0.2", + "org.mockito:mockito-core:5.16.1", "org.openjdk.jmh:jmh-core:1.37", "org.openjdk.jmh:jmh-generator-annprocess:1.37", - "org.projectlombok:lombok:1.18.40", - "org.redisson:redisson:3.51.0", + "org.projectlombok:lombok:1.18.36", + "org.redisson:redisson:3.45.1", "org.slf4j:slf4j-simple:2.0.17", - "org.threeten:threetenbp:1.7.2", - "org.xerial:sqlite-jdbc:3.50.3.0", - "org.yaml:snakeyaml:2.5", + "org.threeten:threetenbp:1.7.0", + "org.xerial:sqlite-jdbc:3.49.1.0", + "org.yaml:snakeyaml:2.4", "redis.clients:jedis:5.2.0", ], boms = [ @@ -157,10 +141,10 @@ maven.install( "com.amazonaws:aws-java-sdk-bom:1.12.544", "com.github.docker-java:docker-java-bom:3.3.3", "com.google.auth:google-auth-library-bom:1.23.0", - "io.grpc:grpc-bom:1.75.0", + "io.grpc:grpc-bom:1.62.2", "io.netty:netty-bom:4.1.108.Final", ], - fail_if_repin_required = True, # TO RE-PIN: REPIN=1 bazel run @buildfarm_maven//:pin + fail_if_repin_required = True, # TO RE-PIN: REPIN=1 bazel run @unpinned_maven//:pin generate_compat_repositories = True, lock_file = "//:maven_install.json", repositories = [ @@ -173,7 +157,6 @@ maven.install( # Test only maven dependencies [ maven.artifact( - name = "buildfarm_maven", testonly = True, artifact = artifact, group = group, @@ -183,16 +166,14 @@ maven.install( "com.github.fppt:jedis-mock:1.0.13", "com.google.jimfs:jimfs:1.3.0", "com.google.truth:truth:1.4.2", - "io.grpc:grpc-inprocess:1.75.0", - "io.grpc:grpc-testing:1.75.0", "junit:junit:4.13.2", ]] ] use_repo( maven, - "buildfarm_maven", - "unpinned_buildfarm_maven", + "maven", + "unpinned_maven", ) oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") @@ -200,7 +181,7 @@ oci = use_extension("@rules_oci//oci:extensions.bzl", "oci") # Server base image oci.pull( name = "amazon_corretto_java_image_base", - digest = "sha256:53463affec61309c62bf118c2d8cbc5d1b4b11e6f456e9f2a71e4f0a4b27f037", + digest = "sha256:efbf26f3d4105684429a74842d742683a99215c6ec304c6ff0faa9e493bb4329", image = "public.ecr.aws/amazoncorretto/amazoncorretto", platforms = [ "linux/amd64", @@ -212,7 +193,7 @@ oci.pull( # Worker base image oci.pull( name = "ubuntu_noble", # aka 24 - digest = "sha256:a1da084ca1429dd84a71e364e88985b4b85934f3d8717bfdc00f2fd06170158a", + digest = "sha256:f18e19d1329c0df38dd5f6bfd931c7764fa1e3acf6b3280da62015a338805ce8", image = "index.docker.io/bazelbuild/buildfarm-worker-base", platforms = [ "linux/amd64", @@ -232,7 +213,7 @@ use_repo( # https://github.com/bazelbuild/rules_python/pull/713#issuecomment-1885628496 # Satisfy running tests in Docker as root. -bazel_dep(name = "rules_python", version = "1.6.1") +bazel_dep(name = "rules_python", version = "1.3.0") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( diff --git a/README.md b/README.md index 2b9f171b73..087cd3f32b 100644 --- a/README.md +++ b/README.md @@ -166,5 +166,5 @@ helm install \ --create-namespace \ bazel-buildfarm \ oci://ghcr.io/buildfarm/buildfarm \ - --version "0.4.1" + --version "0.2.9" ``` diff --git a/_site/docs/configuration/configuration.md b/_site/docs/configuration/configuration.md index 2d1204f19b..3f67519043 100644 --- a/_site/docs/configuration/configuration.md +++ b/_site/docs/configuration/configuration.md @@ -222,7 +222,7 @@ server: | workersHashName | String, _Workers_ | | | Redis key used to store a hash of registered workers | | workerChannel | String, _WorkerChannel_ | | | Redis pubsub channel key where changes of the cluster membership are announced | | actionCachePrefix | String, _ActionCache_ | | | Redis key prefix for all ActionCache entries | -| actionCacheExpire | Integer, _2419200_ | | | The TTL maintained for ActionCache entries, refreshed on getActionResult hit | +| actionCacheExpire | Integer, _2419200_ | | | The TTL maintained for ActionCache entries, not refreshed on getActionResult hit | | actionBlacklistPrefix | String, _ActionBlacklist_ | | | Redis key prefix for all blacklisted actions, which are rejected | | actionBlacklistExpire | Integer, _3600_ | | | The TTL maintained for action blacklist entries | | invocationBlacklistPrefix | String, _InvocationBlacklist_ | | | Redis key prefix for blacklisted invocations, suffixed with a a tool invocation ID | @@ -239,7 +239,7 @@ server: | dispatchedOperationsHashName | String, _DispatchedOperations_ | | | Redis key of a hash of operation names to the worker lease for its execution, which are monitored by the dispatched monitor | | operationChannelPrefix | String, _OperationChannel_ | | | Redis pubsub channel prefix suffixed by an operation name | | casPrefix | String, _ContentAddressableStorage_ | | | Redis key prefix suffixed with a blob digest that maps to a set of workers with that blob's availability | -| casExpire | Integer, _604800_ | | | The TTL maintained for CAS entries, which is refreshed on any read access of the blob | +| casExpire | Integer, _604800_ | | | The TTL maintained for CAS entries, which is not refreshed on any read access of the blob | | subscribeToBackplane | boolean, _true_ | | | Enable an agent of the backplane client which subscribes to worker channel and operation channel events. If disabled, responsiveness of watchers and CAS are reduced | | runFailsafeOperation | boolean, _true_ | | | Enable an agent in the backplane client which monitors watched operations and ensures they are in a known maintained, or expirable state | | maxQueueDepth | Integer, _100000_ | | | Maximum length that the ready to run queue is allowed to reach to control an arrival flow for execution | @@ -248,7 +248,6 @@ server: | timeout | Integer, _10000_ | | | Default timeout | | maxInvocationIdTimeout | Integer, _604800_ | | | Maximum TTL (Time-to-Live in second) of invocationId keys in RedisBackplane | | maxAttempts | Integer, _20_ | | | Maximum number of execution attempts | -| connectionValidatedOnBorrow | boolean, _false_ | | | Whether to validate Redis connections when borrowing from the pool | Example: @@ -309,7 +308,7 @@ backplane: | allowBringYourOwnContainer | boolean, _false_ | | Enable execution in a custom Docker container | | errorOperationRemainingResources | boolean, _false_ | | | | errorOperationOutputSizeExceeded | boolean, _false_ | | Operations which produce single output files which exceed maxEntrySizeBytes will fail with a violation type which implies a user error. When disabled, the violation will indicate a transient error, with the action blacklisted. | -| linkedInputDirectories | List of Strings, _^(?!external$).*$_ | | A list of regular expressions matching input directories which will be subject to the effects of linkInputDirectories setting | +| realInputDirectories | List of Strings, _external_ | | A list of paths that will not be subject to the effects of linkInputDirectories setting, may also be used to provide writable directories as input roots for actions which expect to be able to write to an input location and will fail if they cannot | | gracefulShutdownSeconds | Integer, 0 | | Time in seconds to allow for operations in flight to finish when shutdown signal is received | | createSymlinkOutputs | boolean, _false_ | | Creates SymlinkNodes for symbolic links discovered in output paths for actions. No verification of the symlink target path occurs. Buildstream, for example, requires this. | | zstdBufferPoolSize | Integer, _2048_ | | Specifies the maximum number of zstd data buffers that may be in use concurrently by the filesystem CAS. Increase to improve compressed blob throughput, decrease to reduce memory usage. | @@ -319,8 +318,8 @@ backplane: worker: port: 8981 publicName: "localhost:8981" - linkedInputDirectories: - - "^path/to/common/directory" + realInputDirectories: + - "external" ``` ### Capabilities @@ -460,7 +459,6 @@ worker: | Configuration | Accepted and _Default_ Values | Description | |------------------|------------------------------------------------------------|---------------------------------------------------------------------| | name | String | Execution policy name | -| prioritized | Boolean, _false_ | If true, policy will run before built-in policies | | executionWrapper | Execution wrapper, containing a path and list of arguments | Execution wrapper, its path and a list of arguments for the wrapper | Example: @@ -468,40 +466,15 @@ Example: ```yaml worker: executionPolicies: - - name: as-nobody - prioritized: true - executionWrapper: - path: /app/build_buildfarm/as-nobody - arguments: - - "-u" - - "" - - name: unshare - executionWrapper: - path: /usr/bin/unshare - arguments: - - "-n" - - "-r" - - name: linux-sandbox - executionWrapper: - path: /app/build_buildfarm/linux-sandbox - arguments: - # use "--" to signal the end of linux-sandbox args. "--" should always be last! - - "--" - name: test executionWrapper: - path: /YOUR/WRAPPER + path: / arguments: - arg1 - arg2 - "" ``` -`arg1` and `arg2` are interpreted literally. `` will be substituted with the value of a property named `"platform-property-name"` from a Command's Platform _or_ the requested pool resources for the execution. If a matching property or pool resource is not found for a specified name, the entire wrapper will be discarded and have no effect on the execution. +_arg1_ and _arg2_ are interpreted literally. __ will be substituted with the value of a property named `"platform-property-name"` from a Command's Platform _or_ the requested pool resources for the execution. If a matching property or pool resource is not found for a specified name, the entire wrapper will be discarded and have no effect on the execution. -`` is an automatically provided pool resource when `execOwner` or `execOwners` is specified, and will contain the value of the execution's owner selected for exec tree creation. - -An execution with `as-nobody`, `unshare`, and `linux-sandbox` execution policies enabled would produce a command line like: -```sh -/app/build_buildfarm/as-nobody -u /usr/bin/unshare -n -r /app/build_buildfarm/linux-sandbox -- /YOUR/WRAPPER arg1 arg2 ACTION -``` -where ACTION is the Command from remote execution action. +__ is an automatically provided pool resource when `execOwner` or `execOwners` is specified, and will contain the value of the execution's owner selected for exec tree creation. diff --git a/_site/docs/execution/execution_properties.md b/_site/docs/execution/execution_properties.md index 2996ce1a95..c799579065 100644 --- a/_site/docs/execution/execution_properties.md +++ b/_site/docs/execution/execution_properties.md @@ -132,22 +132,6 @@ PASS ### `debug-after-execution` **description:** Runs the execution, but fails it afterward with important debug information on how the execution was performed. -## Execution Flow: - -This is a special set of action mnemonics (*not* Platform properties) that can be used to test the flow of actions through the execution segments (prequeue/queue). -In all of these cases, the buildfarm agent which observes the operation at the halt state will deliver a `done` operation with no further processing. - -### `buildfarm:halt-on-execute` - -Server halts execution after receiving the execute request. This is the earliest possible halt after creating the operation. - -### `buildfarm:halt-on-deprequeue` - -Server which removes the prequeue entry halts execution. This guarantees flow through the prequeue infrastructure. - -### `buildfarm:halt-on-dequeue` - -Worker which removes the queue entry halts execution. This guarantees flow through the queue and match. ## Additional Information Custom properties can also be added to buildfarm's configuration in order to facilitate queue matching (see [Queues](https://buildfarm.github.io/buildfarm/docs/architecture/queues/)). diff --git a/_site/docs/tools/bf-cat.md b/_site/docs/tools/bf-cat.md index 5d5b9ad194..b643a5dd3f 100644 --- a/_site/docs/tools/bf-cat.md +++ b/_site/docs/tools/bf-cat.md @@ -25,10 +25,6 @@ Its basic usage is: * **`File `**: Downloads a Blob from the CAS and prints it to stdout. This can be safely redirected to a file, with no additional output interceding * **`Missing ...`**: Make a findMissingBlobs request, outputting only the digests in the parameter list that are missing from the CAS * **`Operation ...`**: Retrieves current operation statuses and renders them with field identifiers as able. This uses the Operations API and will include rich information about operations in flight, compared to the 'execute' function -* **`Operations [name]`**: Retrieves a list of operations pertaining to a filter and a name scope: - * `toolInvocationId=`: list executions in a client invocation group - * `correlatedInvocationsId= toolInvocations`: list the invocations in a client correlated list - * `status=dispatched`: list the currently dispatched executions * **`BackplaneStatus`**: Retrieve the status of a shard cluster's operation queues, with discrete information about each provisioned layer of the ready-to-run queue. * **`TreeLayout `**: Retrieves Trees of inputs from a root node. A Tree is printed with indent-levels according to depth in the directory hierarchy with FileNode and DirectoryNode fields with digests for each entry, as well as a weight by byte and % of the sizes of each directory subtree. * **`WorkerProfile`**: Retrieve profile information about a worker's operation, including the size of the CAS and the relative performance of the execution pipeline diff --git a/as-nobody-windows.c b/as-nobody-windows.c index 8c6d922e15..a46e701553 100644 --- a/as-nobody-windows.c +++ b/as-nobody-windows.c @@ -1,18 +1,3 @@ -/* - * Copyright 2021-2025 The Buildfarm Authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ // not implemented for Windows int diff --git a/as-nobody.c b/as-nobody.c index c4c65860b9..467f6b8e3c 100644 --- a/as-nobody.c +++ b/as-nobody.c @@ -1,19 +1,3 @@ -/* - * Copyright 2021-2025 The Buildfarm Authors. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - #include #include #include diff --git a/ci/base-worker-image/jammy/Dockerfile b/ci/base-worker-image/jammy/Dockerfile index bb2087d7ea..372cb92dac 100644 --- a/ci/base-worker-image/jammy/Dockerfile +++ b/ci/base-worker-image/jammy/Dockerfile @@ -1,6 +1,6 @@ # A minimal container for building a base worker image. # Buildfarm public releases are build using this image as a starting point. -FROM ubuntu:22.04@sha256:1aa979d85661c488ce030ac292876cf6ed04535d3a237e49f61542d8e5de5ae0 +FROM ubuntu:22.04@sha256:0e5e4a57c2499249aafc3b40fcd541e9a456aab7296681a3994d631587203f97 RUN apt-get update RUN apt-get -y install default-jre default-jdk build-essential libfuse2 cgroup-tools diff --git a/ci/license-header.txt b/ci/license-header.txt deleted file mode 100644 index cdf9c55764..0000000000 --- a/ci/license-header.txt +++ /dev/null @@ -1,13 +0,0 @@ -Copyright{{ " " }}{%- set created = attrs.git_file_created_year or attrs.disk_file_created_year -%}{%- set modified = attrs.git_file_modified_year or created -%}{%- if created != modified -%} {{created}}-{{modified}}{%- else -%}{{created}}{%- endif -%}{{ " " }}{{ props["copyrightOwner"] }}. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/config/BUILD b/config/BUILD index b9dc776333..bb27399a04 100644 --- a/config/BUILD +++ b/config/BUILD @@ -8,3 +8,15 @@ config_setting( name = "open_telemetry", values = {"define": "open_telemetry=true"}, ) + +filegroup( + name = "base_properties", + srcs = glob(["*.properties"]), + visibility = ["//visibility:public"], +) + +filegroup( + name = "base_configs", + srcs = glob(["*.yml"]), + visibility = ["//visibility:public"], +) diff --git a/examples/backplane.yml b/config/backplane.yml similarity index 100% rename from examples/backplane.yml rename to config/backplane.yml diff --git a/config/config.minimal.yml b/config/config.minimal.yml new file mode 100644 index 0000000000..91f64e4634 --- /dev/null +++ b/config/config.minimal.yml @@ -0,0 +1,3 @@ +backplane: !include "backplane.yml" +server: !include "server.yml" +worker: !include "worker.yml" diff --git a/config/logging.properties b/config/logging.properties new file mode 100644 index 0000000000..974a9cb86b --- /dev/null +++ b/config/logging.properties @@ -0,0 +1,4 @@ +handlers=java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%4$-7s] %2$s - %5$s %6$s %n \ No newline at end of file diff --git a/examples/queues.yml b/config/queues.yml similarity index 100% rename from examples/queues.yml rename to config/queues.yml diff --git a/examples/server.yml b/config/server.yml similarity index 100% rename from examples/server.yml rename to config/server.yml diff --git a/examples/worker.yml b/config/worker.yml similarity index 54% rename from examples/worker.yml rename to config/worker.yml index 0bc2e3221e..b4918c4fb7 100644 --- a/examples/worker.yml +++ b/config/worker.yml @@ -1,4 +1,8 @@ publicName: localhost:8981 +storages: +- type: FILESYSTEM + path: cache + maxSizeBytes: 2147483648 # 2 * 1024 * 1024 * 1024 dequeueMatchSettings: allowUnmatched: true linkedInputDirectories: diff --git a/container/BUILD b/container/BUILD index 25a1d869d3..dde5f6d75f 100644 --- a/container/BUILD +++ b/container/BUILD @@ -164,7 +164,7 @@ pkg_tar( pkg_tar( name = "layer_logging_config", - srcs = ["@build_buildfarm//examples:example_properties"], + srcs = ["@build_buildfarm//config:base_properties"], package_dir = DEFAULT_PACKAGE_DIR + "/src/main/java/build/buildfarm", tags = ["container"], ) diff --git a/delay.sh b/delay.sh index 97bf79274f..1ea1b42639 100755 --- a/delay.sh +++ b/delay.sh @@ -1,18 +1,4 @@ #!/usr/bin/env bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Delay running a program by sleeping. # If you are also skipping sleep syscalls, this will result in timeshifting you into the future. # This can help catch problems by tricking the program into thinking its future time. diff --git a/examples/BUILD b/examples/BUILD index 57fc7465e4..012928e1a6 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -4,9 +4,3 @@ filegroup( name = "example_configs", srcs = glob(["*.yml"]), ) - -filegroup( - name = "example_properties", - srcs = glob(["*.properties"]), - visibility = ["//visibility:public"], -) diff --git a/examples/config.minimal.yml b/examples/config.minimal.yml index 91f64e4634..4e8adda170 100644 --- a/examples/config.minimal.yml +++ b/examples/config.minimal.yml @@ -1,3 +1,11 @@ -backplane: !include "backplane.yml" -server: !include "server.yml" -worker: !include "worker.yml" +backplane: + redisUri: redis://localhost:6379 + queues: + - name: cpu + properties: + - name: '*' + value: '*' +worker: + publicName: localhost:8981 + dequeueMatchSettings: + allowUnmatched: true diff --git a/examples/config.yml b/examples/config.yml index cfb921bafd..ec4fa8e9e9 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -63,7 +63,6 @@ backplane: jedisPoolMaxIdle: 8 jedisPoolMinIdle: 0 jedisTimeBetweenEvictionRunsMillis: 30000 - connectionValidatedOnBorrow: false workersHashName: Workers workerChannel: WorkerChannel actionCachePrefix: ActionCache @@ -135,7 +134,7 @@ worker: linkExecFileSystem: true linkInputDirectories: true linkedInputDirectories: - - ^(?!external$).*$ + - (?!external/)[^/]+ execOwner: defaultMaxCores: 0 limitGlobalExecution: false @@ -157,7 +156,6 @@ worker: zstdBufferPoolSize: 2048 executionPolicies: - name: test - prioritized: false executionWrapper: path: / arguments: diff --git a/examples/development-redis-cluster.sh b/examples/development-redis-cluster.sh index b1a178360c..b239e8d5ae 100755 --- a/examples/development-redis-cluster.sh +++ b/examples/development-redis-cluster.sh @@ -1,18 +1,4 @@ #!/usr/bin/env bash -# Copyright 2021-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # First, start six Redis instances using this script: # ./development-redis-cluster.sh 0 diff --git a/examples/logging.properties b/examples/logging.properties index 974a9cb86b..7d079900bb 100644 --- a/examples/logging.properties +++ b/examples/logging.properties @@ -1,4 +1,6 @@ +build.buildfarm.level=FINE + handlers=java.util.logging.ConsoleHandler -java.util.logging.ConsoleHandler.level=INFO +java.util.logging.ConsoleHandler.level=FINE java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter java.util.logging.SimpleFormatter.format=[%4$-7s] %2$s - %5$s %6$s %n \ No newline at end of file diff --git a/extensions.bzl b/extensions.bzl index 426e0e6af8..d23fde6400 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -9,9 +9,9 @@ def archive_dependencies(third_party): # Bazel is referenced as a dependency so that buildfarm can access the linux-sandbox as a potential execution wrapper. { "name": "bazel", - "sha256": "03c2a5cfaeb7af45666f77bd7b56d768cb684551925ace1edd96d419f3d53260", - "strip_prefix": "bazel-b4216efd8c13c564e92115dae25dd6620423bac1", - "urls": ["https://github.com/bazelbuild/bazel/archive/b4216efd8c13c564e92115dae25dd6620423bac1.tar.gz"], + "sha256": "de54ed570e59445246bed5f44c16863ca83646c9cbd0ceaf4ca1e4ce9581f805", + "strip_prefix": "bazel-7.3.2", + "urls": ["https://github.com/bazelbuild/bazel/archive/refs/tags/7.3.2.tar.gz"], "patch_args": ["-p1"], "patches": ["%s/bazel:bazel_visibility.patch" % third_party], }, diff --git a/generate_coverage.sh b/generate_coverage.sh index f49045c767..11715f8429 100755 --- a/generate_coverage.sh +++ b/generate_coverage.sh @@ -1,18 +1,4 @@ #!/usr/bin/env bash -# Copyright 2020-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # Run on specifc test targets via: ./generate_coverage.sh # Script can be run without to get coverage on all tests. diff --git a/kubernetes/helm-charts/buildfarm/Chart.yaml b/kubernetes/helm-charts/buildfarm/Chart.yaml index 077600fe69..5e197974e6 100644 --- a/kubernetes/helm-charts/buildfarm/Chart.yaml +++ b/kubernetes/helm-charts/buildfarm/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.1 +version: 0.2.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 2.15.0 +appVersion: 2.12.0 dependencies: - condition: redis.enabled diff --git a/kubernetes/helm-charts/buildfarm/templates/_helpers.tpl b/kubernetes/helm-charts/buildfarm/templates/_helpers.tpl index ac80f55661..fe8826bb59 100644 --- a/kubernetes/helm-charts/buildfarm/templates/_helpers.tpl +++ b/kubernetes/helm-charts/buildfarm/templates/_helpers.tpl @@ -82,4 +82,4 @@ autoscaling/v1 {{ required "If `externalRedis.host` is set, then `redis.enabled` should be `false`!" nil }} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/kubernetes/helm-charts/buildfarm/templates/configmap.yaml b/kubernetes/helm-charts/buildfarm/templates/configmap.yaml index 5adb405ed7..6d4dbb4e47 100644 --- a/kubernetes/helm-charts/buildfarm/templates/configmap.yaml +++ b/kubernetes/helm-charts/buildfarm/templates/configmap.yaml @@ -18,56 +18,15 @@ data: {{- with .Values.config.backplane }} {{- toYaml . | nindent 6 }} {{- end }} - {{- with .Values.server.config }} + {{- with .Values.config.server }} server: {{- toYaml . | nindent 6 }} {{- end }} - {{- with .Values.shardWorker.config }} + {{- with .Values.config.worker }} worker: {{- toYaml . | nindent 6 }} {{- end }} --- -{{- if .Values.execWorker.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-config -data: - config.yml: |- - {{- range $key, $value := .Values.config }} - {{- if kindIs "map" $value }} - {{- else }} - {{ $key }}: {{ $value }}{{- end }} - {{- end }} - backplane: - {{- if .Values.redis.enabled }} - redisUri: '{{ printf "redis://%s-redis-master.%s:6379" .Release.Name .Release.Namespace }}' - {{- else }} - redisUri: "{{ .Values.externalRedis.uri }}" - {{- end }} - {{- with .Values.config.backplane }} - {{- toYaml . | nindent 6 }} - {{- end }} - {{- with .Values.server.config }} - server: - {{- toYaml . | nindent 6 }} - {{- end }} - {{- with .Values.execWorker.config }} - worker: - {{- toYaml . | nindent 6 }} - {{- end }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-worker-log-props -data: - logging.properties: |- - {{- range $key, $value := .Values.execWorker.loggingProperties }} - {{ $key }}={{ $value }} - {{- end }} ---- -{{- end }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/kubernetes/helm-charts/buildfarm/templates/exec-worker/deployment.yaml b/kubernetes/helm-charts/buildfarm/templates/exec-worker/deployment.yaml deleted file mode 100644 index 715f8b4ea1..0000000000 --- a/kubernetes/helm-charts/buildfarm/templates/exec-worker/deployment.yaml +++ /dev/null @@ -1,124 +0,0 @@ -{{- if .Values.execWorker.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-worker - labels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.labels" . | nindent 4 }} - app.kubernetes.io/component: exec-worker - annotations: - {{- toYaml .Values.execWorker.annotations | nindent 4 }} -spec: - {{- if .Values.execWorker.autoscaling.enabled }} - replicas: {{ .Values.execWorker.autoscaling.minReplicas }} - {{- else }} - replicas: {{ .Values.execWorker.replicaCount }} - {{- end }} - selector: - matchLabels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - checksum/worker-config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} - {{- with .Values.podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: exec-worker - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "buildfarm.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.execWorker.securityContext | nindent 8 }} - containers: - - name: buildfarm-worker - image: "{{ .Values.execWorker.image.repository }}:{{ .Values.execWorker.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.execWorker.image.pullPolicy }} - args: - - --public_name=$(POD_IP):8982 - env: - - name: CONFIG_PATH - value: /config/config.yml - - name: JAVA_TOOL_OPTIONS - value: -XX:+UseContainerSupport -XX:MaxRAMPercentage=80.0 -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+HeapDumpOnOutOfMemoryError -Djava.util.logging.config.file=/exec-worker-log-props/logging.properties - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - {{- if .Values.execWorker.extraEnv }} - {{- toYaml .Values.execWorker.extraEnv | nindent 12 }} - {{- end }} - ports: - - containerPort: 8982 - name: "worker-comm" - - containerPort: 9090 - name: "metrics" - {{- with .Values.execWorker.livenessProbe }} - livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.execWorker.readinessProbe }} - readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.execWorker.startupProbe }} - startupProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - resources: - {{- toYaml .Values.execWorker.resources | nindent 12 }} - volumeMounts: - - mountPath: /config - name: config - readOnly: true - - mountPath: /exec-worker-log-props - name: exec-worker-log-props - readOnly: true - {{- with .Values.execWorker.extraVolumeMounts }} - {{- tpl (toYaml .) $ | nindent 12 -}} - {{- end }} - {{- with .Values.execWorker.runtimeClassName }} - runtimeClassName: {{ tpl . $ }} - {{- end }} - - {{- with .Values.execWorker.nodeSelector }} - nodeSelector: - {{- tpl (toYaml .) $ | nindent 8 -}} - {{- end }} - - {{- with .Values.execWorker.affinity }} - affinity: - {{- tpl (toYaml .) $ | nindent 8 -}} - {{- end }} - - {{- with .Values.execWorker.topologySpreadConstraints }} - topologySpreadConstraints: - {{- toYaml . | nindent 8 }} - {{- end }} - - {{- with .Values.execWorker.tolerations }} - tolerations: - {{- tpl (toYaml .) $ | nindent 8 -}} - {{- end }} - volumes: - - configMap: - defaultMode: 420 - name: {{ include "buildfarm.fullname" . }}-exec-config - name: config - - configMap: - defaultMode: 420 - name: {{ include "buildfarm.fullname" . }}-exec-worker-log-props - name: exec-worker-log-props - {{- with .Values.execWorker.extraVolumes }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} -{{- end }} diff --git a/kubernetes/helm-charts/buildfarm/templates/exec-worker/hpa.yaml b/kubernetes/helm-charts/buildfarm/templates/exec-worker/hpa.yaml deleted file mode 100644 index 9b239959af..0000000000 --- a/kubernetes/helm-charts/buildfarm/templates/exec-worker/hpa.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- if and (.Values.execWorker.autoscaling.enabled) (.Values.execWorker.enabled) -}} -apiVersion: {{ include "buildfarm.autoscalingVersion" . }} -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-worker - labels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.labels" . | nindent 4 }} - {{- with .Values.execWorker.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - maxReplicas: {{ .Values.execWorker.autoscaling.maxReplicas }} - minReplicas: {{ .Values.execWorker.autoscaling.minReplicas }} - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- if contains "autoscaling/v2" (include "buildfarm.autoscalingVersion" . ) }} - {{- if .Values.execWorker.autoscaling.behavior }} - behavior: - {{- toYaml .Values.execWorker.autoscaling.behavior | nindent 4 }} - {{- end }} - metrics: - {{- toYaml .Values.execWorker.autoscaling.metrics | nindent 4 }} - {{- else }} - targetCPUUtilizationPercentage: {{ .Values.execWorker.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/kubernetes/helm-charts/buildfarm/templates/exec-worker/pdb.yaml b/kubernetes/helm-charts/buildfarm/templates/exec-worker/pdb.yaml deleted file mode 100644 index f35658692d..0000000000 --- a/kubernetes/helm-charts/buildfarm/templates/exec-worker/pdb.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and (.Values.execWorker.pdb.create) (.Values.execWorker.enabled) -}} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-worker - labels: - {{- include "buildfarm.labels" . | nindent 4 }} -spec: - {{- if .Values.execWorker.pdb.minAvailable }} - minAvailable: {{ .Values.execWorker.pdb.minAvailable }} - {{- end }} - {{- if or .Values.execWorker.pdb.maxUnavailable (not .Values.execWorker.pdb.minAvailable)}} - maxUnavailable: {{ .Values.execWorker.pdb.maxUnavailable | default 1 }} - {{- end }} - selector: - matchLabels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.selectorLabels" . | nindent 6 }} -{{- end }} diff --git a/kubernetes/helm-charts/buildfarm/templates/exec-worker/service.yaml b/kubernetes/helm-charts/buildfarm/templates/exec-worker/service.yaml deleted file mode 100644 index c7e6ead126..0000000000 --- a/kubernetes/helm-charts/buildfarm/templates/exec-worker/service.yaml +++ /dev/null @@ -1,27 +0,0 @@ -{{- if .Values.execWorker.enabled }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-worker - labels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.labels" . | nindent 4 }} - {{- with .Values.execWorker.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.execWorker.service.type }} - ports: - - port: {{ .Values.execWorker.service.port }} - targetPort: worker-comm - protocol: TCP - name: gprc - - port: 9090 - targetPort: metrics - protocol: TCP - name: metrics - selector: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.selectorLabels" . | nindent 4 }} -{{- end }} diff --git a/kubernetes/helm-charts/buildfarm/templates/exec-worker/servicemonitor.yaml b/kubernetes/helm-charts/buildfarm/templates/exec-worker/servicemonitor.yaml deleted file mode 100644 index 05c1c2cf9b..0000000000 --- a/kubernetes/helm-charts/buildfarm/templates/exec-worker/servicemonitor.yaml +++ /dev/null @@ -1,40 +0,0 @@ -{{- if and (.Values.execWorker.serviceMonitor.enabled) (.Values.execWorker.enabled) }} ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: {{ include "buildfarm.fullname" . }}-exec-worker - labels: - {{- include "buildfarm.labels" . | nindent 4 }} - {{- with .Values.execWorker.serviceMonitor.additionalLabels -}} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - endpoints: - - port: "metrics" - {{- with .Values.execWorker.serviceMonitor.interval }} - interval: {{ . }} - {{- end }} - {{- with .Values.execWorker.serviceMonitor.scrapeTimeout }} - scrapeTimeout: {{ . }} - {{- end }} - honorLabels: true - path: {{ .Values.execWorker.serviceMonitor.path }} - scheme: {{ .Values.execWorker.serviceMonitor.scheme }} - {{- with .Values.execWorker.serviceMonitor.relabelings }} - relabelings: - {{- toYaml . | nindent 6 }} - {{- end }} - jobLabel: "{{ .Release.Name }}" - selector: - matchLabels: - name: {{ include "buildfarm.fullname" . }}-exec-worker - {{- include "buildfarm.labels" . | nindent 6 }} - namespaceSelector: - matchNames: - - {{ .Release.Namespace }} - {{- with .Values.execWorker.serviceMonitor.targetLabels }} - targetLabels: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- end }} diff --git a/kubernetes/helm-charts/buildfarm/templates/server/deployment.yaml b/kubernetes/helm-charts/buildfarm/templates/server/deployment.yaml index 93583345f7..f5a7382227 100644 --- a/kubernetes/helm-charts/buildfarm/templates/server/deployment.yaml +++ b/kubernetes/helm-charts/buildfarm/templates/server/deployment.yaml @@ -50,18 +50,14 @@ spec: name: "server-comm" - containerPort: 9090 name: "metrics" - {{- with .Values.server.livenessProbe}} livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.server.readinessProbe }} + grpc: + port: 8980 + initialDelaySeconds: 10 readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.server.startupProbe }} - startupProbe: - {{- toYaml . | nindent 12 }} - {{- end }} + grpc: + port: 8980 + initialDelaySeconds: 10 resources: {{- toYaml .Values.server.resources | nindent 12 }} volumeMounts: diff --git a/kubernetes/helm-charts/buildfarm/templates/shard-worker/hpa.yaml b/kubernetes/helm-charts/buildfarm/templates/shard-worker/autoscaler.yaml similarity index 100% rename from kubernetes/helm-charts/buildfarm/templates/shard-worker/hpa.yaml rename to kubernetes/helm-charts/buildfarm/templates/shard-worker/autoscaler.yaml diff --git a/kubernetes/helm-charts/buildfarm/templates/shard-worker/statefulset.yaml b/kubernetes/helm-charts/buildfarm/templates/shard-worker/statefulsets.yaml similarity index 91% rename from kubernetes/helm-charts/buildfarm/templates/shard-worker/statefulset.yaml rename to kubernetes/helm-charts/buildfarm/templates/shard-worker/statefulsets.yaml index 4f57b3e82c..7b41b1d271 100644 --- a/kubernetes/helm-charts/buildfarm/templates/shard-worker/statefulset.yaml +++ b/kubernetes/helm-charts/buildfarm/templates/shard-worker/statefulsets.yaml @@ -9,7 +9,6 @@ metadata: annotations: {{- toYaml .Values.shardWorker.annotations | nindent 4 }} spec: - podManagementPolicy: {{ .Values.shardWorker.podManagementPolicy }} serviceName: {{ include "buildfarm.fullname" . }}-shard-worker {{- if .Values.shardWorker.autoscaling.enabled }} replicas: {{ .Values.shardWorker.autoscaling.minReplicas }} @@ -63,18 +62,14 @@ spec: name: "worker-comm" - containerPort: 9090 name: "metrics" - {{- with .Values.shardWorker.livenessProbe }} livenessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.shardWorker.readinessProbe }} + grpc: + port: 8982 + initialDelaySeconds: 10 readinessProbe: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.shardWorker.startupProbe }} - startupProbe: - {{- toYaml . | nindent 12 }} - {{- end }} + grpc: + port: 8982 + initialDelaySeconds: 10 resources: {{- toYaml .Values.shardWorker.resources | nindent 12 }} volumeMounts: diff --git a/kubernetes/helm-charts/buildfarm/values.yaml b/kubernetes/helm-charts/buildfarm/values.yaml index bf13c0dd47..8bf40813f9 100644 --- a/kubernetes/helm-charts/buildfarm/values.yaml +++ b/kubernetes/helm-charts/buildfarm/values.yaml @@ -6,10 +6,8 @@ fullnameOverride: "" imagePullSecrets: [] -# Configuration shared by all workers and the server. -# see: https://github.com/buildfarm/buildfarm/blob/main/examples/config.yml -# Workers specific configuration is in the `shardWorker` and `execWorker` sections below. config: + # see: https://github.com/buildfarm/buildfarm/blob/main/examples/config.yml digestFunction: SHA256 defaultActionTimeout: 600 maximumActionTimeout: 3600 @@ -24,26 +22,19 @@ config: value: "*" - name: "max-cores" value: "*" + server: + name: "shard" + recordBesEvents: true + worker: + port: 8982 server: - annotations: {} image: repository: bazelbuild/buildfarm-server pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. tag: "" replicaCount: 1 - - livenessProbe: - grpc: - port: 8980 - initialDelaySeconds: 10 - readinessProbe: - grpc: - port: 8980 - initialDelaySeconds: 10 - startupProbe: {} - # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -57,7 +48,6 @@ server: # cpu: 100m # memory: 128Mi service: - annotations: {} type: ClusterIP port: 8980 @@ -127,10 +117,6 @@ server: relabelings: [] targetLabels: [] - config: - name: "shard" - recordBesEvents: true - loggingProperties: build.buildfarm.level: INFO handlers: java.util.logging.ConsoleHandler @@ -138,133 +124,7 @@ server: java.util.logging.ConsoleHandler.formatter: java.util.logging.SimpleFormatter java.util.logging.SimpleFormatter.format: "[%4$-7s] %2$s - %5$s %6$s %n" -# By default a worker handles both CAS and Exec. shardWorker: - image: - repository: bazelbuild/buildfarm-worker - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - annotations: {} - podManagementPolicy: Parallel # Or "OrderedReady" - replicaCount: 2 - autoscaling: - enabled: true - minReplicas: 2 - maxReplicas: 4 - behavior: {} # effective only in `v2*` - metrics: # effective only in `v2*` - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 - targetCPUUtilizationPercentage: 50 # effective only in `v1` - - livenessProbe: - grpc: - port: 8982 - initialDelaySeconds: 10 - readinessProbe: - grpc: - port: 8982 - initialDelaySeconds: 10 - startupProbe: {} - - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - resources: - {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - persistentVolumeClaimRetentionPolicy: - whenDeleted: Retain - whenScaled: Retain - storage: - enabled: true - # the storage class for pv, leave empty will using default - class: "" - size: 50Gi - - service: - annotations: {} - type: ClusterIP - port: 8982 - - runtimeClassName: "" - nodeSelector: {} - tolerations: [] - affinity: {} - - ## Security Context - ## https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ - ## - securityContext: {} - - ## Pod Disruption Budget configuration - ## ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/ - ## - pdb: - create: false - minAvailable: "" - maxUnavailable: "" - - extraVolumes: [] - # - name: additionalSecret - # secret: - # secretName: my-secret - # defaultMode: 0600 - - extraVolumeMounts: [] - # - name: customConfig - # mountPath: /mnt/config - # readOnly: true - - extraEnv: - - name: JAVABIN - value: "/usr/bin/java" - - serviceMonitor: - ## If true, a ServiceMonitor CRD is created for a prometheus operator - ## https://github.com/coreos/prometheus-operator - ## - enabled: false - path: /metrics - # namespace: monitoring (defaults to use the namespace this chart is deployed to) - additionalLabels: {} - interval: 1m - scheme: http - tlsConfig: {} - scrapeTimeout: 30s - relabelings: [] - targetLabels: [] - - config: - port: 8982 - # Specific configuration for this worker. - # see the `worker` section here: https://github.com/buildfarm/buildfarm/blob/main/examples/config.yml - capabilities: - # By default this worker supports both CAS and Exec. - # You can disable this and choose dedicated workers for CAS and Exec by setting the execWorker.enabled to true and disabling Exec in this worker. - execution: true - - loggingProperties: - build.buildfarm.level: INFO - handlers: java.util.logging.ConsoleHandler - java.util.logging.ConsoleHandler.level: INFO - java.util.logging.ConsoleHandler.formatter: java.util.logging.SimpleFormatter - java.util.logging.SimpleFormatter.format: "[%4$-7s] %2$s - %5$s %6$s %n" - -execWorker: - enabled: false - annotations: {} image: repository: bazelbuild/buildfarm-worker pullPolicy: IfNotPresent @@ -285,16 +145,6 @@ execWorker: averageUtilization: 50 targetCPUUtilizationPercentage: 50 # effective only in `v1` - livenessProbe: - grpc: - port: 8982 - initialDelaySeconds: 10 - readinessProbe: - grpc: - port: 8982 - initialDelaySeconds: 10 - startupProbe: {} - # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -317,7 +167,6 @@ execWorker: size: 50Gi service: - annotations: {} type: ClusterIP port: 8982 @@ -369,12 +218,6 @@ execWorker: relabelings: [] targetLabels: [] - config: - port: 8982 - capabilities: - # This is an Exec worker, so we disable CAS. - cas: false - loggingProperties: build.buildfarm.level: INFO handlers: java.util.logging.ConsoleHandler diff --git a/licenserc.toml b/licenserc.toml deleted file mode 100644 index 9d42080eeb..0000000000 --- a/licenserc.toml +++ /dev/null @@ -1,17 +0,0 @@ -headerPath = "ci/license-header.txt" - -includes = [ - "*.c", - # "*.java", # TODO - "*.sh", -] - -[git] -attrs = 'auto' -ignore = 'auto' - -[properties] -copyrightOwner = "The Buildfarm Authors" - -[mapping.DOUBLESLASH_STYLE] -extensions = ["java"] diff --git a/macos-wrapper.sh b/macos-wrapper.sh index acd92391b8..2c30c81aa6 100755 --- a/macos-wrapper.sh +++ b/macos-wrapper.sh @@ -1,18 +1,4 @@ #!/usr/bin/env bash -# Copyright 2022-2025 The Buildfarm Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - # An Apple specific Apple execution wrapper - mainly to set SDKROOT and # DEVELOPER_DIR. # There is no enforcement of where Xcode.app is installed locally or remote so diff --git a/maven_install.json b/maven_install.json index f7cd1a34e7..910ace91c5 100755 --- a/maven_install.json +++ b/maven_install.json @@ -1,8 +1,20 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": -471240470, - "__RESOLVED_ARTIFACTS_HASH": -782609793, + "__INPUT_ARTIFACTS_HASH": 62890358, + "__RESOLVED_ARTIFACTS_HASH": -531703330, "artifacts": { + "biz.aQute.bnd:biz.aQute.bnd.util": { + "shasums": { + "jar": "65b5bd4a0fab16812f1800c98ff74a038f37a38bfe899af382efed4efdc1e3e1" + }, + "version": "6.4.0" + }, + "biz.aQute.bnd:biz.aQute.bndlib": { + "shasums": { + "jar": "357145074872f9dbf67e629fcd237e6152707e575d735df4535282f9f588d2d8" + }, + "version": "6.4.0" + }, "com.amazonaws:aws-java-sdk-core": { "shasums": { "jar": "79682855ea21bd65094ad97109f9b3e4361d3e02926f5ee14ade3411c7ca43da" @@ -53,21 +65,21 @@ }, "com.fasterxml.jackson.core:jackson-annotations": { "shasums": { - "jar": "959a2ffb2d591436f51f183c6a521fc89347912f711bf0cae008cdf045d95319" + "jar": "8aa5740d80b5a5025508b41bbadbaa1fb3772267c628b2e30681a4f45f8b8931" }, - "version": "2.20" + "version": "2.18.3" }, "com.fasterxml.jackson.core:jackson-core": { "shasums": { - "jar": "bc0cf46075877201f8406ee7de2741ae7df6c066f5f0457bd80632a718c06e72" + "jar": "056bc4d3e5e53ce821450fa97b3f9e0f8dde125cf6da6884353bb1f09582e1d9" }, - "version": "2.20.0" + "version": "2.18.3" }, "com.fasterxml.jackson.core:jackson-databind": { "shasums": { - "jar": "a70e146a6bf2cba4f9cd367169787f50adcfbb57122bc2e9c8390cd0b397ac30" + "jar": "510bdda75a7a6186c5bf33b851239488a1450906ae5757121f2e1cc48a7e108f" }, - "version": "2.20.0" + "version": "2.18.3" }, "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor": { "shasums": { @@ -77,15 +89,9 @@ }, "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml": { "shasums": { - "jar": "d0d538c5ba235ac88c8c3c52e146d3d5e1bbcc8b90e3d757cf0dbe39225e8d00" + "jar": "381a1c0711e4bb88561a6c0008b5a945465628ca07764ccd66a0d97ee07ad612" }, - "version": "2.19.1" - }, - "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { - "shasums": { - "jar": "366eb89065d568a7c0ede73c26a343f293f076d2f8efd0dc69cbe574ca01220e" - }, - "version": "2.19.1" + "version": "2.18.2" }, "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base": { "shasums": { @@ -107,9 +113,9 @@ }, "com.github.ben-manes.caffeine:caffeine": { "shasums": { - "jar": "c74a6c72221dfb76eb92f2bb40108ea561a7da2f315dc3b1e64afa8f077f210c" + "jar": "ec411dfdf0c03f25218648ce89861630b71680e5858a9a7278ebac8e55cab3d7" }, - "version": "3.2.2" + "version": "3.2.0" }, "com.github.docker-java:docker-java": { "shasums": { @@ -198,15 +204,15 @@ }, "com.github.luben:zstd-jni": { "shasums": { - "jar": "e7f064bf1eab83fa785cf587f657f09649e3b0af6f27caa5382416953b5a35f8" + "jar": "c5530c6fe99df04f9ac041e1b11fcb5772a7f4218c727d7273e0a6e5948b5d12" }, - "version": "1.5.7-4" + "version": "1.5.7-2" }, "com.github.oshi:oshi-core": { "shasums": { - "jar": "442cc9d0ab1765dc1b90ed0df0c364a6948d8120eeb1b87c6f706b35aafc047e" + "jar": "ea9496658faba427462b6c0b7a0062f6ca7d5cccb949e9e8a41ec55f138cefde" }, - "version": "6.8.3" + "version": "6.8.0" }, "com.github.pcj:google-options": { "shasums": { @@ -228,21 +234,21 @@ }, "com.google.api.grpc:proto-google-common-protos": { "shasums": { - "jar": "dd8f29e813d2b27a4d5e9ae34c90dd17af3e98af3f445556815516b39eeaf366" + "jar": "ee9c751f06b112e92b37f75e4f73a17d03ef2c3302c6e8d986adbcc721b63cb0" }, - "version": "2.61.1" + "version": "2.29.0" }, "com.google.auth:google-auth-library-credentials": { "shasums": { - "jar": "d982eda20835e301dcbeec4d083289a44fdd06e9a35ce18449054f4ffd3f099f" + "jar": "5dbf1207d14e093f67995f457cb69c3cf49bed1364150b23465e09acada65d96" }, - "version": "1.23.0" + "version": "1.24.1" }, "com.google.auth:google-auth-library-oauth2-http": { "shasums": { - "jar": "f2bf739509b5f3697cb1bf33ff9dc27e8fc886cedb2f6376a458263f793ed133" + "jar": "88a75cd4448ea2f3b46e48a89497a6cf0985a5fa4e21274af4940e07f59f6eaf" }, - "version": "1.23.0" + "version": "1.24.1" }, "com.google.auto.service:auto-service-annotations": { "shasums": { @@ -250,11 +256,17 @@ }, "version": "1.0.1" }, + "com.google.auto.value:auto-value": { + "shasums": { + "jar": "aaf8d637bfed3c420436b9facf1b7a88d12c8785374e4202382783005319c2c3" + }, + "version": "1.11.0" + }, "com.google.auto.value:auto-value-annotations": { "shasums": { - "jar": "e1c45e6beadaef9797cb0d9afd5a45621ad061cd8632012f85582853a3887825" + "jar": "5a055ce4255333b3346e1a8703da5bf8ff049532286fdcd31712d624abe111dd" }, - "version": "1.10.4" + "version": "1.11.0" }, "com.google.auto:auto-common": { "shasums": { @@ -262,6 +274,48 @@ }, "version": "1.2.2" }, + "com.google.caliper:caliper": { + "shasums": { + "jar": "d54e1bfdbe9359a79e175c217b8d472555cf20da9c5c0a18bdb1ea7db979ed8e" + }, + "version": "1.0-beta-3" + }, + "com.google.caliper:caliper-api": { + "shasums": { + "jar": "374f0c6c0c1f8784cb69d885e1dcbb7498c34ca20369e0597264568530642928" + }, + "version": "1.0-beta-3" + }, + "com.google.caliper:caliper-core": { + "shasums": { + "jar": "514deac8f8b09dd4262733e0a4406a333208c899dc7ea726b03600b9bb94f192" + }, + "version": "1.0-beta-3" + }, + "com.google.caliper:caliper-runner": { + "shasums": { + "jar": "3ab58890aa01343361adedf859500d280f67813df0cedfcd165b169533b9b1fa" + }, + "version": "1.0-beta-3" + }, + "com.google.caliper:caliper-util": { + "shasums": { + "jar": "6e9af500c7020450dfdb5003d09501d512d395f431c54c7ee8f79e712463fe66" + }, + "version": "1.0-beta-3" + }, + "com.google.caliper:caliper-worker": { + "shasums": { + "jar": "315d8e51df1f60551645a0e3bf2c504d2c79762f688e6f728eb9d1d9cd9a491b" + }, + "version": "1.0-beta-3" + }, + "com.google.caliper:caliper-worker-jvm": { + "shasums": { + "jar": "e14e1ecfdf939c82abdb902105be41ff3f83c18cb968116232015f2662d065a4" + }, + "version": "1.0-beta-3" + }, "com.google.code.findbugs:jsr305": { "shasums": { "jar": "766ad2a0783f2687962c8ad74ceecc38a28b9f72a2d085ee438b7813e928d0c7" @@ -270,39 +324,57 @@ }, "com.google.code.gson:gson": { "shasums": { - "jar": "dd0ce1b55a3ed2080cb70f9c655850cda86c206862310009dcb5e5c95265a5e0" + "jar": "ebee13d5fb7477cd7f1cc010e0c356df8ca80709715248da97f79e35ccb4fbec" + }, + "version": "2.12.1" + }, + "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter": { + "shasums": { + "jar": "1ef5535a8bd41cf3072469f381b9ee6ab28275311a7499f53d6e52adf976fef0" }, - "version": "2.13.2" + "version": "3.3.0" + }, + "com.google.dagger:dagger": { + "shasums": { + "jar": "329d4340f24c4f5717af016c097e90668bfea2a5376e6aa9964b01cef3fd241a" + }, + "version": "2.22.1" + }, + "com.google.dagger:dagger-producers": { + "shasums": { + "jar": "f834a0082014213a68ff06a0f048d750178d02196c58b0b15beb367d32b97e35" + }, + "version": "2.22.1" }, "com.google.errorprone:error_prone_annotation": { "shasums": { - "jar": "192e99ad3be8a0a44fe210cdc9b0265601dcb446e521ba922c793e50add413c3" + "jar": "06ebe7f108f515f4192980580a8d423df57ae0af3d4205069736ca053c1c3f88" }, - "version": "2.41.0" + "version": "2.37.0" }, "com.google.errorprone:error_prone_annotations": { "shasums": { - "jar": "a56e782b5b50811ac204073a355a21d915a2107fce13ec711331ad036f660fcc" + "jar": "d29a6263b48d46d4c7c28b64717a6d105b3e2a3e64256092f845e8e774fca6ba" }, - "version": "2.41.0" + "version": "2.37.0" }, "com.google.errorprone:error_prone_check_api": { "shasums": { - "jar": "841c8abc2655b75df1ac22a66a18d433351a8b76e6af622609b1c4c447810fdb" + "jar": "1bc5968da69009afaca4dcbe4b68a9d1cc3289d5d399d5e369bde369f648c86c" }, - "version": "2.41.0" + "version": "2.37.0" }, "com.google.errorprone:error_prone_core": { "shasums": { - "jar": "60d6ef754a6d2bd4eca2b3d36b32f721fb33e630170d38aab7e07324e0799135" + "jar": "b6e5513fd8c09f4bfd5ff008e636b074ab14544f35d0516a210829af8a98005c" }, - "version": "2.41.0" + "version": "2.37.0" }, "com.google.googlejavaformat:google-java-format": { "shasums": { - "jar": "e0ac64ebc57756b2c724a202aa9b896fd79b61766671ece6f807f84f6630e301" + "jar": "9a4e0b9f4ec4d71a8a1d3641fd481118100fda2eeab712dbdfd4b2a06e9de4ce" }, - "version": "1.27.0" + "version": "1.24.0" }, "com.google.guava:failureaccess": { "shasums": { @@ -312,9 +384,15 @@ }, "com.google.guava:guava": { "shasums": { - "jar": "f3d7f57f67fd622f4d468dfdd692b3a5e3909246c28017ac3263405f0fe617ed" + "jar": "958a035b74ff6c7d0cdff9c384524b645eb618f7117b60e1ee915f9cffd0e716" }, - "version": "33.4.8-jre" + "version": "33.4.6-jre" + }, + "com.google.guava:guava-testlib": { + "shasums": { + "jar": "c97511849a5e085280f106df2b059566febd280b118d33d6a9e068d238100b63" + }, + "version": "32.0.1-jre" }, "com.google.guava:listenablefuture": { "shasums": { @@ -324,21 +402,21 @@ }, "com.google.http-client:google-http-client": { "shasums": { - "jar": "60aca7428c5a1ff3655b70541a98ff3d70dded48ac1324dae1af39f1b61914af" + "jar": "390618d7b51704240b8fd28e1230fa35d220f93f4b4ba80f63e38db00dacb09e" }, - "version": "1.43.3" + "version": "1.44.2" }, "com.google.http-client:google-http-client-gson": { "shasums": { - "jar": "e31a4edcb9c83954a2587e14fa2f3f8f4aad56152381b3321a3bd0bcae03fa26" + "jar": "1119b66685195310375b717de2215d6c5d14fa8ed9f57e07b4fecd461e7b9db7" }, - "version": "1.43.3" + "version": "1.44.2" }, "com.google.j2objc:j2objc-annotations": { "shasums": { - "jar": "84d3a150518485f8140ea99b8a985656749629f6433c92b80c75b36aba3b099b" + "jar": "88241573467ddca44ffd4d74aa04c2bbfd11bf7c17e0c342c94c9de7a70a7c64" }, - "version": "3.1" + "version": "3.0.0" }, "com.google.jimfs:jimfs": { "shasums": { @@ -348,15 +426,21 @@ }, "com.google.protobuf:protobuf-java": { "shasums": { - "jar": "8c99e4d971338bafb0b0b1d1cea9b1bbb3dc9630eb9c25109e4c7c27bca832cb" + "jar": "24f7d0d91797ed230a6d3da93cd80590a4c3aa9a27249f6025b5c6da078edde7" }, - "version": "4.32.1" + "version": "3.19.1" }, "com.google.protobuf:protobuf-java-util": { "shasums": { - "jar": "65bd990b8a3d5625cee9910a4256d00eb780c3961eda5b422a0945ace09dcafd" + "jar": "2fdeab29e8b3296a5347dd3ac2a5e03016cb877d76055d1f19e447ae9a7e90a0" + }, + "version": "3.19.1" + }, + "com.google.re2j:re2j": { + "shasums": { + "jar": "4f657af51ab8bb0909bcc3eb40862d26125af8cbcf92aaaba595fed77f947bc0" }, - "version": "4.32.1" + "version": "1.7" }, "com.google.truth:truth": { "shasums": { @@ -382,12 +466,36 @@ }, "version": "2.0.1" }, + "com.squareup.okhttp:okhttp": { + "shasums": { + "jar": "88ac9fd1bb51f82bcc664cc1eb9c225c90dc4389d660231b4cc737bebfe7d0aa" + }, + "version": "2.7.5" + }, + "com.squareup.okio:okio": { + "shasums": { + "jar": "a27f091d34aa452e37227e2cfa85809f29012a8ef2501a9b5a125a978e4fcbc1" + }, + "version": "2.10.0" + }, "com.sun.activation:jakarta.activation": { "shasums": { "jar": "d84d4ba8b55cdb7fdcbb885e6939386367433f56f5ab8cfdc302a7c3587fa92b" }, "version": "1.2.1" }, + "com.sun.jersey:jersey-client": { + "shasums": { + "jar": "639c825c5db580f8115bf49ffc893093526d2ed1079fbc929b6a5fbd0b2eda40" + }, + "version": "1.19.4" + }, + "com.sun.jersey:jersey-core": { + "shasums": { + "jar": "64b03198e0264849d0fc341857ebcc9c882b1909a2dc35a0972fe7d901b826e5" + }, + "version": "1.19.4" + }, "commons-codec:commons-codec": { "shasums": { "jar": "b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63" @@ -396,9 +504,9 @@ }, "commons-io:commons-io": { "shasums": { - "jar": "df90bba0fe3cb586b7f164e78fe8f8f4da3f2dd5c27fa645f888100ccc25dd72" + "jar": "f3ca0f8d63c40e23a56d54101c60d5edee136b42d84bfb85bc7963093109cf8b" }, - "version": "2.20.0" + "version": "2.18.0" }, "commons-logging:commons-logging": { "shasums": { @@ -406,6 +514,12 @@ }, "version": "1.1.3" }, + "info.picocli:picocli": { + "shasums": { + "jar": "b0a5159e926de8084ff066025142270443533656bc599b8bb31d14d11fd138a4" + }, + "version": "4.6.3" + }, "io.github.eisop:dataflow-errorprone": { "shasums": { "jar": "10434fba4e53f55fa9c76904cde414b918932548c9dfc4e2d634ac05ff7a7d10" @@ -420,93 +534,93 @@ }, "io.grpc:grpc-api": { "shasums": { - "jar": "7f309616691fa655d02512762049ed18bf4ab2b52ced424cab2f527d0bb8e3fc" + "jar": "2e896944cf513e0e5cfd32bcd72c89601a27c6ca56916f84b20f3a13bacf1b1f" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-auth": { "shasums": { - "jar": "216e551be78b4c080edd6ed43aef63af21019a2bd92a81f3c31d4cadb4185520" + "jar": "6a16c43d956c79190486d3d0b951836a6706b3282b5d275a9bc4d33eb79d5618" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-context": { "shasums": { - "jar": "7e1de7847ea621a9ec7cb988a8baa947748d0eaef94bfe457d04d9b57105079f" + "jar": "9959747df6a753119e1c1a3dff01aa766d2455f5e4860acaa305359e1d533a05" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-core": { "shasums": { - "jar": "f10cdbe558378494e4ffc6b1bb328b8a137f3201d46c8b24f0154eb9e51191a1" + "jar": "18439902c473a2c1511e517d13b8ae796378850a8eda43787c6ba778fa90fcc5" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-inprocess": { "shasums": { - "jar": "d2953165a5bf6d236bbd84c5f9c6abd344cc722ad0cf13de24c82883477e983e" + "jar": "f3c28a9d7f13fa995e4dd89e4f6aa08fa3b383665314fdfccb9f87f346625df7" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-netty": { "shasums": { - "jar": "2563004eb1c0a6f676678edb545377f21786144c6f55331ff1b8e6be73eb6a57" + "jar": "6060217fe26a9f8b2d899d02d95c9b52513be774233326ee43a6b8433edb03c8" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-netty-shaded": { "shasums": { - "jar": "d33d47a9720a392512c4e4ceaa17a458a0e3ed6188d17e98d6c4faf0679374d7" + "jar": "b3f1823ef30ca02ac721020f4b6492248efdbd0548c78e893d5d245cbca2cc60" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-protobuf": { "shasums": { - "jar": "f52b53c349b0776815e437636dff3d3844e9b10cbb25be419a786bf3d6f20269" + "jar": "66a0b196318bdfd817d965d2d82b9c81dfced8eb08c0f7510fcb728d2994237a" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-protobuf-lite": { "shasums": { - "jar": "60fafc627aa04bcab328dcc9206f0e7aa71d95f1b612774a18272dfe7dc24cf1" + "jar": "79997989a8c2b5bf4dd18182a2df2e2f668703d68ba7c317e7a07809d33f91f4" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-services": { "shasums": { - "jar": "b6b3db6fecf14220db17d085eb2143f7c68879f6072f2159692c24c607e9391f" + "jar": "72f6eba0670184b634e7dcde0b97cde378a7cd74cdf63300f453d15c23bbbb6a" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-stub": { "shasums": { - "jar": "dc98fe18654206948666e6657e13fe301ad0754751f28c6c10961e1e6c457997" + "jar": "fb4ca679a4214143406c65ac4167b2b5e2ee2cab1fc101566bb1c4695d105e36" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-testing": { "shasums": { - "jar": "2350126abc27ba24e42d04c8e401962f23de62ac011465d3bb9b8c65e693eb7c" + "jar": "a952fed1a1b43569ececd832ed820bd149a6f214905a9c7d4fc8853dfd553df4" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.grpc:grpc-util": { "shasums": { - "jar": "92b5a1195dad4cbd7c405b01cbd9fd2dbbd548d13d4cfabf9d2d6a37bad4cb81" + "jar": "3c7103e6f3738571e3aeda420fe2a6ac68e354534d8b66f41897b6755b48b735" }, - "version": "1.75.0" + "version": "1.62.2" }, "io.netty:netty-buffer": { "shasums": { - "jar": "348e3ff64c7129ca661bc09d4bdda09c824474cfd1f5918368bdc56f5ee17f79" + "jar": "46d74e79125aacc055c31f18152fdc5d4a569aa8d60091203d0baa833973ac3c" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-codec": { "shasums": { - "jar": "32c220dea93756fba28f9302481bc657738cc40d07440daa985a2ba21df226f1" + "jar": "9eccce9a8d827bb8ce84f9c3183fec58bd1c96a51010cf711297746034af3701" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-codec-dns": { "shasums": { @@ -516,45 +630,45 @@ }, "io.netty:netty-codec-http": { "shasums": { - "jar": "2bdb276d40c2293014638a7e065bea977b574fb6a978e1197f514f2e13b695a6" + "jar": "dc0d6af5054630a70ff0ef354f20aa7a6e46738c9fc5636ed3d4fe77e38bd48d" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-codec-http2": { "shasums": { - "jar": "a813e1810f7c1959b90fc7f221e03ce15e66005ee56e29cd0d68312b9679c772" + "jar": "b546c75445a487bb7bcd5a94779caecce33582cf7be31b8b39fc0e65b1ee26fc" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-codec-socks": { "shasums": { - "jar": "6a1340535f24ab997e1e5fec8bf2649108550a0816ee8e3728c14c2e55921bd8" + "jar": "976052a3c9bb280bc6d99f3a29e6404677cf958c3de05b205093d38c006b880c" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-common": { "shasums": { - "jar": "8e3649fc6bab84a88ad47af82e38f9c36ab3725de478632c8a59e4bd74d16e08" + "jar": "9851ec66548b9e0d41164ce98943cdd4bbe305f68ddbd24eae52e4501a0d7b1a" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-handler": { "shasums": { - "jar": "55b2458011527d94abc868086afd039cd00cc3a547e7322569e0fb4f906d9d80" + "jar": "d5a08d7de364912e4285968de4d4cce3f01da4bb048d5c6937e5f2af1f8e148a" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-handler-proxy": { "shasums": { - "jar": "0e11683b57e75393d6ea710e0c6a8ebcb5c0913e7938ab2a8a8667ba740219da" + "jar": "ad54ab4fe9c47ef3e723d71251126db53e8db543871adb9eafc94446539eff52" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-resolver": { "shasums": { - "jar": "55279fdcf6c0e1819b6561cc70b0eb2de1b1cf1ef5635fc46334d7e06faa9dd9" + "jar": "a2e9b4ae7caa92fc5bd747e11d1dec20d81b18fc00959554302244ac5c56ce70" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-resolver-dns": { "shasums": { @@ -562,11 +676,28 @@ }, "version": "4.1.108.Final" }, + "io.netty:netty-tcnative-boringssl-static": { + "shasums": { + "jar": "bb8349bda3ba043c09f5c732c5b2d47f18ac4dafce248cb850cfbd0599e5ba01", + "linux-aarch_64": "9fd6f905bce8472ca9e24f8728962b91b9319c84fcbb618ab87f6b9774c063ed", + "linux-x86_64": "45ce55b49f4c16de65278d9f4608a9f06460f290f1e3b4fc3f2452866519d618", + "osx-aarch_64": "834d3df2f9eb3aa08d49326409c454e2f9c75a49c3f0454b7cab64619b1614dc", + "osx-x86_64": "3a16b7ca4e891f9ad51e9ae9eb167c5e81d7f55a0001edf2471b693f4f8c12e1", + "windows-x86_64": "855d930366fae9bec3872569975959e1e31dd752dffc9f925eae8436f933e11a" + }, + "version": "2.0.65.Final" + }, + "io.netty:netty-tcnative-classes": { + "shasums": { + "jar": "84ef0241ada1b4ed92785e10c16edbeb063348959a3b0ef740712badd09fa128" + }, + "version": "2.0.65.Final" + }, "io.netty:netty-transport": { "shasums": { - "jar": "fef2ec66fe01aa89734db40f292676719da3985786512fc31a9efe1ca4d2e0ff" + "jar": "a42dd68390ca14b4ff2d40628a096c76485b4adb7c19602d5289321a0669e704" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-transport-classes-epoll": { "shasums": { @@ -582,10 +713,10 @@ }, "io.netty:netty-transport-native-epoll": { "shasums": { - "jar": "a825c5320c545360c91249ae8718c56b33135abc8943057479647b4e018dab31", - "linux-x86_64": "452b8f0ae41fab3abb55d7e591be7186068b734c391ea25e8f9af067fd06e83f" + "jar": "64415f53ca5fa222d40caa72e9fd27f09c04489ddbc0a6ad5179b93e75333102", + "linux-x86_64": "dcd60c6b3076af307ab877201a136e1f1066c9be809aaed827391a23909f9135" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.netty:netty-transport-native-kqueue": { "shasums": { @@ -596,15 +727,21 @@ }, "io.netty:netty-transport-native-unix-common": { "shasums": { - "jar": "c3f324a6f526313e432235bf1a3a12e3db283e3b8669e02f26f569c421036bcb" + "jar": "51717bb7471141950390c6713a449fdb1054d07e60737ee7dda7083796cdee48" }, - "version": "4.1.108.Final" + "version": "4.1.110.Final" }, "io.opencensus:opencensus-api": { "shasums": { - "jar": "f1474d47f4b6b001558ad27b952e35eda5cc7146788877fc52938c6eba24b382" + "jar": "702ba55d78f39d55195dcf041fdfaab7a7490a9ac45013542487ed9e4d3a4d23" }, - "version": "0.31.1" + "version": "0.31.0" + }, + "io.opencensus:opencensus-contrib-grpc-metrics": { + "shasums": { + "jar": "b28fc72490edd49c4c40a3c216b709200833fb361f7f602f1c7c9a527f7b7f63" + }, + "version": "0.31.0" }, "io.opencensus:opencensus-contrib-http-util": { "shasums": { @@ -714,6 +851,12 @@ }, "version": "1" }, + "javax.ws.rs:jsr311-api": { + "shasums": { + "jar": "ab1534b73b5fa055808e6598a5e73b599ccda28c3159c3c0908977809422ee4a" + }, + "version": "1.1.1" + }, "joda-time:joda-time": { "shasums": { "jar": "b4670b95f75957c974284c5f3ada966040be2578f643c5c6083d262162061fa2" @@ -734,15 +877,15 @@ }, "net.bytebuddy:byte-buddy": { "shasums": { - "jar": "d26382a839cb26d5c62a0b0f04715bcef55a531f96ac6ce40de452a1c0539e70" + "jar": "fa08998aae1e7bdae83bde0712c50e8444d71c0e0c196bb2247ade8d4ad0eb90" }, - "version": "1.17.6" + "version": "1.15.11" }, "net.bytebuddy:byte-buddy-agent": { "shasums": { - "jar": "8a80959465bb09848cd41a89301490c55b9d940dc6ec30d8abac0f63b39fd5d0" + "jar": "316d2c0795c2a4d4c4756f2e6f9349837c7430ac34e0477ead874d05f5cc19e5" }, - "version": "1.17.6" + "version": "1.15.11" }, "net.java.dev.jna:jna": { "shasums": { @@ -794,15 +937,15 @@ }, "org.apache.commons:commons-compress": { "shasums": { - "jar": "e1522945218456f3649a39bc4afd70ce4bd466221519dba7d378f2141a4642ca" + "jar": "293d80f54b536b74095dcd7ea3cf0a29bbfc3402519281332495f4420d370d16" }, - "version": "1.28.0" + "version": "1.27.1" }, "org.apache.commons:commons-lang3": { "shasums": { - "jar": "4eeeae8d20c078abb64b015ec158add383ac581571cddc45c68f0c9ae0230720" + "jar": "6ee731df5c8e5a2976a1ca023b6bb320ea8d3539fbe64c8a1d5cb765127c33b4" }, - "version": "3.18.0" + "version": "3.17.0" }, "org.apache.commons:commons-math3": { "shasums": { @@ -858,11 +1001,17 @@ }, "version": "1.75" }, + "org.checkerframework:checker-compat-qual": { + "shasums": { + "jar": "d76b9afea61c7c082908023f0cbc1427fab9abd2df915c8b8a3e7a509bccbc6d" + }, + "version": "2.5.3" + }, "org.checkerframework:checker-qual": { "shasums": { - "jar": "8e91676dc916804bd12f7909e7f5dc07646cc98220147a8b5fea7ff42bd6ff4a" + "jar": "1a77cff4a701597c8a848303d84b40d42280ed2824e108397fdcfa07fa51de0d" }, - "version": "3.51.0" + "version": "3.49.1" }, "org.codehaus.mojo:animal-sniffer-annotations": { "shasums": { @@ -942,11 +1091,23 @@ }, "version": "3.28.0-GA" }, + "org.jetbrains.kotlin:kotlin-stdlib": { + "shasums": { + "jar": "b8ab1da5cdc89cb084d41e1f28f20a42bd431538642a5741c52bbfae3fa3e656" + }, + "version": "1.4.20" + }, + "org.jetbrains.kotlin:kotlin-stdlib-common": { + "shasums": { + "jar": "a7112c9b3cefee418286c9c9372f7af992bd1e6e030691d52f60cb36dbec8320" + }, + "version": "1.4.20" + }, "org.jetbrains:annotations": { "shasums": { "jar": "2037be378980d3ba9333e97955f3b2cde392aa124d04ca73ce2eee6657199297" }, - "version": "26.0.2-1" + "version": "26.0.2" }, "org.jodd:jodd-util": { "shasums": { @@ -974,9 +1135,9 @@ }, "org.mockito:mockito-core": { "shasums": { - "jar": "d875ff234a4b72e0eedfe170dd804797a4b87ba4c76207c9659d6f46b527877b" + "jar": "d72bf7d23ec19d0513c680bc7ebbaccef0c643af22c82c5ecfd999b2109fc797" }, - "version": "5.19.0" + "version": "5.16.1" }, "org.objenesis:objenesis": { "shasums": { @@ -996,6 +1157,60 @@ }, "version": "1.37" }, + "org.osgi:org.osgi.dto": { + "shasums": { + "jar": "cb75f3c7e48e5a31a31df22e26873346f5bf659e2dcab2369e031e4850d2ff43" + }, + "version": "1.0.0" + }, + "org.osgi:org.osgi.framework": { + "shasums": { + "jar": "ec194b7871af27681716ff05259319a5c3c9b9727e8000e9e832499b93484b4e" + }, + "version": "1.8.0" + }, + "org.osgi:org.osgi.resource": { + "shasums": { + "jar": "81fc50f1f1d38a4af28e131907d4afe213249aab05060484edca0e60c4af9b4a" + }, + "version": "1.0.0" + }, + "org.osgi:org.osgi.service.log": { + "shasums": { + "jar": "ff6710c4856d32684cf3ebdc45248f41036ff734f2b03bbc08c4609a61fecfa0" + }, + "version": "1.3.0" + }, + "org.osgi:org.osgi.service.repository": { + "shasums": { + "jar": "c5553e95b459529192433486d4c4cc22ff45a2eae4968484f9f717319264a532" + }, + "version": "1.1.0" + }, + "org.osgi:org.osgi.util.function": { + "shasums": { + "jar": "208819c7c71690c15a6bb8b187474e7f9d0147946b680182a62b9f222ae014ec" + }, + "version": "1.2.0" + }, + "org.osgi:org.osgi.util.promise": { + "shasums": { + "jar": "fef86e64f584d012a16a0306160764f6179663b90988a226c4641b920f3a4b36" + }, + "version": "1.2.0" + }, + "org.osgi:org.osgi.util.tracker": { + "shasums": { + "jar": "7d78c2cc9bcb6421c24f17aa097866ce8d9115c219a4f8d6cc753bc4dfb97efa" + }, + "version": "1.5.4" + }, + "org.osgi:osgi.annotation": { + "shasums": { + "jar": "a0e8a4c362bd3600812f37b0ea45fba966c7bc049d01fed56a09ecc74082759e" + }, + "version": "8.0.1" + }, "org.ow2.asm:asm": { "shasums": { "jar": "3c6fac2424db3d4a853b669f4e3d1d9c3c552235e19a319673f887083c2303a1" @@ -1034,9 +1249,9 @@ }, "org.projectlombok:lombok": { "shasums": { - "jar": "b764c3a1b0c86748c9e2e80d64e03d45402ce3edeb631fa81e30a56bffa6daf3" + "jar": "73b6b05b6a2d365b700bab08d30f94de9d336490bc0acce5b6181fef48cbf18e" }, - "version": "1.18.40" + "version": "1.18.36" }, "org.reactivestreams:reactive-streams": { "shasums": { @@ -1046,9 +1261,9 @@ }, "org.redisson:redisson": { "shasums": { - "jar": "7cc15d584ad10d399c6e96713d2c6871c638fb362883679273a9eceb82fc834e" + "jar": "3a0bbc732cd52ccd14ad2192b4afa9ee2415e1965373dbe870c70e0bc51596c1" }, - "version": "3.51.0" + "version": "3.45.1" }, "org.reflections:reflections": { "shasums": { @@ -1076,21 +1291,21 @@ }, "org.threeten:threetenbp": { "shasums": { - "jar": "541c224d8dcf76271e538416575bb81b933deffd69e17e2314142786e481ecb8" + "jar": "857917d2319a4e92dc1c5e3aeb75a0dac84445ed315e7ac3d82bb8d2b298977f" }, - "version": "1.7.2" + "version": "1.7.0" }, "org.xerial:sqlite-jdbc": { "shasums": { - "jar": "a3f53a2aa15ae9425a9e793bbe9c8e5288febeb4b65ef5c1a4e80d4c2045cf08" + "jar": "5c8609d2ca341deb8c6f71778974b5ba4995c7d32d7c7c89d9392a3e72c39291" }, - "version": "3.50.3.0" + "version": "3.49.1.0" }, "org.yaml:snakeyaml": { "shasums": { - "jar": "e6682acf1ace77508ef13649cbf4f8d09d2cf5457bdb61d25ffb6ac0233d78dd" + "jar": "ef779af5d29a9dde8cc70ce0341f5c6f7735e23edff9685ceaa9d35359b7bb7f" }, - "version": "2.5" + "version": "2.4" }, "redis.clients:jedis": { "shasums": { @@ -1106,26 +1321,46 @@ } }, "conflict_resolution": { - "com.fasterxml.jackson.core:jackson-annotations:2.10.3": "com.fasterxml.jackson.core:jackson-annotations:2.20", - "com.fasterxml.jackson.core:jackson-annotations:2.19.1": "com.fasterxml.jackson.core:jackson-annotations:2.20", - "com.fasterxml.jackson.core:jackson-core:2.10.3": "com.fasterxml.jackson.core:jackson-core:2.20.0", - "com.fasterxml.jackson.core:jackson-core:2.12.6": "com.fasterxml.jackson.core:jackson-core:2.20.0", - "com.fasterxml.jackson.core:jackson-core:2.19.1": "com.fasterxml.jackson.core:jackson-core:2.20.0", - "com.google.auto.value:auto-value-annotations:1.9": "com.google.auto.value:auto-value-annotations:1.10.4", + "com.fasterxml.jackson.core:jackson-annotations:2.10.3": "com.fasterxml.jackson.core:jackson-annotations:2.18.3", + "com.fasterxml.jackson.core:jackson-annotations:2.18.2": "com.fasterxml.jackson.core:jackson-annotations:2.18.3", + "com.fasterxml.jackson.core:jackson-core:2.10.3": "com.fasterxml.jackson.core:jackson-core:2.18.3", + "com.fasterxml.jackson.core:jackson-core:2.12.6": "com.fasterxml.jackson.core:jackson-core:2.18.3", + "com.fasterxml.jackson.core:jackson-core:2.18.2": "com.fasterxml.jackson.core:jackson-core:2.18.3", "commons-codec:commons-codec:1.11": "commons-codec:commons-codec:1.15", - "commons-codec:commons-codec:1.19.0": "commons-codec:commons-codec:1.15", + "commons-codec:commons-codec:1.17.1": "commons-codec:commons-codec:1.15", "commons-logging:commons-logging:1.2": "commons-logging:commons-logging:1.1.3", - "net.bytebuddy:byte-buddy:1.17.7": "net.bytebuddy:byte-buddy:1.17.6", + "joda-time:joda-time:2.10.10": "joda-time:joda-time:2.8.1", + "net.bytebuddy:byte-buddy:1.15.3": "net.bytebuddy:byte-buddy:1.15.11", "org.apache.httpcomponents:httpcore:4.4.13": "org.apache.httpcomponents:httpcore:4.4.16", "org.javassist:javassist:3.25.0-GA": "org.javassist:javassist:3.28.0-GA", "org.objenesis:objenesis:3.4": "org.objenesis:objenesis:3.3", + "org.osgi:org.osgi.util.function:1.1.0": "org.osgi:org.osgi.util.function:1.2.0", + "org.ow2.asm:asm-analysis:7.2": "org.ow2.asm:asm-analysis:9.7.1", + "org.ow2.asm:asm-commons:7.2": "org.ow2.asm:asm-commons:9.7.1", + "org.ow2.asm:asm-tree:7.2": "org.ow2.asm:asm-tree:9.7.1", + "org.ow2.asm:asm-util:7.2": "org.ow2.asm:asm-util:9.7.1", + "org.ow2.asm:asm:7.2": "org.ow2.asm:asm:9.6", "org.ow2.asm:asm:9.7.1": "org.ow2.asm:asm:9.6", + "org.slf4j:slf4j-api:1.7.25": "org.slf4j:slf4j-api:2.0.9", "org.slf4j:slf4j-api:1.7.30": "org.slf4j:slf4j-api:2.0.9", "org.slf4j:slf4j-api:1.7.32": "org.slf4j:slf4j-api:2.0.9", "org.slf4j:slf4j-api:1.7.36": "org.slf4j:slf4j-api:2.0.9", + "org.slf4j:slf4j-api:2.0.16": "org.slf4j:slf4j-api:2.0.9", "org.slf4j:slf4j-api:2.0.17": "org.slf4j:slf4j-api:2.0.9" }, "dependencies": { + "biz.aQute.bnd:biz.aQute.bndlib": [ + "biz.aQute.bnd:biz.aQute.bnd.util", + "org.osgi:org.osgi.dto", + "org.osgi:org.osgi.framework", + "org.osgi:org.osgi.resource", + "org.osgi:org.osgi.service.log", + "org.osgi:org.osgi.service.repository", + "org.osgi:org.osgi.util.function", + "org.osgi:org.osgi.util.promise", + "org.osgi:org.osgi.util.tracker", + "org.slf4j:slf4j-api" + ], "com.amazonaws:aws-java-sdk-core": [ "com.fasterxml.jackson.core:jackson-databind", "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", @@ -1169,11 +1404,6 @@ "com.fasterxml.jackson.core:jackson-databind", "org.yaml:snakeyaml" ], - "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": [ - "com.fasterxml.jackson.core:jackson-annotations", - "com.fasterxml.jackson.core:jackson-core", - "com.fasterxml.jackson.core:jackson-databind" - ], "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base": [ "com.fasterxml.jackson.core:jackson-core", "com.fasterxml.jackson.core:jackson-databind" @@ -1283,9 +1513,82 @@ "com.google.auto:auto-common": [ "com.google.guava:guava" ], + "com.google.caliper:caliper": [ + "com.google.caliper:caliper-core", + "com.google.caliper:caliper-runner", + "com.google.caliper:caliper-worker-jvm", + "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter", + "com.google.dagger:dagger", + "com.google.guava:guava", + "com.sun.jersey:jersey-client" + ], + "com.google.caliper:caliper-api": [ + "com.google.guava:guava", + "joda-time:joda-time" + ], + "com.google.caliper:caliper-core": [ + "com.google.auto.value:auto-value-annotations", + "com.google.caliper:caliper-api", + "com.google.caliper:caliper-util", + "com.google.dagger:dagger", + "com.google.guava:guava", + "com.squareup.okhttp:okhttp", + "joda-time:joda-time" + ], + "com.google.caliper:caliper-runner": [ + "com.google.caliper:caliper-api", + "com.google.caliper:caliper-core", + "com.google.caliper:caliper-util", + "com.google.dagger:dagger", + "com.google.dagger:dagger-producers", + "com.google.guava:guava", + "com.squareup.okhttp:okhttp", + "joda-time:joda-time" + ], + "com.google.caliper:caliper-util": [ + "com.google.code.gson:gson", + "com.google.dagger:dagger", + "com.google.guava:guava", + "joda-time:joda-time" + ], + "com.google.caliper:caliper-worker": [ + "com.google.caliper:caliper-api", + "com.google.caliper:caliper-core", + "com.google.caliper:caliper-util", + "com.google.dagger:dagger", + "com.google.guava:guava", + "com.squareup.okhttp:okhttp", + "joda-time:joda-time" + ], + "com.google.caliper:caliper-worker-jvm": [ + "com.google.caliper:caliper-core", + "com.google.caliper:caliper-worker", + "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter", + "com.google.dagger:dagger", + "com.google.guava:guava", + "com.sun.jersey:jersey-client" + ], "com.google.code.gson:gson": [ "com.google.errorprone:error_prone_annotations" ], + "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter": [ + "com.google.guava:guava", + "javax.annotation:javax.annotation-api", + "org.ow2.asm:asm", + "org.ow2.asm:asm-analysis", + "org.ow2.asm:asm-commons", + "org.ow2.asm:asm-tree", + "org.ow2.asm:asm-util" + ], + "com.google.dagger:dagger": [ + "javax.inject:javax.inject" + ], + "com.google.dagger:dagger-producers": [ + "com.google.dagger:dagger", + "com.google.guava:guava", + "javax.inject:javax.inject", + "org.checkerframework:checker-compat-qual" + ], "com.google.errorprone:error_prone_annotation": [ "com.google.guava:guava" ], @@ -1325,11 +1628,20 @@ "com.google.j2objc:j2objc-annotations", "org.jspecify:jspecify" ], + "com.google.guava:guava-testlib": [ + "com.google.code.findbugs:jsr305", + "com.google.errorprone:error_prone_annotations", + "com.google.guava:guava", + "com.google.j2objc:j2objc-annotations", + "junit:junit", + "org.checkerframework:checker-qual" + ], "com.google.http-client:google-http-client": [ "com.google.code.findbugs:jsr305", "com.google.errorprone:error_prone_annotations", "com.google.guava:guava", "com.google.j2objc:j2objc-annotations", + "io.grpc:grpc-context", "io.opencensus:opencensus-api", "io.opencensus:opencensus-contrib-http-util", "org.apache.httpcomponents:httpclient", @@ -1361,6 +1673,19 @@ "com.owteam.engUtils:netrc": [ "org.apache.httpcomponents:httpclient" ], + "com.squareup.okhttp:okhttp": [ + "com.squareup.okio:okio" + ], + "com.squareup.okio:okio": [ + "org.jetbrains.kotlin:kotlin-stdlib", + "org.jetbrains.kotlin:kotlin-stdlib-common" + ], + "com.sun.jersey:jersey-client": [ + "com.sun.jersey:jersey-core" + ], + "com.sun.jersey:jersey-core": [ + "javax.ws.rs:jsr311-api" + ], "io.grpc:grpc-api": [ "com.google.code.findbugs:jsr305", "com.google.errorprone:error_prone_annotations", @@ -1398,8 +1723,7 @@ "io.netty:netty-codec-http2", "io.netty:netty-handler-proxy", "io.netty:netty-transport-native-unix-common", - "io.perfmark:perfmark-api", - "org.codehaus.mojo:animal-sniffer-annotations" + "io.perfmark:perfmark-api" ], "io.grpc:grpc-netty-shaded": [ "com.google.errorprone:error_prone_annotations", @@ -1407,8 +1731,7 @@ "io.grpc:grpc-api", "io.grpc:grpc-core", "io.grpc:grpc-util", - "io.perfmark:perfmark-api", - "org.codehaus.mojo:animal-sniffer-annotations" + "io.perfmark:perfmark-api" ], "io.grpc:grpc-protobuf": [ "com.google.api.grpc:proto-google-common-protos", @@ -1427,6 +1750,7 @@ "com.google.code.gson:gson", "com.google.errorprone:error_prone_annotations", "com.google.guava:guava", + "com.google.j2objc:j2objc-annotations", "com.google.protobuf:protobuf-java-util", "io.grpc:grpc-core", "io.grpc:grpc-protobuf", @@ -1436,8 +1760,7 @@ "io.grpc:grpc-stub": [ "com.google.errorprone:error_prone_annotations", "com.google.guava:guava", - "io.grpc:grpc-api", - "org.codehaus.mojo:animal-sniffer-annotations" + "io.grpc:grpc-api" ], "io.grpc:grpc-testing": [ "io.grpc:grpc-api", @@ -1516,6 +1839,44 @@ "io.netty:netty-resolver", "io.netty:netty-transport" ], + "io.netty:netty-tcnative-boringssl-static": [ + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64", + "io.netty:netty-tcnative-classes" + ], + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64": [ + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64" + ], + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64": [ + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64" + ], + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64": [ + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64" + ], + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64": [ + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64" + ], + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64": [ + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64" + ], "io.netty:netty-transport": [ "io.netty:netty-buffer", "io.netty:netty-common", @@ -1569,6 +1930,10 @@ "io.opencensus:opencensus-api": [ "io.grpc:grpc-context" ], + "io.opencensus:opencensus-contrib-grpc-metrics": [ + "com.google.guava:guava", + "io.opencensus:opencensus-api" + ], "io.opencensus:opencensus-contrib-http-util": [ "com.google.guava:guava", "io.opencensus:opencensus-api" @@ -1681,6 +2046,10 @@ "org.glassfish.jersey.core:jersey-common", "org.javassist:javassist" ], + "org.jetbrains.kotlin:kotlin-stdlib": [ + "org.jetbrains.kotlin:kotlin-stdlib-common", + "org.jetbrains:annotations" + ], "org.mockito:mockito-core": [ "net.bytebuddy:byte-buddy", "net.bytebuddy:byte-buddy-agent", @@ -1693,6 +2062,16 @@ "org.openjdk.jmh:jmh-generator-annprocess": [ "org.openjdk.jmh:jmh-core" ], + "org.osgi:org.osgi.util.function": [ + "org.osgi:osgi.annotation" + ], + "org.osgi:org.osgi.util.promise": [ + "org.osgi:org.osgi.util.function", + "org.osgi:osgi.annotation" + ], + "org.osgi:org.osgi.util.tracker": [ + "org.osgi:osgi.annotation" + ], "org.ow2.asm:asm-analysis": [ "org.ow2.asm:asm-tree" ], @@ -1714,7 +2093,6 @@ "com.fasterxml.jackson.core:jackson-core", "com.fasterxml.jackson.core:jackson-databind", "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", - "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", "io.netty:netty-buffer", "io.netty:netty-codec", "io.netty:netty-common", @@ -1749,6 +2127,161 @@ ] }, "packages": { + "biz.aQute.bnd:biz.aQute.bnd.util": [ + "aQute.bnd.classfile", + "aQute.bnd.classfile.builder", + "aQute.bnd.classfile.preview", + "aQute.bnd.exceptions", + "aQute.bnd.memoize", + "aQute.bnd.result", + "aQute.bnd.signatures", + "aQute.bnd.stream", + "aQute.bnd.unmodifiable", + "aQute.lib.io", + "aQute.lib.stringrover", + "aQute.libg.glob" + ], + "biz.aQute.bnd:biz.aQute.bndlib": [ + "aQute.bnd.annotation", + "aQute.bnd.annotation.baseline", + "aQute.bnd.annotation.component", + "aQute.bnd.annotation.headers", + "aQute.bnd.annotation.jpms", + "aQute.bnd.annotation.licenses", + "aQute.bnd.annotation.metatype", + "aQute.bnd.annotation.plugin", + "aQute.bnd.annotation.service", + "aQute.bnd.annotation.spi", + "aQute.bnd.annotation.xml", + "aQute.bnd.apiguardian.api", + "aQute.bnd.aspectj.plugin", + "aQute.bnd.build", + "aQute.bnd.build.api", + "aQute.bnd.build.model", + "aQute.bnd.build.model.clauses", + "aQute.bnd.build.model.conversions", + "aQute.bnd.buildtool", + "aQute.bnd.bundle.annotations", + "aQute.bnd.cdi", + "aQute.bnd.classindex", + "aQute.bnd.compatibility", + "aQute.bnd.component", + "aQute.bnd.component.annotations", + "aQute.bnd.component.error", + "aQute.bnd.connection.settings", + "aQute.bnd.differ", + "aQute.bnd.exporter.executable", + "aQute.bnd.exporter.runbundles", + "aQute.bnd.filerepo", + "aQute.bnd.header", + "aQute.bnd.help", + "aQute.bnd.help.instructions", + "aQute.bnd.http", + "aQute.bnd.junit", + "aQute.bnd.make", + "aQute.bnd.make.calltree", + "aQute.bnd.make.component", + "aQute.bnd.make.coverage", + "aQute.bnd.make.metatype", + "aQute.bnd.maven", + "aQute.bnd.maven.support", + "aQute.bnd.metatype", + "aQute.bnd.metatype.annotations", + "aQute.bnd.obr", + "aQute.bnd.osgi", + "aQute.bnd.osgi.eclipse", + "aQute.bnd.osgi.repository", + "aQute.bnd.osgi.resource", + "aQute.bnd.plugin.ant", + "aQute.bnd.plugin.eclipse", + "aQute.bnd.plugin.git", + "aQute.bnd.plugin.gradle", + "aQute.bnd.plugin.jpms", + "aQute.bnd.plugin.maven", + "aQute.bnd.plugin.spi", + "aQute.bnd.print", + "aQute.bnd.properties", + "aQute.bnd.remoteworkspace.client", + "aQute.bnd.remoteworkspace.server", + "aQute.bnd.resource.repository", + "aQute.bnd.service", + "aQute.bnd.service.action", + "aQute.bnd.service.classparser", + "aQute.bnd.service.clipboard", + "aQute.bnd.service.diff", + "aQute.bnd.service.export", + "aQute.bnd.service.extension", + "aQute.bnd.service.externalplugin", + "aQute.bnd.service.generate", + "aQute.bnd.service.library", + "aQute.bnd.service.lifecycle", + "aQute.bnd.service.maven", + "aQute.bnd.service.message", + "aQute.bnd.service.progress", + "aQute.bnd.service.release", + "aQute.bnd.service.remotelaunch", + "aQute.bnd.service.remoteworkspace", + "aQute.bnd.service.reporter", + "aQute.bnd.service.repository", + "aQute.bnd.service.resolve.hook", + "aQute.bnd.service.specifications", + "aQute.bnd.service.url", + "aQute.bnd.service.verifier", + "aQute.bnd.signing", + "aQute.bnd.url", + "aQute.bnd.util.dto", + "aQute.bnd.util.home", + "aQute.bnd.util.repository", + "aQute.bnd.version", + "aQute.bnd.version.maven", + "aQute.bnd.xmlattribute", + "aQute.lib.aspects", + "aQute.lib.base64", + "aQute.lib.collections", + "aQute.lib.concurrent.serial", + "aQute.lib.concurrentinit", + "aQute.lib.converter", + "aQute.lib.date", + "aQute.lib.deployer", + "aQute.lib.fileset", + "aQute.lib.filter", + "aQute.lib.formatter", + "aQute.lib.hex", + "aQute.lib.hierarchy", + "aQute.lib.io", + "aQute.lib.json", + "aQute.lib.link", + "aQute.lib.manifest", + "aQute.lib.mavenpasswordobfuscator", + "aQute.lib.persistentmap", + "aQute.lib.redirect", + "aQute.lib.settings", + "aQute.lib.specinterface", + "aQute.lib.spring", + "aQute.lib.stringrover", + "aQute.lib.strings", + "aQute.lib.tag", + "aQute.lib.utf8properties", + "aQute.lib.watcher", + "aQute.lib.xml", + "aQute.lib.xpath", + "aQute.lib.zip", + "aQute.libg.command", + "aQute.libg.cryptography", + "aQute.libg.filelock", + "aQute.libg.generics", + "aQute.libg.glob", + "aQute.libg.gzip", + "aQute.libg.ints", + "aQute.libg.map", + "aQute.libg.qtokens", + "aQute.libg.reporter", + "aQute.libg.reporter.slf4j", + "aQute.libg.sed", + "aQute.libg.tuple", + "aQute.libg.uri", + "aQute.service.reporter" + ], "com.amazonaws:aws-java-sdk-core": [ "com.amazonaws", "com.amazonaws.adapters.types", @@ -1888,9 +2421,7 @@ "com.fasterxml.jackson.core.exc", "com.fasterxml.jackson.core.filter", "com.fasterxml.jackson.core.format", - "com.fasterxml.jackson.core.internal.shaded.fdp.v2_20_0", - "com.fasterxml.jackson.core.internal.shaded.fdp.v2_20_0.bte", - "com.fasterxml.jackson.core.internal.shaded.fdp.v2_20_0.chr", + "com.fasterxml.jackson.core.internal.shaded.fdp.v2_18_3", "com.fasterxml.jackson.core.io", "com.fasterxml.jackson.core.io.schubfach", "com.fasterxml.jackson.core.json", @@ -1933,14 +2464,6 @@ "com.fasterxml.jackson.dataformat.yaml.snakeyaml.error", "com.fasterxml.jackson.dataformat.yaml.util" ], - "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": [ - "com.fasterxml.jackson.datatype.jsr310", - "com.fasterxml.jackson.datatype.jsr310.deser", - "com.fasterxml.jackson.datatype.jsr310.deser.key", - "com.fasterxml.jackson.datatype.jsr310.ser", - "com.fasterxml.jackson.datatype.jsr310.ser.key", - "com.fasterxml.jackson.datatype.jsr310.util" - ], "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base": [ "com.fasterxml.jackson.jaxrs.annotation", "com.fasterxml.jackson.jaxrs.base", @@ -2184,7 +2707,6 @@ ], "com.google.api.grpc:proto-google-common-protos": [ "com.google.api", - "com.google.apps.card.v1", "com.google.cloud", "com.google.cloud.audit", "com.google.cloud.location", @@ -2193,7 +2715,6 @@ "com.google.longrunning", "com.google.rpc", "com.google.rpc.context", - "com.google.shopping.type", "com.google.type" ], "com.google.auth:google-auth-library-credentials": [ @@ -2206,6 +2727,44 @@ "com.google.auto.service:auto-service-annotations": [ "com.google.auto.service" ], + "com.google.auto.value:auto-value": [ + "autovalue.shaded.com.google.auto.common", + "autovalue.shaded.com.google.auto.service", + "autovalue.shaded.com.google.common.annotations", + "autovalue.shaded.com.google.common.base", + "autovalue.shaded.com.google.common.cache", + "autovalue.shaded.com.google.common.collect", + "autovalue.shaded.com.google.common.escape", + "autovalue.shaded.com.google.common.eventbus", + "autovalue.shaded.com.google.common.graph", + "autovalue.shaded.com.google.common.hash", + "autovalue.shaded.com.google.common.html", + "autovalue.shaded.com.google.common.io", + "autovalue.shaded.com.google.common.math", + "autovalue.shaded.com.google.common.net", + "autovalue.shaded.com.google.common.primitives", + "autovalue.shaded.com.google.common.reflect", + "autovalue.shaded.com.google.common.util.concurrent", + "autovalue.shaded.com.google.common.xml", + "autovalue.shaded.com.google.errorprone.annotations", + "autovalue.shaded.com.google.errorprone.annotations.concurrent", + "autovalue.shaded.com.google.escapevelocity", + "autovalue.shaded.com.google.j2objc.annotations", + "autovalue.shaded.com.squareup.javapoet", + "autovalue.shaded.net.ltgt.gradle.incap", + "autovalue.shaded.org.checkerframework.checker.nullness.qual", + "autovalue.shaded.org.checkerframework.framework.qual", + "autovalue.shaded.org.objectweb.asm", + "com.google.auto.value.extension", + "com.google.auto.value.extension.memoized.processor", + "com.google.auto.value.extension.serializable.processor", + "com.google.auto.value.extension.serializable.serializer", + "com.google.auto.value.extension.serializable.serializer.impl", + "com.google.auto.value.extension.serializable.serializer.interfaces", + "com.google.auto.value.extension.serializable.serializer.runtime", + "com.google.auto.value.extension.toprettystring.processor", + "com.google.auto.value.processor" + ], "com.google.auto.value:auto-value-annotations": [ "com.google.auto.value", "com.google.auto.value.extension.memoized", @@ -2215,6 +2774,48 @@ "com.google.auto:auto-common": [ "com.google.auto.common" ], + "com.google.caliper:caliper": [ + "com.google.caliper.runner", + "com.google.caliper.runner.instrument", + "com.google.caliper.runner.resultprocessor" + ], + "com.google.caliper:caliper-api": [ + "com.google.caliper", + "com.google.caliper.api", + "com.google.caliper.model" + ], + "com.google.caliper:caliper-core": [ + "com.google.caliper.bridge", + "com.google.caliper.core", + "com.google.caliper.memory" + ], + "com.google.caliper:caliper-runner": [ + "com.google.caliper.runner", + "com.google.caliper.runner.config", + "com.google.caliper.runner.experiment", + "com.google.caliper.runner.instrument", + "com.google.caliper.runner.options", + "com.google.caliper.runner.resultprocessor", + "com.google.caliper.runner.server", + "com.google.caliper.runner.target", + "com.google.caliper.runner.worker", + "com.google.caliper.runner.worker.dryrun", + "com.google.caliper.runner.worker.targetinfo", + "com.google.caliper.runner.worker.trial" + ], + "com.google.caliper:caliper-util": [ + "com.google.caliper.json", + "com.google.caliper.util" + ], + "com.google.caliper:caliper-worker": [ + "com.google.caliper.worker", + "com.google.caliper.worker.connection", + "com.google.caliper.worker.handler", + "com.google.caliper.worker.instrument" + ], + "com.google.caliper:caliper-worker-jvm": [ + "com.google.caliper.worker" + ], "com.google.code.findbugs:jsr305": [ "javax.annotation", "javax.annotation.concurrent", @@ -2231,6 +2832,48 @@ "com.google.gson.reflect", "com.google.gson.stream" ], + "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter": [ + "com.google.monitoring.runtime.instrumentation", + "com.google.monitoring.runtime.instrumentation.asm", + "com.google.monitoring.runtime.instrumentation.asm.commons", + "com.google.monitoring.runtime.instrumentation.asm.signature", + "com.google.monitoring.runtime.instrumentation.asm.tree", + "com.google.monitoring.runtime.instrumentation.asm.tree.analysis", + "com.google.monitoring.runtime.instrumentation.asm.util", + "com.google.monitoring.runtime.instrumentation.checker.nullness.compatqual", + "com.google.monitoring.runtime.instrumentation.common.annotations", + "com.google.monitoring.runtime.instrumentation.common.base", + "com.google.monitoring.runtime.instrumentation.common.base.internal", + "com.google.monitoring.runtime.instrumentation.common.cache", + "com.google.monitoring.runtime.instrumentation.common.collect", + "com.google.monitoring.runtime.instrumentation.common.escape", + "com.google.monitoring.runtime.instrumentation.common.eventbus", + "com.google.monitoring.runtime.instrumentation.common.graph", + "com.google.monitoring.runtime.instrumentation.common.hash", + "com.google.monitoring.runtime.instrumentation.common.html", + "com.google.monitoring.runtime.instrumentation.common.io", + "com.google.monitoring.runtime.instrumentation.common.math", + "com.google.monitoring.runtime.instrumentation.common.net", + "com.google.monitoring.runtime.instrumentation.common.primitives", + "com.google.monitoring.runtime.instrumentation.common.reflect", + "com.google.monitoring.runtime.instrumentation.common.util.concurrent", + "com.google.monitoring.runtime.instrumentation.common.xml", + "com.google.monitoring.runtime.instrumentation.errorprone.annotations", + "com.google.monitoring.runtime.instrumentation.errorprone.annotations.concurrent", + "com.google.monitoring.runtime.instrumentation.j2objc.annotations", + "com.google.thirdparty.publicsuffix" + ], + "com.google.dagger:dagger": [ + "dagger", + "dagger.internal", + "dagger.multibindings" + ], + "com.google.dagger:dagger-producers": [ + "dagger.producers", + "dagger.producers.internal", + "dagger.producers.monitoring", + "dagger.producers.monitoring.internal" + ], "com.google.errorprone:error_prone_annotation": [ "com.google.errorprone" ], @@ -2282,6 +2925,7 @@ "com.google.googlejavaformat", "com.google.googlejavaformat.java", "com.google.googlejavaformat.java.filer", + "com.google.googlejavaformat.java.java17", "com.google.googlejavaformat.java.java21", "com.google.googlejavaformat.java.javadoc" ], @@ -2308,6 +2952,16 @@ "com.google.common.xml", "com.google.thirdparty.publicsuffix" ], + "com.google.guava:guava-testlib": [ + "com.google.common.collect.testing", + "com.google.common.collect.testing.features", + "com.google.common.collect.testing.google", + "com.google.common.collect.testing.suites", + "com.google.common.collect.testing.testers", + "com.google.common.escape.testing", + "com.google.common.testing", + "com.google.common.util.concurrent.testing" + ], "com.google.http-client:google-http-client": [ "com.google.api.client.http", "com.google.api.client.http.apache", @@ -2342,6 +2996,9 @@ "com.google.protobuf:protobuf-java-util": [ "com.google.protobuf.util" ], + "com.google.re2j:re2j": [ + "com.google.re2j" + ], "com.google.truth:truth": [ "com.google.common.truth" ], @@ -2354,11 +3011,59 @@ "com.owteam.engUtils:netrc": [ "com.owteam.engUtils.netrc" ], + "com.squareup.okhttp:okhttp": [ + "com.squareup.okhttp", + "com.squareup.okhttp.internal", + "com.squareup.okhttp.internal.framed", + "com.squareup.okhttp.internal.http", + "com.squareup.okhttp.internal.io", + "com.squareup.okhttp.internal.tls" + ], + "com.squareup.okio:okio": [ + "okio", + "okio.internal" + ], "com.sun.activation:jakarta.activation": [ "com.sun.activation.registries", "com.sun.activation.viewers", "javax.activation" ], + "com.sun.jersey:jersey-client": [ + "com.sun.jersey.api.client", + "com.sun.jersey.api.client.async", + "com.sun.jersey.api.client.config", + "com.sun.jersey.api.client.filter", + "com.sun.jersey.client.impl", + "com.sun.jersey.client.impl.async", + "com.sun.jersey.client.proxy", + "com.sun.jersey.client.urlconnection", + "com.sun.ws.rs.ext" + ], + "com.sun.jersey:jersey-core": [ + "com.sun.jersey.api.provider.jaxb", + "com.sun.jersey.api.representation", + "com.sun.jersey.api.uri", + "com.sun.jersey.core.header", + "com.sun.jersey.core.header.reader", + "com.sun.jersey.core.impl.provider.entity", + "com.sun.jersey.core.impl.provider.header", + "com.sun.jersey.core.impl.provider.xml", + "com.sun.jersey.core.osgi", + "com.sun.jersey.core.provider", + "com.sun.jersey.core.provider.jaxb", + "com.sun.jersey.core.reflection", + "com.sun.jersey.core.spi.component", + "com.sun.jersey.core.spi.component.ioc", + "com.sun.jersey.core.spi.factory", + "com.sun.jersey.core.spi.scanning", + "com.sun.jersey.core.spi.scanning.uri", + "com.sun.jersey.core.util", + "com.sun.jersey.impl", + "com.sun.jersey.localization", + "com.sun.jersey.spi", + "com.sun.jersey.spi.inject", + "com.sun.jersey.spi.service" + ], "commons-codec:commons-codec": [ "org.apache.commons.codec", "org.apache.commons.codec.binary", @@ -2389,6 +3094,9 @@ "org.apache.commons.logging", "org.apache.commons.logging.impl" ], + "info.picocli:picocli": [ + "picocli" + ], "io.github.eisop:dataflow-errorprone": [ "org.checkerframework.dataflow.qual", "org.checkerframework.errorprone.checker.builder.qual", @@ -2543,12 +3251,8 @@ "io.grpc.netty.shaded.io.netty.util.concurrent", "io.grpc.netty.shaded.io.netty.util.internal", "io.grpc.netty.shaded.io.netty.util.internal.logging", - "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.counters", - "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.maps", "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues", "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic", - "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.atomic.unpadded", - "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.unpadded", "io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util", "io.grpc.netty.shaded.io.netty.util.internal.svm" ], @@ -2564,7 +3268,6 @@ "io.grpc.health.v1", "io.grpc.protobuf.services", "io.grpc.protobuf.services.internal", - "io.grpc.reflection.v1", "io.grpc.reflection.v1alpha", "io.grpc.services" ], @@ -2652,6 +3355,9 @@ "io.netty:netty-resolver-dns": [ "io.netty.resolver.dns" ], + "io.netty:netty-tcnative-classes": [ + "io.netty.internal.tcnative" + ], "io.netty:netty-transport": [ "io.netty.bootstrap", "io.netty.channel", @@ -2694,6 +3400,9 @@ "io.opencensus.trace.samplers", "io.opencensus.trace.unsafe" ], + "io.opencensus:opencensus-contrib-grpc-metrics": [ + "io.opencensus.contrib.grpc.metrics" + ], "io.opencensus:opencensus-contrib-http-util": [ "io.opencensus.contrib.http", "io.opencensus.contrib.http.util" @@ -2813,6 +3522,11 @@ "javax.inject:javax.inject": [ "javax.inject" ], + "javax.ws.rs:jsr311-api": [ + "javax.ws.rs", + "javax.ws.rs.core", + "javax.ws.rs.ext" + ], "joda-time:joda-time": [ "org.joda.time", "org.joda.time.base", @@ -2891,7 +3605,6 @@ "net.bytebuddy.jar.asm", "net.bytebuddy.jar.asm.commons", "net.bytebuddy.jar.asm.signature", - "net.bytebuddy.jar.asmjdkbridge", "net.bytebuddy.matcher", "net.bytebuddy.pool", "net.bytebuddy.utility", @@ -3556,6 +4269,9 @@ "org.bouncycastle.oer.its.template.ieee1609dot2.basetypes", "org.bouncycastle.oer.its.template.ieee1609dot2dot1" ], + "org.checkerframework:checker-compat-qual": [ + "org.checkerframework.checker.nullness.compatqual" + ], "org.checkerframework:checker-qual": [ "org.checkerframework.checker.builder.qual", "org.checkerframework.checker.calledmethods.qual", @@ -3694,6 +4410,38 @@ "javassist.util", "javassist.util.proxy" ], + "org.jetbrains.kotlin:kotlin-stdlib": [ + "kotlin", + "kotlin.annotation", + "kotlin.collections", + "kotlin.collections.builders", + "kotlin.collections.unsigned", + "kotlin.comparisons", + "kotlin.concurrent", + "kotlin.contracts", + "kotlin.coroutines", + "kotlin.coroutines.cancellation", + "kotlin.coroutines.intrinsics", + "kotlin.coroutines.jvm.internal", + "kotlin.experimental", + "kotlin.internal", + "kotlin.io", + "kotlin.js", + "kotlin.jvm", + "kotlin.jvm.functions", + "kotlin.jvm.internal", + "kotlin.jvm.internal.markers", + "kotlin.jvm.internal.unsafe", + "kotlin.math", + "kotlin.properties", + "kotlin.random", + "kotlin.ranges", + "kotlin.reflect", + "kotlin.sequences", + "kotlin.system", + "kotlin.text", + "kotlin.time" + ], "org.jetbrains:annotations": [ "org.intellij.lang.annotations", "org.jetbrains.annotations" @@ -3827,6 +4575,46 @@ "org.openjdk.jmh.generators", "org.openjdk.jmh.generators.annotations" ], + "org.osgi:org.osgi.dto": [ + "org.osgi.dto" + ], + "org.osgi:org.osgi.framework": [ + "org.osgi.framework", + "org.osgi.framework.dto", + "org.osgi.framework.hooks.bundle", + "org.osgi.framework.hooks.resolver", + "org.osgi.framework.hooks.service", + "org.osgi.framework.hooks.weaving", + "org.osgi.framework.launch", + "org.osgi.framework.namespace", + "org.osgi.framework.startlevel", + "org.osgi.framework.startlevel.dto", + "org.osgi.framework.wiring", + "org.osgi.framework.wiring.dto" + ], + "org.osgi:org.osgi.resource": [ + "org.osgi.resource", + "org.osgi.resource.dto" + ], + "org.osgi:org.osgi.service.log": [ + "org.osgi.service.log" + ], + "org.osgi:org.osgi.service.repository": [ + "org.osgi.service.repository" + ], + "org.osgi:org.osgi.util.function": [ + "org.osgi.util.function" + ], + "org.osgi:org.osgi.util.promise": [ + "org.osgi.util.promise" + ], + "org.osgi:org.osgi.util.tracker": [ + "org.osgi.util.tracker" + ], + "org.osgi:osgi.annotation": [ + "org.osgi.annotation.bundle", + "org.osgi.annotation.versioning" + ], "org.ow2.asm:asm": [ "org.objectweb.asm", "org.objectweb.asm.signature" @@ -3869,23 +4657,19 @@ "org.redisson.api.annotation", "org.redisson.api.condition", "org.redisson.api.executor", - "org.redisson.api.fanout", "org.redisson.api.geo", - "org.redisson.api.keys", "org.redisson.api.listener", "org.redisson.api.map", "org.redisson.api.map.event", "org.redisson.api.mapreduce", "org.redisson.api.options", "org.redisson.api.queue", - "org.redisson.api.queue.event", "org.redisson.api.redisnode", "org.redisson.api.search", "org.redisson.api.search.aggregate", "org.redisson.api.search.index", "org.redisson.api.search.query", "org.redisson.api.stream", - "org.redisson.api.vector", "org.redisson.cache", "org.redisson.client", "org.redisson.client.codec", @@ -4037,6 +4821,8 @@ }, "repositories": { "https://repo.maven.apache.org/maven2/": [ + "biz.aQute.bnd:biz.aQute.bnd.util", + "biz.aQute.bnd:biz.aQute.bndlib", "com.amazonaws:aws-java-sdk-core", "com.amazonaws:aws-java-sdk-kms", "com.amazonaws:aws-java-sdk-s3", @@ -4050,7 +4836,6 @@ "com.fasterxml.jackson.core:jackson-databind", "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", - "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", "com.fasterxml.jackson.module:jackson-module-jaxb-annotations", @@ -4079,10 +4864,21 @@ "com.google.auth:google-auth-library-credentials", "com.google.auth:google-auth-library-oauth2-http", "com.google.auto.service:auto-service-annotations", + "com.google.auto.value:auto-value", "com.google.auto.value:auto-value-annotations", "com.google.auto:auto-common", + "com.google.caliper:caliper", + "com.google.caliper:caliper-api", + "com.google.caliper:caliper-core", + "com.google.caliper:caliper-runner", + "com.google.caliper:caliper-util", + "com.google.caliper:caliper-worker", + "com.google.caliper:caliper-worker-jvm", "com.google.code.findbugs:jsr305", "com.google.code.gson:gson", + "com.google.code.java-allocation-instrumenter:java-allocation-instrumenter", + "com.google.dagger:dagger", + "com.google.dagger:dagger-producers", "com.google.errorprone:error_prone_annotation", "com.google.errorprone:error_prone_annotations", "com.google.errorprone:error_prone_check_api", @@ -4090,6 +4886,7 @@ "com.google.googlejavaformat:google-java-format", "com.google.guava:failureaccess", "com.google.guava:guava", + "com.google.guava:guava-testlib", "com.google.guava:listenablefuture", "com.google.http-client:google-http-client", "com.google.http-client:google-http-client-gson", @@ -4097,14 +4894,20 @@ "com.google.jimfs:jimfs", "com.google.protobuf:protobuf-java", "com.google.protobuf:protobuf-java-util", + "com.google.re2j:re2j", "com.google.truth:truth", "com.kohlschutter.junixsocket:junixsocket-common", "com.kohlschutter.junixsocket:junixsocket-native-common", "com.owteam.engUtils:netrc", + "com.squareup.okhttp:okhttp", + "com.squareup.okio:okio", "com.sun.activation:jakarta.activation", + "com.sun.jersey:jersey-client", + "com.sun.jersey:jersey-core", "commons-codec:commons-codec", "commons-io:commons-io", "commons-logging:commons-logging", + "info.picocli:picocli", "io.github.eisop:dataflow-errorprone", "io.github.java-diff-utils:java-diff-utils", "io.grpc:grpc-api", @@ -4131,6 +4934,13 @@ "io.netty:netty-handler-proxy", "io.netty:netty-resolver", "io.netty:netty-resolver-dns", + "io.netty:netty-tcnative-boringssl-static", + "io.netty:netty-tcnative-boringssl-static:jar:linux-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:linux-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-aarch_64", + "io.netty:netty-tcnative-boringssl-static:jar:osx-x86_64", + "io.netty:netty-tcnative-boringssl-static:jar:windows-x86_64", + "io.netty:netty-tcnative-classes", "io.netty:netty-transport", "io.netty:netty-transport-classes-epoll", "io.netty:netty-transport-classes-kqueue", @@ -4140,6 +4950,7 @@ "io.netty:netty-transport-native-kqueue:jar:osx-x86_64", "io.netty:netty-transport-native-unix-common", "io.opencensus:opencensus-api", + "io.opencensus:opencensus-contrib-grpc-metrics", "io.opencensus:opencensus-contrib-http-util", "io.perfmark:perfmark-api", "io.projectreactor:reactor-core", @@ -4158,6 +4969,7 @@ "javax.annotation:javax.annotation-api", "javax.cache:cache-api", "javax.inject:javax.inject", + "javax.ws.rs:jsr311-api", "joda-time:joda-time", "junit:junit", "me.dinowernli:java-grpc-prometheus", @@ -4182,6 +4994,7 @@ "org.bouncycastle:bcprov-jdk15on", "org.bouncycastle:bcprov-jdk18on", "org.bouncycastle:bcutil-jdk18on", + "org.checkerframework:checker-compat-qual", "org.checkerframework:checker-qual", "org.codehaus.mojo:animal-sniffer-annotations", "org.glassfish.hk2.external:aopalliance-repackaged", @@ -4196,6 +5009,8 @@ "org.glassfish.jersey.inject:jersey-hk2", "org.hamcrest:hamcrest-core", "org.javassist:javassist", + "org.jetbrains.kotlin:kotlin-stdlib", + "org.jetbrains.kotlin:kotlin-stdlib-common", "org.jetbrains:annotations", "org.jodd:jodd-util", "org.json:json", @@ -4205,6 +5020,15 @@ "org.objenesis:objenesis", "org.openjdk.jmh:jmh-core", "org.openjdk.jmh:jmh-generator-annprocess", + "org.osgi:org.osgi.dto", + "org.osgi:org.osgi.framework", + "org.osgi:org.osgi.resource", + "org.osgi:org.osgi.service.log", + "org.osgi:org.osgi.service.repository", + "org.osgi:org.osgi.util.function", + "org.osgi:org.osgi.util.promise", + "org.osgi:org.osgi.util.tracker", + "org.osgi:osgi.annotation", "org.ow2.asm:asm", "org.ow2.asm:asm-analysis", "org.ow2.asm:asm-commons", @@ -4249,11 +5073,6 @@ "com.fasterxml.jackson.dataformat.yaml.YAMLMapper" ] }, - "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": { - "com.fasterxml.jackson.databind.Module": [ - "com.fasterxml.jackson.datatype.jsr310.JavaTimeModule" - ] - }, "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider": { "javax.ws.rs.ext.MessageBodyReader": [ "com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider" @@ -4267,6 +5086,27 @@ "com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule" ] }, + "com.google.auto.value:auto-value": { + "com.google.auto.value.extension.AutoValueExtension": [ + "com.google.auto.value.extension.memoized.processor.MemoizeExtension", + "com.google.auto.value.extension.serializable.processor.SerializableAutoValueExtension", + "com.google.auto.value.extension.toprettystring.processor.ToPrettyStringExtension" + ], + "com.google.auto.value.extension.serializable.serializer.interfaces.SerializerExtension": [ + "com.google.auto.value.extension.serializable.serializer.impl.ImmutableListSerializerExtension", + "com.google.auto.value.extension.serializable.serializer.impl.ImmutableMapSerializerExtension", + "com.google.auto.value.extension.serializable.serializer.impl.OptionalSerializerExtension" + ], + "javax.annotation.processing.Processor": [ + "com.google.auto.value.extension.memoized.processor.MemoizedValidator", + "com.google.auto.value.extension.toprettystring.processor.ToPrettyStringValidator", + "com.google.auto.value.processor.AutoAnnotationProcessor", + "com.google.auto.value.processor.AutoBuilderProcessor", + "com.google.auto.value.processor.AutoOneOfProcessor", + "com.google.auto.value.processor.AutoValueBuilderProcessor", + "com.google.auto.value.processor.AutoValueProcessor" + ] + }, "com.google.errorprone:error_prone_core": { "com.sun.source.util.Plugin": [ "com.google.errorprone.ErrorProneJavacPlugin" @@ -4285,6 +5125,78 @@ "com.google.common.jimfs.SystemJimfsFileSystemProvider" ] }, + "com.sun.jersey:jersey-core": { + "com.sun.jersey.spi.HeaderDelegateProvider": [ + "com.sun.jersey.core.impl.provider.header.CacheControlProvider", + "com.sun.jersey.core.impl.provider.header.CookieProvider", + "com.sun.jersey.core.impl.provider.header.DateProvider", + "com.sun.jersey.core.impl.provider.header.EntityTagProvider", + "com.sun.jersey.core.impl.provider.header.LocaleProvider", + "com.sun.jersey.core.impl.provider.header.MediaTypeProvider", + "com.sun.jersey.core.impl.provider.header.NewCookieProvider", + "com.sun.jersey.core.impl.provider.header.StringProvider", + "com.sun.jersey.core.impl.provider.header.URIProvider" + ], + "com.sun.jersey.spi.inject.InjectableProvider": [ + "com.sun.jersey.core.impl.provider.xml.DocumentBuilderFactoryProvider", + "com.sun.jersey.core.impl.provider.xml.SAXParserContextProvider", + "com.sun.jersey.core.impl.provider.xml.TransformerFactoryProvider", + "com.sun.jersey.core.impl.provider.xml.XMLStreamReaderContextProvider" + ], + "javax.ws.rs.ext.MessageBodyReader": [ + "com.sun.jersey.core.impl.provider.entity.ByteArrayProvider", + "com.sun.jersey.core.impl.provider.entity.DataSourceProvider", + "com.sun.jersey.core.impl.provider.entity.DocumentProvider", + "com.sun.jersey.core.impl.provider.entity.EntityHolderReader", + "com.sun.jersey.core.impl.provider.entity.FileProvider", + "com.sun.jersey.core.impl.provider.entity.FormMultivaluedMapProvider", + "com.sun.jersey.core.impl.provider.entity.FormProvider", + "com.sun.jersey.core.impl.provider.entity.InputStreamProvider", + "com.sun.jersey.core.impl.provider.entity.MimeMultipartProvider", + "com.sun.jersey.core.impl.provider.entity.ReaderProvider", + "com.sun.jersey.core.impl.provider.entity.RenderedImageProvider", + "com.sun.jersey.core.impl.provider.entity.SourceProvider$DOMSourceReader", + "com.sun.jersey.core.impl.provider.entity.SourceProvider$SAXSourceReader", + "com.sun.jersey.core.impl.provider.entity.SourceProvider$StreamSourceReader", + "com.sun.jersey.core.impl.provider.entity.StringProvider", + "com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$Text", + "com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$Text", + "com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$Text", + "com.sun.jersey.core.impl.provider.entity.XMLRootObjectProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLRootObjectProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLRootObjectProvider$Text" + ], + "javax.ws.rs.ext.MessageBodyWriter": [ + "com.sun.jersey.core.impl.provider.entity.ByteArrayProvider", + "com.sun.jersey.core.impl.provider.entity.DataSourceProvider", + "com.sun.jersey.core.impl.provider.entity.DocumentProvider", + "com.sun.jersey.core.impl.provider.entity.FileProvider", + "com.sun.jersey.core.impl.provider.entity.FormMultivaluedMapProvider", + "com.sun.jersey.core.impl.provider.entity.FormProvider", + "com.sun.jersey.core.impl.provider.entity.InputStreamProvider", + "com.sun.jersey.core.impl.provider.entity.MimeMultipartProvider", + "com.sun.jersey.core.impl.provider.entity.ReaderProvider", + "com.sun.jersey.core.impl.provider.entity.RenderedImageProvider", + "com.sun.jersey.core.impl.provider.entity.SourceProvider$SourceWriter", + "com.sun.jersey.core.impl.provider.entity.StreamingOutputProvider", + "com.sun.jersey.core.impl.provider.entity.StringProvider", + "com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider$Text", + "com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLListElementProvider$Text", + "com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$App", + "com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$General", + "com.sun.jersey.core.impl.provider.entity.XMLRootElementProvider$Text" + ] + }, "io.grpc:grpc-core": { "io.grpc.LoadBalancerProvider": [ "io.grpc.internal.PickFirstLoadBalancerProvider" diff --git a/persistentworkers/examples/src/main/java/BUILD b/persistentworkers/examples/src/main/java/BUILD index eec58282be..73042580bf 100644 --- a/persistentworkers/examples/src/main/java/BUILD +++ b/persistentworkers/examples/src/main/java/BUILD @@ -8,7 +8,7 @@ java_library( deps = [ "//persistentworkers/src/main/java/persistent/bazel:bazel-persistent-workers", "//persistentworkers/src/main/java/persistent/common:persistent-common", - "@buildfarm_maven//:com_google_guava_guava", + "@maven//:com_google_guava_guava", ], ) diff --git a/persistentworkers/examples/src/test/java/BUILD b/persistentworkers/examples/src/test/java/BUILD index 306ec7fd60..4748acb37c 100644 --- a/persistentworkers/examples/src/test/java/BUILD +++ b/persistentworkers/examples/src/test/java/BUILD @@ -11,7 +11,7 @@ java_test( "//persistentworkers/src/main/java/persistent/common:persistent-common", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", "//persistentworkers/src/test/java/persistent/testutil", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_truth_truth", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/persistentworkers/src/main/java/persistent/bazel/BUILD b/persistentworkers/src/main/java/persistent/bazel/BUILD index 0561e72f07..4ad437ef40 100644 --- a/persistentworkers/src/main/java/persistent/bazel/BUILD +++ b/persistentworkers/src/main/java/persistent/bazel/BUILD @@ -8,20 +8,18 @@ java_library( deps = [ "//persistentworkers/src/main/java/persistent/common:persistent-common", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", - "//src/main/java/build/buildfarm/common", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_apache_commons_commons_pool2", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_apache_commons_commons_pool2", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/persistentworkers/src/main/java/persistent/bazel/client/BasicWorkerKey.java b/persistentworkers/src/main/java/persistent/bazel/client/BasicWorkerKey.java deleted file mode 100644 index 524968c0f1..0000000000 --- a/persistentworkers/src/main/java/persistent/bazel/client/BasicWorkerKey.java +++ /dev/null @@ -1,107 +0,0 @@ -package persistent.bazel.client; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import java.util.Objects; -import lombok.Getter; -import lombok.ToString; - -/** - * Based off of copy-pasting from Bazel's WorkerKey. Has fewer dependencies, but only ProtoBuf and - * non-multiplex support. - * - *

Data container that uniquely identifies a kind of worker process. - * - *

You'll notice that this class is named {@link BasicWorkerKey} and that there's a {@link - * WorkerKey} class as well. The difference is in addition to containing {@link BasicWorkerKey}, - * {@link WorkerKey} also contains the exec owner (the user the worker will be run as) and the - * fields that are derived from it (namely, {@link build.buildfarm.common.config.ExecutionWrapper - * execution wrapper} arguments and the exec root). This sort of separation allows us to, given a - * {@link BasicWorkerKey}, preemptively select an exec owner for which we already have an alive - * worker with that key. That way, we avoid creating too many workers, which would happen by virtue - * of the fact that {@link WorkerKey} is the key of the {@link CommonsWorkerPool worker pool}. - */ -@ToString(onlyExplicitlyIncluded = true) -public final class BasicWorkerKey { - @Getter @ToString.Include private final ImmutableList cmd; - - @Getter @ToString.Include private final ImmutableList args; - - @Getter @ToString.Include private final ImmutableMap env; - - /** Mnemonic of the worker; but we don't actually have the real action mnemonic */ - @Getter @ToString.Include private final String mnemonic; - - /** If true, the workers run inside a sandbox. Returns true if workers are sandboxed. */ - @Getter private final boolean sandboxed; - - /** If true, the workers for this key are able to cancel work requests. */ - @Getter private final boolean cancellable; - - /** - * Cached value for the hash of this key, because the value is expensive to calculate - * (ImmutableMap and ImmutableList do not cache their hashcodes). - */ - private final int hash; - - public BasicWorkerKey( - ImmutableList cmd, - ImmutableList args, - ImmutableMap env, - String mnemonic, - boolean sandboxed, - boolean cancellable) { - this.cmd = Preconditions.checkNotNull(cmd); - this.args = Preconditions.checkNotNull(args); - this.env = Preconditions.checkNotNull(env); - this.mnemonic = Preconditions.checkNotNull(mnemonic); - this.sandboxed = sandboxed; - this.cancellable = cancellable; - - this.hash = calculateHashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - BasicWorkerKey workerKey = (BasicWorkerKey) o; - if (this.hash != workerKey.hash) { - return false; - } - if (!cmd.equals(workerKey.cmd)) { - return false; - } - if (!args.equals(workerKey.args)) { - return false; - } - if (!cancellable == workerKey.cancellable) { - return false; - } - if (!sandboxed == workerKey.sandboxed) { - return false; - } - if (!env.equals(workerKey.env)) { - return false; - } - return mnemonic.equals(workerKey.mnemonic); - } - - /** Since all fields involved in the {@code hashCode} are final, we cache the result. */ - @Override - public int hashCode() { - return hash; - } - - private int calculateHashCode() { - // Use the string representation of the protocolFormat because the hash of the same enum value - // can vary across instances. - return Objects.hash(cmd, args, env, mnemonic, cancellable, sandboxed); - } -} diff --git a/persistentworkers/src/main/java/persistent/bazel/client/CommonsWorkerPool.java b/persistentworkers/src/main/java/persistent/bazel/client/CommonsWorkerPool.java index 24f7398dec..7c39a4a881 100644 --- a/persistentworkers/src/main/java/persistent/bazel/client/CommonsWorkerPool.java +++ b/persistentworkers/src/main/java/persistent/bazel/client/CommonsWorkerPool.java @@ -1,23 +1,10 @@ package persistent.bazel.client; -import org.apache.commons.pool2.DestroyMode; import persistent.common.CommonsPool; /** Specializes CommmonsPool for PersistentWorker */ public class CommonsWorkerPool extends CommonsPool { - private final WorkerIndex workerIndex; - - public CommonsWorkerPool(WorkerIndex workerIndex, WorkerSupervisor supervisor, int maxPerKey) { + public CommonsWorkerPool(WorkerSupervisor supervisor, int maxPerKey) { super(supervisor, maxPerKey); - - this.workerIndex = workerIndex; - } - - @Override - public void invalidateObject(WorkerKey key, PersistentWorker worker, DestroyMode destroyMode) - throws Exception { - super.invalidateObject(key, worker, destroyMode); - - workerIndex.removeWorker(worker); } } diff --git a/persistentworkers/src/main/java/persistent/bazel/client/PersistentWorker.java b/persistentworkers/src/main/java/persistent/bazel/client/PersistentWorker.java index 0fe60dc80c..662c0e5129 100644 --- a/persistentworkers/src/main/java/persistent/bazel/client/PersistentWorker.java +++ b/persistentworkers/src/main/java/persistent/bazel/client/PersistentWorker.java @@ -40,16 +40,12 @@ public PersistentWorker(WorkerKey key, String workerDir) throws IOException { this.key = key; this.execRoot = key.getExecRoot().resolve(workerDir); this.initCmd = - ImmutableList.builder() - .addAll(key.getWrapperArguments()) - .addAll(key.getCmd()) - .addAll(key.getArgs()) - .build(); + ImmutableList.builder().addAll(key.getCmd()).addAll(key.getArgs()).build(); Files.createDirectories(execRoot); final var logLevel = Level.FINE; - if (logger.isLoggable(logLevel)) { + if (logger.isLoggable(logLevel)){ Set workerFiles = ImmutableSet.copyOf(key.getWorkerFilesWithHashes().keySet()); StringBuilder msg = new StringBuilder(); msg.append("Starting Worker["); diff --git a/persistentworkers/src/main/java/persistent/bazel/client/WorkerIndex.java b/persistentworkers/src/main/java/persistent/bazel/client/WorkerIndex.java deleted file mode 100644 index 6c43661312..0000000000 --- a/persistentworkers/src/main/java/persistent/bazel/client/WorkerIndex.java +++ /dev/null @@ -1,55 +0,0 @@ -package persistent.bazel.client; - -import java.nio.file.attribute.UserPrincipal; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import javax.annotation.Nullable; - -/** - * This class augments the {@link CommonsWorkerPool worker pool} with a persistent worker map whose - * key is the worker's {@link BasicWorkerKey}, not its {@link WorkerKey}. - * - *

This allows us to, given a {@link BasicWorkerKey}, preemptively select an exec owner for which - * we already have an alive worker with that key. That way, we avoid creating too many workers, - * which would happen by virtue of the fact that {@link WorkerKey} is the key of the {@link - * CommonsWorkerPool worker pool}. - * - *

A worker is inserted into the worker index when it's created and removed when it's destroyed. - */ -public class WorkerIndex { - private final Map> workerIndex; - - public WorkerIndex() { - workerIndex = new ConcurrentHashMap<>(); - } - - public @Nullable UserPrincipal getOwnerForIdleWorker(BasicWorkerKey workerKey) { - Set idleWorkers = workerIndex.get(workerKey); - - if (idleWorkers == null) { - return null; - } - - try { - return idleWorkers.iterator().next().getKey().getOwner(); - } catch (NoSuchElementException exception) { - return null; - } - } - - public void registerWorker(PersistentWorker worker) { - workerIndex - .computeIfAbsent(worker.getKey().getBasicWorkerKey(), key -> ConcurrentHashMap.newKeySet()) - .add(worker); - } - - public void removeWorker(PersistentWorker worker) { - Set workersForKey = workerIndex.get(worker.getKey().getBasicWorkerKey()); - - if (workersForKey != null) { - workersForKey.remove(worker); - } - } -} diff --git a/persistentworkers/src/main/java/persistent/bazel/client/WorkerKey.java b/persistentworkers/src/main/java/persistent/bazel/client/WorkerKey.java index 6ad7f7a0c1..f7dc515986 100644 --- a/persistentworkers/src/main/java/persistent/bazel/client/WorkerKey.java +++ b/persistentworkers/src/main/java/persistent/bazel/client/WorkerKey.java @@ -5,40 +5,38 @@ import com.google.common.collect.ImmutableMap; import com.google.common.hash.HashCode; import java.nio.file.Path; -import java.nio.file.attribute.UserPrincipal; import java.util.Objects; import java.util.SortedMap; -import javax.annotation.Nullable; import lombok.Getter; import lombok.ToString; /** - * The key of the {@link CommonsWorkerPool worker pool}. + * Based off of copy-pasting from Bazel's WorkerKey. Has less dependencies, but only ProtoBuf and + * non-multiplex support. * - *

See {@link BasicWorkerKey} for more information between this class and that one. + *

Data container that uniquely identifies a kind of worker process. */ @ToString(onlyExplicitlyIncluded = true) -public class WorkerKey { - @Getter @ToString.Include private final BasicWorkerKey basicWorkerKey; +public final class WorkerKey { + @Getter @ToString.Include private final ImmutableList cmd; - /** The user the worker process is running under and the owner of the worker's files. */ - @Getter @Nullable private final UserPrincipal owner; + @Getter @ToString.Include private final ImmutableList args; - /** Execution wrapper arguments to be prepended to the worker command. */ - @Getter private final ImmutableList wrapperArguments; + @Getter @ToString.Include private final ImmutableMap env; @Getter @ToString.Include private final Path execRoot; + /** Mnemonic of the worker; but we don't actually have the real action mnemonic */ + @Getter @ToString.Include private final String mnemonic; + /** * In a remote persistent worker we don't want to eagerly throw away an existing worker if two * different clients have two different workers, so we include the tool inputs hash in the - * WorkerKey so that different clients with different implementations can each have their own - * remote persistent workers. + * WorkerKey so that different clients with different implementations can each have their + * own remote persistent workers. */ @Getter @ToString.Include private final HashCode workerFilesCombinedHash; - @Getter private final Path toolRoot; - /** * Worker files with the corresponding hash code. * @@ -47,6 +45,14 @@ public class WorkerKey { */ @Getter private final SortedMap workerFilesWithHashes; + @Getter private final Path toolRoot; + + /** If true, the workers run inside a sandbox. Returns true if workers are sandboxed. */ + @Getter private final boolean sandboxed; + + /** If true, the workers for this key are able to cancel work requests. */ + @Getter private final boolean cancellable; + /** * Cached value for the hash of this key, because the value is expensive to calculate * (ImmutableMap and ImmutableList do not cache their hashcodes). @@ -54,19 +60,24 @@ public class WorkerKey { private final int hash; public WorkerKey( - BasicWorkerKey basicWorkerKey, - @Nullable UserPrincipal owner, - ImmutableList wrapperArguments, + ImmutableList cmd, + ImmutableList args, + ImmutableMap env, Path execRoot, + String mnemonic, HashCode workerFilesCombinedHash, - SortedMap workerFilesWithHashes) { + SortedMap workerFilesWithHashes, + boolean sandboxed, + boolean cancellable) { // Part of hash - this.basicWorkerKey = Preconditions.checkNotNull(basicWorkerKey); - this.owner = owner; - this.wrapperArguments = Preconditions.checkNotNull(wrapperArguments); + this.cmd = Preconditions.checkNotNull(cmd); + this.args = Preconditions.checkNotNull(args); + this.env = Preconditions.checkNotNull(env); this.execRoot = Preconditions.checkNotNull(execRoot); + this.mnemonic = Preconditions.checkNotNull(mnemonic); + this.sandboxed = sandboxed; + this.cancellable = cancellable; this.workerFilesCombinedHash = Preconditions.checkNotNull(workerFilesCombinedHash); - // Not part of hash this.workerFilesWithHashes = Preconditions.checkNotNull(workerFilesWithHashes); this.toolRoot = execRoot.resolve(workerFilesCombinedHash.toString()); @@ -75,67 +86,51 @@ public WorkerKey( } @Override - public boolean equals(Object other) { - if (this == other) { + public boolean equals(Object o) { + if (this == o) { return true; } - - if (other == null || getClass() != other.getClass()) { + if (o == null || getClass() != o.getClass()) { return false; } - WorkerKey otherWorkerKey = (WorkerKey) other; - - if (!basicWorkerKey.equals(otherWorkerKey.basicWorkerKey)) { + WorkerKey workerKey = (WorkerKey) o; + if (this.hash != workerKey.hash) { return false; } - - if (!((owner == null && otherWorkerKey.owner == null) - || (owner != null && otherWorkerKey.owner != null && owner.equals(otherWorkerKey.owner)))) { + if (!cmd.equals(workerKey.cmd)) { return false; } - - if (!wrapperArguments.equals(otherWorkerKey.wrapperArguments)) { + if (!args.equals(workerKey.args)) { return false; } - - if (!execRoot.equals(otherWorkerKey.execRoot)) { + if (!cancellable == workerKey.cancellable) { return false; } - - return workerFilesCombinedHash.equals(otherWorkerKey.workerFilesCombinedHash); - } - - public ImmutableList getArgs() { - return basicWorkerKey.getArgs(); - } - - public ImmutableList getCmd() { - return basicWorkerKey.getCmd(); - } - - public ImmutableMap getEnv() { - return basicWorkerKey.getEnv(); - } - - public String getMnemonic() { - return basicWorkerKey.getMnemonic(); + if (!sandboxed == workerKey.sandboxed) { + return false; + } + if (!env.equals(workerKey.env)) { + return false; + } + if (!execRoot.equals(workerKey.execRoot)) { + return false; + } + if (!workerFilesCombinedHash.equals(workerKey.workerFilesCombinedHash)) { + return false; + } + return mnemonic.equals(workerKey.mnemonic); } + /** Since all fields involved in the {@code hashCode} are final, we cache the result. */ @Override public int hashCode() { return hash; } - public boolean isCancellable() { - return basicWorkerKey.isCancellable(); - } - - public boolean isSandboxed() { - return basicWorkerKey.isSandboxed(); - } - private int calculateHashCode() { - return Objects.hash(basicWorkerKey, owner, wrapperArguments, execRoot, workerFilesCombinedHash); + // Use the string representation of the protocolFormat because the hash of the same enum value + // can vary across instances. + return Objects.hash(cmd, args, env, execRoot, mnemonic, cancellable, sandboxed, workerFilesCombinedHash); } } diff --git a/persistentworkers/src/main/java/persistent/bazel/client/WorkerSupervisor.java b/persistentworkers/src/main/java/persistent/bazel/client/WorkerSupervisor.java index ce7204c60e..cc993558d5 100644 --- a/persistentworkers/src/main/java/persistent/bazel/client/WorkerSupervisor.java +++ b/persistentworkers/src/main/java/persistent/bazel/client/WorkerSupervisor.java @@ -8,23 +8,18 @@ import persistent.common.CommonsSupervisor; public abstract class WorkerSupervisor extends CommonsSupervisor { - private final Logger logger = Logger.getLogger(this.getClass().getName()); - private final WorkerIndex workerIndex; - - public WorkerSupervisor(WorkerIndex workerIndex) { - this.workerIndex = workerIndex; + public static WorkerSupervisor simple() { + return new WorkerSupervisor() { + @Override + public PersistentWorker create(WorkerKey workerKey) throws Exception { + return new PersistentWorker(workerKey, ""); + } + }; } - public abstract PersistentWorker createUnderlying(WorkerKey workerKey) throws Exception; - - public final PersistentWorker create(WorkerKey key) throws Exception { - PersistentWorker worker = createUnderlying(key); - - // The worker is about to be entered into the pool, so add it to the index - workerIndex.registerWorker(worker); + private final Logger logger = Logger.getLogger(this.getClass().getName()); - return worker; - } + public abstract PersistentWorker create(WorkerKey workerKey) throws Exception; @Override public PooledObject wrap(PersistentWorker persistentWorker) { diff --git a/persistentworkers/src/main/java/persistent/common/BUILD b/persistentworkers/src/main/java/persistent/common/BUILD index 8e29de21d6..fd9087ec44 100644 --- a/persistentworkers/src/main/java/persistent/common/BUILD +++ b/persistentworkers/src/main/java/persistent/common/BUILD @@ -6,19 +6,20 @@ java_library( plugins = ["//src/main/java/build/buildfarm/common:lombok"], visibility = ["//visibility:public"], deps = [ + "//persistentworkers/src/main/java/persistent/common/util", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_apache_commons_commons_pool2", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_apache_commons_commons_pool2", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/persistentworkers/src/main/protobuf/BUILD b/persistentworkers/src/main/protobuf/BUILD index bd81403992..cff0f0a3c7 100755 --- a/persistentworkers/src/main/protobuf/BUILD +++ b/persistentworkers/src/main/protobuf/BUILD @@ -1,6 +1,6 @@ load("@bazel_tools//tools/build_rules:utilities.bzl", "java_library_srcs") -load("@com_google_protobuf//bazel:java_proto_library.bzl", "java_proto_library") -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@protobuf//bazel:java_proto_library.bzl", "java_proto_library") +load("@protobuf//bazel:proto_library.bzl", "proto_library") package(default_visibility = ["//visibility:public"]) diff --git a/persistentworkers/src/test/java/persistent/bazel/BUILD b/persistentworkers/src/test/java/persistent/bazel/BUILD index 50cc9c3a89..ce8179ef82 100644 --- a/persistentworkers/src/test/java/persistent/bazel/BUILD +++ b/persistentworkers/src/test/java/persistent/bazel/BUILD @@ -5,11 +5,11 @@ COMMON_DEPS = [ "//persistentworkers/src/main/java/persistent/bazel:bazel-persistent-workers", "//persistentworkers/src/test/java/persistent/testutil:testutil", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:commons_io_commons_io", + "@maven//:org_mockito_mockito_core", ] java_test( diff --git a/persistentworkers/src/test/java/persistent/common/BUILD b/persistentworkers/src/test/java/persistent/common/BUILD index 02a431f763..1332baff28 100644 --- a/persistentworkers/src/test/java/persistent/common/BUILD +++ b/persistentworkers/src/test/java/persistent/common/BUILD @@ -7,8 +7,9 @@ java_test( deps = [ "//persistentworkers/src/main/java/persistent/common:persistent-common", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:org_mockito_mockito_core", + "//persistentworkers/src/test/java/persistent/testutil", + "@maven//:com_google_truth_truth", + "@maven//:commons_io_commons_io", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/persistentworkers/src/test/java/persistent/testutil/BUILD b/persistentworkers/src/test/java/persistent/testutil/BUILD index cb72176260..53edb3a831 100644 --- a/persistentworkers/src/test/java/persistent/testutil/BUILD +++ b/persistentworkers/src/test/java/persistent/testutil/BUILD @@ -9,10 +9,10 @@ java_library( "//persistentworkers/src/main/java/persistent/bazel:bazel-persistent-workers", "//persistentworkers/src/main/java/persistent/common:persistent-common", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:commons_io_commons_io", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/persistentworkers/src/test/java/persistent/testutil/WorkerUtils.java b/persistentworkers/src/test/java/persistent/testutil/WorkerUtils.java index 9931c342b5..a3499209da 100644 --- a/persistentworkers/src/test/java/persistent/testutil/WorkerUtils.java +++ b/persistentworkers/src/test/java/persistent/testutil/WorkerUtils.java @@ -5,23 +5,19 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.hash.HashCode; import java.nio.file.Path; -import persistent.bazel.client.BasicWorkerKey; import persistent.bazel.client.WorkerKey; public class WorkerUtils { public static WorkerKey emptyWorkerKey(Path execDir, ImmutableList initCmd) { return new WorkerKey( - new BasicWorkerKey( - initCmd, - ImmutableList.of(), - ImmutableMap.of(), - "TestOp-Adder", - false, - false), - null, + initCmd, ImmutableList.of(), + ImmutableMap.of(), execDir, + "TestOp-Adder", HashCode.fromInt(0), - ImmutableSortedMap.of()); + ImmutableSortedMap.of(), + false, + false); } } diff --git a/renovate.json b/renovate.json index 9686e30b2e..5218115223 100644 --- a/renovate.json +++ b/renovate.json @@ -36,11 +36,6 @@ ], "enabled": false }, - { - "description": "disable helm-values", - "matchManagers": ["helm-values"], - "enabled": false - }, { "description": "Maven dependencies need to be pinned by 'REPIN=1 bazelisk run @maven//:pin && git add maven_install.json && git commit' to pass CI, then they can be auto-merged after approval", "matchManagers": ["bazel-module"], @@ -71,9 +66,7 @@ "matchPackageNames": [ "com.google.errorprone:*", "net.jcip:jcip-annotations", - "org.checkerframework:checker-qual", - "org.jetbrains:annotations", - "org.jspecify:jspecify" + "org.checkerframework:checker-qual" ] }, { diff --git a/run_server b/run_server index d7c11fbeb4..ad57cd4243 100755 --- a/run_server +++ b/run_server @@ -1,24 +1,18 @@ -#!/usr/bin/env bash - -set -euo pipefail +#!/bin/bash +set -e +set -o pipefail # Run Redis docker container docker start buildfarm-redis || docker run -d --rm --name buildfarm-redis -p 6379:6379 redis:7.2.4 - -if command -v redis-cli &>/dev/null -then - redis-cli config set stop-writes-on-bgsave-error no -else - printf 'WARNING: redis-cli not on PATH; can’t unset stop-writes-on-bgsave-error\n' >&2 -fi +redis-cli config set stop-writes-on-bgsave-error no # Determine which configuration file to use - default or user provided -if [[ -z ${1:-} ]] +if [[ -z "$1" ]] then - config="$PWD/examples/config.minimal.yml" + config=$PWD/examples/config.minimal.yml else config=$1 fi # Run Server -bazelisk run //src/main/java/build/buildfarm:buildfarm-server -- --jvm_flag=-Djava.util.logging.config.file="$PWD/examples/logging.properties" "$config" +bazelisk run //src/main/java/build/buildfarm:buildfarm-server -- --jvm_flag=-Djava.util.logging.config.file=$PWD/examples/logging.properties $config diff --git a/run_worker b/run_worker index d854639c15..6ffacb8bce 100755 --- a/run_worker +++ b/run_worker @@ -1,21 +1,15 @@ -#!/usr/bin/env bash - -set -euo pipefail +#!/bin/bash +set -e +set -o pipefail # Run Redis docker container docker start buildfarm-redis || docker run -d --rm --name buildfarm-redis -p 6379:6379 redis:7.2.4 - -if command -v redis-cli &>/dev/null -then - redis-cli config set stop-writes-on-bgsave-error no -else - printf 'WARNING: redis-cli not on PATH; can’t unset stop-writes-on-bgsave-error\n' >&2 -fi +redis-cli config set stop-writes-on-bgsave-error no # Determine which configuration file to use - default or user provided -if [[ -z ${1:-} ]] +if [[ -z "$1" ]] then - config="$PWD/examples/config.minimal.yml" + config=$PWD/examples/config.minimal.yml else config=$1 fi diff --git a/src/main/java/build/buildfarm/BUILD b/src/main/java/build/buildfarm/BUILD index 121cddd5f9..e9649a0718 100644 --- a/src/main/java/build/buildfarm/BUILD +++ b/src/main/java/build/buildfarm/BUILD @@ -8,23 +8,23 @@ package( java_binary( name = "buildfarm-server", data = [ - "//examples:example_configs", - "//examples:example_properties", + "//config:base_configs", + "//config:base_properties", ], jvm_flags = ensure_accurate_metadata() + add_opens_sun_nio_fs(), main_class = "build.buildfarm.server.BuildFarmServer", visibility = ["//visibility:public"], runtime_deps = [ "//src/main/java/build/buildfarm/server", - "@buildfarm_maven//:org_slf4j_slf4j_simple", + "@maven//:org_slf4j_slf4j_simple", ], ) java_binary( name = "buildfarm-shard-worker", data = [ - "//examples:example_configs", - "//examples:example_properties", + "//config:base_configs", + "//config:base_properties", ], jvm_flags = ensure_accurate_metadata() + add_opens_sun_nio_fs(), main_class = "build.buildfarm.worker.shard.Worker", diff --git a/src/main/java/build/buildfarm/actioncache/BUILD b/src/main/java/build/buildfarm/actioncache/BUILD index c4a144cd3b..5488785623 100644 --- a/src/main/java/build/buildfarm/actioncache/BUILD +++ b/src/main/java/build/buildfarm/actioncache/BUILD @@ -9,13 +9,15 @@ java_library( "//src/main/java/build/buildfarm/backplane", "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/config", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:net_javacrumbs_future_converter_future_converter_java8_guava", - "@buildfarm_maven//:org_projectlombok_lombok", + "//src/main/java/build/buildfarm/common/resources", + "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:net_javacrumbs_future_converter_future_converter_java8_guava", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/backplane/BUILD b/src/main/java/build/buildfarm/backplane/BUILD index d55198d321..fb122317dc 100644 --- a/src/main/java/build/buildfarm/backplane/BUILD +++ b/src/main/java/build/buildfarm/backplane/BUILD @@ -7,13 +7,14 @@ java_library( visibility = ["//visibility:public"], deps = [ "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:net_jcip_jcip_annotations", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/longrunning:longrunning_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:net_jcip_jcip_annotations", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/backplane/Backplane.java b/src/main/java/build/buildfarm/backplane/Backplane.java index 98df995f55..3e53a890d7 100644 --- a/src/main/java/build/buildfarm/backplane/Backplane.java +++ b/src/main/java/build/buildfarm/backplane/Backplane.java @@ -113,11 +113,6 @@ ScanResult scanCorrelatedInvocations(String scope, String value, String ScanResult scanCorrelatedInvocationIndexEntries(String cursor, int count, String keyMatch) throws IOException; - ScanResult> scanQueuedOperations(String cursor, int count) - throws IOException; - - ScanResult scanPrequeuedOperations(String cursor, int count) throws IOException; - /** Returns a map of the worker name and its start time for given workers. */ Map getWorkersStartTimeInEpochSecs(Set workerNames) throws IOException; diff --git a/src/main/java/build/buildfarm/cas/BUILD b/src/main/java/build/buildfarm/cas/BUILD index 79ea798983..3f3144eb8d 100644 --- a/src/main/java/build/buildfarm/cas/BUILD +++ b/src/main/java/build/buildfarm/cas/BUILD @@ -6,7 +6,7 @@ java_library( plugins = ["//src/main/java/build/buildfarm/common:lombok"], visibility = ["//visibility:public"], runtime_deps = [ - "@buildfarm_maven//:org_xerial_sqlite_jdbc", + "@maven//:org_xerial_sqlite_jdbc", ], deps = [ "//src/main/java/build/buildfarm/common", @@ -18,24 +18,23 @@ java_library( "//src/main/java/build/buildfarm/instance/stub", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_code_gson_gson", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_netty_netty_codec_http", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:net_jcip_jcip_annotations", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_code_gson_gson", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_netty_netty_codec_http", + "@maven//:io_prometheus_simpleclient", + "@maven//:net_jcip_jcip_annotations", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/cas/ContentAddressableStorage.java b/src/main/java/build/buildfarm/cas/ContentAddressableStorage.java index 9ddc38bd8c..4e5bcb1017 100644 --- a/src/main/java/build/buildfarm/cas/ContentAddressableStorage.java +++ b/src/main/java/build/buildfarm/cas/ContentAddressableStorage.java @@ -28,7 +28,6 @@ import com.google.rpc.Code; import com.google.rpc.Status; import io.grpc.stub.ServerCallStreamObserver; -import java.time.Duration; import java.util.List; import java.util.UUID; import javax.annotation.Nullable; @@ -101,11 +100,6 @@ void get( ServerCallStreamObserver blobObserver, RequestMetadata requestMetadata); - /** Whether attempts to write will fail */ - boolean isReadOnly(); - - void waitForWritable(Duration timeout) throws InterruptedException; - /** Acquire a write associated with the blob, may return null if readonly */ @Nullable Write getWrite( diff --git a/src/main/java/build/buildfarm/cas/GrpcCAS.java b/src/main/java/build/buildfarm/cas/GrpcCAS.java index 77e46ca8a3..63b696b122 100644 --- a/src/main/java/build/buildfarm/cas/GrpcCAS.java +++ b/src/main/java/build/buildfarm/cas/GrpcCAS.java @@ -60,7 +60,6 @@ import io.grpc.stub.ServerCallStreamObserver; import java.io.IOException; import java.io.InputStream; -import java.time.Duration; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -297,18 +296,6 @@ public static Write newWrite( /* autoflush= */ false); } - @Override - public boolean isReadOnly() { - return readonly; - } - - @Override - public void waitForWritable(Duration timeout) { - if (readonly) { - throw new UnsupportedOperationException(); - } - } - @Override @Nullable public Write getWrite( diff --git a/src/main/java/build/buildfarm/cas/MemoryCAS.java b/src/main/java/build/buildfarm/cas/MemoryCAS.java index cad5d04e3d..0f2269723c 100644 --- a/src/main/java/build/buildfarm/cas/MemoryCAS.java +++ b/src/main/java/build/buildfarm/cas/MemoryCAS.java @@ -36,7 +36,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.NoSuchFileException; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -259,14 +258,6 @@ private long size() { return count; } - @Override - public boolean isReadOnly() { - return false; - } - - @Override - public void waitForWritable(Duration timeout) {} - @Override public Write getWrite( Compressor.Value compressor, Digest digest, UUID uuid, RequestMetadata requestMetadata) { diff --git a/src/main/java/build/buildfarm/cas/cfc/CASFileCache.java b/src/main/java/build/buildfarm/cas/cfc/CASFileCache.java index 1f1758dc3d..2ab93dbf5f 100644 --- a/src/main/java/build/buildfarm/cas/cfc/CASFileCache.java +++ b/src/main/java/build/buildfarm/cas/cfc/CASFileCache.java @@ -16,6 +16,7 @@ import static build.buildfarm.common.DigestUtil.OMITTED_DIGEST_FUNCTIONS; import static build.buildfarm.common.io.EvenMoreFiles.setReadOnlyPerms; +import static build.buildfarm.common.io.Utils.getFileKey; import static build.buildfarm.common.io.Utils.getOrIOException; import static build.buildfarm.common.io.Utils.listDir; import static build.buildfarm.common.io.Utils.stat; @@ -50,7 +51,6 @@ import build.bazel.remote.execution.v2.SymlinkNode; import build.buildfarm.cas.ContentAddressableStorage; import build.buildfarm.cas.DigestMismatchException; -import build.buildfarm.cas.cfc.LRUDB.SizeEntry; import build.buildfarm.common.BuildfarmExecutors; import build.buildfarm.common.DigestUtil; import build.buildfarm.common.DigestUtil.HashFunction; @@ -79,6 +79,7 @@ import com.google.common.cache.LoadingCache; import com.google.common.cache.RemovalListener; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.hash.HashingOutputStream; @@ -96,7 +97,6 @@ import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import io.prometheus.client.Histogram; -import java.io.BufferedReader; import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; @@ -113,9 +113,8 @@ import java.time.Duration; import java.time.Instant; import java.util.AbstractMap; -import java.util.ArrayList; +import java.util.Collections; import java.util.Date; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -174,9 +173,6 @@ public abstract class CASFileCache implements ContentAddressableStorage { private final Consumer> onExpire; private final Executor accessRecorder; private final ExecutorService expireService; - private final LRUDB db = new TextLRUDB(); - private volatile Deadline saveLRUAfter = Deadline.after(10, MINUTES); - private final Path lru; private final FixedBufferPool zstdBufferPool; @Nullable private final ContentAddressableStorage delegate; @@ -224,16 +220,6 @@ public SettableFuture load(Digest digest) { protected final transient Entry header = new SentinelEntry(); protected volatile long unreferencedEntryCount = 0; - enum State { - STOPPED, - STARTING, // files are checked for existence/read - WRITING, // only writable state - READ_ONLY, - } - - @GuardedBy("this") - private State state = State.STOPPED; - @GuardedBy("this") private long removedEntrySize = 0; @@ -319,8 +305,6 @@ public CASFileCache( new FailoverInputStreamFactory(this::newTransparentInput, externalInputStreamFactory)); this.zstdBufferPool = zstdBufferPool; - lru = root.resolve("lru.txt"); - writes = CacheBuilder.newBuilder() .expireAfterAccess(1, HOURS) @@ -411,14 +395,7 @@ private boolean contains( @Nullable build.bazel.remote.execution.v2.Digest.Builder result, Consumer onContains) { String key = getKey(digest, isExecutable); - Entry entry = getEntry(key); - if (entry != null && entry.referenceCount < 0) { - try { - entry = new Entry(key, Files.size(getPath(key)), null); - } catch (IOException e) { - return false; - } - } + Entry entry = storage.get(key); if (entry != null && (digest.getSize() < 0 || digest.getSize() == entry.size)) { if (result != null) { result.mergeFrom(DigestUtil.toDigest(digest)).setSizeBytes(entry.size); @@ -429,41 +406,10 @@ private boolean contains( return false; } - private void saveLRU() { - List list = lruSizeEntryList(); - try { - synchronized (lru) { - db.save(list.iterator(), lru); - } - } catch (Exception e) { - log.log(Level.SEVERE, "error saving lru state", e); - } - } - - /** Not reentrant, not thread safe */ - private void maybeSaveLRU() { - if (saveLRUAfter == null || !saveLRUAfter.isExpired()) { - return; - } - saveLRUAfter = null; - expireService.execute( - () -> { - try { - saveLRU(); - } finally { - saveLRUAfter = Deadline.after(10, MINUTES); - } - }); - } - private void accessed(Iterable keys) { /* could also bucket these */ try { - accessRecorder.execute( - () -> { - recordAccess(keys); - maybeSaveLRU(); - }); + accessRecorder.execute(() -> recordAccess(keys)); } catch (RejectedExecutionException e) { log.log(Level.SEVERE, format("could not record access for %d keys", Iterables.size(keys)), e); } @@ -516,15 +462,9 @@ public Iterable findMissingBlobs( builder.add(result.build()); } } - boolean recordAccess; - synchronized (this) { - recordAccess = state == State.WRITING || state == State.READ_ONLY; - } - if (recordAccess) { - List foundDigests = found.build(); - if (!foundDigests.isEmpty()) { - accessed(foundDigests); - } + List foundDigests = found.build(); + if (!foundDigests.isEmpty()) { + accessed(foundDigests); } ImmutableList missingDigests = builder.build(); return CasFallbackDelegate.findMissingBlobs(delegate, missingDigests, digestFunction); @@ -561,30 +501,16 @@ private InputStream compressorInputStream(Compressor.Value compressor, InputStre return new ZstdCompressingInputStream(identity); } - private Entry getEntry(String key) { - Entry entry = storage.get(key); - if (entry != null) { - return entry; - } - synchronized (this) { - if (state != State.WRITING && state != State.READ_ONLY) { - return new Entry(); - } - } - return null; - } - @SuppressWarnings({"ResultOfMethodCallIgnored", "PMD.CompareObjectsWithEquals"}) InputStream newLocalInput(Compressor.Value compressor, Digest digest, long offset) throws IOException { - // branch here or above for STARTING log.log(Level.FINER, format("getting input stream for %s", DigestUtil.toString(digest))); boolean isExecutable = false; do { String key = getKey(digest, isExecutable); - Entry e = getEntry(key); + Entry e = storage.get(key); if (e != null) { - InputStream input = null; + InputStream input; try { input = compressorInputStream(compressor, Files.newInputStream(getPath(key))); input.skip(offset); @@ -594,37 +520,31 @@ InputStream newLocalInput(Compressor.Value compressor, Digest digest, long offse log.log( Level.WARNING, format("error opening %s at %d", DigestUtil.toString(digest), offset), - ioEx); + e); } - if (e.referenceCount >= 0) { - boolean removed = false; - synchronized (this) { - invalidateWrite(digest); - Entry removedEntry = safeStorageRemoval(key); - if (removedEntry == e) { // Intentional reference comparison - unlinkEntry(removedEntry); - removed = true; - } else if (removedEntry != null) { - log.severe( - format( - "nonexistent entry %s did not match last unreferenced entry, restoring it", - key)); - storage.put(key, removedEntry); - } - } - if (removed && isExecutable) { - onExpire.accept(ImmutableList.of(digest)); + boolean removed = false; + synchronized (this) { + invalidateWrite(digest); + Entry removedEntry = safeStorageRemoval(key); + if (removedEntry == e) { // Intentional reference comparison + unlinkEntry(removedEntry); + removed = true; + } else if (removedEntry != null) { + log.severe( + format( + "nonexistent entry %s did not match last unreferenced entry, restoring it", + key)); + storage.put(key, removedEntry); } - e = null; } + if (removed && isExecutable) { + onExpire.accept(ImmutableList.of(digest)); + } + continue; } - if (e != null && e.referenceCount >= 0) { - accessed(ImmutableList.of(key)); - } - if (input != null) { - return input; - } + accessed(ImmutableList.of(key)); + return input; } isExecutable = !isExecutable; } while (isExecutable); @@ -832,29 +752,6 @@ public void put(Blob blob, Runnable onExpiration) throws InterruptedException { } } - @Override - public synchronized boolean isReadOnly() { - return state != State.WRITING; - } - - public synchronized boolean setReadOnly(boolean value) { - if (value && state == State.WRITING) { - state = State.READ_ONLY; - return true; - } - if (!value && state == State.READ_ONLY) { - state = State.WRITING; - notifyAll(); // ensure that we wake up any waits for this state - return true; - } - return false; - } - - @Override - public synchronized void waitForWritable(Duration timeout) throws InterruptedException { - wait(timeout.toMillis(), (int) (timeout.toNanos() % 1000000)); - } - @Override public Write getWrite( Compressor.Value compressor, Digest digest, UUID uuid, RequestMetadata requestMetadata) @@ -1299,81 +1196,48 @@ public void initializeRootDirectory() throws IOException { fileStore = Files.getFileStore(root); } - @SuppressWarnings({"PMD.CompareObjectsWithEquals"}) - private synchronized List lruSizeEntryList() { - /** - * Steps the entries in order from oldest to newest access The order is used here to insert into - * the lru on load - */ - List list = new ArrayList<>(storage.size()); - for (Entry current = header.after; current != header; current = checkNotNull(current.after)) { - list.add(new SizeEntry(current.key, current.size)); - } - return list; - } - - public synchronized void stop() throws IOException, InterruptedException { + public void stop() throws InterruptedException { if (prometheusMetricsThread != null) { prometheusMetricsThread.interrupt(); prometheusMetricsThread.join(); } - // lock ordering, [this] -> [lru] - // path used as lock due to isolation by filename - saveLRU(); - state = State.STOPPED; } - public ListenableFuture start(boolean skipLoad) { - return start(newDirectExecutorService(), skipLoad, /* writable= */ true); + public StartupCacheResults start(boolean skipLoad) throws IOException, InterruptedException { + return start(newDirectExecutorService(), skipLoad); } - public ListenableFuture start( - ExecutorService removeDirectoryService, boolean skipLoad, boolean writable) { - return start(onPut, removeDirectoryService, skipLoad, writable); + public StartupCacheResults start(ExecutorService removeDirectoryService, boolean skipLoad) + throws IOException, InterruptedException { + return start(onPut, removeDirectoryService, skipLoad); } - public ListenableFuture start( - Consumer onStartPut, - ExecutorService removeDirectoryService, - boolean skipLoad, - boolean writable) { - synchronized (this) { - checkState(state == State.STOPPED); - state = State.STARTING; - } - CasFallbackDelegate.start( - delegate, onStartPut, removeDirectoryService, delegateSkipLoad, writable); - - return listeningDecorator(expireService) - .submit( - () -> { - startRoutine(onStartPut, removeDirectoryService, skipLoad); - synchronized (this) { - saveLRUAfter = Deadline.after(10, MINUTES); - state = writable ? State.WRITING : State.READ_ONLY; - if (writable) { - notifyAll(); - } - } - return null; - }); - } + protected void afterCacheLoaded() {} /** * initialize the cache for persistent storage and inject any consistent entries which already * exist under the root into the storage map. This call will create the root if it does not exist, * and will scale in cost with the number of files already present. */ - private void startRoutine( + public StartupCacheResults start( Consumer onStartPut, ExecutorService removeDirectoryService, boolean skipLoad) throws IOException, InterruptedException { + CasFallbackDelegate.start(delegate, onStartPut, removeDirectoryService, delegateSkipLoad); + log.log(Level.INFO, "Initializing cache at: " + root); Instant startTime = Instant.now(); + CacheLoadResults loadResults = + new CacheLoadResults( + skipLoad, + new CacheScanResults( + Collections.emptyList(), Collections.emptyList(), Collections.emptyMap()), + Collections.emptyList()); + // Load the cache if (!skipLoad) { initializeRootDirectory(); - loadCache(onStartPut, removeDirectoryService); + loadResults = loadCache(onStartPut, removeDirectoryService); } else { // Skip loading the cache and ensure it is empty fileStore = Files.getFileStore(root); @@ -1405,6 +1269,9 @@ private void startRoutine( }, "Prometheus CAS Metrics Collector"); prometheusMetricsThread.start(); + + // return information about the cache startup. + return new StartupCacheResults(root, loadResults, startupTime); } protected CacheLoadResults loadCache( @@ -1426,16 +1293,16 @@ protected CacheLoadResults loadCache( } private void deleteInvalidFileContent(List files, ExecutorService removeDirectoryService) { - for (Path path : files) { - try { + try { + for (Path path : files) { if (Files.isDirectory(path)) { Directories.remove(path, fileStore, removeDirectoryService); } else { Files.delete(path); } - } catch (Exception e) { - log.log(Level.SEVERE, "failure to delete CAS content: ", e); } + } catch (Exception e) { + log.log(Level.SEVERE, "failure to delete CAS content: ", e); } } @@ -1444,6 +1311,7 @@ private void logCacheScanResults(CacheScanResults cacheScanResults) { Map map = Map.of( "dirs", cacheScanResults.computeDirs.size(), + "keys", cacheScanResults.fileKeys.size(), "delete", cacheScanResults.deleteFiles.size()); log.log(Level.INFO, new Gson().toJson(map)); } @@ -1466,11 +1334,12 @@ private CacheScanResults scanRoot(Consumer onStartPut) // collect keys from cache root. ImmutableList.Builder computeDirsBuilder = new ImmutableList.Builder<>(); ImmutableList.Builder deleteFilesBuilder = new ImmutableList.Builder<>(); + ImmutableMap.Builder fileKeysBuilder = new ImmutableMap.Builder<>(); // TODO invalidate mismatched hash prefix - Set files = new HashSet<>(); + Iterable files = ImmutableList.of(); for (Path path : entryPathStrategy) { - files.addAll(listDir(path)); + files = Iterables.concat(files, listDir(path)); } for (Path branchDir : entryPathStrategy.branchDirectories()) { @@ -1483,32 +1352,12 @@ private CacheScanResults scanRoot(Consumer onStartPut) } } - // TODO test for hex bins - try (BufferedReader br = Files.newBufferedReader(lru)) { - for (SizeEntry entry : db.entries(br)) { - // ignore files in the lru that are not present in the directories - Path path = entryPathStrategy.getPath(entry.key()); - if (files.remove(path)) { - processRootFile(onStartPut, path, entry, computeDirsBuilder, deleteFilesBuilder); - } - } - // prevent the lru db from being processed -> removed in the purge below - files.remove(lru); - } catch (NoSuchFileException e) { - // ignore - } for (Path file : files) { - String basename = file.getFileName().toString(); pool.execute( () -> { try { - FileStatus stat = stat(file, false, fileStore); processRootFile( - onStartPut, - file, - new SizeEntry(basename, stat.getSize()), - computeDirsBuilder, - deleteFilesBuilder); + onStartPut, file, computeDirsBuilder, deleteFilesBuilder, fileKeysBuilder); } catch (Exception e) { log.log(Level.SEVERE, "error reading file " + file.toString(), e); } @@ -1518,45 +1367,63 @@ private CacheScanResults scanRoot(Consumer onStartPut) joinThreads(pool, "Scanning Cache Root..."); // log information from scanning cache root. - return new CacheScanResults(computeDirsBuilder.build(), deleteFilesBuilder.build(), null); + return new CacheScanResults( + computeDirsBuilder.build(), deleteFilesBuilder.build(), fileKeysBuilder.build()); } @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") private void processRootFile( Consumer onStartPut, - Path path, - SizeEntry entry, + Path file, ImmutableList.Builder computeDirs, - ImmutableList.Builder deleteFiles) + ImmutableList.Builder deleteFiles, + ImmutableMap.Builder fileKeys) throws IOException { - String basename = entry.key(); + String basename = file.getFileName().toString(); + + FileStatus stat = stat(file, false, fileStore); // mark directory for later key compute - if (basename.endsWith("_dir")) { - synchronized (computeDirs) { - computeDirs.add(path); + if (file.toString().endsWith("_dir")) { + if (stat.isDirectory()) { + synchronized (computeDirs) { + computeDirs.add(file); + } + } else { + synchronized (deleteFiles) { + deleteFiles.add(file); + } + } + } else if (stat.isDirectory()) { + synchronized (deleteFiles) { + deleteFiles.add(file); } } else { // if cas is full or entry is oversized or empty, mark file for later deletion. - long size = entry.size(); + long size = stat.getSize(); if (sizeInBytes + size > maxSizeInBytes || size > maxEntrySizeInBytes || size == 0) { synchronized (deleteFiles) { - deleteFiles.add(path); + deleteFiles.add(file); } } else { // get the key entry from the file name. - FileEntryKey fileEntryKey = parseFileEntryKey(basename, size); + FileEntryKey fileEntryKey = parseFileEntryKey(basename, stat.getSize()); // if key entry file name cannot be parsed, mark file for later deletion. - if (fileEntryKey == null) { + if (fileEntryKey == null || stat.isReadOnlyExecutable() != fileEntryKey.isExecutable()) { synchronized (deleteFiles) { - deleteFiles.add(path); + deleteFiles.add(file); } } else { String key = fileEntryKey.key(); + Path keyPath = getPath(key); // populate key if it is not currently stored. Entry e = new Entry(key, size, Deadline.after(10, SECONDS)); - checkState(storage.put(e.key, e) == null, key); + Object fileKey = getFileKey(keyPath, stat); + synchronized (fileKeys) { + fileKeys.put(fileKey, e); + } + storage.put(e.key, e); onStartPut.accept(fileEntryKey.digest()); synchronized (this) { if (e.decrementReference(header)) { @@ -1851,8 +1718,6 @@ private Entry safeStorageRemoval(String key) throws IOException { Files.delete(path); deleteExpiredPath = false; } - } catch (NoSuchFileException e) { - // ignore, already removed } finally { entry = storage.remove(key); if (deleteExpiredPath) { @@ -1960,8 +1825,7 @@ private void putDirectoryFiles( put(digest, isExecutable, service), cacheFilePath -> { try { - onFileContent.call( - filePath, cacheFilePath.path(), digest.getSize(), isExecutable); + onFileContent.call(filePath, cacheFilePath, digest.getSize(), isExecutable); } catch (Exception e) { return immediateFailedFuture(e); } @@ -2071,8 +1935,7 @@ public abstract ListenableFuture putDirectory( ExecutorService service); @VisibleForTesting - public PathResult put(Digest digest, boolean isExecutable) - throws IOException, InterruptedException { + public Path put(Digest digest, boolean isExecutable) throws IOException, InterruptedException { checkState(digest.getSize() > 0, "file entries may not be empty"); return putAndCopy(digest, isExecutable); @@ -2080,7 +1943,7 @@ public PathResult put(Digest digest, boolean isExecutable) // This can result in deadlock if called with a direct executor. I'm unsure how to guard // against it, until we can get to using a current-download future - public ListenableFuture put(Digest digest, boolean isExecutable, Executor executor) { + public ListenableFuture put(Digest digest, boolean isExecutable, Executor executor) { checkState(digest.getSize() > 0, "file entries may not be empty"); return transformAsync( @@ -2090,10 +1953,8 @@ public ListenableFuture put(Digest digest, boolean isExecutable, Exe } @SuppressWarnings("ThrowFromFinallyBlock") - PathResult putAndCopy(Digest digest, boolean isExecutable) - throws IOException, InterruptedException { + Path putAndCopy(Digest digest, boolean isExecutable) throws IOException, InterruptedException { String key = getKey(digest, isExecutable); - boolean downloadComplete = false; CancellableOutputStream out = putImpl( key, @@ -2105,13 +1966,14 @@ PathResult putAndCopy(Digest digest, boolean isExecutable) () -> invalidateWrite(digest), /* isReset= */ true); if (out != null) { + boolean complete = false; try { copyExternalInput(digest, out); - downloadComplete = true; + complete = true; } finally { try { log.log(Level.FINER, format("closing output stream for %s", DigestUtil.toString(digest))); - if (downloadComplete) { + if (complete) { out.close(); } else { out.cancel(); @@ -2136,7 +1998,7 @@ PathResult putAndCopy(Digest digest, boolean isExecutable) } } } - return new PathResult(getPath(key), downloadComplete); + return getPath(key); } private void copyExternalInputProgressive(Digest digest, CancellableOutputStream out) @@ -2177,10 +2039,8 @@ private void copyExternalInput(Digest digest, CancellableOutputStream out) out.cancel(); log.log( Level.WARNING, - format( - "error downloading %s: %s", - DigestUtil.toString(digest), - e.getMessage())); // prevent burial by early end of stream during close + format("error downloading %s", DigestUtil.toString(digest)), + e); // prevent burial by early end of stream during close throw e; } log.log(Level.FINER, format("download of %s complete", DigestUtil.toString(digest))); diff --git a/src/main/java/build/buildfarm/cas/cfc/CasFallbackDelegate.java b/src/main/java/build/buildfarm/cas/cfc/CasFallbackDelegate.java index 2886f04a1b..7d9365e11a 100644 --- a/src/main/java/build/buildfarm/cas/cfc/CasFallbackDelegate.java +++ b/src/main/java/build/buildfarm/cas/cfc/CasFallbackDelegate.java @@ -45,11 +45,11 @@ public static void start( @Nullable ContentAddressableStorage delegate, Consumer onStartPut, ExecutorService removeDirectoryService, - boolean skipLoad, - boolean readonly) { + boolean skipLoad) + throws IOException, InterruptedException { // start delegate if we specifically have a CASFileCache if (delegate instanceof CASFileCache fileCacheDelegate) { - fileCacheDelegate.start(onStartPut, removeDirectoryService, skipLoad, readonly); + fileCacheDelegate.start(onStartPut, removeDirectoryService, skipLoad); } } diff --git a/src/main/java/build/buildfarm/cas/cfc/DirectoryEntryCFC.java b/src/main/java/build/buildfarm/cas/cfc/DirectoryEntryCFC.java index 0916c1e788..5990d47f08 100644 --- a/src/main/java/build/buildfarm/cas/cfc/DirectoryEntryCFC.java +++ b/src/main/java/build/buildfarm/cas/cfc/DirectoryEntryCFC.java @@ -98,6 +98,8 @@ public DirectoryEntryCFC( externalInputStreamFactory); } + // needs to delete directories which have multiple references (link count > 1) + private void computeDirectory(Path path, ImmutableList.Builder invalidDirectories) { String key = path.getFileName().toString(); try { @@ -107,6 +109,9 @@ private void computeDirectory(Path path, ImmutableList.Builder invalidDire new SimpleFileVisitor<>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + // a bold option here: if we're dealing with a Legacy multi-link directory + // there will never be an overcharge on the committed size, and this directory + // can be deleted here or through expiration due to size, with no possible leaks if (attrs.isRegularFile()) { blobSizeInBytes.addAndGet(attrs.size()); } diff --git a/src/main/java/build/buildfarm/cas/cfc/LRUDB.java b/src/main/java/build/buildfarm/cas/cfc/LRUDB.java deleted file mode 100644 index 7ce6838dd1..0000000000 --- a/src/main/java/build/buildfarm/cas/cfc/LRUDB.java +++ /dev/null @@ -1,14 +0,0 @@ -package build.buildfarm.cas.cfc; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Iterator; - -interface LRUDB { - record SizeEntry(String key, long size) {} - - Iterable entries(BufferedReader br) throws IOException; - - void save(Iterator head, Path path) throws IOException; -} diff --git a/src/main/java/build/buildfarm/cas/cfc/LegacyDirectoryCFC.java b/src/main/java/build/buildfarm/cas/cfc/LegacyDirectoryCFC.java index 920a9121fc..eef33b8e9c 100644 --- a/src/main/java/build/buildfarm/cas/cfc/LegacyDirectoryCFC.java +++ b/src/main/java/build/buildfarm/cas/cfc/LegacyDirectoryCFC.java @@ -39,8 +39,6 @@ import build.buildfarm.common.io.Directories; import build.buildfarm.common.io.NamedFileKey; import build.buildfarm.v1test.Digest; -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.util.concurrent.ListenableFuture; @@ -202,13 +200,15 @@ public void unlock() { } private static final class LockMap { - /* - * Using weak values is important to avoid memory leaks, and avoid the caller from cleaning up. - */ - private final Cache mutexes = Caffeine.newBuilder().weakValues().build(); + private final Map mutexes = Maps.newHashMap(); private synchronized Lock acquire(Path key) { - return mutexes.get(key, path -> new SharedLock()); + Lock mutex = mutexes.get(key); + if (mutex == null) { + mutex = new SharedLock(); + mutexes.put(key, mutex); + } + return mutex; } } diff --git a/src/main/java/build/buildfarm/cas/cfc/TextLRUDB.java b/src/main/java/build/buildfarm/cas/cfc/TextLRUDB.java deleted file mode 100644 index 1b07e50989..0000000000 --- a/src/main/java/build/buildfarm/cas/cfc/TextLRUDB.java +++ /dev/null @@ -1,71 +0,0 @@ -package build.buildfarm.cas.cfc; - -import static com.google.common.base.Preconditions.checkState; -import static java.lang.String.format; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Iterator; -import java.util.NoSuchElementException; - -class TextLRUDB implements LRUDB { - private static class EntriesIterator implements Iterator { - private String next = null; - private final BufferedReader br; - - EntriesIterator(BufferedReader br) { - this.br = br; - } - - @Override - public boolean hasNext() { - if (next == null) { - try { - next = br.readLine(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - return next != null; - } - - private SizeEntry parse(String line) { - int sep = line.indexOf(','); - checkState(sep != -1); - return new SizeEntry(line.substring(0, sep), Long.parseLong(line.substring(sep + 1))); - } - - @Override - public SizeEntry next() { - if (next == null) { - throw new NoSuchElementException(); - } - SizeEntry sizeEntry = parse(next); - next = null; - return sizeEntry; - } - } - - @Override - public Iterable entries(BufferedReader br) throws IOException { - return new Iterable<>() { - @Override - public Iterator iterator() { - return new EntriesIterator(br); - } - }; - } - - @Override - public void save(Iterator entries, Path path) throws IOException { - try (BufferedWriter writer = Files.newBufferedWriter(path)) { - while (entries.hasNext()) { - SizeEntry entry = entries.next(); - writer.write(format("%s,%d\n", entry.key(), entry.size())); - } - } - } -} diff --git a/src/main/java/build/buildfarm/common/BUILD b/src/main/java/build/buildfarm/common/BUILD index 0f41dbd643..1eb5d07eaa 100644 --- a/src/main/java/build/buildfarm/common/BUILD +++ b/src/main/java/build/buildfarm/common/BUILD @@ -19,28 +19,28 @@ java_library( "//src/main/java/build/buildfarm/common/blake3", "//src/main/java/build/buildfarm/common/resources:resource_java_proto", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_jnr_jnr_posix", - "@buildfarm_maven//:com_github_luben_zstd_jni", - "@buildfarm_maven//:com_github_oshi_oshi_core", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_failureaccess", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_apache_commons_commons_compress", - "@buildfarm_maven//:org_apache_commons_commons_pool2", - "@buildfarm_maven//:org_projectlombok_lombok", - "@buildfarm_maven//:org_threeten_threetenbp", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_jnr_jnr_posix", + "@maven//:com_github_luben_zstd_jni", + "@maven//:com_github_oshi_oshi_core", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_failureaccess", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_apache_commons_commons_compress", + "@maven//:org_apache_commons_commons_pool2", + "@maven//:org_projectlombok_lombok", + "@maven//:org_threeten_threetenbp", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -54,29 +54,31 @@ java_library( visibility = ["//visibility:public"], deps = [ "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/common/config", + "//src/main/java/build/buildfarm/common/resources", "//src/main/java/build/buildfarm/common/resources:resource_java_proto", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_jnr_jnr_posix", - "@buildfarm_maven//:com_github_luben_zstd_jni", - "@buildfarm_maven//:com_github_oshi_oshi_core", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_failureaccess", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_apache_commons_commons_compress", - "@buildfarm_maven//:org_projectlombok_lombok", - "@buildfarm_maven//:org_threeten_threetenbp", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_jnr_jnr_posix", + "@maven//:com_github_luben_zstd_jni", + "@maven//:com_github_oshi_oshi_core", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_failureaccess", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_apache_commons_commons_compress", + "@maven//:org_projectlombok_lombok", + "@maven//:org_threeten_threetenbp", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -86,5 +88,5 @@ java_plugin( generates_api = True, processor_class = "lombok.launch.AnnotationProcessorHider$AnnotationProcessor", visibility = ["//visibility:public"], - deps = ["@buildfarm_maven//:org_projectlombok_lombok"], + deps = ["@maven//:org_projectlombok_lombok"], ) diff --git a/src/main/java/build/buildfarm/common/Claim.java b/src/main/java/build/buildfarm/common/Claim.java index 76cec988e4..a40d3d5b7f 100644 --- a/src/main/java/build/buildfarm/common/Claim.java +++ b/src/main/java/build/buildfarm/common/Claim.java @@ -20,14 +20,6 @@ import javax.annotation.Nullable; public interface Claim { - interface Lease { - int getAmount(); - - Stage getStage(); - - void release(); - } - enum Stage { EXECUTE_ACTION_STAGE, REPORT_RESULT_STAGE, @@ -37,12 +29,8 @@ enum Stage { void release(); - void replace(String resourceName, Lease lease); - @Nullable - UserPrincipal getOwner(); - - void setOwner(@Nullable UserPrincipal owner); + UserPrincipal owner(); - Iterable>> getPools(); + Iterable>> getPools(); } diff --git a/src/main/java/build/buildfarm/common/DigestUtil.java b/src/main/java/build/buildfarm/common/DigestUtil.java index 142bbd4c59..a07e8c983d 100644 --- a/src/main/java/build/buildfarm/common/DigestUtil.java +++ b/src/main/java/build/buildfarm/common/DigestUtil.java @@ -14,8 +14,6 @@ package build.buildfarm.common; -import static com.google.protobuf.ByteString.EMPTY; - import build.bazel.remote.execution.v2.Action; import build.bazel.remote.execution.v2.DigestFunction; import build.buildfarm.common.blake3.Blake3HashFunction; @@ -61,12 +59,13 @@ public class DigestUtil { public static final Map empty = ImmutableMap.of( - DigestFunction.Value.MD5, DigestUtil.forHash("MD5").computeImpl(EMPTY), - DigestFunction.Value.SHA1, DigestUtil.forHash("SHA1").computeImpl(EMPTY), - DigestFunction.Value.SHA256, DigestUtil.forHash("SHA256").computeImpl(EMPTY), - DigestFunction.Value.SHA384, DigestUtil.forHash("SHA384").computeImpl(EMPTY), - DigestFunction.Value.SHA512, DigestUtil.forHash("SHA512").computeImpl(EMPTY), - DigestFunction.Value.BLAKE3, DigestUtil.forHash("BLAKE3").computeImpl(EMPTY)); + DigestFunction.Value.MD5, DigestUtil.forHash("MD5").computeImpl(ByteString.empty()), + DigestFunction.Value.SHA1, DigestUtil.forHash("SHA1").computeImpl(ByteString.empty()), + DigestFunction.Value.SHA256, DigestUtil.forHash("SHA256").computeImpl(ByteString.empty()), + DigestFunction.Value.SHA384, DigestUtil.forHash("SHA384").computeImpl(ByteString.empty()), + DigestFunction.Value.SHA512, DigestUtil.forHash("SHA512").computeImpl(ByteString.empty()), + DigestFunction.Value.BLAKE3, + DigestUtil.forHash("BLAKE3").computeImpl(ByteString.empty())); /** Type of hash function to use for digesting blobs. */ // The underlying HashFunctions are immutable and thread safe. diff --git a/src/main/java/build/buildfarm/common/Dispenser.java b/src/main/java/build/buildfarm/common/Dispenser.java new file mode 100644 index 0000000000..536b926ca6 --- /dev/null +++ b/src/main/java/build/buildfarm/common/Dispenser.java @@ -0,0 +1,90 @@ +// Copyright 2025 The Buildfarm Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build.buildfarm.common; + +import static com.google.common.base.Preconditions.checkState; + +import java.util.AbstractCollection; +import java.util.Iterator; +import java.util.Queue; + +/** + * @class Dispenser + * @brief A queue which delivers a single element repeatedly without delay or regard for size. + * @details For compatibility with POOL resources, this Queue delivers an infinite immediate + * sequence of an identical element. + */ +public final class Dispenser extends AbstractCollection implements Queue { + private final T element; + + public Dispenser(T element) { + this.element = element; + } + + // used methods + @Override + public T poll() { + return element; + } + + @Override + public boolean add(T o) { + checkState(o.equals(element)); + return true; + } + + @Override + public boolean isEmpty() { + return false; + } + + // unused methods + // Queue + @Override + public T peek() { + throw new UnsupportedOperationException(); + } + + @Override + public T element() { + throw new UnsupportedOperationException(); + } + + @Override + public T remove() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean offer(T o) { + throw new UnsupportedOperationException(); + } + + @Override + public void clear() { + throw new UnsupportedOperationException(); + } + + // Collection + @Override + public int size() { + throw new UnsupportedOperationException(); + } + + @Override + public Iterator iterator() { + throw new UnsupportedOperationException(); + } +} diff --git a/src/main/java/build/buildfarm/common/Queue.java b/src/main/java/build/buildfarm/common/Queue.java index af4b079b8e..7aa642b9a9 100644 --- a/src/main/java/build/buildfarm/common/Queue.java +++ b/src/main/java/build/buildfarm/common/Queue.java @@ -3,7 +3,6 @@ import java.time.Duration; import java.util.function.Supplier; import redis.clients.jedis.AbstractPipeline; -import redis.clients.jedis.resps.ScanResult; public interface Queue { // java.util.BlockingQueue-ish @@ -30,6 +29,4 @@ public interface Queue { boolean removeFromDequeue(E e); void removeFromDequeue(AbstractPipeline pipeline, E e); - - ScanResult scan(String cursor, int count, String match); } diff --git a/src/main/java/build/buildfarm/common/SystemProcessors.java b/src/main/java/build/buildfarm/common/SystemProcessors.java index 7556b6d229..bab9f0ea7c 100644 --- a/src/main/java/build/buildfarm/common/SystemProcessors.java +++ b/src/main/java/build/buildfarm/common/SystemProcessors.java @@ -14,8 +14,6 @@ package build.buildfarm.common; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import oshi.SystemInfo; import oshi.hardware.CentralProcessor; import oshi.hardware.HardwareAbstractionLayer; @@ -44,22 +42,14 @@ public enum PROCESSOR_DERIVE { OSHI } - /** - * @brief Memoized supplier for processor count. - * @details Uses Guava's memoize to cache the processor count since it won't change during - * runtime. - */ - private static final Supplier processorCount = - Suppliers.memoize( - () -> Math.max(get(PROCESSOR_DERIVE.JAVA_RUNTIME), get(PROCESSOR_DERIVE.OSHI))); - /** * @brief Get the number of logical processors on the system. * @details Buildfarm will choose the best implementation. * @return Number of logical processors on the system. */ public static int get() { - return processorCount.get(); + // Have buildfarm choose the best value. + return Math.max(get(PROCESSOR_DERIVE.JAVA_RUNTIME), get(PROCESSOR_DERIVE.OSHI)); } /** diff --git a/src/main/java/build/buildfarm/common/blake3/BUILD b/src/main/java/build/buildfarm/common/blake3/BUILD index 28cceec299..a31baa6050 100644 --- a/src/main/java/build/buildfarm/common/blake3/BUILD +++ b/src/main/java/build/buildfarm/common/blake3/BUILD @@ -18,9 +18,9 @@ java_library( visibility = ["//src/main/java/build/buildfarm/common:__pkg__"], deps = [ "//src/main/java/build/buildfarm/common/base", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_errorprone_error_prone_annotations", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/common/config/BUILD b/src/main/java/build/buildfarm/common/config/BUILD index 99f70c4e7f..b3e6080d31 100644 --- a/src/main/java/build/buildfarm/common/config/BUILD +++ b/src/main/java/build/buildfarm/common/config/BUILD @@ -11,20 +11,20 @@ java_library( "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/base", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_oshi_oshi_core", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_auth_google_auth_library_oauth2_http", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:me_dinowernli_java_grpc_prometheus", - "@buildfarm_maven//:org_projectlombok_lombok", - "@buildfarm_maven//:org_redisson_redisson", - "@buildfarm_maven//:org_yaml_snakeyaml", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_oshi_oshi_core", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_auth_google_auth_library_oauth2_http", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:me_dinowernli_java_grpc_prometheus", + "@maven//:org_projectlombok_lombok", + "@maven//:org_redisson_redisson", + "@maven//:org_yaml_snakeyaml", + "@maven//:redis_clients_jedis", ], ) diff --git a/src/main/java/build/buildfarm/common/config/Backplane.java b/src/main/java/build/buildfarm/common/config/Backplane.java index 0be70f2222..41c159368b 100644 --- a/src/main/java/build/buildfarm/common/config/Backplane.java +++ b/src/main/java/build/buildfarm/common/config/Backplane.java @@ -22,7 +22,6 @@ public enum BACKPLANE_TYPE { private int jedisPoolMaxIdle = 8; private int jedisPoolMinIdle = 0; private long jedisTimeBetweenEvictionRunsMillis = 30000L; - private boolean connectionValidatedOnBorrow = false; private String workersHashName = "Workers"; private String workerChannel = "WorkerChannel"; private String actionCachePrefix = "ActionCache"; diff --git a/src/main/java/build/buildfarm/common/config/ExecutionPolicy.java b/src/main/java/build/buildfarm/common/config/ExecutionPolicy.java index 245b6376c5..0e83f4b0d3 100644 --- a/src/main/java/build/buildfarm/common/config/ExecutionPolicy.java +++ b/src/main/java/build/buildfarm/common/config/ExecutionPolicy.java @@ -5,7 +5,6 @@ @Data public class ExecutionPolicy { private String name; - private boolean prioritized; private ExecutionWrapper executionWrapper; /** Required for snakeyaml to parse correctly */ diff --git a/src/main/java/build/buildfarm/common/config/Worker.java b/src/main/java/build/buildfarm/common/config/Worker.java index 00aa8cc414..1f2f07555b 100644 --- a/src/main/java/build/buildfarm/common/config/Worker.java +++ b/src/main/java/build/buildfarm/common/config/Worker.java @@ -48,7 +48,7 @@ public class Worker { private int reportResultStageWidth = 1; private boolean linkExecFileSystem = true; private boolean linkInputDirectories = true; - private List linkedInputDirectories = Arrays.asList("^(?!external$).*$"); + private List linkedInputDirectories = Arrays.asList("(?!external/)[^/]+"); private String execOwner; private List execOwners = new ArrayList<>(); private int defaultMaxCores = 0; diff --git a/src/main/java/build/buildfarm/common/grpc/BUILD b/src/main/java/build/buildfarm/common/grpc/BUILD index c97c534b8f..4ba78e3292 100644 --- a/src/main/java/build/buildfarm/common/grpc/BUILD +++ b/src/main/java/build/buildfarm/common/grpc/BUILD @@ -6,22 +6,22 @@ java_library( plugins = ["//src/main/java/build/buildfarm/common:lombok"], visibility = ["//visibility:public"], runtime_deps = [ - "@buildfarm_maven//:org_slf4j_slf4j_simple", + "@maven//:org_slf4j_slf4j_simple", ], deps = [ "//src/main/java/build/buildfarm/common", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/common/grpc/NullObserver.java b/src/main/java/build/buildfarm/common/grpc/NullObserver.java deleted file mode 100644 index aeb0a7b6f5..0000000000 --- a/src/main/java/build/buildfarm/common/grpc/NullObserver.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2025 The Buildfarm Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package build.buildfarm.common.grpc; - -import io.grpc.stub.StreamObserver; - -public final class NullObserver implements StreamObserver { - @Override - public void onNext(T request) {} - - @Override - public void onError(Throwable t) {} - - @Override - public void onCompleted() {} -} diff --git a/src/main/java/build/buildfarm/common/redis/BUILD b/src/main/java/build/buildfarm/common/redis/BUILD index 81c911d1d2..7ba04a4088 100644 --- a/src/main/java/build/buildfarm/common/redis/BUILD +++ b/src/main/java/build/buildfarm/common/redis/BUILD @@ -10,13 +10,13 @@ java_library( "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_auth_google_auth_library_oauth2_http", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:org_apache_commons_commons_pool2", - "@buildfarm_maven//:org_projectlombok_lombok", - "@buildfarm_maven//:org_redisson_redisson", - "@buildfarm_maven//:redis_clients_jedis", + "@maven//:com_google_auth_google_auth_library_oauth2_http", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:io_grpc_grpc_api", + "@maven//:org_apache_commons_commons_pool2", + "@maven//:org_projectlombok_lombok", + "@maven//:org_redisson_redisson", + "@maven//:redis_clients_jedis", ], ) diff --git a/src/main/java/build/buildfarm/common/redis/BalancedRedisQueue.java b/src/main/java/build/buildfarm/common/redis/BalancedRedisQueue.java index 419444d4f9..e1b7df6cfb 100644 --- a/src/main/java/build/buildfarm/common/redis/BalancedRedisQueue.java +++ b/src/main/java/build/buildfarm/common/redis/BalancedRedisQueue.java @@ -15,8 +15,6 @@ package build.buildfarm.common.redis; import static com.google.common.collect.Iterables.transform; -import static com.google.common.collect.Lists.newArrayList; -import static redis.clients.jedis.params.ScanParams.SCAN_POINTER_START; import build.buildfarm.common.Queue; import build.buildfarm.common.Visitor; @@ -26,7 +24,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -35,6 +32,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import javax.annotation.Nullable; +import lombok.Data; import lombok.Getter; import redis.clients.jedis.AbstractPipeline; import redis.clients.jedis.Connection; @@ -42,7 +40,6 @@ import redis.clients.jedis.JedisCluster; import redis.clients.jedis.JedisPooled; import redis.clients.jedis.UnifiedJedis; -import redis.clients.jedis.resps.ScanResult; import redis.clients.jedis.util.JedisClusterCRC16; /** @@ -59,7 +56,16 @@ public class BalancedRedisQueue { private static final Duration MAX_TIMEOUT = Duration.ofSeconds(8); - public record BalancedQueueEntry(String queue, String value) {} + @Data + public static final class BalancedQueueEntry { + private final String queue; + private final String value; + + BalancedQueueEntry(String queue, String value) { + this.queue = queue; + this.value = value; + } + } private static Visitor createBalancedQueueVisitor( String queue, Visitor visitor) { @@ -190,9 +196,9 @@ public boolean offer(UnifiedJedis unified, String val, double priority) { * @note Suggested return identifier: wasRemoved. */ public boolean removeFromDequeue(UnifiedJedis unified, BalancedQueueEntry balancedQueueEntry) { - String queue = balancedQueueEntry.queue(); + String queue = balancedQueueEntry.getQueue(); try (Jedis jedis = getJedisFromKey(unified, queue)) { - if (queueDecorator.decorate(jedis, queue).removeFromDequeue(balancedQueueEntry.value())) { + if (queueDecorator.decorate(jedis, queue).removeFromDequeue(balancedQueueEntry.getValue())) { return true; } } @@ -201,8 +207,8 @@ public boolean removeFromDequeue(UnifiedJedis unified, BalancedQueueEntry balanc public void removeFromDequeue(AbstractPipeline pipeline, BalancedQueueEntry balancedQueueEntry) { queueDecorator - .decorate(null, balancedQueueEntry.queue()) - .removeFromDequeue(pipeline, balancedQueueEntry.value()); + .decorate(null, balancedQueueEntry.getQueue()) + .removeFromDequeue(pipeline, balancedQueueEntry.getValue()); } private String take(Jedis jedis, Queue queue, Duration timeout, ExecutorService service) @@ -626,61 +632,4 @@ private List partialIterationQueueOrder() { Collections.shuffle(randomQueues); return randomQueues; } - - public ScanResult scan( - UnifiedJedis unified, String queueCursor, int count, String match) { - int queueIndex = queueCursor.indexOf('[') + 1; - String currentQueue = null; - if (queueIndex > 0) { - int queueEnd = queueCursor.indexOf(']'); - currentQueue = queueCursor.substring(queueIndex, queueEnd); - queueCursor = queueCursor.substring(queueEnd + 1); - } - Iterator queueIter = queues.iterator(); - - String queue = null; - while (currentQueue != null && !currentQueue.equals(queue) && queueIter.hasNext()) { - queue = queueIter.next(); - } - - if (currentQueue != null && !currentQueue.equals(queue)) { - return new ScanResult<>(SCAN_POINTER_START, new ArrayList<>()); - } - - List result = new ArrayList<>(count); - while (result.size() < count) { - if (currentQueue == null || queueCursor.equals(SCAN_POINTER_START)) { - if (!queueIter.hasNext()) { - break; - } - currentQueue = queueIter.next(); - } - // should we put the source hash in the result set? - // should we be trying to use the same jedis connection for each cycle? - final String entryQueue = currentQueue; - try (Jedis jedis = getJedisFromKey(unified, entryQueue)) { - ScanResult scanResult = - queueDecorator - .decorate(jedis, currentQueue) - .scan(queueCursor, count - result.size(), match); - queueCursor = scanResult.getCursor(); - result.addAll( - newArrayList( - transform( - scanResult.getResult(), entry -> new BalancedQueueEntry(entryQueue, entry)))); - } - } - - if (queueCursor.equals(SCAN_POINTER_START)) { - currentQueue = null; - if (queueIter.hasNext()) { - currentQueue = queueIter.next(); - } - } - String nextCursor = SCAN_POINTER_START; - if (currentQueue != null) { - nextCursor = "[" + currentQueue + "]" + queueCursor; - } - return new ScanResult(nextCursor, result); - } } diff --git a/src/main/java/build/buildfarm/common/redis/RedisMap.java b/src/main/java/build/buildfarm/common/redis/RedisMap.java index 922a564476..44e28ed0d9 100644 --- a/src/main/java/build/buildfarm/common/redis/RedisMap.java +++ b/src/main/java/build/buildfarm/common/redis/RedisMap.java @@ -28,7 +28,6 @@ import redis.clients.jedis.JedisCluster; import redis.clients.jedis.Response; import redis.clients.jedis.UnifiedJedis; -import redis.clients.jedis.params.GetExParams; import redis.clients.jedis.params.ScanParams; import redis.clients.jedis.params.SetParams; import redis.clients.jedis.resps.ScanResult; @@ -170,20 +169,6 @@ public String get(UnifiedJedis jedis, String key) { return jedis.get(createKeyName(key)); } - /** - * @brief Get the value of the key and update expiration. - * @details If the key does not exist, null is returned. - * @param jedis Jedis cluster client. - * @param key The name of the key. - * @return The value of the key. null if key does not exist. - * @note Overloaded. - * @note Suggested return identifier: value. - */ - public String getex(UnifiedJedis jedis, String key, long timeout_s) { - GetExParams params = GetExParams.getExParams().ex(timeout_s); - return jedis.getEx(createKeyName(key), params); - } - /** * @brief Get the values of the keys. * @details If the key does not exist, null is returned. diff --git a/src/main/java/build/buildfarm/common/redis/RedisPriorityQueue.java b/src/main/java/build/buildfarm/common/redis/RedisPriorityQueue.java index 6c89021bd2..61b190c2f2 100644 --- a/src/main/java/build/buildfarm/common/redis/RedisPriorityQueue.java +++ b/src/main/java/build/buildfarm/common/redis/RedisPriorityQueue.java @@ -14,12 +14,9 @@ package build.buildfarm.common.redis; -import static com.google.common.collect.Lists.newArrayList; - import build.buildfarm.common.Queue; import build.buildfarm.common.Visitor; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import java.time.Clock; import java.time.Duration; import java.util.List; @@ -27,9 +24,6 @@ import lombok.Getter; import redis.clients.jedis.AbstractPipeline; import redis.clients.jedis.Jedis; -import redis.clients.jedis.params.ScanParams; -import redis.clients.jedis.resps.ScanResult; -import redis.clients.jedis.resps.Tuple; /** * @class RedisPriorityQueue @@ -322,31 +316,4 @@ private String getLuaScript() { private boolean isEmpty(String val) { return val == null || val.isEmpty() || val.equalsIgnoreCase("null"); } - - /** responses from zscan are score:value */ - private String entryValue(String entry) { - int sepIndex = entry.indexOf(':'); - return entry.substring(sepIndex + 1); - } - - public ScanResult scan(String queueCursor, int count, String match) { - // redis has much worse performance when scanning for small counts - // break even is around 5k - int scanCount = 5000; - // maybe use type regular? - // TODO might be some optimization around unspecified match, look into this - ScanParams scanParams = new ScanParams().count(scanCount).match(match); - return new OffsetScanner() { - @Override - protected ScanResult scan(String cursor, int remaining) { - // do we strip here or in transform?? - ScanResult scanResult = jedis.zscan(name, cursor, scanParams); - return new ScanResult<>( - scanResult.getCursor(), - newArrayList( - Iterables.transform( - scanResult.getResult(), entry -> entryValue(entry.getElement())))); - } - }.fill(queueCursor, count); - } } diff --git a/src/main/java/build/buildfarm/common/redis/RedisQueue.java b/src/main/java/build/buildfarm/common/redis/RedisQueue.java index ca2bd5e52a..7a842c5add 100644 --- a/src/main/java/build/buildfarm/common/redis/RedisQueue.java +++ b/src/main/java/build/buildfarm/common/redis/RedisQueue.java @@ -16,17 +16,14 @@ import static redis.clients.jedis.args.ListDirection.LEFT; import static redis.clients.jedis.args.ListDirection.RIGHT; -import static redis.clients.jedis.params.ScanParams.SCAN_POINTER_START; import build.buildfarm.common.Queue; import build.buildfarm.common.Visitor; -import com.google.common.collect.ImmutableList; import java.time.Duration; import java.util.List; import java.util.function.Supplier; import redis.clients.jedis.AbstractPipeline; import redis.clients.jedis.Jedis; -import redis.clients.jedis.resps.ScanResult; /** * @class RedisQueue @@ -216,18 +213,4 @@ private void visit(String queueName, Visitor visitor) { nextIndex += entries.size(); } while (entries.size() == listPageSize); } - - public ScanResult scan(String cursor, int count, String match) { - int start = cursor.isEmpty() ? 0 : Integer.parseInt(cursor); - int stop = start + count - 1; - - if (stop < start) { - // FIXME no support for the -1 reversal - return new ScanResult<>(SCAN_POINTER_START, ImmutableList.of()); - } - - List range = jedis.lrange(name, start, stop); - return new ScanResult<>( - range.isEmpty() ? SCAN_POINTER_START : String.valueOf(start + range.size()), range); - } } diff --git a/src/main/java/build/buildfarm/common/resources/BUILD b/src/main/java/build/buildfarm/common/resources/BUILD index 2f3c3fbe45..913fb4063c 100644 --- a/src/main/java/build/buildfarm/common/resources/BUILD +++ b/src/main/java/build/buildfarm/common/resources/BUILD @@ -1,5 +1,5 @@ -load("@com_google_protobuf//bazel:java_proto_library.bzl", "java_proto_library") -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@protobuf//bazel:java_proto_library.bzl", "java_proto_library") +load("@protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_java//java:java_library.bzl", "java_library") package(default_visibility = ["//visibility:public"]) @@ -9,11 +9,11 @@ proto_library( srcs = ["resource.proto"], deps = [ "//src/main/protobuf/build/buildfarm/v1test:buildfarm_proto", - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:timestamp_proto", "@googleapis//google/api:annotations_proto", "@googleapis//google/longrunning:operations_proto", "@googleapis//google/rpc:status_proto", + "@protobuf//:duration_proto", + "@protobuf//:timestamp_proto", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_proto", ], ) @@ -34,25 +34,25 @@ java_library( "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/resources:resource_java_proto", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_jnr_jnr_posix", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_failureaccess", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:org_apache_commons_commons_compress", - "@buildfarm_maven//:org_apache_commons_commons_lang3", - "@buildfarm_maven//:org_projectlombok_lombok", - "@buildfarm_maven//:org_threeten_threetenbp", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_jnr_jnr_posix", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_failureaccess", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:org_apache_commons_commons_compress", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_projectlombok_lombok", + "@maven//:org_threeten_threetenbp", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/common/s3/BUILD b/src/main/java/build/buildfarm/common/s3/BUILD index eaf97f6687..02946d1697 100644 --- a/src/main/java/build/buildfarm/common/s3/BUILD +++ b/src/main/java/build/buildfarm/common/s3/BUILD @@ -7,10 +7,10 @@ java_library( visibility = ["//visibility:public"], deps = [ "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_amazonaws_aws_java_sdk_core", - "@buildfarm_maven//:com_amazonaws_aws_java_sdk_s3", - "@buildfarm_maven//:com_amazonaws_aws_java_sdk_secretsmanager", - "@buildfarm_maven//:com_fasterxml_jackson_core_jackson_databind", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_amazonaws_aws_java_sdk_core", + "@maven//:com_amazonaws_aws_java_sdk_s3", + "@maven//:com_amazonaws_aws_java_sdk_secretsmanager", + "@maven//:com_fasterxml_jackson_core_jackson_databind", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/common/services/BUILD b/src/main/java/build/buildfarm/common/services/BUILD index 0d87a5b86e..9a0ad477a3 100644 --- a/src/main/java/build/buildfarm/common/services/BUILD +++ b/src/main/java/build/buildfarm/common/services/BUILD @@ -16,21 +16,21 @@ java_library( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//third_party/remote-apis:build_bazel_remote_asset_v1_remote_asset_java_grpc", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_services", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/devtools/build/v1:build_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_services", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/asset/v1:remote_asset_java_proto", "@remoteapis//build/bazel/semver:semver_java_proto", ], diff --git a/src/main/java/build/buildfarm/common/services/ByteStreamService.java b/src/main/java/build/buildfarm/common/services/ByteStreamService.java index 17db723e7a..384cd0159f 100644 --- a/src/main/java/build/buildfarm/common/services/ByteStreamService.java +++ b/src/main/java/build/buildfarm/common/services/ByteStreamService.java @@ -21,7 +21,6 @@ import static io.grpc.Status.INVALID_ARGUMENT; import static io.grpc.Status.NOT_FOUND; import static io.grpc.Status.OUT_OF_RANGE; -import static io.grpc.Status.UNAVAILABLE; import static java.lang.String.format; import build.bazel.remote.execution.v2.Compressor; @@ -31,7 +30,6 @@ import build.buildfarm.common.Write.CompleteWrite; import build.buildfarm.common.config.BuildfarmConfigs; import build.buildfarm.common.grpc.DelegateServerCallStreamObserver; -import build.buildfarm.common.grpc.NullObserver; import build.buildfarm.common.grpc.TracingMetadataUtils; import build.buildfarm.common.grpc.UniformDelegateServerCallStreamObserver; import build.buildfarm.common.io.FeedbackOutputStream; @@ -470,16 +468,6 @@ private ServerCallStreamObserver initializeBackPressure( @Override public StreamObserver write(StreamObserver responseObserver) { - if (instance.isReadOnly()) { - responseObserver.onError( - UNAVAILABLE.withDescription("instance is in read-only mode").asException()); - return new NullObserver<>(); - } - return observeWrite(responseObserver); - } - - private StreamObserver observeWrite( - StreamObserver responseObserver) { ServerCallStreamObserver serverCallStreamObserver = initializeBackPressure(responseObserver); diff --git a/src/main/java/build/buildfarm/instance/BUILD b/src/main/java/build/buildfarm/instance/BUILD index 383d0cc655..1e58a8518a 100644 --- a/src/main/java/build/buildfarm/instance/BUILD +++ b/src/main/java/build/buildfarm/instance/BUILD @@ -7,16 +7,16 @@ java_library( deps = [ "//src/main/java/build/buildfarm/common", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_failureaccess", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/longrunning:longrunning_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_failureaccess", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/instance/Instance.java b/src/main/java/build/buildfarm/instance/Instance.java index f2fa922f11..656f8e86fc 100644 --- a/src/main/java/build/buildfarm/instance/Instance.java +++ b/src/main/java/build/buildfarm/instance/Instance.java @@ -32,11 +32,11 @@ import build.buildfarm.common.Watcher; import build.buildfarm.common.Write; import build.buildfarm.v1test.BackplaneStatus; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.GetClientStartTimeRequest; import build.buildfarm.v1test.GetClientStartTimeResult; import build.buildfarm.v1test.PrepareWorkerForGracefulShutDownRequestResults; import build.buildfarm.v1test.Tree; +import build.buildfarm.v1test.WorkerListMessage; import build.buildfarm.v1test.WorkerProfileMessage; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -99,8 +99,6 @@ ListenableFuture> getAllBlobsFuture( String getTree( build.buildfarm.v1test.Digest rootDigest, int pageSize, String pageToken, Tree.Builder tree); - boolean isReadOnly(); - Write getBlobWrite( Compressor.Value compressor, build.buildfarm.v1test.Digest digest, @@ -161,9 +159,9 @@ String listOperations( ServerCapabilities getCapabilities(); - ListenableFuture getWorkerProfile(String name); + WorkerProfileMessage getWorkerProfile(); - ListenableFuture batchWorkerProfiles(Iterable names); + WorkerListMessage getWorkerList(); PrepareWorkerForGracefulShutDownRequestResults shutDownWorkerGracefully(); diff --git a/src/main/java/build/buildfarm/instance/server/BUILD b/src/main/java/build/buildfarm/instance/server/BUILD index c5f4da211c..46289c09b4 100644 --- a/src/main/java/build/buildfarm/instance/server/BUILD +++ b/src/main/java/build/buildfarm/instance/server/BUILD @@ -14,20 +14,20 @@ java_library( "//src/main/java/build/buildfarm/common/resources:resource_java_proto", "//src/main/java/build/buildfarm/instance", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_owteam_engUtils_netrc", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_netty_netty_codec_http", - "@buildfarm_maven//:org_apache_httpcomponents_httpclient", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_owteam_engUtils_netrc", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_netty_netty_codec_http", + "@maven//:org_apache_httpcomponents_httpclient", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/instance/server/NodeInstance.java b/src/main/java/build/buildfarm/instance/server/NodeInstance.java index d51ee27b1c..cc39d8db86 100644 --- a/src/main/java/build/buildfarm/instance/server/NodeInstance.java +++ b/src/main/java/build/buildfarm/instance/server/NodeInstance.java @@ -83,7 +83,6 @@ import build.buildfarm.common.resources.ResourceParser; import build.buildfarm.instance.Instance; import build.buildfarm.instance.InstanceBase; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.Digest; import build.buildfarm.v1test.GetClientStartTimeRequest; import build.buildfarm.v1test.GetClientStartTimeResult; @@ -91,6 +90,7 @@ import build.buildfarm.v1test.QueuedOperation; import build.buildfarm.v1test.QueuedOperationMetadata; import build.buildfarm.v1test.Tree; +import build.buildfarm.v1test.WorkerListMessage; import build.buildfarm.v1test.WorkerProfileMessage; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -415,11 +415,6 @@ public InputStream newBlobInput( return contentAddressableStorage.newInput(compressor, digest, offset); } - @Override - public boolean isReadOnly() { - return contentAddressableStorage.isReadOnly(); - } - @Override public Write getBlobWrite( Compressor.Value compressor, @@ -1734,7 +1729,6 @@ protected CacheCapabilities getCacheCapabilities() { ActionCacheUpdateCapabilities.newBuilder().setUpdateEnabled(true)) .setMaxBatchTotalSizeBytes(Size.mbToBytes(4)) .setSymlinkAbsolutePathStrategy(SymlinkAbsolutePathStrategy.Value.DISALLOWED) - .setMaxCasBlobSizeBytes(configs.getMaxEntrySizeBytes()) // Compression support .addSupportedCompressors(Compressor.Value.IDENTITY) @@ -1775,21 +1769,20 @@ public ServerCapabilities getCapabilities() { } @Override - public ListenableFuture getWorkerProfile(String name) { + public WorkerProfileMessage getWorkerProfile() { throw new UnsupportedOperationException( "NodeInstance doesn't support getWorkerProfile() method."); } @Override - public ListenableFuture batchWorkerProfiles(Iterable names) { - throw new UnsupportedOperationException( - "NodeInstance doesn't support batchWorkerProfiles() method."); + public WorkerListMessage getWorkerList() { + throw new UnsupportedOperationException("NodeInstance doesn't support getWorkerList() method."); } @Override public PrepareWorkerForGracefulShutDownRequestResults shutDownWorkerGracefully() { throw new UnsupportedOperationException( - "NodeInstance doesn't support shutDownWorkerGracefully() method."); + "NodeInstance doesn't support drainWorkerPipeline() method."); } @Override diff --git a/src/main/java/build/buildfarm/instance/shard/BUILD b/src/main/java/build/buildfarm/instance/shard/BUILD index 2c8101a55e..bb9c880f97 100644 --- a/src/main/java/build/buildfarm/instance/shard/BUILD +++ b/src/main/java/build/buildfarm/instance/shard/BUILD @@ -19,23 +19,23 @@ java_library( "//src/main/java/build/buildfarm/instance/stub", "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_netty_netty_codec_http", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:net_javacrumbs_future_converter_future_converter_java8_guava", - "@buildfarm_maven//:org_apache_commons_commons_pool2", - "@buildfarm_maven//:org_projectlombok_lombok", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_netty_netty_codec_http", + "@maven//:io_prometheus_simpleclient", + "@maven//:net_javacrumbs_future_converter_future_converter_java8_guava", + "@maven//:org_apache_commons_commons_pool2", + "@maven//:org_projectlombok_lombok", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/instance/shard/ExecutionQueue.java b/src/main/java/build/buildfarm/instance/shard/ExecutionQueue.java index ec663b3932..d0871456d8 100644 --- a/src/main/java/build/buildfarm/instance/shard/ExecutionQueue.java +++ b/src/main/java/build/buildfarm/instance/shard/ExecutionQueue.java @@ -15,9 +15,6 @@ package build.buildfarm.instance.shard; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.Iterables.transform; -import static com.google.common.collect.Lists.newArrayList; -import static redis.clients.jedis.params.ScanParams.SCAN_POINTER_START; import build.bazel.remote.execution.v2.Platform; import build.buildfarm.common.Visitor; @@ -34,17 +31,15 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import java.time.Duration; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.function.Supplier; import java.util.logging.Level; import java.util.stream.Collectors; +import lombok.Data; import lombok.extern.java.Log; import redis.clients.jedis.AbstractPipeline; import redis.clients.jedis.UnifiedJedis; -import redis.clients.jedis.resps.ScanResult; /** * @class ExecutionQueue @@ -58,8 +53,19 @@ public class ExecutionQueue { private static final Duration MAX_TIMEOUT = Duration.ofSeconds(8); - public record ExecutionQueueEntry( - BalancedRedisQueue queue, BalancedQueueEntry balancedQueueEntry, QueueEntry queueEntry) {} + @Data + public static final class ExecutionQueueEntry { + private final BalancedRedisQueue queue; + private final BalancedQueueEntry balancedQueueEntry; + private final QueueEntry queueEntry; + + ExecutionQueueEntry( + BalancedRedisQueue queue, BalancedQueueEntry balancedQueueEntry, QueueEntry queueEntry) { + this.queue = queue; + this.balancedQueueEntry = balancedQueueEntry; + this.queueEntry = queueEntry; + } + } /** * @field maxQueueSize @@ -110,7 +116,7 @@ private static Visitor createExecutionQueueVisitor( return new Visitor<>() { @Override public void visit(BalancedQueueEntry balancedQueueEntry) { - String entry = balancedQueueEntry.value(); + String entry = balancedQueueEntry.getValue(); QueueEntry.Builder queueEntry = QueueEntry.newBuilder(); try { JsonFormat.parser().merge(entry, queueEntry); @@ -145,11 +151,11 @@ public void visitDequeue(UnifiedJedis jedis, Visitor visito * @note Suggested return identifier: wasRemoved. */ public static boolean removeFromDequeue(UnifiedJedis jedis, ExecutionQueueEntry entry) { - return entry.queue().removeFromDequeue(jedis, entry.balancedQueueEntry()); + return entry.getQueue().removeFromDequeue(jedis, entry.getBalancedQueueEntry()); } public static void removeFromDequeue(AbstractPipeline pipeline, ExecutionQueueEntry entry) { - entry.queue().removeFromDequeue(pipeline, entry.balancedQueueEntry()); + entry.getQueue().removeFromDequeue(pipeline, entry.getBalancedQueueEntry()); } /** @@ -286,7 +292,7 @@ public ExecutionQueueEntry take( if (balancedQueueEntry != null) { try { QueueEntry.Builder queueEntryBuilder = QueueEntry.newBuilder(); - JsonFormat.parser().merge(balancedQueueEntry.value(), queueEntryBuilder); + JsonFormat.parser().merge(balancedQueueEntry.getValue(), queueEntryBuilder); QueueEntry queueEntry = queueEntryBuilder.build(); return new ExecutionQueueEntry(queue, balancedQueueEntry, queueEntry); @@ -500,72 +506,4 @@ private static SetMultimap toMultimap(List pr } return set; } - - private static QueueEntry parse(String json) { - QueueEntry.Builder queueEntry = QueueEntry.newBuilder(); - try { - JsonFormat.parser().merge(json, queueEntry); - } catch (InvalidProtocolBufferException e) { - log.log(Level.SEVERE, "invalid QueueEntry json: " + json, e); - } - return queueEntry.build(); - } - - public ScanResult scan( - UnifiedJedis jedis, String queueCursor, int count, String match) { - int queueIndex = queueCursor.indexOf('{') + 1; - String currentQueue = null; - if (queueIndex > 0) { - int queueEnd = queueCursor.indexOf('}'); - currentQueue = queueCursor.substring(queueIndex, queueEnd); - queueCursor = queueCursor.substring(queueEnd + 1); - } - Iterator queueIter = queues.iterator(); - - BalancedRedisQueue queue = null; - while (currentQueue != null && queueIter.hasNext()) { - queue = queueIter.next().queue(); - if (queue.getName().equals(currentQueue)) { - break; - } - } - - if (currentQueue != null && !currentQueue.equals(queue.getName())) { - return new ScanResult<>(SCAN_POINTER_START, new ArrayList<>()); - } - - List result = new ArrayList<>(count); - while (result.size() < count) { - if (currentQueue == null || queueCursor.equals(SCAN_POINTER_START)) { - if (!queueIter.hasNext()) { - break; - } - queue = queueIter.next().queue(); - currentQueue = queue.getName(); - } - final BalancedRedisQueue entryQueue = queue; - ScanResult scanResult = - queue.scan(jedis, queueCursor, count - result.size(), match); - queueCursor = scanResult.getCursor(); - result.addAll( - newArrayList( - transform( - scanResult.getResult(), - balancedQueueEntry -> - new ExecutionQueueEntry( - entryQueue, balancedQueueEntry, parse(balancedQueueEntry.value()))))); - } - - if (queueCursor.equals(SCAN_POINTER_START)) { - currentQueue = null; - if (queueIter.hasNext()) { - currentQueue = queueIter.next().queue().getName(); - } - } - String nextCursor = SCAN_POINTER_START; - if (currentQueue != null) { - nextCursor = "{" + currentQueue + "}" + queueCursor; - } - return new ScanResult<>(nextCursor, result); - } } diff --git a/src/main/java/build/buildfarm/instance/shard/JedisCasWorkerMap.java b/src/main/java/build/buildfarm/instance/shard/JedisCasWorkerMap.java index 0353d6cd5d..03aa763c66 100644 --- a/src/main/java/build/buildfarm/instance/shard/JedisCasWorkerMap.java +++ b/src/main/java/build/buildfarm/instance/shard/JedisCasWorkerMap.java @@ -159,7 +159,6 @@ public void removeAll(Iterable blobDigests, String workerName) { @Override public String getAny(Digest blobDigest) { String key = redisCasKey(blobDigest); - jedis.expire(key, keyExpiration_s); return jedis.srandmember(key); } @@ -174,7 +173,6 @@ public String getAny(Digest blobDigest) { @Override public Set get(Digest blobDigest) { String key = redisCasKey(blobDigest); - jedis.expire(key, keyExpiration_s); return jedis.smembers(key); } diff --git a/src/main/java/build/buildfarm/instance/shard/JedisClusterFactory.java b/src/main/java/build/buildfarm/instance/shard/JedisClusterFactory.java index 90999ca7f2..2290ba3271 100644 --- a/src/main/java/build/buildfarm/instance/shard/JedisClusterFactory.java +++ b/src/main/java/build/buildfarm/instance/shard/JedisClusterFactory.java @@ -213,7 +213,6 @@ private static ConnectionPoolConfig createConnectionPoolConfig() { connectionPoolConfig.setMinIdle(configs.getBackplane().getJedisPoolMinIdle()); connectionPoolConfig.setTimeBetweenEvictionRuns( Duration.ofMillis(configs.getBackplane().getJedisTimeBetweenEvictionRunsMillis())); - connectionPoolConfig.setTestOnBorrow(configs.getBackplane().isConnectionValidatedOnBorrow()); return connectionPoolConfig; } diff --git a/src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareOutputStream.java b/src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareOutputStream.java deleted file mode 100644 index e9f86da890..0000000000 --- a/src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareOutputStream.java +++ /dev/null @@ -1,80 +0,0 @@ -package build.buildfarm.instance.shard; - -import build.buildfarm.common.function.IOSupplier; -import build.buildfarm.common.io.FeedbackOutputStream; -import io.grpc.Status; -import io.grpc.Status.Code; -import java.io.IOException; - -class ReadOnlyAwareOutputStream extends FeedbackOutputStream { - @FunctionalInterface - private interface OnStream { - void accept(FeedbackOutputStream out) throws IOException; - } - - private final IOSupplier supplier; - private FeedbackOutputStream feedbackOut; - - ReadOnlyAwareOutputStream(IOSupplier supplier) throws IOException { - this(supplier, supplier.get()); - } - - private ReadOnlyAwareOutputStream( - IOSupplier supplier, FeedbackOutputStream feedbackOut) { - super(); - this.feedbackOut = feedbackOut; - this.supplier = supplier; - } - - private synchronized FeedbackOutputStream getOutput() throws IOException { - if (feedbackOut == null) { - feedbackOut = supplier.get(); - } - return feedbackOut; - } - - private void withReadOnlyRetry(OnStream onStream) throws IOException { - // may be overeager here - need more metrics as well - for (; ; ) { - try { - onStream.accept(getOutput()); - return; - } catch (IOException e) { - if (Status.fromThrowable(e).getCode() != Code.FAILED_PRECONDITION) { - throw e; - } - feedbackOut = null; - } - } - } - - @Override - public void write(int b) throws IOException { - withReadOnlyRetry(out -> out.write(b)); - } - - @Override - public void write(byte[] b) throws IOException { - withReadOnlyRetry(out -> out.write(b)); - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - withReadOnlyRetry(out -> out.write(b, off, len)); - } - - @Override - public void flush() throws IOException { - withReadOnlyRetry(out -> out.flush()); - } - - @Override - public void close() throws IOException { - withReadOnlyRetry(out -> out.close()); - } - - @Override - public boolean isReady() { - return feedbackOut.isReady(); - } -} diff --git a/src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareWrite.java b/src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareWrite.java deleted file mode 100644 index 22183cea5c..0000000000 --- a/src/main/java/build/buildfarm/instance/shard/ReadOnlyAwareWrite.java +++ /dev/null @@ -1,115 +0,0 @@ -package build.buildfarm.instance.shard; - -import static com.google.common.util.concurrent.Futures.addCallback; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; - -import build.buildfarm.common.Write; -import build.buildfarm.common.function.IOSupplier; -import build.buildfarm.common.io.FeedbackOutputStream; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.SettableFuture; -import io.grpc.Status; -import io.grpc.Status.Code; -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -/** - * The response to any write may be FAILED_PRECONDITION, which indicates that the backend is in a - * readonly state (currently only implementation). - * - *

Here, and in the resulting output stream, we interpret that status from the future and attempt - * to retry on different workers. The worker should have a backoff as well before being considered - * for any writes. - * - *

If any of the underlying stream operations fail on the new worker, that is conveyed to the - * caller. - */ -class ReadOnlyAwareWrite implements Write { - private final IOSupplier delegateSupplier; - private final SettableFuture future = SettableFuture.create(); - private Write delegate; - - ReadOnlyAwareWrite(IOSupplier delegateSupplier) { - this.delegateSupplier = delegateSupplier; - } - - // our undelegated future - @Override - public ListenableFuture getFuture() { - return future; - } - - private FutureCallback delegateCallback() { - return new FutureCallback<>() { - @Override - public void onSuccess(Long committedSize) { - future.set(committedSize); - } - - @Override - public void onFailure(Throwable t) { - // workers return FAILED_PRECONDITION for read only mode - if (Status.fromThrowable(t).getCode() == Code.FAILED_PRECONDITION) { - synchronized (ReadOnlyAwareWrite.this) { - delegate = null; - } - } else { - future.setException(t); - } - } - }; - } - - private synchronized Write getDelegate() throws IOException { - if (delegate == null) { - delegate = delegateSupplier.get(); - addCallback(delegate.getFuture(), delegateCallback(), directExecutor()); - } - return delegate; - } - - @Override - public long getCommittedSize() { - try { - return getDelegate().getCommittedSize(); - } catch (IOException e) { - // unlikely - return 0; - } - } - - @Override - public boolean isComplete() { - try { - return getDelegate().isComplete(); - } catch (IOException e) { - // unlikely - return false; - } - } - - @Override - public FeedbackOutputStream getOutput( - long offset, long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) - throws IOException { - return new ReadOnlyAwareOutputStream( - () -> getDelegate().getOutput(offset, deadlineAfter, deadlineAfterUnits, onReadyHandler)); - } - - @Override - public ListenableFuture getOutputFuture( - long offset, long deadlineAfter, TimeUnit deadlineAfterUnits, Runnable onReadyHandler) { - // unused interface call on servers - throw new UnsupportedOperationException(); - } - - @Override - public void reset() { - try { - getDelegate().reset(); - } catch (IOException e) { - // unlikely, ignore - } - } -} diff --git a/src/main/java/build/buildfarm/instance/shard/RedisShardBackplane.java b/src/main/java/build/buildfarm/instance/shard/RedisShardBackplane.java index 33291ce620..0750b08f8e 100644 --- a/src/main/java/build/buildfarm/instance/shard/RedisShardBackplane.java +++ b/src/main/java/build/buildfarm/instance/shard/RedisShardBackplane.java @@ -22,7 +22,6 @@ import build.bazel.remote.execution.v2.ActionResult; import build.bazel.remote.execution.v2.ExecuteOperationMetadata; -import build.bazel.remote.execution.v2.ExecutedActionMetadata; import build.bazel.remote.execution.v2.ExecutionStage; import build.bazel.remote.execution.v2.Platform; import build.bazel.remote.execution.v2.RequestMetadata; @@ -58,7 +57,6 @@ import build.buildfarm.v1test.QueuedOperationMetadata; import build.buildfarm.v1test.ShardWorker; import build.buildfarm.v1test.WorkerChange; -import build.buildfarm.v1test.WorkerExecutedMetadata; import build.buildfarm.v1test.WorkerType; import build.buildfarm.worker.resources.LocalResourceSet; import com.google.common.annotations.VisibleForTesting; @@ -73,7 +71,6 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.longrunning.Operation; -import com.google.protobuf.Any; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; import com.google.protobuf.util.JsonFormat; @@ -84,7 +81,6 @@ import io.grpc.Deadline; import java.io.IOException; import java.time.Instant; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -113,7 +109,7 @@ public class RedisShardBackplane implements Backplane { private static final int workerSetMaxAge = 3; // seconds - static final JsonFormat.Printer executionPrinter = + static final JsonFormat.Printer operationPrinter = JsonFormat.printer() .usingTypeRegistry( JsonFormat.TypeRegistry.newBuilder() @@ -122,20 +118,6 @@ public class RedisShardBackplane implements Backplane { .add(PreconditionFailure.getDescriptor()) .build()); - static final JsonFormat.Printer actionResultPrinter = - JsonFormat.printer() - .usingTypeRegistry( - JsonFormat.TypeRegistry.newBuilder() - .add(WorkerExecutedMetadata.getDescriptor()) - .build()); - - static final JsonFormat.Parser actionResultParser = - JsonFormat.parser() - .usingTypeRegistry( - JsonFormat.TypeRegistry.newBuilder() - .add(WorkerExecutedMetadata.getDescriptor()) - .build()); - private final String source; // used in operation change publication private final boolean subscribeToBackplane; private final boolean runFailsafeOperation; @@ -206,7 +188,7 @@ abstract static class ExecuteEntryListVisitor implements Visitor onOperationNam new Visitor<>() { @Override public void visit(ExecutionQueueEntry executionQueueEntry) { - QueueEntry queueEntry = executionQueueEntry.queueEntry(); + QueueEntry queueEntry = executionQueueEntry.getQueueEntry(); String executionName = queueEntry.getExecuteEntry().getOperationName(); String value = state.dispatchingExecutions.get(jedis, executionName); long dispatchingTimeout_ms = configs.getBackplane().getDispatchingTimeoutMillis(); @@ -319,7 +301,7 @@ private void scanQueue(UnifiedJedis jedis, Consumer onOperationName) { new Visitor<>() { @Override public void visit(ExecutionQueueEntry executionQueueEntry) { - QueueEntry queueEntry = executionQueueEntry.queueEntry(); + QueueEntry queueEntry = executionQueueEntry.getQueueEntry(); onOperationName.accept(queueEntry.getExecuteEntry().getOperationName()); } }); @@ -387,7 +369,7 @@ private void updateWatchers(UnifiedJedis jedis) { static String printOperationChange(OperationChange operationChange) throws InvalidProtocolBufferException { - return executionPrinter.print(operationChange); + return operationPrinter.print(operationChange); } void publish( @@ -903,10 +885,10 @@ private Map fetchAndExpireWorkers( return returnWorkers; } - public static ActionResult parseActionResult(String json) { + private static ActionResult parseActionResult(String json) { try { ActionResult.Builder builder = ActionResult.newBuilder(); - actionResultParser.merge(json, builder); + JsonFormat.parser().merge(json, builder); return builder.build(); } catch (InvalidProtocolBufferException e) { return null; @@ -916,11 +898,7 @@ public static ActionResult parseActionResult(String json) { @SuppressWarnings("ConstantConditions") @Override public ActionResult getActionResult(ActionKey actionKey) throws IOException { - String json = - client.call( - jedis -> - state.actionCache.getex( - jedis, actionKey.toString(), configs.getBackplane().getActionCacheExpire())); + String json = client.call(jedis -> state.actionCache.get(jedis, actionKey.toString())); if (json == null) { return null; } @@ -942,45 +920,10 @@ public void blacklistAction(String actionId) throws IOException { jedis, actionId, "", configs.getBackplane().getActionBlacklistExpire())); } - private String printActionResult(ActionResult actionResult) - throws InvalidProtocolBufferException { - InvalidProtocolBufferException cause; - try { - return actionResultPrinter.print(actionResult); - } catch (InvalidProtocolBufferException e) { - // can happen with unknown types in auxiliary_metadata - // this is extremely brittle and will have issues with any novel introduction of Any in remote - // apis releases - cause = e; - } - - ActionResult.Builder builder = actionResult.toBuilder(); - ExecutedActionMetadata.Builder metadata = - builder.getExecutionMetadataBuilder().clearAuxiliaryMetadata(); - for (Any auxiliaryMetadata : actionResult.getExecutionMetadata().getAuxiliaryMetadataList()) { - try { - // test the serialization capacity of this any - actionResultPrinter.print(auxiliaryMetadata); - // serialization passed, re-add it - metadata.addAuxiliaryMetadata(auxiliaryMetadata); - } catch (InvalidProtocolBufferException e) { - // ignore - } - } - - String json = actionResultPrinter.print(builder.build()); - // purge must have succeeded, indicate as much to the server log - log.log( - Level.WARNING, - "error printing auxiliary_metadata for key %s, unrecognized content purged", - cause); - return json; - } - @SuppressWarnings("ConstantConditions") @Override public void putActionResult(ActionKey actionKey, ActionResult actionResult) throws IOException { - String json = printActionResult(actionResult); + String json = JsonFormat.printer().print(actionResult); client.run( jedis -> state.actionCache.insert( @@ -1083,7 +1026,7 @@ public boolean putOperation(Operation operation, ExecutionStage.Value stage) thr String json; try { - json = executionPrinter.print(operation); + json = operationPrinter.print(operation); } catch (InvalidProtocolBufferException e) { log.log(Level.SEVERE, "error printing operation " + operation.getName(), e); return false; @@ -1126,7 +1069,7 @@ private void queue( @Override public void queue(QueueEntry queueEntry, Operation operation) throws IOException { String executionName = operation.getName(); - String operationJson = executionPrinter.print(operation); + String operationJson = operationPrinter.print(operation); String queueEntryJson = JsonFormat.printer().print(queueEntry); Operation publishOperation = onPublish.apply(operation); int priority = queueEntry.getExecuteEntry().getExecutionPolicy().getPriority(); @@ -1192,37 +1135,6 @@ public ScanResult scanDispatchedOperations(String cursor, i return new ScanResult(tokenFromRedisCursor(scanResult.getCursor()), builder.build()); } - @Override - public ScanResult> scanQueuedOperations(String cursor, int count) - throws IOException { - ImmutableList.Builder> builder = ImmutableList.builder(); - redis.clients.jedis.resps.ScanResult scanResult = - client.call(jedis -> state.executionQueue.scan(jedis, cursor, count, /* match= */ "*")); - for (ExecutionQueueEntry entry : scanResult.getResult()) { - builder.add(new AbstractMap.SimpleEntry<>(entry.queue().getName(), entry.queueEntry())); - } - return new ScanResult(tokenFromRedisCursor(scanResult.getCursor()), builder.build()); - } - - @Override - public ScanResult scanPrequeuedOperations(String cursor, int count) - throws IOException { - ImmutableList.Builder builder = ImmutableList.builder(); - redis.clients.jedis.resps.ScanResult scanResult = - client.call(jedis -> state.prequeue.scan(jedis, cursor, count, /* match= */ "*")); - for (BalancedQueueEntry entry : scanResult.getResult()) { - ExecuteEntry.Builder executeEntryBuilder = ExecuteEntry.newBuilder(); - try { - JsonFormat.parser().merge(entry.value(), executeEntryBuilder); - ExecuteEntry executeEntry = executeEntryBuilder.build(); - builder.add(executeEntry); - } catch (InvalidProtocolBufferException e) { - log.log(Level.SEVERE, "error parsing execute entry", e); - } - } - return new ScanResult(tokenFromRedisCursor(scanResult.getCursor()), builder.build()); - } - private synchronized ExecutorService getDequeueService() { if (dequeueService == null) { dequeueService = BuildfarmExecutors.getDequeuePool(); @@ -1238,7 +1150,7 @@ private ExecuteEntry deprequeueOperation(UnifiedJedis jedis) throws InterruptedE ExecuteEntry.Builder executeEntryBuilder = ExecuteEntry.newBuilder(); try { - JsonFormat.parser().merge(balancedQueueEntry.value(), executeEntryBuilder); + JsonFormat.parser().merge(balancedQueueEntry.getValue(), executeEntryBuilder); ExecuteEntry executeEntry = executeEntryBuilder.build(); String executionName = executeEntry.getOperationName(); @@ -1276,7 +1188,7 @@ public ExecuteEntry deprequeueOperation() throws IOException, InterruptedExcepti return null; } - QueueEntry queueEntry = executionQueueEntry.queueEntry(); + QueueEntry queueEntry = executionQueueEntry.getQueueEntry(); String executionName = queueEntry.getExecuteEntry().getOperationName(); Operation operation = keepaliveExecution(executionName); Unified unified = (Unified) jedis; @@ -1391,7 +1303,7 @@ public boolean prequeue(ExecuteEntry executeEntry, Operation execution, boolean throws IOException { String toolInvocationId = executeEntry.getRequestMetadata().getToolInvocationId(); String executionName = execution.getName(); - String operationJson = executionPrinter.print(execution); + String operationJson = operationPrinter.print(execution); String executeEntryJson = JsonFormat.printer().print(executeEntry); Operation publishExecution = onPublish.apply(execution); int priority = executeEntry.getExecutionPolicy().getPriority(); diff --git a/src/main/java/build/buildfarm/instance/shard/ServerInstance.java b/src/main/java/build/buildfarm/instance/shard/ServerInstance.java index 706efd1d7b..d6f768af1b 100644 --- a/src/main/java/build/buildfarm/instance/shard/ServerInstance.java +++ b/src/main/java/build/buildfarm/instance/shard/ServerInstance.java @@ -34,9 +34,7 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.transform; import static com.google.common.util.concurrent.Futures.transformAsync; -import static com.google.common.util.concurrent.Futures.whenAllComplete; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; -import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; import static java.lang.String.format; import static java.util.concurrent.Executors.newFixedThreadPool; import static java.util.concurrent.Executors.newSingleThreadExecutor; @@ -94,7 +92,6 @@ import build.buildfarm.instance.server.NodeInstance; import build.buildfarm.instance.stub.StubInstance; import build.buildfarm.v1test.BackplaneStatus; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.DispatchedOperation; import build.buildfarm.v1test.ExecuteEntry; import build.buildfarm.v1test.GetClientStartTimeRequest; @@ -105,7 +102,6 @@ import build.buildfarm.v1test.QueuedOperation; import build.buildfarm.v1test.QueuedOperationMetadata; import build.buildfarm.v1test.Tree; -import build.buildfarm.v1test.WorkerProfileMessage; import com.github.benmanes.caffeine.cache.AsyncCache; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -283,8 +279,6 @@ public class ServerInstance extends NodeInstance { private final boolean mergeExecutions; private final Scannable indexKeys; private final Scannable operations; - private final Scannable prequeuedOperations; - private final Scannable> queuedOperations; private final Scannable dispatchedOperations; private final Scannable correlatedInvocations; private final Scannable toolInvocations; @@ -447,7 +441,6 @@ public String scan(int limit, String pageToken, Consumer onOperation) return result.getToken(); } }; - // we can probably just construct the scanner for this once this.dispatchedOperations = new Scannable<>() { @Override @@ -465,39 +458,6 @@ public String scan( return scanResult.getToken(); } }; - this.queuedOperations = - new Scannable<>() { - @Override - public String getName() { - return "queued"; - } - - @Override - public String scan( - int limit, String pageToken, Consumer> onQueueEntry) - throws IOException { - Backplane.ScanResult> scanResult = - backplane.scanQueuedOperations(pageToken, limit); - scanResult.getResult().forEach(onQueueEntry); - return scanResult.getToken(); - } - }; - this.prequeuedOperations = - new Scannable<>() { - @Override - public String getName() { - return "prequeued"; - } - - @Override - public String scan(int limit, String pageToken, Consumer onExecuteEntry) - throws IOException { - Backplane.ScanResult scanResult = - backplane.scanPrequeuedOperations(pageToken, limit); - scanResult.getResult().forEach(onExecuteEntry); - return scanResult.getToken(); - } - }; this.indexKeys = new OperationNameScannable( new Scannable<>() { @@ -554,29 +514,6 @@ ListenableFuture iterate() throws IOException, InterruptedException { log.log(Level.SEVERE, "OperationQueuer: Got null from deprequeue..."); return immediateFuture(null); } - if (executeEntry - .getRequestMetadata() - .getActionMnemonic() - .equals("buildfarm:halt-on-deprequeue")) { - return listeningDecorator(operationTransformService) - .submit( - () -> { - try { - backplane.putOperation( - Operation.newBuilder() - .setName(executeEntry.getOperationName()) - .setDone(true) - .setMetadata( - Any.pack(ExecuteOperationMetadata.getDefaultInstance())) - .setResponse(Any.pack(ExecuteResponse.getDefaultInstance())) - .build(), - ExecutionStage.Value.COMPLETED); - } catch (IOException e) { - throw Status.fromThrowable(e).asRuntimeException(); - } - return null; - }); - } // half the watcher expiry, need to expose this from backplane Poller poller = new Poller(Durations.fromSeconds(5)); String operationName = executeEntry.getOperationName(); @@ -1500,11 +1437,6 @@ public InputStream newBlobInput( compressor, digest, offset, deadlineAfter, deadlineAfterUnits, requestMetadata); } - @Override - public boolean isReadOnly() { - return false; - } - @Override public Write getBlobWrite( Compressor.Value compressor, @@ -1863,36 +1795,12 @@ private ListenableFuture uploadQueuedOperation( .setPlatform(queuedOperation.getCommand().getPlatform()) .build(); return transform( - retryWriteBlobFuture( - queuedOperationDigest, - queuedOperationBlob, - executeEntry.getRequestMetadata(), - timeout, - 5), + writeBlobFuture( + queuedOperationDigest, queuedOperationBlob, executeEntry.getRequestMetadata(), timeout), (committedSize) -> new QueuedOperationResult(entry, metadata), service); } - private ListenableFuture retryWriteBlobFuture( - build.buildfarm.v1test.Digest digest, - ByteString content, - RequestMetadata requestMetadata, - Duration timeout, - int maxRetries) - throws EntryLimitException { - ListenableFuture future = writeBlobFuture(digest, content, requestMetadata, timeout); - return catchingAsync( - future, - Throwable.class, - t -> { - if (maxRetries == 0 && SHARD_IS_RETRIABLE.test(Status.fromThrowable(t))) { - return immediateFailedFuture(t); - } - return retryWriteBlobFuture(digest, content, requestMetadata, timeout, maxRetries - 1); - }, - directExecutor()); - } - private ListenableFuture writeBlobFuture( build.buildfarm.v1test.Digest digest, ByteString content, @@ -2484,14 +2392,7 @@ private Operation schedule( .build(); Operation operation = Operation.newBuilder().setName(executionName).setMetadata(Any.pack(metadata)).build(); - if (requestMetadata.getActionMnemonic().equals("buildfarm:halt-on-execute")) { - operation = - operation.toBuilder() - .setDone(true) - .setMetadata(Any.pack(ExecuteOperationMetadata.getDefaultInstance())) - .setResponse(Any.pack(ExecuteResponse.getDefaultInstance())) - .build(); - } else if (inDenyList(requestMetadata)) { + if (inDenyList(requestMetadata)) { operation = operation.toBuilder() .setDone(true) @@ -2836,8 +2737,8 @@ private ListenableFuture transformAndQueue( profiledQueuedMetadata.getQueuedOperationMetadata().getQueuedOperationDigest(); long startUploadUSecs = stopwatch.elapsed(MICROSECONDS); return transform( - retryWriteBlobFuture( - queuedOperationDigest, queuedOperationBlob, requestMetadata, timeout, 5), + writeBlobFuture( + queuedOperationDigest, queuedOperationBlob, requestMetadata, timeout), (committedSize) -> profiledQueuedMetadata .setUploadedIn( @@ -2935,13 +2836,6 @@ public BackplaneStatus backplaneStatus() { public boolean putOperation(Operation operation) { if (isErrored(operation)) { try { - ExecuteOperationMetadata metadata = expectExecuteOperationMetadata(operation); - if (metadata != null) { - ActionKey actionKey = - DigestUtil.asActionKey( - DigestUtil.fromDigest(metadata.getActionDigest(), metadata.getDigestFunction())); - backplane.unmergeExecution(actionKey); - } return backplane.putOperation(operation, ExecutionStage.Value.COMPLETED); } catch (IOException e) { throw Status.fromThrowable(e).asRuntimeException(); @@ -3121,100 +3015,34 @@ static Operation toOperation(String name) { } } - private Scannable dispatchedOperationsScannable() { - return new Scannable<>() { - @Override - public String getName() { - return dispatchedOperations.getName(); - } - - @Override - public String scan(int limit, String pageToken, Consumer onOperation) - throws IOException { - return dispatchedOperations.scan( - limit, - pageToken, - dispatchedOperation -> { - ExecuteEntry executeEntry = dispatchedOperation.getQueueEntry().getExecuteEntry(); - onOperation.accept( - Operation.newBuilder() - .setName(executeEntry.getOperationName()) - .setMetadata(Any.pack(dispatchedOperation)) - .build()); - }); - } - }; - } - - private Scannable prequeuedOperationsScannable() { - return new Scannable<>() { - @Override - public String getName() { - return prequeuedOperations.getName(); - } - - @Override - public String scan(int limit, String pageToken, Consumer onOperation) - throws IOException { - // fancier things later with queue - return prequeuedOperations.scan( - limit, - pageToken, - executeEntry -> { - onOperation.accept( - Operation.newBuilder() - .setName(executeEntry.getOperationName()) - .setMetadata(Any.pack(executeEntry)) - .build()); - }); - } - }; - } - - private Scannable queuedOperationsScannable() { - return new Scannable<>() { - @Override - public String getName() { - return queuedOperations.getName(); - } - - @Override - public String scan(int limit, String pageToken, Consumer onOperation) - throws IOException { - // fancier things later with queue - return queuedOperations.scan( - limit, - pageToken, - entry -> { - ExecuteEntry executeEntry = entry.getValue().getExecuteEntry(); - onOperation.accept( - Operation.newBuilder() - .setName(executeEntry.getOperationName()) - .setMetadata(Any.pack(entry.getValue())) - .build()); - }); - } - }; - } - Filter parseOperationsFilter(String filter) { if (filter.startsWith("toolInvocationId=")) { return new Filter<>( ImmutableList.of(new ToolInvocationExecutionsBounds(filter.split("=")[1]))); } - if (filter.startsWith("status=")) { - Scannable scannable = null; - if (filter.equals("status=dispatched")) { - // could create this once per object - scannable = dispatchedOperationsScannable(); - } else if (filter.equals("status=queued")) { - scannable = queuedOperationsScannable(); - } else if (filter.equals("status=prequeued")) { - scannable = prequeuedOperationsScannable(); - } - if (scannable != null) { - return new Filter<>(ImmutableList.of(scannable)); - } + if (filter.equals("status=dispatched")) { + return new Filter<>( + ImmutableList.of( + new Scannable<>() { + @Override + public String getName() { + return dispatchedOperations.getName(); + } + + @Override + public String scan(int limit, String pageToken, Consumer onOperation) + throws IOException { + return dispatchedOperations.scan( + limit, + pageToken, + dispatchedOperation -> { + ExecuteEntry executeEntry = + dispatchedOperation.getQueueEntry().getExecuteEntry(); + onOperation.accept( + OperationNameScannable.toOperation(executeEntry.getOperationName())); + }); + } + })); } // more? return new Filter<>(ImmutableList.of(operations)); @@ -3297,10 +3125,7 @@ private String listFilter( CountingConsumer onCounting = new CountingConsumer<>(onResult); token = location.scan(pageSize, token, onCounting); checkState( - token.equals(Scannable.SENTINEL_PAGE_TOKEN) || onCounting.getCount() == pageSize, - format( - "%s was not %s or %d != %d", - token, Scannable.SENTINEL_PAGE_TOKEN, onCounting.getCount(), pageSize)); + token.equals(Scannable.SENTINEL_PAGE_TOKEN) || onCounting.getCount() == pageSize); pageSize -= onCounting.getCount(); if (pageSize > 0) { locationName = null; @@ -3378,37 +3203,6 @@ protected CacheCapabilities getCacheCapabilities() { .build(); } - @Override - public ListenableFuture getWorkerProfile(String name) { - return workerStub(name).getWorkerProfile(name); - } - - @Override - public ListenableFuture batchWorkerProfiles(Iterable names) { - Iterable> profiles = - Iterables.transform(names, this::getWorkerProfile); - return whenAllComplete(profiles) - .call( - () -> { - Iterator iter = names.iterator(); - Iterator> profileIter = profiles.iterator(); - BatchWorkerProfilesResponse.Builder response = - BatchWorkerProfilesResponse.newBuilder(); - while (iter.hasNext() && profileIter.hasNext()) { - ListenableFuture profileFuture = profileIter.next(); - BatchWorkerProfilesResponse.Response.Builder builder = - response.addResponsesBuilder().setWorkerName(iter.next()); - try { - builder.setProfile(profileFuture.get()); - } catch (Exception e) { - builder.setStatus(StatusProto.fromThrowable(e)); - } - } - return response.build(); - }, - directExecutor()); - } - public String indexCorrelatedInvocations(URI uri) throws IOException { // policy might not be right to apply outside of here // definitely correct for directing backplane though diff --git a/src/main/java/build/buildfarm/instance/stub/BUILD b/src/main/java/build/buildfarm/instance/stub/BUILD index c22b13dd69..0a57d01b04 100644 --- a/src/main/java/build/buildfarm/instance/stub/BUILD +++ b/src/main/java/build/buildfarm/instance/stub/BUILD @@ -15,21 +15,21 @@ java_library( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//third_party/remote-apis:build_bazel_remote_asset_v1_remote_asset_java_grpc", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/longrunning:longrunning_java_grpc", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/asset/v1:remote_asset_java_proto", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], diff --git a/src/main/java/build/buildfarm/instance/stub/StubInstance.java b/src/main/java/build/buildfarm/instance/stub/StubInstance.java index c110c2c235..2a11bf505d 100644 --- a/src/main/java/build/buildfarm/instance/stub/StubInstance.java +++ b/src/main/java/build/buildfarm/instance/stub/StubInstance.java @@ -84,8 +84,6 @@ import build.buildfarm.v1test.AdminGrpc.AdminBlockingStub; import build.buildfarm.v1test.BackplaneStatus; import build.buildfarm.v1test.BackplaneStatusRequest; -import build.buildfarm.v1test.BatchWorkerProfilesRequest; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.GetClientStartTimeRequest; import build.buildfarm.v1test.GetClientStartTimeResult; import build.buildfarm.v1test.OperationQueueGrpc; @@ -96,11 +94,13 @@ import build.buildfarm.v1test.ReindexCasRequest; import build.buildfarm.v1test.ReindexCasRequestResults; import build.buildfarm.v1test.ShutDownWorkerGracefullyRequest; +import build.buildfarm.v1test.ShutDownWorkerGrpc; +import build.buildfarm.v1test.ShutDownWorkerGrpc.ShutDownWorkerBlockingStub; import build.buildfarm.v1test.Tree; -import build.buildfarm.v1test.WorkerControlGrpc; -import build.buildfarm.v1test.WorkerControlGrpc.WorkerControlBlockingStub; +import build.buildfarm.v1test.WorkerListMessage; +import build.buildfarm.v1test.WorkerListRequest; import build.buildfarm.v1test.WorkerProfileGrpc; -import build.buildfarm.v1test.WorkerProfileGrpc.WorkerProfileFutureStub; +import build.buildfarm.v1test.WorkerProfileGrpc.WorkerProfileBlockingStub; import build.buildfarm.v1test.WorkerProfileMessage; import build.buildfarm.v1test.WorkerProfileRequest; import com.google.bytestream.ByteStreamGrpc; @@ -344,22 +344,22 @@ public OperationQueueBlockingStub get() { }); @SuppressWarnings("Guava") - private final Supplier workerProfileFutureStub = + private final Supplier workerProfileBlockingStub = Suppliers.memoize( new Supplier<>() { @Override - public WorkerProfileFutureStub get() { - return WorkerProfileGrpc.newFutureStub(channel); + public WorkerProfileBlockingStub get() { + return WorkerProfileGrpc.newBlockingStub(channel); } }); @SuppressWarnings("Guava") - private final Supplier workerControlBlockingStub = + private final Supplier shutDownWorkerBlockingStub = Suppliers.memoize( new Supplier<>() { @Override - public WorkerControlBlockingStub get() { - return WorkerControlGrpc.newBlockingStub(channel); + public ShutDownWorkerBlockingStub get() { + return ShutDownWorkerGrpc.newBlockingStub(channel); } }); @@ -541,11 +541,6 @@ public ListenableFuture fetchBlob( directExecutor()); } - @Override - public boolean isReadOnly() { - return false; - } - @Override public Write getOperationStreamWrite(String name) { return getWrite( @@ -943,19 +938,14 @@ public ServerCapabilities getCapabilities() { } @Override - public ListenableFuture getWorkerProfile(String name) { - return deadlined(workerProfileFutureStub) - .getWorkerProfile(WorkerProfileRequest.newBuilder().setWorkerName(name).build()); + public WorkerProfileMessage getWorkerProfile() { + return deadlined(workerProfileBlockingStub) + .getWorkerProfile(WorkerProfileRequest.newBuilder().build()); } @Override - public ListenableFuture batchWorkerProfiles(Iterable names) { - return deadlined(workerProfileFutureStub) - .batchWorkerProfiles( - BatchWorkerProfilesRequest.newBuilder() - .setInstanceName(getName()) - .addAllWorkerNames(names) - .build()); + public WorkerListMessage getWorkerList() { + return workerProfileBlockingStub.get().getWorkerList(WorkerListRequest.newBuilder().build()); } @Override @@ -988,7 +978,7 @@ public void deregisterWorker(String workerName) { @Override public PrepareWorkerForGracefulShutDownRequestResults shutDownWorkerGracefully() { throwIfStopped(); - return workerControlBlockingStub + return shutDownWorkerBlockingStub .get() .prepareWorkerForGracefulShutdown( PrepareWorkerForGracefulShutDownRequest.newBuilder().build()); diff --git a/src/main/java/build/buildfarm/metrics/AbstractMetricsPublisher.java b/src/main/java/build/buildfarm/metrics/AbstractMetricsPublisher.java index 1a0ec1bcbf..011cae0812 100644 --- a/src/main/java/build/buildfarm/metrics/AbstractMetricsPublisher.java +++ b/src/main/java/build/buildfarm/metrics/AbstractMetricsPublisher.java @@ -20,7 +20,6 @@ import build.bazel.remote.execution.v2.RequestMetadata; import build.buildfarm.common.Time; import build.buildfarm.v1test.OperationRequestMetadata; -import build.buildfarm.v1test.WorkerExecutedMetadata; import com.google.common.annotations.VisibleForTesting; import com.google.longrunning.Operation; import com.google.protobuf.InvalidProtocolBufferException; @@ -122,7 +121,7 @@ protected OperationRequestMetadata populateRequestMetadata( queuedTime.observe( Time.toDurationSeconds( executionMetadata.getQueuedTimestamp(), - executionMetadata.getWorkerStartTimestamp())); + executionMetadata.getExecutionStartTimestamp())); outputUploadTime.observe( Time.toDurationSeconds( executionMetadata.getOutputUploadStartTimestamp(), @@ -156,7 +155,6 @@ protected static String formatRequestMetadataToJson( .add(ExecuteResponse.getDescriptor()) .add(ExecuteOperationMetadata.getDescriptor()) .add(PreconditionFailure.getDescriptor()) - .add(WorkerExecutedMetadata.getDescriptor()) .build(); String formattedRequestMetadata = diff --git a/src/main/java/build/buildfarm/metrics/BUILD b/src/main/java/build/buildfarm/metrics/BUILD index 4e3a31945f..6b0ca5cbe5 100644 --- a/src/main/java/build/buildfarm/metrics/BUILD +++ b/src/main/java/build/buildfarm/metrics/BUILD @@ -8,12 +8,12 @@ java_library( deps = [ "//src/main/java/build/buildfarm/common", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/metrics/log/BUILD b/src/main/java/build/buildfarm/metrics/log/BUILD index ecd0b31463..79a576e476 100644 --- a/src/main/java/build/buildfarm/metrics/log/BUILD +++ b/src/main/java/build/buildfarm/metrics/log/BUILD @@ -9,10 +9,10 @@ java_library( "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/metrics", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/metrics/prometheus/BUILD b/src/main/java/build/buildfarm/metrics/prometheus/BUILD index b45a5a2c64..12d49b7cd1 100644 --- a/src/main/java/build/buildfarm/metrics/prometheus/BUILD +++ b/src/main/java/build/buildfarm/metrics/prometheus/BUILD @@ -8,9 +8,9 @@ java_library( plugins = ["//src/main/java/build/buildfarm/common:lombok"], visibility = ["//visibility:public"], deps = [ - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:io_prometheus_simpleclient_hotspot", - "@buildfarm_maven//:io_prometheus_simpleclient_httpserver", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:io_prometheus_simpleclient", + "@maven//:io_prometheus_simpleclient_hotspot", + "@maven//:io_prometheus_simpleclient_httpserver", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/proxy/http/BUILD b/src/main/java/build/buildfarm/proxy/http/BUILD index 0fa443a73d..c048ba9c3c 100644 --- a/src/main/java/build/buildfarm/proxy/http/BUILD +++ b/src/main/java/build/buildfarm/proxy/http/BUILD @@ -11,36 +11,36 @@ java_library( "//src/main/java/build/buildfarm/common/resources:resource_java_proto", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_auth_google_auth_library_credentials", - "@buildfarm_maven//:com_google_auth_google_auth_library_oauth2_http", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_auth", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_services", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_util", - "@buildfarm_maven//:io_netty_netty_buffer", - "@buildfarm_maven//:io_netty_netty_codec", - "@buildfarm_maven//:io_netty_netty_codec_http", - "@buildfarm_maven//:io_netty_netty_common", - "@buildfarm_maven//:io_netty_netty_handler", - "@buildfarm_maven//:io_netty_netty_transport", - "@buildfarm_maven//:io_netty_netty_transport_classes_epoll", - "@buildfarm_maven//:io_netty_netty_transport_classes_kqueue", - "@buildfarm_maven//:io_netty_netty_transport_native_epoll", - "@buildfarm_maven//:io_netty_netty_transport_native_kqueue", - "@buildfarm_maven//:io_netty_netty_transport_native_unix_common", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/rpc:rpc_java_proto", "@io_grpc_grpc_proto//:health_java_proto", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_auth_google_auth_library_credentials", + "@maven//:com_google_auth_google_auth_library_oauth2_http", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_auth", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_services", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_util", + "@maven//:io_netty_netty_buffer", + "@maven//:io_netty_netty_codec", + "@maven//:io_netty_netty_codec_http", + "@maven//:io_netty_netty_common", + "@maven//:io_netty_netty_handler", + "@maven//:io_netty_netty_transport", + "@maven//:io_netty_netty_transport_classes_epoll", + "@maven//:io_netty_netty_transport_classes_kqueue", + "@maven//:io_netty_netty_transport_native_epoll", + "@maven//:io_netty_netty_transport_native_kqueue", + "@maven//:io_netty_netty_transport_native_unix_common", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/main/java/build/buildfarm/rpms/server/BUILD b/src/main/java/build/buildfarm/rpms/server/BUILD index ceb5217789..934ecd35ea 100644 --- a/src/main/java/build/buildfarm/rpms/server/BUILD +++ b/src/main/java/build/buildfarm/rpms/server/BUILD @@ -4,8 +4,8 @@ pkg_rpm( name = "buildfarm-server-rpm", data = [ ":server-service", + "//config:base_properties", "//examples:example_configs", - "//examples:example_properties", "//src/main/java/build/buildfarm:buildfarm-server_deploy.jar", ], release = "0", diff --git a/src/main/java/build/buildfarm/rpms/worker/BUILD b/src/main/java/build/buildfarm/rpms/worker/BUILD index 29684801d2..3edfe9be76 100644 --- a/src/main/java/build/buildfarm/rpms/worker/BUILD +++ b/src/main/java/build/buildfarm/rpms/worker/BUILD @@ -4,8 +4,8 @@ pkg_rpm( name = "buildfarm-worker-rpm", data = [ ":worker-service", + "//config:base_properties", "//examples:example_configs", - "//examples:example_properties", "//src/main/java/build/buildfarm:buildfarm-shard-worker_deploy.jar", ], release = "0", diff --git a/src/main/java/build/buildfarm/server/BUILD b/src/main/java/build/buildfarm/server/BUILD index db7ee8ae3c..017f773cf2 100644 --- a/src/main/java/build/buildfarm/server/BUILD +++ b/src/main/java/build/buildfarm/server/BUILD @@ -17,18 +17,18 @@ java_library( "//src/main/java/build/buildfarm/metrics/prometheus", "//src/main/java/build/buildfarm/server/services", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_api_grpc_proto_google_common_protos", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_services", - "@buildfarm_maven//:io_grpc_grpc_util", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:javax_annotation_javax_annotation_api", - "@buildfarm_maven//:org_bouncycastle_bcprov_jdk15on", - "@buildfarm_maven//:org_projectlombok_lombok", "@io_grpc_grpc_proto//:health_java_proto", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_api_grpc_proto_google_common_protos", + "@maven//:com_google_guava_guava", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_services", + "@maven//:io_grpc_grpc_util", + "@maven//:io_prometheus_simpleclient", + "@maven//:javax_annotation_javax_annotation_api", + "@maven//:org_bouncycastle_bcprov_jdk15on", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/server/BuildFarmServer.java b/src/main/java/build/buildfarm/server/BuildFarmServer.java index 1181d6d44f..0d63d6f151 100644 --- a/src/main/java/build/buildfarm/server/BuildFarmServer.java +++ b/src/main/java/build/buildfarm/server/BuildFarmServer.java @@ -37,7 +37,6 @@ import build.buildfarm.server.services.OperationQueueService; import build.buildfarm.server.services.OperationsService; import build.buildfarm.server.services.PublishBuildEventService; -import build.buildfarm.server.services.WorkerProfileService; import io.grpc.ServerBuilder; import io.grpc.ServerInterceptor; import io.grpc.health.v1.HealthCheckResponse.ServingStatus; @@ -162,7 +161,6 @@ public synchronized void start(ServerBuilder serverBuilder, String publicName .addService(new FetchService(instance)) .addService(ProtoReflectionService.newInstance()) .addService(new PublishBuildEventService()) - .addService(new WorkerProfileService(instance)) .intercept(TransmitStatusRuntimeExceptionInterceptor.instance()) .intercept(headersInterceptor); GrpcMetrics.handleGrpcMetricIntercepts(serverBuilder, configs.getServer().getGrpcMetrics()); diff --git a/src/main/java/build/buildfarm/server/services/BUILD b/src/main/java/build/buildfarm/server/services/BUILD index 1619816691..fc610d1e1d 100644 --- a/src/main/java/build/buildfarm/server/services/BUILD +++ b/src/main/java/build/buildfarm/server/services/BUILD @@ -17,22 +17,22 @@ java_library( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//third_party/remote-apis:build_bazel_remote_asset_v1_remote_asset_java_grpc", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_services", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/devtools/build/v1:build_java_grpc", "@googleapis//google/devtools/build/v1:build_java_proto", "@googleapis//google/longrunning:longrunning_java_grpc", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_services", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/asset/v1:remote_asset_java_proto", "@remoteapis//build/bazel/semver:semver_java_proto", ], diff --git a/src/main/java/build/buildfarm/server/services/WorkerProfileService.java b/src/main/java/build/buildfarm/server/services/WorkerProfileService.java deleted file mode 100644 index cfc6ce87d7..0000000000 --- a/src/main/java/build/buildfarm/server/services/WorkerProfileService.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2020 The Buildfarm Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package build.buildfarm.server.services; - -import static com.google.common.util.concurrent.Futures.addCallback; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; - -import build.buildfarm.instance.Instance; -import build.buildfarm.v1test.BatchWorkerProfilesRequest; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; -import build.buildfarm.v1test.WorkerProfileGrpc; -import build.buildfarm.v1test.WorkerProfileMessage; -import build.buildfarm.v1test.WorkerProfileRequest; -import com.google.common.util.concurrent.FutureCallback; -import io.grpc.stub.StreamObserver; - -public class WorkerProfileService extends WorkerProfileGrpc.WorkerProfileImplBase { - private final Instance instance; - - public WorkerProfileService(Instance instance) { - this.instance = instance; - } - - @Override - public void getWorkerProfile( - WorkerProfileRequest request, StreamObserver responseObserver) { - addCallback( - instance.getWorkerProfile(request.getWorkerName()), - new FutureCallback<>() { - @Override - public void onSuccess(WorkerProfileMessage profile) { - responseObserver.onNext(profile); - responseObserver.onCompleted(); - } - - @Override - public void onFailure(Throwable t) { - responseObserver.onError(t); - } - }, - directExecutor()); - } - - @Override - public void batchWorkerProfiles( - BatchWorkerProfilesRequest request, - StreamObserver responseObserver) { - addCallback( - instance.batchWorkerProfiles(request.getWorkerNamesList()), - new FutureCallback<>() { - @Override - public void onSuccess(BatchWorkerProfilesResponse response) { - responseObserver.onNext(response); - responseObserver.onCompleted(); - } - - @Override - public void onFailure(Throwable t) { - responseObserver.onError(t); - } - }, - directExecutor()); - } -} diff --git a/src/main/java/build/buildfarm/tools/BUILD b/src/main/java/build/buildfarm/tools/BUILD index 59b982e4e6..7f52799583 100644 --- a/src/main/java/build/buildfarm/tools/BUILD +++ b/src/main/java/build/buildfarm/tools/BUILD @@ -12,16 +12,16 @@ java_binary( "//src/main/java/build/buildfarm/common/grpc", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", - "//src/main/java/build/buildfarm/worker/filesystem", + "//src/main/java/build/buildfarm/worker", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", ], ) @@ -29,8 +29,8 @@ java_plugin( name = "io-benchmark-plugins", processor_class = "org.openjdk.jmh.generators.BenchmarkProcessor", deps = [ - "@buildfarm_maven//:org_openjdk_jmh_jmh_core", - "@buildfarm_maven//:org_openjdk_jmh_jmh_generator_annprocess", + "@maven//:org_openjdk_jmh_jmh_core", + "@maven//:org_openjdk_jmh_jmh_generator_annprocess", ], ) @@ -45,18 +45,18 @@ java_binary( "//src/main/java/build/buildfarm/instance/stub", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/longrunning:operations_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -70,15 +70,15 @@ java_binary( "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/grpc", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -94,9 +94,9 @@ java_binary( "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -105,11 +105,12 @@ java_binary( name = "bf-cat", srcs = ["Cat.java"], data = [ - "//examples:example_properties", + "//config:base_properties", ], main_class = "build.buildfarm.tools.Cat", visibility = ["//visibility:public"], deps = [ + ":worker-profiler-printer", "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/grpc", "//src/main/java/build/buildfarm/common/resources", @@ -118,15 +119,15 @@ java_binary( "//src/main/java/build/buildfarm/instance/stub", "//src/main/java/build/buildfarm/server/services", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", ], ) @@ -134,27 +135,31 @@ java_binary( name = "bf-workerprofile", srcs = ["WorkerProfile.java"], data = [ - "//examples:example_properties", + "//config:base_properties", ], main_class = "build.buildfarm.tools.WorkerProfile", visibility = ["//visibility:public"], deps = [ ":worker-profiler-printer", "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/common/grpc", + "//src/main/java/build/buildfarm/common/redis", "//src/main/java/build/buildfarm/instance", + "//src/main/java/build/buildfarm/instance/shard", "//src/main/java/build/buildfarm/instance/stub", + "//src/main/java/build/buildfarm/worker/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:redis_clients_jedis", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:redis_clients_jedis", ], ) @@ -162,25 +167,26 @@ java_binary( name = "bf-ac", srcs = ["Ac.java"], data = [ - "//examples:example_properties", + "//config:base_properties", ], main_class = "build.buildfarm.tools.Ac", visibility = ["//visibility:public"], deps = [ + ":worker-profiler-printer", "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/grpc", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", ], ) @@ -193,7 +199,7 @@ java_binary( "//src/main/java/build/buildfarm/common/grpc", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_grpc", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_api", ], ) @@ -208,15 +214,15 @@ java_binary( "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", "//src/main/java/build/buildfarm/server/services", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -231,15 +237,15 @@ java_binary( "//src/main/java/build/buildfarm/common/grpc", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -261,11 +267,11 @@ java_binary( "//src/main/java/build/buildfarm/common/grpc", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", ], ) @@ -274,16 +280,23 @@ java_library( srcs = ["WorkerProfilePrinter.java"], visibility = ["//visibility:public"], deps = [ + "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/common/config", + "//src/main/java/build/buildfarm/common/redis", + "//src/main/java/build/buildfarm/instance", + "//src/main/java/build/buildfarm/instance/shard", + "//src/main/java/build/buildfarm/instance/stub", + "//src/main/java/build/buildfarm/worker/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:redis_clients_jedis", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:redis_clients_jedis", ], ) diff --git a/src/main/java/build/buildfarm/tools/Cat.java b/src/main/java/build/buildfarm/tools/Cat.java index c606907bf4..9d03604cbd 100644 --- a/src/main/java/build/buildfarm/tools/Cat.java +++ b/src/main/java/build/buildfarm/tools/Cat.java @@ -33,7 +33,6 @@ import build.bazel.remote.execution.v2.ExecuteOperationMetadata; import build.bazel.remote.execution.v2.ExecuteResponse; import build.bazel.remote.execution.v2.ExecutedActionMetadata; -import build.bazel.remote.execution.v2.ExecutionStage; import build.bazel.remote.execution.v2.FileNode; import build.bazel.remote.execution.v2.OutputDirectory; import build.bazel.remote.execution.v2.OutputFile; @@ -46,17 +45,12 @@ import build.buildfarm.common.resources.UploadBlobRequest; import build.buildfarm.instance.Instance; import build.buildfarm.instance.stub.StubInstance; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.Digest; -import build.buildfarm.v1test.DispatchedOperation; -import build.buildfarm.v1test.ExecuteEntry; import build.buildfarm.v1test.OperationTimesBetweenStages; -import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.QueuedOperation; import build.buildfarm.v1test.QueuedOperationMetadata; import build.buildfarm.v1test.StageInformation; import build.buildfarm.v1test.Tree; -import build.buildfarm.v1test.WorkerExecutedMetadata; import build.buildfarm.v1test.WorkerProfileMessage; import com.google.common.base.Stopwatch; import com.google.common.base.Strings; @@ -223,16 +217,6 @@ private static void printActionResult( } } - private static void printWorkerMetadata(WorkerExecutedMetadata metadata, int indentLevel) { - indentOut(indentLevel, format("Fetched Bytes: %d", metadata.getFetchedBytes())); - if (metadata.getLinkedInputDirectoriesCount() > 0) { - indentOut(indentLevel, "Linked Input Directories:"); - } - for (String linkedInputDirectory : metadata.getLinkedInputDirectoriesList()) { - indentOut(indentLevel + 1, linkedInputDirectory); - } - } - private static void printExecutedActionMetadata( ExecutedActionMetadata metadata, int indentLevel) { if (!metadata.getWorker().isEmpty()) { @@ -251,7 +235,7 @@ private static void printExecutedActionMetadata( indentLevel, "Input Fetch Start: " + Timestamps.toString(metadata.getInputFetchStartTimestamp())); } - if (metadata.hasInputFetchCompletedTimestamp()) { + if (metadata.hasQueuedTimestamp()) { indentOut( indentLevel, "Input Fetch Completed: " @@ -283,21 +267,6 @@ private static void printExecutedActionMetadata( indentLevel, "Worker Completed: " + Timestamps.toString(metadata.getWorkerCompletedTimestamp())); } - if (metadata.getAuxiliaryMetadataCount() > 0) { - indentOut(indentLevel, "Auxiliary Metadata:"); - } - for (Any auxiliary : metadata.getAuxiliaryMetadataList()) { - if (auxiliary.is(WorkerExecutedMetadata.class)) { - try { - printWorkerMetadata(auxiliary.unpack(WorkerExecutedMetadata.class), indentLevel + 1); - } catch (InvalidProtocolBufferException e) { - // unlikely - e.printStackTrace(); - } - } else { - indentOut(indentLevel + 1, "Unrecognized Metadata: " + auxiliary); - } - } } private static void printFindMissing( @@ -626,15 +595,6 @@ private static void printExecuteResponse( // FIXME server_logs } - private static ExecuteOperationMetadata executeEntryMetadata( - ExecuteEntry executeEntry, ExecutionStage.Value stage) { - return ExecuteOperationMetadata.newBuilder() - .setStage(stage) - .setActionDigest(DigestUtil.toDigest(executeEntry.getActionDigest())) - .setDigestFunction(executeEntry.getActionDigest().getDigestFunction()) - .build(); - } - private static void printOperation(Operation operation) { System.out.println("Operation: " + operation.getName()); if (operation.getDone()) { @@ -649,39 +609,20 @@ private static void printOperation(Operation operation) { operation.getMetadata().unpack(QueuedOperationMetadata.class); metadata = queuedOperationMetadata.getExecuteOperationMetadata(); requestMetadata = queuedOperationMetadata.getRequestMetadata(); - } else if (operation.getMetadata().is(ExecuteEntry.class)) { - ExecuteEntry executeEntry = operation.getMetadata().unpack(ExecuteEntry.class); - metadata = executeEntryMetadata(executeEntry, ExecutionStage.Value.UNKNOWN); - requestMetadata = executeEntry.getRequestMetadata(); - } else if (operation.getMetadata().is(QueueEntry.class)) { - QueueEntry queueEntry = operation.getMetadata().unpack(QueueEntry.class); - ExecuteEntry executeEntry = queueEntry.getExecuteEntry(); - metadata = executeEntryMetadata(executeEntry, ExecutionStage.Value.QUEUED); - requestMetadata = executeEntry.getRequestMetadata(); - } else if (operation.getMetadata().is(DispatchedOperation.class)) { - DispatchedOperation dispatchedOperation = - operation.getMetadata().unpack(DispatchedOperation.class); - ExecuteEntry executeEntry = dispatchedOperation.getQueueEntry().getExecuteEntry(); - metadata = - executeEntryMetadata( - executeEntry, ExecutionStage.Value.QUEUED); // latest we can know about here - requestMetadata = executeEntry.getRequestMetadata(); } else { metadata = operation.getMetadata().unpack(ExecuteOperationMetadata.class); requestMetadata = null; } - if (metadata != null) { - printExecutedActionMetadata(metadata.getPartialExecutionMetadata(), 1); - System.out.println("Metadata:"); - System.out.println(" Stage: " + metadata.getStage()); - digestFunction = metadata.getDigestFunction(); - System.out.println( - " Action: " - + DigestUtil.toString( - DigestUtil.fromDigest(metadata.getActionDigest(), digestFunction))); - System.out.println(" Stdout Stream: " + metadata.getStdoutStreamName()); - System.out.println(" Stderr Stream: " + metadata.getStderrStreamName()); - } + printExecutedActionMetadata(metadata.getPartialExecutionMetadata(), 1); + System.out.println("Metadata:"); + System.out.println(" Stage: " + metadata.getStage()); + digestFunction = metadata.getDigestFunction(); + System.out.println( + " Action: " + + DigestUtil.toString( + DigestUtil.fromDigest(metadata.getActionDigest(), digestFunction))); + System.out.println(" Stdout Stream: " + metadata.getStdoutStreamName()); + System.out.println(" Stderr Stream: " + metadata.getStderrStreamName()); if (requestMetadata != null) { printRequestMetadata(requestMetadata); } @@ -746,29 +687,16 @@ private static void fetch(Instance instance, Iterable uris) throws Excep System.out.println(DigestUtil.toString(digest.get())); } - private static void batchWorkerProfiles(Instance instance, Iterable names) - throws Exception { - BatchWorkerProfilesResponse responses = instance.batchWorkerProfiles(names).get(); - for (BatchWorkerProfilesResponse.Response response : responses.getResponsesList()) { - System.out.println("Worker: " + response.getWorkerName()); - int code = response.getStatus().getCode(); - if (code == Code.OK.getNumber()) { - printWorkerProfile(response.getProfile()); - } else { - System.out.println("Error: " + Code.forNumber(code)); - } - } - } - - private static void printWorkerProfile(WorkerProfileMessage workerProfile) { + private static void getWorkerProfile(Instance instance) { + WorkerProfileMessage response = instance.getWorkerProfile(); System.out.println("\nWorkerProfile:"); String strIntFormat = "%-50s : %d"; - long entryCount = workerProfile.getCasEntryCount(); - long unreferencedEntryCount = workerProfile.getCasUnreferencedEntryCount(); + long entryCount = response.getCasEntryCount(); + long unreferencedEntryCount = response.getCasUnreferencedEntryCount(); System.out.printf((strIntFormat) + "%n", "Current Total Entry Count", entryCount); - System.out.printf((strIntFormat) + "%n", "Current Total Size", workerProfile.getCasSize()); - System.out.printf((strIntFormat) + "%n", "Max Size", workerProfile.getCasMaxSize()); - System.out.printf((strIntFormat) + "%n", "Max Entry Size", workerProfile.getCasMaxEntrySize()); + System.out.printf((strIntFormat) + "%n", "Current Total Size", response.getCasSize()); + System.out.printf((strIntFormat) + "%n", "Max Size", response.getCasMaxSize()); + System.out.printf((strIntFormat) + "%n", "Max Entry Size", response.getCasMaxEntrySize()); System.out.printf( (strIntFormat) + "%n", "Current Unreferenced Entry Count", unreferencedEntryCount); if (entryCount != 0) { @@ -780,22 +708,20 @@ private static void printWorkerProfile(WorkerProfileMessage workerProfile) { System.out.printf( (strIntFormat) + "%n", "Current DirectoryEntry Count", - workerProfile.getCasDirectoryEntryCount()); + response.getCasDirectoryEntryCount()); System.out.printf( - (strIntFormat) + "%n", - "Number of Evicted Entries", - workerProfile.getCasEvictedEntryCount()); + (strIntFormat) + "%n", "Number of Evicted Entries", response.getCasEvictedEntryCount()); System.out.printf( (strIntFormat) + "%n", "Total Evicted Entries size in Bytes", - workerProfile.getCasEvictedEntrySize()); + response.getCasEvictedEntrySize()); - List stages = workerProfile.getStagesList(); + List stages = response.getStagesList(); for (StageInformation stage : stages) { printStageInformation(stage); } - List times = workerProfile.getTimesList(); + List times = response.getTimesList(); for (OperationTimesBetweenStages time : times) { printOperationTime(time); } @@ -962,8 +888,8 @@ public String description() { } @Override - public void run(Instance instance, Iterable args) throws Exception { - batchWorkerProfiles(instance, args); + public void run(Instance instance, Iterable args) { + getWorkerProfile(instance); } } diff --git a/src/main/java/build/buildfarm/tools/GracefulShutdown.java b/src/main/java/build/buildfarm/tools/GracefulShutdown.java index f97fc757d2..d79b62d891 100644 --- a/src/main/java/build/buildfarm/tools/GracefulShutdown.java +++ b/src/main/java/build/buildfarm/tools/GracefulShutdown.java @@ -20,7 +20,7 @@ import build.buildfarm.v1test.DisableScaleInProtectionRequest; import build.buildfarm.v1test.PrepareWorkerForGracefulShutDownRequest; import build.buildfarm.v1test.ShutDownWorkerGracefullyRequest; -import build.buildfarm.v1test.WorkerControlGrpc; +import build.buildfarm.v1test.ShutDownWorkerGrpc; import io.grpc.ManagedChannel; class GracefulShutdown { @@ -57,9 +57,9 @@ private static void prepareWorkerForShutDown(String[] args) { String workerIpWithPort = args[1]; System.out.println("Inform worker " + workerIpWithPort + " to prepare for shutdown!"); ManagedChannel channel = createChannel(workerIpWithPort); - WorkerControlGrpc.WorkerControlBlockingStub workerControlBlockingStub = - WorkerControlGrpc.newBlockingStub(channel); - workerControlBlockingStub.prepareWorkerForGracefulShutdown( + ShutDownWorkerGrpc.ShutDownWorkerBlockingStub shutDownWorkerBlockingStub = + ShutDownWorkerGrpc.newBlockingStub(channel); + shutDownWorkerBlockingStub.prepareWorkerForGracefulShutdown( PrepareWorkerForGracefulShutDownRequest.newBuilder().build()); System.out.println("Worker " + workerIpWithPort + " informed!"); } diff --git a/src/main/java/build/buildfarm/tools/Hist.java b/src/main/java/build/buildfarm/tools/Hist.java index eb02b6da9e..50f94219cf 100644 --- a/src/main/java/build/buildfarm/tools/Hist.java +++ b/src/main/java/build/buildfarm/tools/Hist.java @@ -51,7 +51,7 @@ private static void printHistogram(Instance instance) throws IOException { do { pageToken = instance.listOperations( - "executions", + instance.getName() + "/executions", LIST_OPERATIONS_MAXIMUM_PAGE_SIZE, pageToken, "status=dispatched", diff --git a/src/main/java/build/buildfarm/tools/Mount.java b/src/main/java/build/buildfarm/tools/Mount.java index b3b77acdaa..046b1090d2 100644 --- a/src/main/java/build/buildfarm/tools/Mount.java +++ b/src/main/java/build/buildfarm/tools/Mount.java @@ -25,7 +25,7 @@ import build.buildfarm.instance.Instance; import build.buildfarm.instance.stub.StubInstance; import build.buildfarm.v1test.Digest; -import build.buildfarm.worker.filesystem.FuseCAS; +import build.buildfarm.worker.FuseCAS; import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; import java.io.IOException; @@ -88,7 +88,7 @@ public synchronized InputStream newInput( } public static void main(String[] args) throws Exception { - if (args.length != 5) { + if (args.length != 6) { System.err.println("Usage: bf-mount "); System.err.println("\nMount an REAPI directory specified by 'digest' at 'name' under 'root'"); System.exit(1); diff --git a/src/main/java/build/buildfarm/tools/WorkerProfile.java b/src/main/java/build/buildfarm/tools/WorkerProfile.java index 8b78e7e622..c155097a75 100644 --- a/src/main/java/build/buildfarm/tools/WorkerProfile.java +++ b/src/main/java/build/buildfarm/tools/WorkerProfile.java @@ -16,28 +16,51 @@ import static build.buildfarm.common.grpc.Channels.createChannel; +import build.buildfarm.common.config.BuildfarmConfigs; +import build.buildfarm.common.config.ShardWorkerOptions; +import build.buildfarm.common.redis.RedisClient; import build.buildfarm.instance.Instance; +import build.buildfarm.instance.shard.JedisClusterFactory; import build.buildfarm.instance.stub.StubInstance; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.OperationTimesBetweenStages; +import build.buildfarm.v1test.ShardWorker; import build.buildfarm.v1test.StageInformation; import build.buildfarm.v1test.WorkerProfileMessage; +import com.google.common.collect.Sets; +import com.google.devtools.common.options.OptionsParser; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.Durations; +import com.google.protobuf.util.JsonFormat; +import io.grpc.StatusRuntimeException; import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import javax.naming.ConfigurationException; +import redis.clients.jedis.UnifiedJedis; class WorkerProfile { - private static Instance instance; + private static BuildfarmConfigs configs = BuildfarmConfigs.getInstance(); + + /** + * Transform worker string from "ip-10-135-31-210.ec2:8981" to "10.135.31.210". + * + * @param worker + * @return + */ + @SuppressWarnings("JavaDoc") + private static String workerStringTransformation(String worker) { + return worker.split("\\.")[0].substring("ip-".length()).replaceAll("-", ".") + + ':' + + worker.split(":")[1]; + } private static void analyzeMessage(String worker, WorkerProfileMessage response) { System.out.println("\nWorkerProfile:"); System.out.println(worker); String strIntFormat = "%-50s : %d"; - long entryCount = response.getCasEntryCount(); long unreferencedEntryCount = response.getCasUnreferencedEntryCount(); System.out.printf((strIntFormat) + "%n", "Current Total Entry Count", entryCount); @@ -59,10 +82,6 @@ private static void analyzeMessage(String worker, WorkerProfileMessage response) (strIntFormat) + "%n", "Total Evicted Entries size in Bytes", response.getCasEvictedEntrySize()); - System.out.printf( - ("%-50s : %2.1f%%") + "%n", - "Total size in Bytes", - (100.0f * response.getCasSize() / response.getCasMaxSize())); List stages = response.getStagesList(); for (StageInformation stage : stages) { @@ -76,35 +95,88 @@ private static void analyzeMessage(String worker, WorkerProfileMessage response) } @SuppressWarnings("ConstantConditions") - private static Set getWorkers() throws ConfigurationException, IOException { - return instance.backplaneStatus().getActiveExecuteWorkersList().stream() - .collect(Collectors.toUnmodifiableSet()); + private static Set getWorkers(String[] args) throws ConfigurationException, IOException { + OptionsParser parser = OptionsParser.newOptionsParser(ShardWorkerOptions.class); + parser.parseAndExitUponError(args); + List residue = parser.getResidue(); + if (residue.isEmpty()) { + throw new IllegalArgumentException("Missing Config_PATH"); + } + try { + configs.loadConfigs(Path.of(residue.get(3))); + } catch (IOException e) { + System.out.println("Could not parse yml configuration file." + e); + } + RedisClient client = new RedisClient(JedisClusterFactory.create("worker-profile").get()); + return client.call(jedis -> fetchWorkers(jedis, System.currentTimeMillis())); + } + + private static Set fetchWorkers(UnifiedJedis jedis, long now) { + Set workers = Sets.newConcurrentHashSet(); + for (Map.Entry entry : + jedis.hgetAll(configs.getBackplane().getWorkersHashName() + "_storage").entrySet()) { + String json = entry.getValue(); + try { + if (json != null) { + ShardWorker.Builder builder = ShardWorker.newBuilder(); + JsonFormat.parser().merge(json, builder); + ShardWorker worker = builder.build(); + if (worker.getExpireAt() > now) { + workers.add(worker.getEndpoint()); + } + } + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); + } + } + return workers; } @SuppressWarnings("BusyWait") - private static void workerProfile() throws IOException, ExecutionException, InterruptedException { + private static void workerProfile(String[] args) throws IOException { Set workers = null; + WorkerProfileMessage currentWorkerMessage; + HashMap workersToChannels = new HashMap<>(); + String type = args[1]; //noinspection InfiniteLoopStatement while (true) { // update worker list if (workers == null) { try { - workers = getWorkers(); + workers = getWorkers(args); } catch (ConfigurationException e) { e.printStackTrace(); } } if (workers == null || workers.isEmpty()) { System.out.println( - "cannot find any workers, make sure there are workers in the" + " cluster"); + "cannot find any workers, check the redis url and make sure there are workers in the" + + " cluster"); } else { - BatchWorkerProfilesResponse responses = instance.batchWorkerProfiles(workers).get(); + // remove the unregistered workers + for (String existingWorker : workersToChannels.keySet()) { + if (!workers.contains(existingWorker)) { + workersToChannels.remove(existingWorker); + } + } + // add new registered workers and profile - for (BatchWorkerProfilesResponse.Response worker : responses.getResponsesList()) { + for (String worker : workers) { + if (!workersToChannels.containsKey(worker)) { + workersToChannels.put( + worker, + new StubInstance( + type, + "bf-workerprofile", + createChannel(workerStringTransformation(worker)), + Durations.fromMinutes(1))); + } try { - analyzeMessage(worker.getWorkerName(), worker.getProfile()); - } catch (Exception e) { + currentWorkerMessage = workersToChannels.get(worker).getWorkerProfile(); + System.out.println(worker); + analyzeMessage(worker, currentWorkerMessage); + } catch (StatusRuntimeException e) { e.printStackTrace(); System.out.println("==============TIMEOUT"); } @@ -121,11 +193,9 @@ private static void workerProfile() throws IOException, ExecutionException, Inte } } - // how to run the binary: bf-workerprofile + // how to run the binary: bf-workerprofile WorkerProfile shard SHA256 + // examples/config.yml public static void main(String[] args) throws Exception { - instance = - new StubInstance( - args[0], "bf-workerprofile", createChannel(args[1]), Durations.fromMinutes(1)); - workerProfile(); + workerProfile(args); } } diff --git a/src/main/java/build/buildfarm/tools/WorkerProfilePrinter.java b/src/main/java/build/buildfarm/tools/WorkerProfilePrinter.java index 0359d29adb..9cd28fe000 100644 --- a/src/main/java/build/buildfarm/tools/WorkerProfilePrinter.java +++ b/src/main/java/build/buildfarm/tools/WorkerProfilePrinter.java @@ -14,12 +14,57 @@ package build.buildfarm.tools; +import static java.lang.String.format; + +import build.buildfarm.instance.Instance; import build.buildfarm.v1test.OperationTimesBetweenStages; import build.buildfarm.v1test.StageInformation; +import build.buildfarm.v1test.WorkerProfileMessage; import com.google.protobuf.Duration; import com.google.protobuf.util.Durations; +import java.util.List; class WorkerProfilePrinter { + public static void getWorkerProfile(Instance instance) { + WorkerProfileMessage response = instance.getWorkerProfile(); + System.out.println("\nWorkerProfile:"); + String strIntFormat = "%-50s : %d"; + long entryCount = response.getCasEntryCount(); + long unreferencedEntryCount = response.getCasUnreferencedEntryCount(); + System.out.printf((strIntFormat) + "%n", "Current Total Entry Count", entryCount); + System.out.printf((strIntFormat) + "%n", "Current Total Size", response.getCasSize()); + System.out.printf((strIntFormat) + "%n", "Max Size", response.getCasMaxSize()); + System.out.printf((strIntFormat) + "%n", "Max Entry Size", response.getCasMaxEntrySize()); + System.out.printf( + (strIntFormat) + "%n", "Current Unreferenced Entry Count", unreferencedEntryCount); + if (entryCount != 0) { + System.out.println( + format( + "%-50s : %2.1f%%", + "Percentage of Unreferenced Entry", (100.0f * unreferencedEntryCount) / entryCount)); + } + System.out.printf( + (strIntFormat) + "%n", + "Current DirectoryEntry Count", + response.getCasDirectoryEntryCount()); + System.out.printf( + (strIntFormat) + "%n", "Number of Evicted Entries", response.getCasEvictedEntryCount()); + System.out.printf( + (strIntFormat) + "%n", + "Total Evicted Entries size in Bytes", + response.getCasEvictedEntrySize()); + + List stages = response.getStagesList(); + for (StageInformation stage : stages) { + printStageInformation(stage); + } + + List times = response.getTimesList(); + for (OperationTimesBetweenStages time : times) { + printOperationTime(time); + } + } + public static void printStageInformation(StageInformation stage) { System.out.printf("%s slots configured: %d%n", stage.getName(), stage.getSlotsConfigured()); System.out.printf("%s slots used %d%n", stage.getName(), stage.getSlotsUsed()); diff --git a/src/main/java/build/buildfarm/worker/BUILD b/src/main/java/build/buildfarm/worker/BUILD index f2c20a8b74..98a9a48262 100644 --- a/src/main/java/build/buildfarm/worker/BUILD +++ b/src/main/java/build/buildfarm/worker/BUILD @@ -8,42 +8,41 @@ java_library( deps = [ "//persistentworkers/src/main/java/persistent/bazel:bazel-persistent-workers", "//persistentworkers/src/main/java/persistent/common:persistent-common", + "//src/main/java/build/buildfarm/cas", "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/instance/stub", - "//src/main/java/build/buildfarm/worker/executionwrappers", - "//src/main/java/build/buildfarm/worker/filesystem", "//src/main/java/build/buildfarm/worker/persistent", "//src/main/java/build/buildfarm/worker/resources", "//src/main/java/build/buildfarm/worker/util", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "@bazel//src/main/protobuf:execution_statistics_java_proto", - "@buildfarm_maven//:com_github_docker_java_docker_java", - "@buildfarm_maven//:com_github_docker_java_docker_java_api", - "@buildfarm_maven//:com_github_docker_java_docker_java_core", - "@buildfarm_maven//:com_github_jnr_jffi_native", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_jnr_jnr_posix", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_code_gson_gson", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_apache_commons_commons_compress", - "@buildfarm_maven//:org_apache_commons_commons_lang3", - "@buildfarm_maven//:org_jetbrains_annotations", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_docker_java_docker_java", + "@maven//:com_github_docker_java_docker_java_api", + "@maven//:com_github_docker_java_docker_java_core", + "@maven//:com_github_jnr_jffi_native", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_jnr_jnr_posix", + "@maven//:com_github_pcj_google_options", + "@maven//:com_github_serceman_jnr_fuse", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_code_gson_gson", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_apache_commons_commons_compress", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_jetbrains_annotations", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/worker/filesystem/CFCExecFileSystem.java b/src/main/java/build/buildfarm/worker/CFCExecFileSystem.java similarity index 83% rename from src/main/java/build/buildfarm/worker/filesystem/CFCExecFileSystem.java rename to src/main/java/build/buildfarm/worker/CFCExecFileSystem.java index 3fec4b5523..6234994988 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/CFCExecFileSystem.java +++ b/src/main/java/build/buildfarm/worker/CFCExecFileSystem.java @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; +import static build.buildfarm.common.io.Utils.getInterruptiblyOrIOException; import static build.buildfarm.common.io.Utils.readdir; import static com.google.common.base.Preconditions.checkState; import static com.google.common.util.concurrent.Futures.allAsList; import static com.google.common.util.concurrent.Futures.catchingAsync; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; -import static com.google.common.util.concurrent.Futures.transform; import static com.google.common.util.concurrent.Futures.transformAsync; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; @@ -34,13 +34,11 @@ import build.bazel.remote.execution.v2.Directory; import build.buildfarm.cas.ContentAddressableStorage; import build.buildfarm.cas.cfc.CASFileCache; -import build.buildfarm.cas.cfc.CASFileCache.PathResult; import build.buildfarm.common.DigestUtil; import build.buildfarm.common.io.Directories; import build.buildfarm.common.io.Dirent; import build.buildfarm.v1test.Digest; -import build.buildfarm.v1test.WorkerExecutedMetadata; -import build.buildfarm.worker.filesystem.ExecDirException.ViolationException; +import build.buildfarm.worker.ExecDirException.ViolationException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -104,8 +102,7 @@ public UserPrincipal getOwner(String name) { @SuppressWarnings("ConstantConditions") @Override - public ListenableFuture start( - Consumer> onDigests, boolean skipLoad, boolean writable) + public void start(Consumer> onDigests, boolean skipLoad) throws IOException, InterruptedException { fileStore = Files.getFileStore(root); List dirents = null; @@ -128,31 +125,21 @@ public ListenableFuture start( } ImmutableList.Builder blobDigests = ImmutableList.builder(); - ListenableFuture fileCacheWritable = - fileCache.start( - digest -> { - synchronized (blobDigests) { - blobDigests.add(digest); - } - }, - removeDirectoryService, - skipLoad, - writable); - fileCacheWritable = - transformAsync( - fileCacheWritable, - result -> { - onDigests.accept(blobDigests.build()); - return immediateFuture(null); - }, - directExecutor()); - removeDirectoryFutures.add(fileCacheWritable); - - return transform(allAsList(removeDirectoryFutures.build()), results -> null, directExecutor()); + fileCache.start( + digest -> { + synchronized (blobDigests) { + blobDigests.add(digest); + } + }, + removeDirectoryService, + skipLoad); + onDigests.accept(blobDigests.build()); + + getInterruptiblyOrIOException(allAsList(removeDirectoryFutures.build())); } @Override - public void stop() throws IOException, InterruptedException { + public void stop() throws InterruptedException { fileCache.stop(); if (!shutdownAndAwaitTermination(fetchService, 1, MINUTES)) { log.log(Level.SEVERE, "could not terminate fetchService"); @@ -194,43 +181,43 @@ protected ListenableFuture putSymlink(Path symlinkPath, String target) { } @SuppressWarnings("ConstantConditions") - private ListenableFuture put(Digest digest, Path path, boolean isExecutable) { + private ListenableFuture put(Digest digest, Path path, boolean isExecutable) { if (digest.getSize() == 0) { return listeningDecorator(fetchService) .submit( () -> { Files.createFile(path); // ignore executable - return new PathResult(path, false); + return null; }); } String key = fileCache.getKey(digest, isExecutable); return transformAsync( fileCache.put(digest, isExecutable, fetchService), - pathResult -> { + (fileCachePath) -> { if (digest.getSize() != 0) { try { - Files.copy(pathResult.path(), path); + Files.copy(fileCachePath, path); } catch (IOException e) { return immediateFailedFuture(e); } finally { fileCache.decrementReference(key); } } - return immediateFuture(pathResult); + return immediateFuture(null); }, fetchService); } - private ListenableFuture catchingPut( + private ListenableFuture catchingPut( Digest digest, Path root, Path path, boolean isExecutable) { return catching( put(digest, path, isExecutable), e -> new ViolationException(digest, root.relativize(path), isExecutable, e)); } - protected ListenableFuture catching( - ListenableFuture future, Function onIOError) { + protected ListenableFuture catching( + ListenableFuture future, Function onIOError) { return catchingAsync( future, Throwable.class, // required per docs @@ -270,11 +257,6 @@ static OutputDirectory createOutputDirectory(Command command) { class ExecFileVisitor implements FileVisitor { protected final List> futures = new ArrayList<>(); protected Path root = null; - protected final WorkerExecutedMetadata.Builder workerExecutedMetadata; - - ExecFileVisitor(WorkerExecutedMetadata.Builder workerExecutedMetadata) { - this.workerExecutedMetadata = workerExecutedMetadata; - } Iterable> futures() { return futures; @@ -296,12 +278,6 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { return FileVisitResult.CONTINUE; } - protected void fetchedBytes(long size) { - synchronized (workerExecutedMetadata) { - workerExecutedMetadata.setFetchedBytes(size + workerExecutedMetadata.getFetchedBytes()); - } - } - @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { ListenableFuture populate; @@ -312,16 +288,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { } else if (attrs.isRegularFile()) { Digest digest = (Digest) attrs.fileKey(); ExecFileAttributes fileAttrs = (ExecFileAttributes) attrs; - populate = - transform( - catchingPut(digest, root, file, fileAttrs.isExecutable()), - pathResult -> { - if (pathResult.isMissed()) { - fetchedBytes(digest.getSize()); - } - return null; - }, - directExecutor()); + populate = catchingPut(digest, root, file, fileAttrs.isExecutable()); } else { populate = immediateFailedFuture(new IOException("unknown file type for " + file)); terminate = true; @@ -343,8 +310,7 @@ public Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - @Nullable UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + @Nullable UserPrincipal owner) throws IOException, InterruptedException { OutputDirectory outputDirectory = createOutputDirectory(command); @@ -353,7 +319,7 @@ public Path createExecDir( log.log(Level.FINER, operationName + " walking execTree"); ExecTree execTree = new ExecTree(directoriesIndex); - ExecFileVisitor visitor = new ExecFileVisitor(workerExecutedMetadata); + ExecFileVisitor visitor = new ExecFileVisitor(); execTree.walk( execDir, DigestUtil.fromDigest(action.getInputRootDigest(), digestFunction), visitor); diff --git a/src/main/java/build/buildfarm/worker/filesystem/CFCLinkExecFileSystem.java b/src/main/java/build/buildfarm/worker/CFCLinkExecFileSystem.java similarity index 90% rename from src/main/java/build/buildfarm/worker/filesystem/CFCLinkExecFileSystem.java rename to src/main/java/build/buildfarm/worker/CFCLinkExecFileSystem.java index a4670ce535..c61a4729dd 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/CFCLinkExecFileSystem.java +++ b/src/main/java/build/buildfarm/worker/CFCLinkExecFileSystem.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; @@ -27,14 +27,11 @@ import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.Directory; import build.bazel.remote.execution.v2.DirectoryNode; -import build.bazel.remote.execution.v2.FileNode; import build.buildfarm.cas.cfc.CASFileCache; -import build.buildfarm.cas.cfc.CASFileCache.PathResult; import build.buildfarm.common.DigestUtil; import build.buildfarm.common.io.Directories; import build.buildfarm.v1test.Digest; -import build.buildfarm.v1test.WorkerExecutedMetadata; -import build.buildfarm.worker.filesystem.ExecDirException.ViolationException; +import build.buildfarm.worker.ExecDirException.ViolationException; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -112,13 +109,13 @@ private ListenableFuture put( String key = fileCache.getKey(digest, isExecutable); return transformAsync( fileCache.put(digest, isExecutable, fetchService), - pathResult -> { + (fileCachePath) -> { checkNotNull(key); // we saw null entries in the built immutable list without synchronization onKey.accept(key); if (digest.getSize() != 0) { try { - Files.createLink(path, pathResult.path()); + Files.createLink(path, fileCachePath); } catch (IOException e) { return immediateFailedFuture(e); } @@ -136,7 +133,7 @@ private ListenableFuture catchingPut( } @SuppressWarnings("ConstantConditions") - private ListenableFuture linkDirectory( + private ListenableFuture linkDirectory( Path execPath, Digest digest, Map directoriesIndex) { @@ -150,7 +147,7 @@ private ListenableFuture linkDirectory( "putDirectory(%s, %s) created", execPath, DigestUtil.toString(digest))); } Files.createSymbolicLink(execPath, path); - return immediateFuture(pathResult); + return immediateFuture(null); }, fetchService); } @@ -241,7 +238,6 @@ static OutputDirectory createOutputDirectory(Command command) { } class LinkExecFileVisitor extends ExecFileVisitor { - private final Path root; private final Set linkedDirectories; // only need contains private final Map index; // only need retrieve @@ -253,14 +249,10 @@ class LinkExecFileVisitor extends ExecFileVisitor { synchronizedList(new ArrayList<>()); LinkExecFileVisitor( - WorkerExecutedMetadata.Builder workerExecutedMetadata, - Path root, Set linkedDirectories, Map index, DigestFunction.Value digestFunction, OutputDirectory outputDirectoryRoot) { - super(workerExecutedMetadata); - this.root = root; this.linkedDirectories = linkedDirectories; this.index = index; this.digestFunction = digestFunction; @@ -275,22 +267,6 @@ List inputDirectories() { return inputDirectories; } - private long sumDirectorySize(build.bazel.remote.execution.v2.Digest root) { - long size = 0; - List digests = new ArrayList<>(); - digests.add(root); - while (!digests.isEmpty()) { - Directory directory = index.get(digests.removeFirst()); - for (FileNode fileNode : directory.getFilesList()) { - size += fileNode.getDigest().getSizeBytes(); - } - Iterables.addAll( - digests, - Iterables.transform(directory.getDirectoriesList(), dirNode -> dirNode.getDigest())); - } - return size; - } - @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { @@ -307,16 +283,12 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) // this is scary, given the switch build.bazel.remote.execution.v2.Digest digest = (build.bazel.remote.execution.v2.Digest) attrs.fileKey(); - workerExecutedMetadata.addLinkedInputDirectories(root.relativize(dir).toString()); futures.add( transform( linkDirectory(dir, DigestUtil.fromDigest(digest, digestFunction), index), - pathResult -> { + result -> { inputDirectories.add(digest); - if (pathResult.isMissed()) { - fetchedBytes(sumDirectorySize(digest)); - } - return null; + return result; }, fetchService)); return FileVisitResult.SKIP_SUBTREE; @@ -372,8 +344,7 @@ public Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - @Nullable UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + @Nullable UserPrincipal owner) throws IOException, InterruptedException { Digest inputRootDigest = DigestUtil.fromDigest(action.getInputRootDigest(), digestFunction); OutputDirectory outputDirectory = createOutputDirectory(command); @@ -396,12 +367,7 @@ public Path createExecDir( ExecTree execTree = new ExecTree(directoriesIndex); LinkExecFileVisitor visitor = new LinkExecFileVisitor( - workerExecutedMetadata, - execDir, - linkedInputDirectories, - directoriesIndex, - digestFunction, - outputDirectory); + linkedInputDirectories, directoriesIndex, digestFunction, outputDirectory); execTree.walk(execDir, inputRootDigest, visitor); Iterable> fetchedFutures = visitor.futures(); boolean success = false; diff --git a/src/main/java/build/buildfarm/worker/DequeueMatchEvaluator.java b/src/main/java/build/buildfarm/worker/DequeueMatchEvaluator.java index d532cc412e..391a6c9d14 100644 --- a/src/main/java/build/buildfarm/worker/DequeueMatchEvaluator.java +++ b/src/main/java/build/buildfarm/worker/DequeueMatchEvaluator.java @@ -49,11 +49,14 @@ public class DequeueMatchEvaluator { * @details Compares the platform properties of the worker to the operation's platform properties. * @param workerProvisions The provisions of the worker. * @param resourceSet The limited resources that the worker has available. + * @param queueEntry An entry recently removed from the queue. * @return An acquired claim on resources, or null if the platform could not be satisfied or * resourced. * @note Overloaded. * @note Suggested return identifier: claim. */ + @SuppressWarnings("NullableProblems") + @NotNull public static @Nullable Claim acquireClaim( SetMultimap workerProvisions, LocalResourceSet resourceSet, diff --git a/src/main/java/build/buildfarm/worker/filesystem/ExecDirException.java b/src/main/java/build/buildfarm/worker/ExecDirException.java similarity index 98% rename from src/main/java/build/buildfarm/worker/filesystem/ExecDirException.java rename to src/main/java/build/buildfarm/worker/ExecDirException.java index b380b36186..7d19a74497 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/ExecDirException.java +++ b/src/main/java/build/buildfarm/worker/ExecDirException.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static build.buildfarm.common.Errors.MISSING_INPUT; import static build.buildfarm.common.Errors.VIOLATION_TYPE_INVALID; @@ -99,7 +99,7 @@ List getExceptions() { return exceptions; } - public Status.Builder toStatus(Status.Builder status) { + Status.Builder toStatus(Status.Builder status) { status.setCode(Code.FAILED_PRECONDITION.getNumber()); // aggregate into a single preconditionFailure diff --git a/src/main/java/build/buildfarm/worker/filesystem/ExecDirectoryStream.java b/src/main/java/build/buildfarm/worker/ExecDirectoryStream.java similarity index 84% rename from src/main/java/build/buildfarm/worker/filesystem/ExecDirectoryStream.java rename to src/main/java/build/buildfarm/worker/ExecDirectoryStream.java index 863d537cdc..ad127c552f 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/ExecDirectoryStream.java +++ b/src/main/java/build/buildfarm/worker/ExecDirectoryStream.java @@ -1,4 +1,4 @@ -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.collect.Iterators.concat; import static com.google.common.collect.Iterators.transform; @@ -7,10 +7,10 @@ import build.bazel.remote.execution.v2.DirectoryNode; import build.bazel.remote.execution.v2.FileNode; import build.bazel.remote.execution.v2.SymlinkNode; -import build.buildfarm.worker.filesystem.ExecFileSystem.ExecDirectoryAttributes; -import build.buildfarm.worker.filesystem.ExecFileSystem.ExecFileAttributes; -import build.buildfarm.worker.filesystem.ExecFileSystem.ExecSymlinkAttributes; -import build.buildfarm.worker.filesystem.ExecTreeWalker.Entry; +import build.buildfarm.worker.ExecFileSystem.ExecDirectoryAttributes; +import build.buildfarm.worker.ExecFileSystem.ExecFileAttributes; +import build.buildfarm.worker.ExecFileSystem.ExecSymlinkAttributes; +import build.buildfarm.worker.ExecTreeWalker.Entry; import java.nio.file.DirectoryStream; import java.nio.file.Path; import java.util.Iterator; diff --git a/src/main/java/build/buildfarm/worker/filesystem/ExecFileSystem.java b/src/main/java/build/buildfarm/worker/ExecFileSystem.java similarity index 91% rename from src/main/java/build/buildfarm/worker/filesystem/ExecFileSystem.java rename to src/main/java/build/buildfarm/worker/ExecFileSystem.java index 805c10369f..5d63a28f03 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/ExecFileSystem.java +++ b/src/main/java/build/buildfarm/worker/ExecFileSystem.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import build.bazel.remote.execution.v2.Action; import build.bazel.remote.execution.v2.Command; @@ -21,8 +21,6 @@ import build.buildfarm.cas.ContentAddressableStorage; import build.buildfarm.common.InputStreamFactory; import build.buildfarm.v1test.Digest; -import build.buildfarm.v1test.WorkerExecutedMetadata; -import com.google.common.util.concurrent.ListenableFuture; import java.io.IOException; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; @@ -35,10 +33,10 @@ import javax.annotation.Nullable; public interface ExecFileSystem extends InputStreamFactory { - ListenableFuture start(Consumer> onDigests, boolean skipLoad, boolean writable) + void start(Consumer> onDigests, boolean skipLoad) throws IOException, InterruptedException; - void stop() throws IOException, InterruptedException; + void stop() throws InterruptedException; Path root(); @@ -52,8 +50,7 @@ Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - @Nullable UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + @Nullable UserPrincipal owner) throws IOException, InterruptedException; void destroyExecDir(Path execDir) throws IOException, InterruptedException; diff --git a/src/main/java/build/buildfarm/worker/filesystem/ExecTree.java b/src/main/java/build/buildfarm/worker/ExecTree.java similarity index 95% rename from src/main/java/build/buildfarm/worker/filesystem/ExecTree.java rename to src/main/java/build/buildfarm/worker/ExecTree.java index a07a2f37f6..67b893b4a8 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/ExecTree.java +++ b/src/main/java/build/buildfarm/worker/ExecTree.java @@ -1,11 +1,11 @@ -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.base.Preconditions.checkNotNull; import build.bazel.remote.execution.v2.Digest; import build.bazel.remote.execution.v2.Directory; import build.buildfarm.common.DigestUtil; -import build.buildfarm.worker.filesystem.ExecTreeWalker.Event; +import build.buildfarm.worker.ExecTreeWalker.Event; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; diff --git a/src/main/java/build/buildfarm/worker/filesystem/ExecTreeWalker.java b/src/main/java/build/buildfarm/worker/ExecTreeWalker.java similarity index 97% rename from src/main/java/build/buildfarm/worker/filesystem/ExecTreeWalker.java rename to src/main/java/build/buildfarm/worker/ExecTreeWalker.java index 5c1fa937d6..832c375348 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/ExecTreeWalker.java +++ b/src/main/java/build/buildfarm/worker/ExecTreeWalker.java @@ -1,4 +1,4 @@ -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.base.Preconditions.checkNotNull; @@ -6,7 +6,7 @@ import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.Directory; import build.buildfarm.common.DigestUtil; -import build.buildfarm.worker.filesystem.ExecFileSystem.ExecDirectoryAttributes; +import build.buildfarm.worker.ExecFileSystem.ExecDirectoryAttributes; import java.io.Closeable; import java.io.IOException; import java.nio.file.DirectoryIteratorException; diff --git a/src/main/java/build/buildfarm/worker/ExecuteActionStage.java b/src/main/java/build/buildfarm/worker/ExecuteActionStage.java index 8e9a987774..2a9e5d941d 100644 --- a/src/main/java/build/buildfarm/worker/ExecuteActionStage.java +++ b/src/main/java/build/buildfarm/worker/ExecuteActionStage.java @@ -14,8 +14,6 @@ package build.buildfarm.worker; -import build.buildfarm.worker.filesystem.ExecFileSystem; -import build.buildfarm.worker.persistent.PersistentWorkerAwareExecOwnerPool; import build.buildfarm.worker.resources.ResourceLimits; import io.prometheus.client.Gauge; import io.prometheus.client.Histogram; @@ -23,7 +21,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; -import javax.annotation.Nullable; import lombok.extern.java.Log; @Log @@ -38,32 +35,15 @@ public class ExecuteActionStage extends SuperscalarPipelineStage { .help("Execution stall time in ms.") .register(); - private final ExecFileSystem execFileSystem; - private final @Nullable PersistentWorkerAwareExecOwnerPool execOwnerPool; private final AtomicInteger executorClaims = new AtomicInteger(0); public ExecuteActionStage( - WorkerContext workerContext, - PipelineStage output, - PipelineStage error, - ExecFileSystem execFileSystem, - @Nullable PersistentWorkerAwareExecOwnerPool execOwnerPool) { - this( - workerContext, - output, - error, - workerContext.getExecuteStageWidth(), - execFileSystem, - execOwnerPool); + WorkerContext workerContext, PipelineStage output, PipelineStage error) { + this(workerContext, output, error, workerContext.getExecuteStageWidth()); } public ExecuteActionStage( - WorkerContext workerContext, - PipelineStage output, - PipelineStage error, - int width, - ExecFileSystem execFileSystem, - @Nullable PersistentWorkerAwareExecOwnerPool execOwnerPool) { + WorkerContext workerContext, PipelineStage output, PipelineStage error, int width) { super( "ExecuteActionStage", "executor", @@ -71,9 +51,6 @@ public ExecuteActionStage( output, createDestroyExecDirStage(workerContext, error), width); - - this.execFileSystem = execFileSystem; - this.execOwnerPool = execOwnerPool; } static PipelineStage createDestroyExecDirStage( @@ -129,14 +106,9 @@ protected int claimsRequired(ExecutionContext executionContext) { @Override protected void iterate() throws InterruptedException { - if (!workerContext.inGracefulShutdown() && isPaused()) { - return; - } ExecutionContext executionContext = take(); ResourceLimits limits = workerContext.commandExecutionSettings(executionContext.command); - Executor actionExecutor = - new Executor( - workerContext, executionContext, this, pollerExecutor, execFileSystem, execOwnerPool); + Executor actionExecutor = new Executor(workerContext, executionContext, this, pollerExecutor); synchronized (this) { int slotUsage = executorClaims.addAndGet(limits.cpu.claimed); diff --git a/src/main/java/build/buildfarm/worker/ExecutionContext.java b/src/main/java/build/buildfarm/worker/ExecutionContext.java index 01382c9df0..900d0e181a 100644 --- a/src/main/java/build/buildfarm/worker/ExecutionContext.java +++ b/src/main/java/build/buildfarm/worker/ExecutionContext.java @@ -22,7 +22,6 @@ import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.QueuedOperationMetadata; import build.buildfarm.v1test.Tree; -import build.buildfarm.v1test.WorkerExecutedMetadata; import com.google.longrunning.Operation; import java.nio.file.Path; @@ -37,7 +36,6 @@ public final class ExecutionContext { final Tree tree; final QueueEntry queueEntry; public final Claim claim; - final WorkerExecutedMetadata.Builder workerExecutedMetadata; private ExecutionContext( ExecuteResponse.Builder executeResponse, @@ -49,8 +47,7 @@ private ExecutionContext( Command command, Tree tree, QueueEntry queueEntry, - Claim claim, - WorkerExecutedMetadata.Builder workerExecutedMetadata) { + Claim claim) { this.executeResponse = executeResponse; this.operation = operation; this.metadata = metadata; @@ -61,7 +58,6 @@ private ExecutionContext( this.tree = tree; this.queueEntry = queueEntry; this.claim = claim; - this.workerExecutedMetadata = workerExecutedMetadata; } public static final class Builder { @@ -75,7 +71,6 @@ public static final class Builder { private Tree tree; private QueueEntry queueEntry; private Claim claim; - private WorkerExecutedMetadata.Builder workerExecutedMetadata; private Builder( ExecuteResponse.Builder executeResponse, @@ -87,8 +82,7 @@ private Builder( Command command, Tree tree, QueueEntry queueEntry, - Claim claim, - WorkerExecutedMetadata.Builder workerExecutedMetadata) { + Claim claim) { this.executeResponse = executeResponse; this.operation = operation; this.metadata = metadata; @@ -99,7 +93,6 @@ private Builder( this.tree = tree; this.queueEntry = queueEntry; this.claim = claim; - this.workerExecutedMetadata = workerExecutedMetadata; } public Builder setOperation(Operation operation) { @@ -158,8 +151,7 @@ public ExecutionContext build() { command, tree, queueEntry, - claim, - workerExecutedMetadata); + claim); } } @@ -174,8 +166,7 @@ public static Builder newBuilder() { /* command= */ null, /* tree= */ null, /* queueEntry= */ null, - /* claim= */ null, - /* workerExecutedMetadata= */ WorkerExecutedMetadata.newBuilder()); + /* claim= */ null); } public Builder toBuilder() { @@ -189,7 +180,6 @@ public Builder toBuilder() { command, tree, queueEntry, - claim, - workerExecutedMetadata); + claim); } } diff --git a/src/main/java/build/buildfarm/worker/Executor.java b/src/main/java/build/buildfarm/worker/Executor.java index e8c273ddaf..2b0647104a 100644 --- a/src/main/java/build/buildfarm/worker/Executor.java +++ b/src/main/java/build/buildfarm/worker/Executor.java @@ -15,6 +15,8 @@ package build.buildfarm.worker; import static build.buildfarm.common.Claim.Stage.EXECUTE_ACTION_STAGE; +import static com.google.common.collect.Maps.transformValues; +import static com.google.common.collect.Maps.uniqueIndex; import static com.google.protobuf.util.Durations.add; import static com.google.protobuf.util.Durations.compare; import static com.google.protobuf.util.Durations.fromSeconds; @@ -27,6 +29,7 @@ import build.bazel.remote.execution.v2.Command; import build.bazel.remote.execution.v2.Command.EnvironmentVariable; import build.bazel.remote.execution.v2.ExecutionStage; +import build.bazel.remote.execution.v2.Platform.Property; import build.buildfarm.common.Claim; import build.buildfarm.common.ProcessUtils; import build.buildfarm.common.Time; @@ -34,12 +37,10 @@ import build.buildfarm.common.Write.NullWrite; import build.buildfarm.common.config.BuildfarmConfigs; import build.buildfarm.common.config.ExecutionPolicy; +import build.buildfarm.common.config.ExecutionWrapper; import build.buildfarm.v1test.Tree; import build.buildfarm.worker.WorkerContext.IOResource; -import build.buildfarm.worker.executionwrappers.ExecutionWrapperUtils; -import build.buildfarm.worker.filesystem.ExecFileSystem; import build.buildfarm.worker.persistent.PersistentExecutor; -import build.buildfarm.worker.persistent.PersistentWorkerAwareExecOwnerPool; import build.buildfarm.worker.persistent.WorkFilesContext; import build.buildfarm.worker.resources.ResourceLimits; import com.github.dockerjava.api.DockerClient; @@ -64,12 +65,13 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.logging.Level; -import javax.annotation.Nullable; import lombok.extern.java.Log; @Log @@ -80,25 +82,18 @@ class Executor { private final ExecutionContext executionContext; private final ExecuteActionStage owner; private final java.util.concurrent.Executor pollerExecutor; - private final ExecFileSystem execFileSystem; - private final @Nullable PersistentWorkerAwareExecOwnerPool execOwnerPool; private int exitCode = INCOMPLETE_EXIT_CODE; private boolean wasErrored = false; - private boolean polling = false; Executor( WorkerContext workerContext, ExecutionContext executionContext, ExecuteActionStage owner, - java.util.concurrent.Executor pollerExecutor, - ExecFileSystem execFileSystem, - @Nullable PersistentWorkerAwareExecOwnerPool execOwnerPool) { + java.util.concurrent.Executor pollerExecutor) { this.workerContext = workerContext; this.executionContext = executionContext; this.owner = owner; this.pollerExecutor = pollerExecutor; - this.execFileSystem = execFileSystem; - this.execOwnerPool = execOwnerPool; } // ensure that only one error put attempt occurs @@ -180,13 +175,10 @@ private long runInterruptible(Stopwatch stopwatch, ResourceLimits limits) policies = Iterables.concat(policies, workerContext.getExecutionPolicies("pool-" + pool)); } - polling = true; try { return executePolled(limits, policies, timeout, stopwatch); } finally { - if (polling) { - executionContext.poller.pause(); - } + executionContext.poller.pause(); } } @@ -216,6 +208,72 @@ private static Duration decideTimeout(TimeoutSettings settings, Action action) { return timeout; } + private static final class Interpolator { + private final Iterable value; + + Interpolator(String value) { + this(ImmutableList.of(value)); + } + + Interpolator(Iterable value) { + this.value = value; + } + + void inject(Consumer consumer) { + Iterables.transform(value, String::valueOf).forEach(consumer); + } + } + + private static Map createInterpolations( + Claim claim, Iterable properties) { + Map interpolations = new HashMap<>(); + if (claim != null) { + for (Map.Entry> pool : claim.getPools()) { + List ids = pool.getValue(); + interpolations.put(pool.getKey(), new Interpolator(ids)); + for (int i = 0; i < ids.size(); i++) { + interpolations.put(pool.getKey() + "-" + i, new Interpolator(String.valueOf(ids.get(i)))); + } + } + } + interpolations.putAll( + transformValues( + uniqueIndex(properties, Property::getName), + property -> new Interpolator(property.getValue()))); + return interpolations; + } + + private static Iterable transformWrapper( + ExecutionWrapper wrapper, Map interpolations) { + ImmutableList.Builder arguments = ImmutableList.builder(); + + arguments.add(wrapper.getPath()); + + if (wrapper.getArguments() != null) { + for (String argument : wrapper.getArguments()) { + // If the argument is of the form , substitute the value of + // the property from the platform specification. + if (!argument.equals("<>") + && argument.charAt(0) == '<' + && argument.charAt(argument.length() - 1) == '>') { + // substitute with matching interpolation + // if this property is not present, the wrapper is ignored + String name = argument.substring(1, argument.length() - 1); + Interpolator interpolator = interpolations.get(name); + if (interpolator == null) { + return ImmutableList.of(); + } + interpolator.inject(arguments::add); + } else { + // If the argument isn't of the form , add the argument directly: + arguments.add(argument); + } + } + } + + return arguments.build(); + } + private long executePolled( ResourceLimits limits, Iterable policies, @@ -232,43 +290,27 @@ private long executePolled( workingDirectory = workingDirectory.resolve(command.getWorkingDirectory()); } - boolean usePersistentWorker = shouldRunOnPersistentWorker(limits); + // similar to the policy selection here + Map interpolations = + createInterpolations( + executionContext.claim, executionContext.command.getPlatform().getPropertiesList()); ImmutableList.Builder arguments = ImmutableList.builder(); - /* - * If we're using a persistent worker, `PersistentExecutor` will assume responsibility of evaluating - * execution wrappers. Because `PersistentExecutor` swaps out the exec owner last-minute and the exec owner can be - * an argument to an execution wrapper, we need to evaluate execution wrappers after the swapping has taken place. - */ - if (!usePersistentWorker) { - // Apply custom PRIORITIZED execution policies BEFORE built-in wrappers - ExecutionWrapperUtils.applyExecutionWrappers( - arguments, - policies, - executionContext.claim, - executionContext.command.getPlatform().getPropertiesList(), - true); + for (ExecutionPolicy policy : policies) { + if (policy.getExecutionWrapper() != null) { + arguments.addAll(transformWrapper(policy.getExecutionWrapper(), interpolations)); + } } Code statusCode; try (IOResource resource = workerContext.limitExecution( operationName, - executionContext.claim.getOwner(), + executionContext.claim.owner(), arguments, executionContext.command, workingDirectory)) { - if (!usePersistentWorker) { - // Apply all other custom execution policies AFTER built-in wrappers - ExecutionWrapperUtils.applyExecutionWrappers( - arguments, - policies, - executionContext.claim, - executionContext.command.getPlatform().getPropertiesList(), - false); - } - // Windows requires that relative command programs are absolutized Iterator argumentItr = command.getArgumentsList().iterator(); boolean absolutizeExe = @@ -293,9 +335,7 @@ private long executePolled( timeout, // executingMetadata.getStdoutStreamName(), // executingMetadata.getStderrStreamName(), - executionContext.executeResponse.getResultBuilder(), - executionContext.claim, - policies); + executionContext.executeResponse.getResultBuilder()); // From Bazel Test Encyclopedia: // If the main process of a test exits, but some of its children are still running, @@ -370,7 +410,6 @@ private long executePolled( } } } - polling = false; return stopwatch.elapsed(MICROSECONDS) - executeUSecs; } @@ -458,9 +497,7 @@ private Code executeCommand( List environmentVariables, ResourceLimits limits, Duration timeout, - ActionResult.Builder resultBuilder, - Claim claim, - Iterable policies) + ActionResult.Builder resultBuilder) throws IOException, InterruptedException { ProcessBuilder processBuilder = new ProcessBuilder(arguments).directory(execDir.toAbsolutePath().toFile()); @@ -496,12 +533,7 @@ private Code executeCommand( limits, timeout, PersistentExecutor.defaultWorkRootsDir, - resultBuilder, - execFileSystem, - execOwnerPool, - claim, - policies, - executionContext.command.getPlatform().getPropertiesList()); + resultBuilder); } // run the action under docker diff --git a/src/main/java/build/buildfarm/worker/filesystem/FuseCAS.java b/src/main/java/build/buildfarm/worker/FuseCAS.java similarity index 99% rename from src/main/java/build/buildfarm/worker/filesystem/FuseCAS.java rename to src/main/java/build/buildfarm/worker/FuseCAS.java index 494688f694..e49737a280 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/FuseCAS.java +++ b/src/main/java/build/buildfarm/worker/FuseCAS.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import build.bazel.remote.execution.v2.Compressor; import build.bazel.remote.execution.v2.Directory; diff --git a/src/main/java/build/buildfarm/worker/InputFetchStage.java b/src/main/java/build/buildfarm/worker/InputFetchStage.java index 10d6cfa7ae..20659a527b 100644 --- a/src/main/java/build/buildfarm/worker/InputFetchStage.java +++ b/src/main/java/build/buildfarm/worker/InputFetchStage.java @@ -14,11 +14,8 @@ package build.buildfarm.worker; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; import io.prometheus.client.Gauge; import io.prometheus.client.Histogram; -import java.util.concurrent.ConcurrentMap; import java.util.logging.Logger; import javax.annotation.concurrent.GuardedBy; import lombok.extern.java.Log; @@ -34,7 +31,6 @@ public class InputFetchStage extends SuperscalarPipelineStage { .name("input_fetch_stall_time_ms") .help("Input fetch stall time in ms.") .register(); - private final ConcurrentMap inputFetchers = Maps.newConcurrentMap(); @GuardedBy("this") private int slotUsage; @@ -64,7 +60,6 @@ synchronized int removeAndRelease(String operationName) { public void releaseInputFetcher( String operationName, long usecs, long stallUSecs, boolean success) { int size = removeAndRelease(operationName); - inputFetchers.remove(operationName); inputFetchTime.observe(usecs / 1000.0); inputFetchStallTime.observe(stallUSecs / 1000.0); complete( @@ -79,21 +74,11 @@ protected int claimsRequired(ExecutionContext executionContext) { return 1; } - @Override - public boolean isStalled() { - // true iff any of the current fetchers are waiting to advance to execute - return Iterables.any(inputFetchers.values(), inputFetcher -> inputFetcher.isStalled()); - } - @Override protected void iterate() throws InterruptedException { - if (!workerContext.inGracefulShutdown() && isPaused()) { - return; - } ExecutionContext executionContext = take(); InputFetcher inputFetcher = new InputFetcher(workerContext, executionContext, this, pollerExecutor); - inputFetchers.put(executionContext.operation.getName(), inputFetcher); synchronized (this) { slotUsage++; diff --git a/src/main/java/build/buildfarm/worker/InputFetcher.java b/src/main/java/build/buildfarm/worker/InputFetcher.java index 9b9d520859..fd34237d6c 100644 --- a/src/main/java/build/buildfarm/worker/InputFetcher.java +++ b/src/main/java/build/buildfarm/worker/InputFetcher.java @@ -33,7 +33,6 @@ import build.buildfarm.v1test.ExecuteEntry; import build.buildfarm.v1test.QueuedOperation; import build.buildfarm.v1test.Tree; -import build.buildfarm.worker.filesystem.ExecDirException; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; import com.google.common.collect.Iterables; @@ -63,19 +62,6 @@ public class InputFetcher implements Runnable { private final InputFetchStage owner; private final Executor pollerExecutor; private boolean success = false; - private boolean polling = false; - private volatile boolean stalled = false; - - private class StallState implements AutoCloseable { - StallState() { - stalled = true; - } - - @Override - public void close() { - stalled = false; - } - } InputFetcher( WorkerContext workerContext, @@ -126,13 +112,10 @@ private long runInterruptibly(Stopwatch stopwatch) throws InterruptedException { Thread.currentThread()::interrupt, Deadline.after(workerContext.getInputFetchDeadline(), SECONDS), pollerExecutor); - polling = true; try { return fetchPolled(stopwatch); } finally { - if (polling) { - executionContext.poller.pause(); - } + executionContext.poller.pause(); } } @@ -250,8 +233,7 @@ long fetchPolled(Stopwatch stopwatch) throws InterruptedException { executionContext.queueEntry.getExecuteEntry().getActionDigest().getDigestFunction(), queuedOperation.getAction(), queuedOperation.getCommand(), - executionContext.claim.getOwner(), - executionContext.workerExecutedMetadata); + executionContext.claim.owner()); } catch (IOException e) { Status.Builder status = Status.newBuilder().setMessage("Error creating exec dir"); if (e instanceof ExecDirException execDirEx) { @@ -301,13 +283,6 @@ long fetchPolled(Stopwatch stopwatch) throws InterruptedException { } } - public boolean isStalled() { - return stalled; - } - - @SuppressWarnings( - "PMD.UnusedLocalVariable") // PMD thinks that the try-with-resources is not used. See - // https://github.com/pmd/pmd/issues/5747 private void proceedToOutput(Action action, Command command, Path execDir, Tree tree) throws InterruptedException { // switch poller to disable deadline @@ -320,6 +295,7 @@ private void proceedToOutput(Action action, Command command, Path execDir, Tree Thread.currentThread()::interrupt, Deadline.after(10, DAYS), pollerExecutor); + ExecutionContext fetchedExecutionContext = executionContext.toBuilder() .setExecDir(execDir) @@ -327,11 +303,7 @@ private void proceedToOutput(Action action, Command command, Path execDir, Tree .setCommand(command) .setTree(tree) .build(); - boolean claimed; - // PMD exemption false positive: unused variable - try (StallState state = new StallState()) { - claimed = owner.output().claim(fetchedExecutionContext); - } + boolean claimed = owner.output().claim(fetchedExecutionContext); executionContext.poller.pause(); if (claimed) { try { @@ -346,7 +318,6 @@ private void proceedToOutput(Action action, Command command, Path execDir, Tree owner.error().put(fetchedExecutionContext); } - polling = false; } @Override diff --git a/src/main/java/build/buildfarm/worker/MatchStage.java b/src/main/java/build/buildfarm/worker/MatchStage.java index b407e8cf96..f2559f0bbe 100644 --- a/src/main/java/build/buildfarm/worker/MatchStage.java +++ b/src/main/java/build/buildfarm/worker/MatchStage.java @@ -20,10 +20,7 @@ import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.MINUTES; -import build.bazel.remote.execution.v2.ExecuteOperationMetadata; -import build.bazel.remote.execution.v2.ExecuteResponse; import build.bazel.remote.execution.v2.ExecutedActionMetadata; -import build.bazel.remote.execution.v2.RequestMetadata; import build.buildfarm.common.Claim; import build.buildfarm.common.DigestUtil; import build.buildfarm.common.Poller; @@ -45,6 +42,7 @@ @Log public class MatchStage extends PipelineStage { + private boolean inGracefulShutdown = false; private ExecutorService pollerExecutor = newSingleThreadExecutor(); public MatchStage(WorkerContext workerContext, PipelineStage output, PipelineStage error) { @@ -72,7 +70,7 @@ boolean wasMatched() { @Override public boolean onWaitStart() { waitStart = stopwatch.elapsed(MICROSECONDS); - return !workerContext.inGracefulShutdown(); + return !inGracefulShutdown; } @Override @@ -86,7 +84,7 @@ public void onWaitEnd() { @Override public boolean onEntry(@Nullable QueueEntry queueEntry, Claim claim) throws InterruptedException { - if (workerContext.inGracefulShutdown()) { + if (inGracefulShutdown) { throw new InterruptedException(); } @@ -94,23 +92,10 @@ public boolean onEntry(@Nullable QueueEntry queueEntry, Claim claim) return false; } - ExecuteEntry executeEntry = queueEntry.getExecuteEntry(); - RequestMetadata requestMetadata = executeEntry.getRequestMetadata(); - if (requestMetadata.getActionMnemonic().equals("buildfarm:halt-on-dequeue")) { - putOperation( - Operation.newBuilder() - .setName(executeEntry.getOperationName()) - .setDone(true) - .setMetadata(Any.pack(ExecuteOperationMetadata.getDefaultInstance())) - .setResponse(Any.pack(ExecuteResponse.getDefaultInstance())) - .build()); - return false; - } - executionContext .metadata .setQueuedOperationDigest(queueEntry.getQueuedOperationDigest()) - .setRequestMetadata(requestMetadata) + .setRequestMetadata(queueEntry.getExecuteEntry().getRequestMetadata()) .getExecuteOperationMetadataBuilder() .setDigestFunction(queueEntry.getExecuteEntry().getActionDigest().getDigestFunction()); @@ -161,7 +146,7 @@ protected Logger getLogger() { protected void iterate() throws InterruptedException { start(); // clear any previous operation // stop matching and picking up any works if the worker is in graceful shutdown. - if (workerContext.inGracefulShutdown() || isPaused() || output.isStalled()) { + if (inGracefulShutdown) { return; } Stopwatch stopwatch = Stopwatch.createStarted(); @@ -179,6 +164,10 @@ protected void iterate() throws InterruptedException { } } + void prepareForGracefulShutdown() { + inGracefulShutdown = true; + } + private void putOperation(Operation operation) throws InterruptedException { boolean operationUpdateSuccess = false; try { diff --git a/src/main/java/build/buildfarm/worker/filesystem/OutputDirectory.java b/src/main/java/build/buildfarm/worker/OutputDirectory.java similarity index 99% rename from src/main/java/build/buildfarm/worker/filesystem/OutputDirectory.java rename to src/main/java/build/buildfarm/worker/OutputDirectory.java index 11b1a5e417..3cf7517c39 100644 --- a/src/main/java/build/buildfarm/worker/filesystem/OutputDirectory.java +++ b/src/main/java/build/buildfarm/worker/OutputDirectory.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import build.bazel.remote.execution.v2.Command.EnvironmentVariable; import com.google.common.base.Preconditions; diff --git a/src/main/java/build/buildfarm/worker/Pipeline.java b/src/main/java/build/buildfarm/worker/Pipeline.java index 142dbb58a8..d865ff4dd6 100644 --- a/src/main/java/build/buildfarm/worker/Pipeline.java +++ b/src/main/java/build/buildfarm/worker/Pipeline.java @@ -16,14 +16,13 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.logging.Level; import lombok.extern.java.Log; @Log -public class Pipeline implements Iterable { +public class Pipeline { private final Map stageThreads; private final Map stageClosePriorities; private Thread joiningThread = null; @@ -36,11 +35,6 @@ public Pipeline() { stageClosePriorities = new HashMap<>(); } - @Override - public Iterator iterator() { - return stageClosePriorities.keySet().iterator(); - } - public void add(PipelineStage stage, int closePriority) { stageThreads.put(stage, new Thread(stage)); if (closePriority < 0) { @@ -49,10 +43,6 @@ public void add(PipelineStage stage, int closePriority) { stageClosePriorities.put(stage, closePriority); } - public void interrupt(PipelineStage stage) { - stageThreads.get(stage).interrupt(); - } - public void start() { for (Thread stageThread : stageThreads.values()) { stageThread.start(); @@ -70,6 +60,18 @@ public void close() throws InterruptedException { join(true); } + /** Inform MatchStage to stop matching or picking up new Operations from queue. */ + public void stopMatchingOperations() { + for (Map.Entry entry : stageThreads.entrySet()) { + PipelineStage stage = entry.getKey(); + if (stage instanceof MatchStage matchStage) { + matchStage.prepareForGracefulShutdown(); + entry.getValue().interrupt(); + return; + } + } + } + /** * Checking if there is any ongoing actions in any stages of the pipeline. * @@ -162,7 +164,7 @@ private void join(boolean closeStage) throws InterruptedException { log.log( Level.FINER, "Stage " - + stage.getName() + + stage.name() + " has exited at priority " + stageClosePriorities.get(stage)); inactiveStages.add(stage); @@ -170,7 +172,7 @@ private void join(boolean closeStage) throws InterruptedException { log.log( Level.INFO, "Interrupting unterminated closed thread in stage " - + stage.getName() + + stage.name() + " at priority " + stageClosePriorities.get(stage)); thread.interrupt(); diff --git a/src/main/java/build/buildfarm/worker/PipelineStage.java b/src/main/java/build/buildfarm/worker/PipelineStage.java index 7d576e5abb..fb93d49cde 100644 --- a/src/main/java/build/buildfarm/worker/PipelineStage.java +++ b/src/main/java/build/buildfarm/worker/PipelineStage.java @@ -22,7 +22,6 @@ import java.util.logging.Logger; import javax.annotation.Nullable; import lombok.Getter; -import lombok.Setter; public abstract class PipelineStage implements Runnable { @Getter protected final String name; @@ -35,7 +34,6 @@ public abstract class PipelineStage implements Runnable { private Thread tickThread = null; private boolean tickCancelledFlag = false; private String operationName = null; - @Getter @Setter private boolean paused = false; PipelineStage( String name, WorkerContext workerContext, PipelineStage output, PipelineStage error) { @@ -45,10 +43,6 @@ public abstract class PipelineStage implements Runnable { this.error = error; } - public boolean isStalled() { - return false; - } - protected void runInterruptible() throws InterruptedException { while (!output.isClosed() || isClaimed()) { iterate(); @@ -101,12 +95,10 @@ private boolean decideTermination(Exception e) { return true; } - public int getWidth() { - return 1; + public String name() { + return name; } - public void setWidth(int width) {} - protected void cancelTick() { // if we are not in a tick, this has no effect // if we are in a tick, set the cancel flag, interrupt the tick thread diff --git a/src/main/java/build/buildfarm/worker/ReportResultStage.java b/src/main/java/build/buildfarm/worker/ReportResultStage.java index 991dee35c5..57aeba3614 100644 --- a/src/main/java/build/buildfarm/worker/ReportResultStage.java +++ b/src/main/java/build/buildfarm/worker/ReportResultStage.java @@ -76,9 +76,6 @@ protected int claimsRequired(ExecutionContext executionContext) { @Override protected void iterate() throws InterruptedException { - if (!workerContext.inGracefulShutdown() && isPaused()) { - return; - } ExecutionContext executionContext = take(); ResultReporter reporter = new ResultReporter(workerContext, executionContext, this, pollerExecutor); diff --git a/src/main/java/build/buildfarm/worker/ResultReporter.java b/src/main/java/build/buildfarm/worker/ResultReporter.java index fa0ee7a183..7bcf7e5412 100644 --- a/src/main/java/build/buildfarm/worker/ResultReporter.java +++ b/src/main/java/build/buildfarm/worker/ResultReporter.java @@ -149,8 +149,7 @@ private long reportPolled(Stopwatch stopwatch) throws InterruptedException { .metadata .getExecuteOperationMetadataBuilder() .getPartialExecutionMetadataBuilder() - .setOutputUploadStartTimestamp(Timestamps.now()) - .addAuxiliaryMetadata(Any.pack(executionContext.workerExecutedMetadata.build())); + .setOutputUploadStartTimestamp(Timestamps.now()); putOperation(executionContext); boolean blacklist = false; diff --git a/src/main/java/build/buildfarm/worker/SuperscalarPipelineStage.java b/src/main/java/build/buildfarm/worker/SuperscalarPipelineStage.java index 3799cdcc89..d5addbb809 100644 --- a/src/main/java/build/buildfarm/worker/SuperscalarPipelineStage.java +++ b/src/main/java/build/buildfarm/worker/SuperscalarPipelineStage.java @@ -14,8 +14,6 @@ package build.buildfarm.worker; -import static com.google.common.base.Preconditions.checkState; - import java.util.HashSet; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; @@ -29,10 +27,9 @@ public abstract class SuperscalarPipelineStage extends PipelineStage { private final BlockingQueue queue = new ArrayBlockingQueue<>(1); protected Set operationNames = new HashSet<>(); - @Getter protected int width; + @Getter protected final int width; protected final BlockingQueue claims; - private int suppressReleases = 0; protected final ThreadPoolExecutor executor; protected final ThreadPoolExecutor pollerExecutor; @@ -51,12 +48,7 @@ public SuperscalarPipelineStage( int width) { super(name, workerContext, output, error); this.width = width; - claims = new LinkedBlockingQueue<>(); - Object handle = new Object(); - for (int i = 0; i < width; i++) { - claims.offer(handle); - } - checkState(claims.size() == width); + claims = new ArrayBlockingQueue<>(width); executor = new ThreadPoolExecutor( width, @@ -169,16 +161,11 @@ public ExecutionContext take() throws InterruptedException { } protected synchronized void releaseClaim(String operationName, int slots) { - Object handle = new Object(); // clear interrupted flag for take boolean interrupted = Thread.interrupted(); try { for (int i = 0; i < slots; i++) { - if (suppressReleases > 0) { - suppressReleases--; - } else { - claims.put(handle); - } + claims.take(); } } catch (InterruptedException e) { catastrophic = true; @@ -198,38 +185,27 @@ protected synchronized void releaseClaim(String operationName, int slots) { } } - @Override - public synchronized void setWidth(int width) { - int difference = width - this.width; - if (difference > 0) { - releaseClaim("stage width increase", difference); - } else { - suppressReleases -= difference; - } - this.width = width; - } - protected String getUsage(int size) { return String.format("%s/%d", size, width); } @SuppressWarnings("unchecked") private boolean claim(int count) throws InterruptedException { + Object handle = new Object(); int claimed = 0; synchronized (claimLock) { while (count > 0 && !isClosed()) { try { - if (claims.poll(10, TimeUnit.MILLISECONDS) != null) { + if (claims.offer(handle, 10, TimeUnit.MILLISECONDS)) { claimed++; count--; } } catch (InterruptedException e) { boolean interrupted = Thread.interrupted(); - Object handle = new Object(); while (claimed != 0) { interrupted = Thread.interrupted() || interrupted; try { - claims.put(handle); + claims.take(); claimed--; } catch (InterruptedException intEx) { // ignore, we must release our claims @@ -258,7 +234,7 @@ public void release() { @Override protected boolean isClaimed() { - return claims.size() + suppressReleases != width; + return !claims.isEmpty(); } @Override diff --git a/src/main/java/build/buildfarm/worker/WorkerContext.java b/src/main/java/build/buildfarm/worker/WorkerContext.java index 06cf449f23..d36852cad8 100644 --- a/src/main/java/build/buildfarm/worker/WorkerContext.java +++ b/src/main/java/build/buildfarm/worker/WorkerContext.java @@ -27,7 +27,6 @@ import build.buildfarm.common.config.ExecutionPolicy; import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.QueuedOperation; -import build.buildfarm.v1test.WorkerExecutedMetadata; import build.buildfarm.worker.resources.ResourceLimits; import com.google.common.collect.ImmutableList; import com.google.longrunning.Operation; @@ -66,10 +65,6 @@ void resumePoller( Deadline deadline, Executor executor); - boolean inGracefulShutdown(); - - void prepareForGracefulShutdown(); - void match(MatchListener listener) throws InterruptedException; List getExecutionPolicies(String name); @@ -103,8 +98,7 @@ Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - @Nullable UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + @Nullable UserPrincipal owner) throws IOException, InterruptedException; void destroyExecDir(Path execDir) throws IOException, InterruptedException; diff --git a/src/main/java/build/buildfarm/worker/cgroup/BUILD b/src/main/java/build/buildfarm/worker/cgroup/BUILD index a81e39b546..3d26569c82 100644 --- a/src/main/java/build/buildfarm/worker/cgroup/BUILD +++ b/src/main/java/build/buildfarm/worker/cgroup/BUILD @@ -7,11 +7,11 @@ java_library( visibility = ["//visibility:public"], deps = [ "//src/main/java/build/buildfarm/worker", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_posix", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_posix", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:io_grpc_grpc_api", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/worker/cgroup/CGroupVersionProvider.java b/src/main/java/build/buildfarm/worker/cgroup/CGroupVersionProvider.java index c65c92c7ab..ad0019f650 100644 --- a/src/main/java/build/buildfarm/worker/cgroup/CGroupVersionProvider.java +++ b/src/main/java/build/buildfarm/worker/cgroup/CGroupVersionProvider.java @@ -24,6 +24,7 @@ @Log public class CGroupVersionProvider implements Supplier { + @Override public CGroupVersion get() { /* Try to figure out which version of CGroups is available. */ diff --git a/src/main/java/build/buildfarm/worker/cgroup/Cpu.java b/src/main/java/build/buildfarm/worker/cgroup/Cpu.java index 7ec5cc9343..8c92ced706 100644 --- a/src/main/java/build/buildfarm/worker/cgroup/Cpu.java +++ b/src/main/java/build/buildfarm/worker/cgroup/Cpu.java @@ -19,6 +19,7 @@ import java.io.IOException; public class Cpu extends Controller { + private static final int CPU_GRANULARITY = 100_000; // microseconds (μS) Cpu(Group group) { diff --git a/src/main/java/build/buildfarm/worker/cgroup/Group.java b/src/main/java/build/buildfarm/worker/cgroup/Group.java index b78b861b67..77e7129e66 100644 --- a/src/main/java/build/buildfarm/worker/cgroup/Group.java +++ b/src/main/java/build/buildfarm/worker/cgroup/Group.java @@ -394,6 +394,7 @@ static void ensureControllerIsEnabled( "There are no controllers at all in CGroup at path " + cgroupPath); } if (!availableControllers.containsAll(controllerNames)) { + throw new IllegalStateException( "Some controllers not enabled in CGroup at path " + cgroupPath diff --git a/src/main/java/build/buildfarm/worker/executionwrappers/BUILD b/src/main/java/build/buildfarm/worker/executionwrappers/BUILD deleted file mode 100644 index 08ca21adaf..0000000000 --- a/src/main/java/build/buildfarm/worker/executionwrappers/BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_java//java:java_library.bzl", "java_library") - -java_library( - name = "executionwrappers", - srcs = glob(["*.java"]), - visibility = ["//visibility:public"], - deps = [ - "//src/main/java/build/buildfarm/common", - "//src/main/java/build/buildfarm/common/config", - "@buildfarm_maven//:com_google_guava_guava", - "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", - ], -) diff --git a/src/main/java/build/buildfarm/worker/executionwrappers/ExecutionWrapperUtils.java b/src/main/java/build/buildfarm/worker/executionwrappers/ExecutionWrapperUtils.java deleted file mode 100644 index e644d5b9e9..0000000000 --- a/src/main/java/build/buildfarm/worker/executionwrappers/ExecutionWrapperUtils.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2023 The Buildfarm Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package build.buildfarm.worker.executionwrappers; - -import static com.google.common.collect.Maps.transformValues; -import static com.google.common.collect.Maps.uniqueIndex; - -import build.bazel.remote.execution.v2.Platform.Property; -import build.buildfarm.common.Claim; -import build.buildfarm.common.config.ExecutionPolicy; -import build.buildfarm.common.config.ExecutionWrapper; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -public class ExecutionWrapperUtils { - private static final class Interpolator { - private final Iterable value; - - Interpolator(String value) { - this(ImmutableList.of(value)); - } - - Interpolator(Iterable value) { - this.value = value; - } - - void inject(Consumer consumer) { - Iterables.transform(value, String::valueOf).forEach(consumer); - } - } - - private static Map createInterpolations( - Claim claim, Iterable properties) { - Map interpolations = new HashMap<>(); - if (claim != null) { - for (Map.Entry> pool : claim.getPools()) { - List ids = pool.getValue(); - interpolations.put(pool.getKey(), new Interpolator(ids)); - for (int i = 0; i < ids.size(); i++) { - interpolations.put(pool.getKey() + "-" + i, new Interpolator(String.valueOf(ids.get(i)))); - } - } - } - interpolations.putAll( - transformValues( - uniqueIndex(properties, Property::getName), - property -> new Interpolator(property.getValue()))); - return interpolations; - } - - private static Iterable transformWrapper( - ExecutionWrapper wrapper, Map interpolations) { - ImmutableList.Builder arguments = ImmutableList.builder(); - - arguments.add(wrapper.getPath()); - - if (wrapper.getArguments() != null) { - for (String argument : wrapper.getArguments()) { - // If the argument is of the form , substitute the value of - // the property from the platform specification. - if (!argument.equals("<>") - && argument.charAt(0) == '<' - && argument.charAt(argument.length() - 1) == '>') { - // substitute with matching interpolation - // if this property is not present, the wrapper is ignored - String name = argument.substring(1, argument.length() - 1); - Interpolator interpolator = interpolations.get(name); - if (interpolator == null) { - return ImmutableList.of(); - } - interpolator.inject(arguments::add); - } else { - // If the argument isn't of the form , add the argument directly: - arguments.add(argument); - } - } - } - - return arguments.build(); - } - - public static void applyExecutionWrappers( - ImmutableList.Builder arguments, - Iterable policies, - Claim claim, - Iterable platformProperties, - boolean applyPrioritized) { - Map interpolations = createInterpolations(claim, platformProperties); - - for (ExecutionPolicy policy : policies) { - if (policy.isPrioritized() == applyPrioritized && policy.getExecutionWrapper() != null) { - arguments.addAll(transformWrapper(policy.getExecutionWrapper(), interpolations)); - } - } - } -} diff --git a/src/main/java/build/buildfarm/worker/filesystem/BUILD.bazel b/src/main/java/build/buildfarm/worker/filesystem/BUILD.bazel deleted file mode 100644 index c6661cc35e..0000000000 --- a/src/main/java/build/buildfarm/worker/filesystem/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@rules_java//java:java_library.bzl", "java_library") - -java_library( - name = "filesystem", - srcs = glob(["*.java"]), - plugins = ["//src/main/java/build/buildfarm/common:lombok"], - visibility = ["//visibility:public"], - deps = [ - "//src/main/java/build/buildfarm/cas", - "//src/main/java/build/buildfarm/common", - "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:org_projectlombok_lombok", - "@googleapis//google/rpc:rpc_java_proto", - ], -) diff --git a/src/main/java/build/buildfarm/worker/persistent/BUILD b/src/main/java/build/buildfarm/worker/persistent/BUILD index 722b1b1abc..929463793a 100644 --- a/src/main/java/build/buildfarm/worker/persistent/BUILD +++ b/src/main/java/build/buildfarm/worker/persistent/BUILD @@ -8,29 +8,26 @@ java_library( deps = [ "//persistentworkers/src/main/java/persistent/bazel:bazel-persistent-workers", "//persistentworkers/src/main/java/persistent/common:persistent-common", + "//persistentworkers/src/main/java/persistent/common/util", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", "//src/main/java/build/buildfarm/common", - "//src/main/java/build/buildfarm/common/config", - "//src/main/java/build/buildfarm/worker/executionwrappers", - "//src/main/java/build/buildfarm/worker/filesystem", "//src/main/java/build/buildfarm/worker/resources", "//src/main/java/build/buildfarm/worker/util", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_api_grpc_proto_google_common_protos", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_apache_commons_commons_compress", - "@buildfarm_maven//:org_jetbrains_annotations", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_google_api_grpc_proto_google_common_protos", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_apache_commons_commons_compress", + "@maven//:org_jetbrains_annotations", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/worker/persistent/FileAccessUtils.java b/src/main/java/build/buildfarm/worker/persistent/FileAccessUtils.java index 43e0aceb52..462462a734 100644 --- a/src/main/java/build/buildfarm/worker/persistent/FileAccessUtils.java +++ b/src/main/java/build/buildfarm/worker/persistent/FileAccessUtils.java @@ -19,15 +19,12 @@ import com.google.common.collect.ImmutableSet; import java.io.IOException; -import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.UserPrincipal; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; -import javax.annotation.Nullable; import lombok.extern.java.Log; /** @@ -68,8 +65,7 @@ private PathLock() {} * @param to * @throws IOException */ - public static void copyFile(Path from, Path to, @Nullable UserPrincipal owner) - throws IOException { + public static void copyFile(Path from, Path to) throws IOException { Path absTo = to.toAbsolutePath(); log.finer("copyFile: " + from + " to " + absTo); if (!Files.exists(from)) { @@ -78,15 +74,9 @@ public static void copyFile(Path from, Path to, @Nullable UserPrincipal owner) IOException ioException = writeFileSafe( to, - owner, () -> { try { Files.copy(from, absTo, REPLACE_EXISTING, COPY_ATTRIBUTES); - - if (owner != null) { - Files.setOwner(absTo, owner); - } - addPosixOwnerWrite(absTo); return null; } catch (IOException e) { @@ -116,7 +106,6 @@ public static void moveFile(Path from, Path to) throws IOException { IOException ioException = writeFileSafe( absTo, - null, () -> { try { Files.move(from, absTo, REPLACE_EXISTING); @@ -149,32 +138,6 @@ public static void deleteFileIfExists(Path toDelete) throws IOException { } } - /** - * Create a directory and all necessary ancestor directories, setting the owner of each created - * directory. - */ - private static void createDirectoriesWithOwner(Path directory, @Nullable UserPrincipal owner) - throws IOException { - if (Files.exists(directory)) { - return; - } - - Path parent = directory.getParent(); - - if (parent != null && !Files.exists(parent)) { - createDirectoriesWithOwner(parent, owner); - } - - try { - Files.createDirectory(directory); - - if (owner != null) { - Files.setOwner(directory, owner); - } - } catch (FileAlreadyExistsException ignored) { - } - } - /** * Thread-safe (not multi-process-safe) wrapper for locking paths before a write operation. * @@ -183,13 +146,12 @@ private static void createDirectoriesWithOwner(Path directory, @Nullable UserPri *

It is up to the write operation to specify whether or not to overwrite existing files. */ @SuppressWarnings("PMD.UnnecessaryLocalBeforeReturn") - private static IOException writeFileSafe( - Path absTo, @Nullable UserPrincipal owner, Supplier writeOp) { + private static IOException writeFileSafe(Path absTo, Supplier writeOp) { PathLock toLock = fileLock(absTo); synchronized (toLock) { try { // If 'absTo' is a symlink, checks if its target file exists - createDirectoriesWithOwner(absTo.getParent(), owner); + Files.createDirectories(absTo.getParent()); return writeOp.get(); } catch (IOException e) { // PMD will complain about UnnecessaryLocalBeforeReturn diff --git a/src/main/java/build/buildfarm/worker/persistent/Keymaker.java b/src/main/java/build/buildfarm/worker/persistent/Keymaker.java index aaad221f8f..834e06c24e 100644 --- a/src/main/java/build/buildfarm/worker/persistent/Keymaker.java +++ b/src/main/java/build/buildfarm/worker/persistent/Keymaker.java @@ -22,61 +22,62 @@ import com.google.common.hash.Hashing; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.attribute.UserPrincipal; import java.util.Objects; import java.util.SortedMap; -import javax.annotation.Nullable; -import persistent.bazel.client.BasicWorkerKey; import persistent.bazel.client.WorkerKey; /** Much of the logic (hashing) is from Bazel itself (private library/methods, i.e. WorkerKey). */ public class Keymaker { // Constructs a key with its worker tool input files being relative paths - public static BasicWorkerKey makeBasicKey( + public static WorkerKey make( + Path opRoot, + Path workRootsDir, ImmutableList workerInitCmd, ImmutableList workerInitArgs, ImmutableMap workerEnv, - String executionName) { + String executionName, + WorkerInputs workerFiles) { // Cancellation not yet supported; can change in the future, // Presumably, following how Bazel's own persistent workers work boolean sandboxed = true; boolean cancellable = false; - return new BasicWorkerKey( - workerInitCmd, workerInitArgs, workerEnv, executionName, sandboxed, cancellable); - } - - public static WorkerKey makeKey( - BasicWorkerKey basicWorkerKey, - @Nullable UserPrincipal owner, - ImmutableList wrapperArguments, - Path workRootsDir, - WorkerInputs workerFiles) { - Path workRoot = calculateWorkRoot(basicWorkerKey, owner, wrapperArguments, workRootsDir); + Path workRoot = + calculateWorkRoot( + workRootsDir, + workerInitCmd, + workerInitArgs, + workerEnv, + executionName, + sandboxed, + cancellable); SortedMap hashedTools = workerFilesWithHashes(workerFiles); HashCode combinedToolsHash = workerFilesCombinedHash(hashedTools); return new WorkerKey( - basicWorkerKey, owner, wrapperArguments, workRoot, combinedToolsHash, hashedTools); + workerInitCmd, + workerInitArgs, + workerEnv, + workRoot, + executionName, + combinedToolsHash, + hashedTools, + sandboxed, + cancellable); } // Hash of a subset of the WorkerKey private static Path calculateWorkRoot( - BasicWorkerKey basicWorkerKey, - @Nullable UserPrincipal owner, - ImmutableList wrapperArguments, - Path workRootsDir) { - int workRootId = - Objects.hash( - basicWorkerKey.getCmd(), - basicWorkerKey.getArgs(), - basicWorkerKey.getEnv(), - basicWorkerKey.isSandboxed(), - basicWorkerKey.isCancellable(), - owner, - wrapperArguments); - String workRootDirName = "work-root_" + basicWorkerKey.getMnemonic() + "_" + workRootId; + Path workRootsDir, + ImmutableList workerInitCmd, + ImmutableList workerInitArgs, + ImmutableMap workerEnv, + String executionName, + boolean sandboxed, + boolean cancellable) { + int workRootId = Objects.hash(workerInitCmd, workerInitArgs, workerEnv, sandboxed, cancellable); + String workRootDirName = "work-root_" + executionName + "_" + workRootId; return workRootsDir.resolve(workRootDirName); } diff --git a/src/main/java/build/buildfarm/worker/persistent/PersistentExecutor.java b/src/main/java/build/buildfarm/worker/persistent/PersistentExecutor.java index 3c734d43a9..f4eb28fa52 100644 --- a/src/main/java/build/buildfarm/worker/persistent/PersistentExecutor.java +++ b/src/main/java/build/buildfarm/worker/persistent/PersistentExecutor.java @@ -17,17 +17,10 @@ import static java.lang.String.join; import build.bazel.remote.execution.v2.ActionResult; -import build.bazel.remote.execution.v2.Platform; -import build.buildfarm.common.Claim; -import build.buildfarm.common.config.ExecutionPolicy; -import build.buildfarm.worker.executionwrappers.ExecutionWrapperUtils; -import build.buildfarm.worker.filesystem.ExecFileSystem; -import build.buildfarm.worker.resources.LocalResourceSet; import build.buildfarm.worker.resources.ResourceLimits; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.devtools.build.lib.worker.WorkerProtocol.Input; import com.google.devtools.build.lib.worker.WorkerProtocol.WorkRequest; import com.google.devtools.build.lib.worker.WorkerProtocol.WorkResponse; @@ -37,16 +30,11 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.attribute.UserPrincipal; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.logging.Level; -import javax.annotation.Nullable; import lombok.extern.java.Log; -import persistent.bazel.client.BasicWorkerKey; -import persistent.bazel.client.WorkerIndex; import persistent.bazel.client.WorkerKey; /** @@ -58,10 +46,8 @@ */ @Log public class PersistentExecutor { - public static final WorkerIndex workerIndex = new WorkerIndex(); - private static final ProtoCoordinator coordinator = - ProtoCoordinator.ofCommonsPool(workerIndex, getMaxWorkersPerKey()); + ProtoCoordinator.ofCommonsPool(getMaxWorkersPerKey()); // TODO load from config (i.e. {worker_root}/persistent) public static final Path defaultWorkRootsDir = Path.of("/tmp/worker/persistent/"); @@ -110,7 +96,6 @@ private static int getMaxWorkersPerKey() { * @param timeout * @param workRootsDir * @param resultBuilder - * @param claim * @return */ public static Code runOnPersistentWorker( @@ -121,12 +106,7 @@ public static Code runOnPersistentWorker( ResourceLimits limits, Duration timeout, Path workRootsDir, - ActionResult.Builder resultBuilder, - ExecFileSystem execFileSystem, - @Nullable PersistentWorkerAwareExecOwnerPool execOwnerPool, - Claim claim, - Iterable executionPolicies, - Iterable platformProperties) + ActionResult.Builder resultBuilder) throws IOException { // Pull out persistent worker start command from the overall action request @@ -167,49 +147,17 @@ public static Code runOnPersistentWorker( "Binary wasn't a tool input nor an absolute path: " + binary); } - BasicWorkerKey basicKey = - Keymaker.makeBasicKey(workerExecCmd, workerInitArgs, env, executionName); - - /* - * As with any other action, an exec owner is reserved for an action executed by a persistent worker. However, - * because the exec owner is part of the worker key, we need to be careful about which exec owner we use. If we use - * any random exec owner, we could end up creating a new persistent worker when one already exists with practically - * the same worker key (except for the exec owner). - * - * To prevent runaway persistent worker creation, we attempt to swap the already-reserved exec owner for a more - * strategic one. We do this in `PersistentExecutor` instead of in `MatchStage` (where the claim is acquired) - * because we don't have enough information in `MatchStage` to know if a persistent worker will be used and - * construct a `BasicWorkerKey`, and therefore which exec owner to lease. - */ - if (execOwnerPool != null) { - Optional newExecOwnerLease = - execOwnerPool.tryAcquireForPersistentWorker(basicKey); - - if (newExecOwnerLease.isPresent()) { - PersistentWorkerAwareExecOwnerPool.Lease lease = newExecOwnerLease.get(); - - claim.replace(LocalResourceSet.EXEC_OWNER_RESOURCE_NAME, lease); - - String newExecOwnerName = Iterables.getOnlyElement(lease.ownerNames); - UserPrincipal newExecOwner = execFileSystem.getOwner(newExecOwnerName); - - claim.setOwner(newExecOwner); - } - } - - // Evaluate execution wrappers after exec owner swapping to ensure correct context - ImmutableList.Builder wrapperArguments = ImmutableList.builder(); - ExecutionWrapperUtils.applyExecutionWrappers( - wrapperArguments, executionPolicies, claim, platformProperties, true); - - ExecutionWrapperUtils.applyExecutionWrappers( - wrapperArguments, executionPolicies, claim, platformProperties, false); - WorkerKey key = - Keymaker.makeKey( - basicKey, claim.getOwner(), wrapperArguments.build(), workRootsDir, workerFiles); - - coordinator.copyToolInputsIntoWorkerToolRoot(key, workerFiles, key.getOwner()); + Keymaker.make( + context.opRoot, + workRootsDir, + workerExecCmd, + workerInitArgs, + env, + executionName, + workerFiles); + + coordinator.copyToolInputsIntoWorkerToolRoot(key, workerFiles); // Make request diff --git a/src/main/java/build/buildfarm/worker/persistent/PersistentWorkerAwareExecOwnerPool.java b/src/main/java/build/buildfarm/worker/persistent/PersistentWorkerAwareExecOwnerPool.java deleted file mode 100644 index d055d8ab45..0000000000 --- a/src/main/java/build/buildfarm/worker/persistent/PersistentWorkerAwareExecOwnerPool.java +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2023 The Buildfarm Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package build.buildfarm.worker.persistent; - -import build.buildfarm.common.Claim.Stage; -import build.buildfarm.worker.resources.LocalResource; -import build.buildfarm.worker.resources.LocalResourceSet; -import java.nio.file.attribute.UserPrincipal; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Queue; -import java.util.Set; -import persistent.bazel.client.BasicWorkerKey; -import persistent.bazel.client.WorkerIndex; - -public class PersistentWorkerAwareExecOwnerPool implements LocalResource { - public static class Lease extends LocalResourceSet.PoolLease { - public final List ownerNames; - - public Lease(Queue pool, Stage stage, List ownerNames) { - super(pool, stage, ownerNames); - - this.ownerNames = ownerNames; - } - } - - private final Queue pool; - private final Set takenForPersistentWorkers; - private final Stage stage; - private final WorkerIndex workerIndex; - - public PersistentWorkerAwareExecOwnerPool( - WorkerIndex workerIndex, Iterable execOwners, Stage stage) { - this.workerIndex = workerIndex; - this.pool = new ArrayDeque<>(); - - for (String execOwner : execOwners) { - pool.add(execOwner); - } - - this.takenForPersistentWorkers = new HashSet<>(); - this.stage = stage; - } - - @Override - public int available() { - return pool.size(); - } - - @Override - public Optional tryAcquire(int amount) { - List ownerNames = new ArrayList<>(amount); - - for (int i = 0; i < amount; i++) { - String ownerName; - - while (true) { - ownerName = pool.poll(); - - if (ownerName == null || !takenForPersistentWorkers.contains(ownerName)) { - break; - } - - takenForPersistentWorkers.remove(ownerName); - } - - if (ownerName == null) { - pool.addAll(ownerNames); - - return Optional.empty(); - } - ownerNames.add(ownerName); - } - - return Optional.of(new Lease(pool, stage, ownerNames)); - } - - public Optional tryAcquireForPersistentWorker(BasicWorkerKey workerKey) { - UserPrincipal owner = workerIndex.getOwnerForIdleWorker(workerKey); - - if (owner == null) { - // Ensure this stays in sync with the value of `EXEC_OWNER_PROPERTY` - return tryAcquire(1); - } - - String ownerName = owner.getName(); - - takenForPersistentWorkers.add(ownerName); - - return Optional.of( - new Lease(pool, stage, List.of(ownerName)) { - @Override - public void release() { - super.release(); - - takenForPersistentWorkers.remove(ownerName); - } - }); - } -} diff --git a/src/main/java/build/buildfarm/worker/persistent/ProtoCoordinator.java b/src/main/java/build/buildfarm/worker/persistent/ProtoCoordinator.java index 5bcee75633..57404f5d8c 100644 --- a/src/main/java/build/buildfarm/worker/persistent/ProtoCoordinator.java +++ b/src/main/java/build/buildfarm/worker/persistent/ProtoCoordinator.java @@ -22,18 +22,15 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.UserPrincipal; import java.util.Timer; import java.util.TimerTask; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; -import javax.annotation.Nullable; import lombok.extern.java.Log; import persistent.bazel.client.CommonsWorkerPool; import persistent.bazel.client.PersistentWorker; import persistent.bazel.client.WorkCoordinator; -import persistent.bazel.client.WorkerIndex; import persistent.bazel.client.WorkerKey; import persistent.bazel.client.WorkerSupervisor; @@ -81,20 +78,19 @@ public void run() { 10000); } - private ProtoCoordinator( - WorkerIndex workerIndex, WorkerSupervisor supervisor, int maxWorkersPerKey) { - this(new CommonsWorkerPool(workerIndex, supervisor, maxWorkersPerKey)); + private ProtoCoordinator(WorkerSupervisor supervisor, int maxWorkersPerKey) { + super(new CommonsWorkerPool(supervisor, maxWorkersPerKey)); } // We copy tool inputs from the shared WorkerKey tools directory into our worker exec root, // since there are multiple workers per key, // and presumably there might be writes to tool inputs? // Tool inputs which are absolute-paths (e.g. /usr/bin/...) are not affected - public static ProtoCoordinator ofCommonsPool(WorkerIndex workerIndex, int maxWorkersPerKey) { + public static ProtoCoordinator ofCommonsPool(int maxWorkersPerKey) { WorkerSupervisor loadToolsOnCreate = - new WorkerSupervisor(workerIndex) { + new WorkerSupervisor() { @Override - public PersistentWorker createUnderlying(WorkerKey workerKey) throws Exception { + public PersistentWorker create(WorkerKey workerKey) throws Exception { Path keyExecRoot = workerKey.getExecRoot(); String workerExecDir = getUniqueSubdir(keyExecRoot); Path workerExecRoot = keyExecRoot.resolve(workerExecDir); @@ -115,11 +111,11 @@ public PersistentWorker createUnderlying(WorkerKey workerKey) throws Exception { return new PersistentWorker(workerKey, workerExecDir); } }; - return new ProtoCoordinator(workerIndex, loadToolsOnCreate, maxWorkersPerKey); + return new ProtoCoordinator(loadToolsOnCreate, maxWorkersPerKey); } - public void copyToolInputsIntoWorkerToolRoot( - WorkerKey key, WorkerInputs workerFiles, @Nullable UserPrincipal owner) throws IOException { + public void copyToolInputsIntoWorkerToolRoot(WorkerKey key, WorkerInputs workerFiles) + throws IOException { WorkerKey lock = keyLock(key); synchronized (lock) { try { @@ -128,7 +124,7 @@ public void copyToolInputsIntoWorkerToolRoot( for (Path opToolPath : workerFiles.opToolInputs) { Path workToolPath = workerFiles.relativizeInput(workToolRoot, opToolPath); if (!Files.exists(workToolPath)) { - workerFiles.copyInputFile(opToolPath, workToolPath, owner); + workerFiles.copyInputFile(opToolPath, workToolPath); } } } finally { @@ -155,7 +151,7 @@ private static void copyToolsIntoWorkerExecRoot(WorkerKey key, Path workerExecRo Path toolInputPath = toolInputRoot.resolve(relPath); Path execRootPath = workerExecRoot.resolve(relPath); - FileAccessUtils.copyFile(toolInputPath, execRootPath, key.getOwner()); + FileAccessUtils.copyFile(toolInputPath, execRootPath); } } @@ -178,7 +174,7 @@ public WorkRequest preWorkInit(WorkerKey key, RequestCtx request, PersistentWork timeoutScheduler.schedule(pendingRequest.task, Durations.toMillis(request.timeout)); // Symlinking should hypothetically be faster+leaner than copying inputs, but it's buggy. - copyNontoolInputs(request.workerInputs, worker.getExecRoot(), key.getOwner()); + copyNontoolInputs(request.workerInputs, worker.getExecRoot()); return request.request; } @@ -228,13 +224,12 @@ private IOException logBadCleanup(RequestCtx request, IOException e) { return new IOException("Response was OK but failed on postWorkCleanup", e); } - private void copyNontoolInputs( - WorkerInputs workerInputs, Path workerExecRoot, @Nullable UserPrincipal owner) + private void copyNontoolInputs(WorkerInputs workerInputs, Path workerExecRoot) throws IOException { for (Path opPath : workerInputs.allInputs.keySet()) { if (!workerInputs.allToolInputs.contains(opPath)) { Path execPath = workerInputs.relativizeInput(workerExecRoot, opPath); - workerInputs.copyInputFile(opPath, execPath, owner); + workerInputs.copyInputFile(opPath, execPath); } } } diff --git a/src/main/java/build/buildfarm/worker/persistent/WorkerInputs.java b/src/main/java/build/buildfarm/worker/persistent/WorkerInputs.java index f8e42ec576..1df0bbe10e 100644 --- a/src/main/java/build/buildfarm/worker/persistent/WorkerInputs.java +++ b/src/main/java/build/buildfarm/worker/persistent/WorkerInputs.java @@ -20,9 +20,7 @@ import com.google.protobuf.ByteString; import java.io.IOException; import java.nio.file.Path; -import java.nio.file.attribute.UserPrincipal; import java.util.List; -import javax.annotation.Nullable; import lombok.extern.java.Log; @Log @@ -67,9 +65,9 @@ public Path relativizeInput(Path newRoot, Path input) { return newRoot.resolve(opRoot.relativize(input)); } - public void copyInputFile(Path from, Path to, @Nullable UserPrincipal owner) throws IOException { + public void copyInputFile(Path from, Path to) throws IOException { checkFileIsInput("copyInputFile()", from); - FileAccessUtils.copyFile(from, to, owner); + FileAccessUtils.copyFile(from, to); } public void deleteInputFileIfExists(Path workerExecRoot, Path opPathInput) throws IOException { diff --git a/src/main/java/build/buildfarm/worker/resources/BUILD b/src/main/java/build/buildfarm/worker/resources/BUILD index b9f297bfd8..3122266844 100644 --- a/src/main/java/build/buildfarm/worker/resources/BUILD +++ b/src/main/java/build/buildfarm/worker/resources/BUILD @@ -8,12 +8,13 @@ java_library( deps = [ "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/config", + "//src/main/java/build/buildfarm/instance", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_code_gson_gson", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:org_apache_commons_commons_lang3", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_code_gson_gson", + "@maven//:com_google_guava_guava", + "@maven//:io_prometheus_simpleclient", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/worker/resources/LocalResource.java b/src/main/java/build/buildfarm/worker/resources/LocalResource.java deleted file mode 100644 index d57037deff..0000000000 --- a/src/main/java/build/buildfarm/worker/resources/LocalResource.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2023 The Buildfarm Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package build.buildfarm.worker.resources; - -import build.buildfarm.common.Claim.Lease; -import java.util.Optional; - -public interface LocalResource { - int available(); - - Optional tryAcquire(int amount); -} diff --git a/src/main/java/build/buildfarm/worker/resources/LocalResourceSet.java b/src/main/java/build/buildfarm/worker/resources/LocalResourceSet.java index ff8f5af731..55cf051b7a 100644 --- a/src/main/java/build/buildfarm/worker/resources/LocalResourceSet.java +++ b/src/main/java/build/buildfarm/worker/resources/LocalResourceSet.java @@ -14,14 +14,9 @@ package build.buildfarm.worker.resources; -import build.bazel.remote.execution.v2.Platform; -import build.buildfarm.common.Claim.Lease; import build.buildfarm.common.Claim.Stage; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Queue; import java.util.concurrent.Semaphore; @@ -35,101 +30,16 @@ * resources are specific to the individual worker. */ public class LocalResourceSet { - public static final String EXEC_OWNER_RESOURCE_NAME = "exec-owner"; - public static final Platform.Property EXEC_OWNER_PROPERTY = - Platform.Property.newBuilder().setName(EXEC_OWNER_RESOURCE_NAME).setValue("1").build(); + public record SemaphoreResource(Semaphore semaphore, Stage stage) {} - public record SemaphoreLease(Semaphore semaphore, Stage stage, int amount) implements Lease { - @Override - public int getAmount() { - return amount; - } - - @Override - public Stage getStage() { - return stage; - } - - @Override - public void release() { - semaphore.release(amount); - } - } - - public record SemaphoreResource(Semaphore semaphore, Stage stage) implements LocalResource { - @Override - public int available() { - return semaphore.availablePermits(); - } - - @Override - public Optional tryAcquire(int amount) { - if (semaphore.tryAcquire(amount)) { - return Optional.of(new SemaphoreLease(semaphore, stage, amount)); - } - - return Optional.empty(); - } - } - - public static class PoolLease implements Lease { - private final Queue pool; - private final Stage stage; - - public final List claims; - - public PoolLease(Queue pool, Stage stage, List claims) { - this.pool = pool; - this.stage = stage; - this.claims = claims; - } - - @Override - public int getAmount() { - return claims.size(); - } - - @Override - public Stage getStage() { - return stage; - } - - @Override - public void release() { - pool.addAll(claims); - } - } - - public record PoolResource(Queue pool, Stage stage) implements LocalResource { - @Override - public int available() { - return pool.size(); - } - - @Override - public Optional tryAcquire(int amount) { - List claimedIds = new ArrayList<>(amount); - - for (int i = 0; i < amount; i++) { - Object id = pool.poll(); - - claimedIds.add(id); - - if (id == null) { - pool.addAll(claimedIds); - - return Optional.empty(); - } - } - - return Optional.of(new PoolLease(pool, stage, claimedIds)); - } - } + public record PoolResource(Queue pool, Stage stage) {} /** * @field resources - * @brief A set containing resources organized by name. + * @brief A set containing resource semaphores organized by name. * @details Key is name, and value contains current usage amount. */ - public Map resources = new HashMap<>(); + public Map resources = new HashMap<>(); + + public Map poolResources = new HashMap<>(); } diff --git a/src/main/java/build/buildfarm/worker/resources/LocalResourceSetUtils.java b/src/main/java/build/buildfarm/worker/resources/LocalResourceSetUtils.java index 3b77d30aa3..8a220b8b1e 100644 --- a/src/main/java/build/buildfarm/worker/resources/LocalResourceSetUtils.java +++ b/src/main/java/build/buildfarm/worker/resources/LocalResourceSetUtils.java @@ -14,24 +14,25 @@ package build.buildfarm.worker.resources; +import static com.google.common.collect.Iterables.transform; + import build.bazel.remote.execution.v2.Platform; import build.buildfarm.common.Claim; -import build.buildfarm.common.Claim.Lease; import build.buildfarm.common.config.LimitedResource; import build.buildfarm.worker.resources.LocalResourceSet.PoolResource; import build.buildfarm.worker.resources.LocalResourceSet.SemaphoreResource; import java.nio.file.attribute.UserPrincipal; import java.util.AbstractMap.SimpleEntry; import java.util.ArrayDeque; -import java.util.HashMap; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Map.Entry; -import java.util.Optional; +import java.util.Queue; import java.util.Set; import java.util.concurrent.Semaphore; +import java.util.function.Consumer; import java.util.stream.IntStream; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; @@ -44,105 +45,99 @@ public class LocalResourceSetUtils { private static final LocalResourceSetMetrics metrics = new LocalResourceSetMetrics(); - public static final class LocalResourceSetClaim implements Claim { - private final Map claimed; - private @Nullable UserPrincipal owner; + private record SemaphoreLease(String name, int amount, Claim.Stage stage) {} + + private record PoolLease(String name, List claims, Claim.Stage stage) {} - LocalResourceSetClaim(Map claimed) { - this.claimed = claimed; + public static final class LocalResourceSetClaim implements Claim { + private final List semaphores; + private final List pools; + private final LocalResourceSet resourceSet; + + LocalResourceSetClaim( + List semaphores, List pools, LocalResourceSet resourceSet) { + this.semaphores = semaphores; + this.pools = pools; + this.resourceSet = resourceSet; } @Override public synchronized void release(Stage stage) { // only remove resources that match the specified stage // O(n) search for staged resources - Iterator> leaseIterator = claimed.entrySet().iterator(); - while (leaseIterator.hasNext()) { - Lease lease = leaseIterator.next().getValue(); - if (lease.getStage() != stage) { + Iterator semaphoreIter = semaphores.iterator(); + while (semaphoreIter.hasNext()) { + SemaphoreLease semaphore = semaphoreIter.next(); + if (semaphore.stage() != stage) { + continue; + } + semaphoreIter.remove(); + String resourceName = semaphore.name(); + SemaphoreResource resource = resourceSet.resources.get(resourceName); + semaphoreRelease(resource.semaphore(), resourceName, semaphore.amount()); + } + Iterator poolIter = pools.iterator(); + while (poolIter.hasNext()) { + PoolLease pool = poolIter.next(); + if (pool.stage() != stage) { continue; } - leaseIterator.remove(); - lease.release(); + poolIter.remove(); + String resourceName = pool.name(); + PoolResource resource = resourceSet.poolResources.get(resourceName); + poolRelease(resource.pool(), resourceName, pool.claims()); } } // safe for multiple sequential calls with removal to release @Override public synchronized void release() { - Iterator> leaseIterator = claimed.entrySet().iterator(); - - while (leaseIterator.hasNext()) { - Lease lease = leaseIterator.next().getValue(); - - leaseIterator.remove(); - lease.release(); + while (!semaphores.isEmpty()) { + SemaphoreLease semaphore = semaphores.removeFirst(); + String resourceName = semaphore.name(); + SemaphoreResource resource = resourceSet.resources.get(resourceName); + semaphoreRelease(resource.semaphore(), resourceName, semaphore.amount()); } - } - @Override - public synchronized void replace(String resourceName, Lease lease) { - Lease oldLease = claimed.get(resourceName); - - if (oldLease != null) { - oldLease.release(); - - metrics.resourceUsageMetric.labels(resourceName).dec(oldLease.getAmount()); - metrics.requestersMetric.labels(resourceName).dec(); + while (!pools.isEmpty()) { + PoolLease pool = pools.removeFirst(); + String resourceName = pool.name(); + PoolResource resource = resourceSet.poolResources.get(resourceName); + poolRelease(resource.pool(), resourceName, pool.claims()); } - - claimed.put(resourceName, lease); - - metrics.resourceUsageMetric.labels(resourceName).inc(lease.getAmount()); - metrics.requestersMetric.labels(resourceName).inc(); } @Override - public @Nullable UserPrincipal getOwner() { - return owner; + public UserPrincipal owner() { + return null; } @Override - public void setOwner(@Nullable UserPrincipal owner) { - this.owner = owner; - } - - @Override - public Iterable>> getPools() { - return claimed.entrySet().stream() - .filter(entry -> entry.getValue() instanceof LocalResourceSet.PoolLease) - .>>map( - entry -> { - LocalResourceSet.PoolLease poolLease = - (LocalResourceSet.PoolLease) entry.getValue(); - - return new SimpleEntry<>(entry.getKey(), poolLease.claims); - }) - .toList(); + public Iterable>> getPools() { + return transform(pools, pool -> new SimpleEntry<>(pool.name(), pool.claims())); } } public static LocalResourceSet create(Iterable resources) { LocalResourceSet resourceSet = new LocalResourceSet(); for (LimitedResource resource : resources) { - LocalResource localResource; - switch (resource.getType()) { case SEMAPHORE: - localResource = + resourceSet.resources.put( + resource.getName(), new SemaphoreResource( - new Semaphore(resource.getAmount()), resource.getReleaseStage()); + new Semaphore(resource.getAmount()), resource.getReleaseStage())); break; case POOL: - localResource = + resourceSet.poolResources.put( + resource.getName(), new PoolResource( new ArrayDeque(IntStream.range(0, resource.getAmount()).boxed().toList()), - resource.getReleaseStage()); + resource.getReleaseStage())); break; default: throw new IllegalStateException("unrecognized resource type: " + resource.getType()); } - resourceSet.resources.put(resource.getName(), localResource); metrics.resourceTotalMetric.labels(resource.getName()).set(resource.getAmount()); } return resourceSet; @@ -150,8 +145,13 @@ public static LocalResourceSet create(Iterable resources) { public static Set exhausted(LocalResourceSet resourceSet) { Set exhausted = new HashSet<>(); - for (Entry resource : resourceSet.resources.entrySet()) { - if (resource.getValue().available() == 0) { + for (Entry resource : resourceSet.resources.entrySet()) { + if (resource.getValue().semaphore().availablePermits() == 0) { + exhausted.add(resource.getKey()); + } + } + for (Entry resource : resourceSet.poolResources.entrySet()) { + if (resource.getValue().pool().isEmpty()) { exhausted.add(resource.getKey()); } } @@ -159,44 +159,91 @@ public static Set exhausted(LocalResourceSet resourceSet) { } public static @Nullable Claim claimResources(Platform platform, LocalResourceSet resourceSet) { - Map claimed = new HashMap<>(); + List semaphoreClaimed = new ArrayList<>(); + List poolClaimed = new ArrayList<>(); boolean allClaimed = true; for (Platform.Property property : platform.getPropertiesList()) { String resourceName = getResourceName(property.getName()); int requestAmount = getResourceRequestAmount(property); if (resourceSet.resources.containsKey(resourceName)) { - LocalResource resource = resourceSet.resources.get(resourceName); + SemaphoreResource resource = resourceSet.resources.get(resourceName); // Attempt to claim. If claiming fails, we must return all other claims. - Optional lease = resource.tryAcquire(requestAmount); - if (lease.isPresent()) { - claimed.put(resourceName, lease.get()); - - metrics.resourceUsageMetric.labels(resourceName).inc(requestAmount); - metrics.requestersMetric.labels(resourceName).inc(); + boolean wasAcquired = semaphoreAquire(resource.semaphore(), resourceName, requestAmount); + if (wasAcquired) { + semaphoreClaimed.add(new SemaphoreLease(resourceName, requestAmount, resource.stage())); } else { allClaimed = false; break; } + } else if (resourceSet.poolResources.containsKey(resourceName)) { + PoolResource resource = resourceSet.poolResources.get(resourceName); + List claimed = new ArrayList<>(); + poolClaimed.add(new PoolLease(resourceName, claimed, resource.stage())); + + // Attempt to claim. If claiming fails, we must return all other claims. + if (!poolAcquire(resource.pool(), resourceName, requestAmount, claimed::add)) { + allClaimed = false; + break; + } } } // cleanup remaining resources if they were not all claimed. if (!allClaimed) { - for (Map.Entry entry : claimed.entrySet()) { - String resourceName = entry.getKey(); + for (SemaphoreLease semaphore : semaphoreClaimed) { + semaphoreRelease( + resourceSet.resources.get(semaphore.name()).semaphore(), + semaphore.name(), + semaphore.amount()); + } + for (PoolLease pool : poolClaimed) { + poolRelease(resourceSet.poolResources.get(pool.name()).pool(), pool.name(), pool.claims()); + } + } - Lease lease = entry.getValue(); + return allClaimed + ? new LocalResourceSetClaim(semaphoreClaimed, poolClaimed, resourceSet) + : null; + } - lease.release(); + private static boolean semaphoreAquire(Semaphore resource, String resourceName, int amount) { + boolean wasAcquired = resource.tryAcquire(amount); + if (wasAcquired) { + metrics.resourceUsageMetric.labels(resourceName).inc(amount); + metrics.requestersMetric.labels(resourceName).inc(); + } + return wasAcquired; + } - metrics.resourceUsageMetric.labels(resourceName).dec(lease.getAmount()); - metrics.requestersMetric.labels(resourceName).dec(); + private static boolean poolAcquire( + Queue resource, String resourceName, int amount, Consumer onClaimed) { + for (int i = 0; i < amount; i++) { + Object id = resource.poll(); + if (id == null) { + return false; } + // still don't like this + onClaimed.accept(id); } + // only records when fully acquired + metrics.resourceUsageMetric.labels(resourceName).inc(amount); + metrics.requestersMetric.labels(resourceName).inc(); + return true; + } + + private static void semaphoreRelease(Semaphore resource, String resourceName, int amount) { + resource.release(amount); + metrics.resourceUsageMetric.labels(resourceName).dec(amount); + metrics.requestersMetric.labels(resourceName).dec(); + } - return allClaimed ? new LocalResourceSetClaim(claimed) : null; + private static void poolRelease( + Queue resource, String resourceName, List claims) { + claims.forEach(resource::add); + metrics.resourceUsageMetric.labels(resourceName).dec(claims.size()); + metrics.requestersMetric.labels(resourceName).dec(); } private static int getResourceRequestAmount(Platform.Property property) { @@ -218,6 +265,8 @@ public static String getResourceName(String propertyName) { public static boolean satisfies(LocalResourceSet resourceSet, Platform.Property property) { String name = property.getName(); - return name.startsWith("resource:") && resourceSet.resources.containsKey(getResourceName(name)); + return name.startsWith("resource:") + && (resourceSet.resources.containsKey(getResourceName(name)) + || resourceSet.poolResources.containsKey(getResourceName(name))); } } diff --git a/src/main/java/build/buildfarm/worker/shard/BUILD b/src/main/java/build/buildfarm/worker/shard/BUILD index 29a2ee519c..e3f3cc16c7 100644 --- a/src/main/java/build/buildfarm/worker/shard/BUILD +++ b/src/main/java/build/buildfarm/worker/shard/BUILD @@ -20,27 +20,25 @@ java_library( "//src/main/java/build/buildfarm/metrics/prometheus", "//src/main/java/build/buildfarm/worker", "//src/main/java/build/buildfarm/worker/cgroup", - "//src/main/java/build/buildfarm/worker/filesystem", - "//src/main/java/build/buildfarm/worker/persistent", "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_grpc", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_github_pcj_google_options", - "@buildfarm_maven//:com_google_code_findbugs_jsr305", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_services", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_prometheus_simpleclient", - "@buildfarm_maven//:javax_annotation_javax_annotation_api", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", "@io_grpc_grpc_proto//:health_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_github_pcj_google_options", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_services", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_prometheus_simpleclient", + "@maven//:javax_annotation_javax_annotation_api", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/main/java/build/buildfarm/worker/shard/FuseExecFileSystem.java b/src/main/java/build/buildfarm/worker/shard/FuseExecFileSystem.java index 96cd4a2592..3028273612 100644 --- a/src/main/java/build/buildfarm/worker/shard/FuseExecFileSystem.java +++ b/src/main/java/build/buildfarm/worker/shard/FuseExecFileSystem.java @@ -14,8 +14,6 @@ package build.buildfarm.worker.shard; -import static com.google.common.util.concurrent.Futures.immediateFuture; - import build.bazel.remote.execution.v2.Action; import build.bazel.remote.execution.v2.Command; import build.bazel.remote.execution.v2.Compressor; @@ -24,10 +22,8 @@ import build.buildfarm.cas.ContentAddressableStorage; import build.buildfarm.common.DigestUtil; import build.buildfarm.v1test.Digest; -import build.buildfarm.v1test.WorkerExecutedMetadata; -import build.buildfarm.worker.filesystem.ExecFileSystem; -import build.buildfarm.worker.filesystem.FuseCAS; -import com.google.common.util.concurrent.ListenableFuture; +import build.buildfarm.worker.ExecFileSystem; +import build.buildfarm.worker.FuseCAS; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; @@ -53,10 +49,8 @@ public UserPrincipal getOwner(String name) { } @Override - public ListenableFuture start( - Consumer> onDigests, boolean skipLoad, boolean writable) { + public void start(Consumer> onDigests, boolean skipLoad) { // onDigests.accept(storage.getAllDigests()); - return immediateFuture(null); } @Override @@ -87,8 +81,7 @@ public Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + UserPrincipal owner) throws IOException, InterruptedException { // FIXME insist that the FUSE perform permission checks against the owner fuseCAS.createInputRoot( diff --git a/src/main/java/build/buildfarm/worker/shard/LocalCasWriter.java b/src/main/java/build/buildfarm/worker/shard/LocalCasWriter.java index b48aec3de7..e73539745e 100644 --- a/src/main/java/build/buildfarm/worker/shard/LocalCasWriter.java +++ b/src/main/java/build/buildfarm/worker/shard/LocalCasWriter.java @@ -21,7 +21,7 @@ import build.buildfarm.common.Write; import build.buildfarm.common.function.IOSupplier; import build.buildfarm.v1test.Digest; -import build.buildfarm.worker.filesystem.ExecFileSystem; +import build.buildfarm.worker.ExecFileSystem; import com.google.common.io.ByteStreams; import com.google.protobuf.ByteString; import io.grpc.Status; diff --git a/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java b/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java index db895d13f5..9deb30f122 100644 --- a/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java +++ b/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java @@ -30,7 +30,6 @@ import build.bazel.remote.execution.v2.ExecutionStage; import build.bazel.remote.execution.v2.Platform; import build.buildfarm.backplane.Backplane; -import build.buildfarm.cas.ContentAddressableStorage; import build.buildfarm.common.Claim; import build.buildfarm.common.CommandUtils; import build.buildfarm.common.DigestPath; @@ -55,8 +54,8 @@ import build.buildfarm.v1test.Digest; import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.QueuedOperation; -import build.buildfarm.v1test.WorkerExecutedMetadata; import build.buildfarm.worker.DequeueMatchEvaluator; +import build.buildfarm.worker.ExecFileSystem; import build.buildfarm.worker.ExecutionPolicies; import build.buildfarm.worker.MatchListener; import build.buildfarm.worker.RetryingMatchListener; @@ -65,7 +64,6 @@ import build.buildfarm.worker.cgroup.Cpu; import build.buildfarm.worker.cgroup.Group; import build.buildfarm.worker.cgroup.Mem; -import build.buildfarm.worker.filesystem.ExecFileSystem; import build.buildfarm.worker.resources.LocalResourceSet; import build.buildfarm.worker.resources.ResourceDecider; import build.buildfarm.worker.resources.ResourceLimits; @@ -104,6 +102,9 @@ @Log class ShardWorkerContext implements WorkerContext { + static final String EXEC_OWNER_RESOURCE_NAME = "exec-owner"; + private static final Platform.Property EXEC_OWNER_PROPERTY = + Platform.Property.newBuilder().setName(EXEC_OWNER_RESOURCE_NAME).setValue("1").build(); private static final String PROVISION_CORES_NAME = "cores"; private static final Counter completedOperations = @@ -140,11 +141,6 @@ class ShardWorkerContext implements WorkerContext { private final LocalResourceSet resourceSet; private final boolean errorOperationOutputSizeExceeded; private final boolean provideOwnedClaim; - private boolean inGracefulShutdown = false; - private boolean pauseMatch = false; - private boolean pauseInputFetch = false; - private boolean pauseExecute = false; - private boolean pauseReportResult = false; static SetMultimap getMatchProvisions( Iterable policies, String name, int executeStageWidth) { @@ -207,8 +203,7 @@ static SetMultimap getMatchProvisions( this.resourceSet = resourceSet; this.writer = writer; - provideOwnedClaim = - this.resourceSet.resources.containsKey(LocalResourceSet.EXEC_OWNER_RESOURCE_NAME); + provideOwnedClaim = this.resourceSet.poolResources.containsKey(EXEC_OWNER_RESOURCE_NAME); } private static Retrier createBackplaneRetrier() { @@ -222,16 +217,6 @@ private static Retrier createBackplaneRetrier() { Retrier.REDIS_IS_RETRIABLE); } - @Override - public boolean inGracefulShutdown() { - return inGracefulShutdown; - } - - @Override - public void prepareForGracefulShutdown() { - inGracefulShutdown = true; - } - @Override public String getName() { return name; @@ -307,10 +292,11 @@ public QueuedOperation getQueuedOperation(QueueEntry queueEntry) } // FIXME make OwnedClaim with owner + // how will this play out with persistent workers, should we have one per user? private @Nullable Claim acquireClaim(Platform platform) { // expand platform requirements with exec owner if (provideOwnedClaim) { - platform = platform.toBuilder().addProperties(LocalResourceSet.EXEC_OWNER_PROPERTY).build(); + platform = platform.toBuilder().addProperties(EXEC_OWNER_PROPERTY).build(); } Claim claim = DequeueMatchEvaluator.acquireClaim(matchProvisions, resourceSet, platform); @@ -318,13 +304,39 @@ public QueuedOperation getQueuedOperation(QueueEntry queueEntry) // a little awkward wrapping with the early return here to preserve effective final if (claim != null && provideOwnedClaim) { // enforced by exec owner property value of "1" - for (Entry> pool : claim.getPools()) { - if (!pool.getKey().equals(LocalResourceSet.EXEC_OWNER_RESOURCE_NAME)) { + for (Entry> pool : claim.getPools()) { + if (!pool.getKey().equals(EXEC_OWNER_RESOURCE_NAME)) { continue; } String name = (String) Iterables.getOnlyElement(pool.getValue()); - claim.setOwner(execFileSystem.getOwner(name)); + return new Claim() { + UserPrincipal owner = execFileSystem.getOwner(name); + + @Override + public void release(Claim.Stage stage) { + claim.release(stage); + } + + @Override + public void release() { + owner = null; + claim.release(); + } + + @Override + public UserPrincipal owner() { + if (owner != null) { + return owner; + } + return claim.owner(); + } + + @Override + public Iterable>> getPools() { + return claim.getPools(); + } + }; } // claim was not provided with owner name value } @@ -372,18 +384,6 @@ private void matchInterruptible(MatchListener listener) throws IOException, Inte return queueEntry; } - /** wait until matching should occur, false return indicates that we are shutting down */ - private boolean waitToMatch() throws InterruptedException { - ContentAddressableStorage storage = execFileSystem.getStorage(); - synchronized (storage) { - // we may want to get interrupted when match is paused - while (!inGracefulShutdown && storage.isReadOnly()) { - storage.waitForWritable(java.time.Duration.ofSeconds(1)); - } - } - return !inGracefulShutdown; - } - @Override public void match(MatchListener listener) throws InterruptedException { RetryingMatchListener dedupMatchListener = @@ -443,7 +443,7 @@ public void onError(Throwable t) { throw new RuntimeException(t); } }; - while (waitToMatch() && !dedupMatchListener.isMatched()) { + while (!dedupMatchListener.isMatched()) { try { matchInterruptible(dedupMatchListener); } catch (IOException e) { @@ -758,17 +758,10 @@ public Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - @Nullable UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + @Nullable UserPrincipal owner) throws IOException, InterruptedException { return execFileSystem.createExecDir( - operationName, - directoriesIndex, - digestFunction, - action, - command, - owner, - workerExecutedMetadata); + operationName, directoriesIndex, digestFunction, action, command, owner); } // might want to split for removeDirectory and decrement references to avoid removing for streamed @@ -827,7 +820,10 @@ public void createExecutionLimits() { void createOperationExecutionLimits() { try { - if (executeStageWidth < SystemProcessors.get()) { + int availableProcessors = SystemProcessors.get(); + Preconditions.checkState(availableProcessors >= executeStageWidth); + executionsGroup.getCpu().setMaxCpu(executeStageWidth); + if (executeStageWidth < availableProcessors) { /* only divide up our cfs quota if we need to limit below the available processors for executions */ executionsGroup.getCpu().setMaxCpu(executeStageWidth); } diff --git a/src/main/java/build/buildfarm/worker/shard/WorkerControl.java b/src/main/java/build/buildfarm/worker/shard/ShutDownWorkerGracefully.java similarity index 66% rename from src/main/java/build/buildfarm/worker/shard/WorkerControl.java rename to src/main/java/build/buildfarm/worker/shard/ShutDownWorkerGracefully.java index c8212c5c41..2dda82fc6b 100644 --- a/src/main/java/build/buildfarm/worker/shard/WorkerControl.java +++ b/src/main/java/build/buildfarm/worker/shard/ShutDownWorkerGracefully.java @@ -16,17 +16,15 @@ import build.buildfarm.v1test.PrepareWorkerForGracefulShutDownRequest; import build.buildfarm.v1test.PrepareWorkerForGracefulShutDownRequestResults; -import build.buildfarm.v1test.WorkerControlGrpc; -import build.buildfarm.v1test.WorkerPipelineChangeRequest; -import build.buildfarm.v1test.WorkerPipelineChangeResponse; +import build.buildfarm.v1test.ShutDownWorkerGrpc; import io.grpc.stub.StreamObserver; import lombok.extern.java.Log; @Log -public class WorkerControl extends WorkerControlGrpc.WorkerControlImplBase { +public class ShutDownWorkerGracefully extends ShutDownWorkerGrpc.ShutDownWorkerImplBase { private final Worker worker; - public WorkerControl(Worker worker) { + public ShutDownWorkerGracefully(Worker worker) { this.worker = worker; } @@ -49,20 +47,4 @@ public void prepareWorkerForGracefulShutdown( responseObserver.onError(e); } } - - @Override - public void pipelineChange( - WorkerPipelineChangeRequest request, - StreamObserver responseObserver) { - try { - WorkerPipelineChangeResponse response = - WorkerPipelineChangeResponse.newBuilder() - .addAllChanges(worker.pipelineChange(request.getChangesList())) - .build(); - responseObserver.onNext(response); - responseObserver.onCompleted(); - } catch (Exception e) { - responseObserver.onError(e); - } - } } diff --git a/src/main/java/build/buildfarm/worker/shard/Worker.java b/src/main/java/build/buildfarm/worker/shard/Worker.java index 3bbc25b595..e9947358fc 100644 --- a/src/main/java/build/buildfarm/worker/shard/Worker.java +++ b/src/main/java/build/buildfarm/worker/shard/Worker.java @@ -21,7 +21,6 @@ import static build.buildfarm.common.io.Utils.getUser; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.concurrent.Executors.newSingleThreadExecutor; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.logging.Level.INFO; @@ -38,6 +37,7 @@ import build.buildfarm.common.BuildfarmExecutors; import build.buildfarm.common.DigestUtil; import build.buildfarm.common.DigestUtil.HashFunction; +import build.buildfarm.common.Dispenser; import build.buildfarm.common.EmptyInputStreamFactory; import build.buildfarm.common.FailoverInputStreamFactory; import build.buildfarm.common.InputStreamFactory; @@ -59,10 +59,13 @@ import build.buildfarm.instance.stub.StubInstance; import build.buildfarm.metrics.prometheus.PrometheusPublisher; import build.buildfarm.v1test.Digest; -import build.buildfarm.v1test.PipelineChange; import build.buildfarm.v1test.ShardWorker; +import build.buildfarm.worker.CFCExecFileSystem; +import build.buildfarm.worker.CFCLinkExecFileSystem; +import build.buildfarm.worker.ExecFileSystem; import build.buildfarm.worker.ExecuteActionStage; import build.buildfarm.worker.ExecutionContext; +import build.buildfarm.worker.FuseCAS; import build.buildfarm.worker.InputFetchStage; import build.buildfarm.worker.MatchStage; import build.buildfarm.worker.Pipeline; @@ -71,23 +74,14 @@ import build.buildfarm.worker.ReportResultStage; import build.buildfarm.worker.SuperscalarPipelineStage; import build.buildfarm.worker.cgroup.Group; -import build.buildfarm.worker.filesystem.CFCExecFileSystem; -import build.buildfarm.worker.filesystem.CFCLinkExecFileSystem; -import build.buildfarm.worker.filesystem.ExecFileSystem; -import build.buildfarm.worker.filesystem.FuseCAS; -import build.buildfarm.worker.persistent.PersistentExecutor; -import build.buildfarm.worker.persistent.PersistentWorkerAwareExecOwnerPool; import build.buildfarm.worker.resources.LocalResourceSet; +import build.buildfarm.worker.resources.LocalResourceSet.PoolResource; import build.buildfarm.worker.resources.LocalResourceSetUtils; import com.google.common.base.Strings; import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; import com.google.longrunning.Operation; import com.google.protobuf.ByteString; import com.google.protobuf.Duration; @@ -97,18 +91,17 @@ import io.grpc.Status.Code; import io.grpc.health.v1.HealthCheckResponse.ServingStatus; import io.grpc.protobuf.services.HealthStatusManager; -import io.grpc.protobuf.services.ProtoReflectionServiceV1; +import io.grpc.protobuf.services.ProtoReflectionService; import io.prometheus.client.Counter; import io.prometheus.client.Gauge; import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.UserPrincipal; -import java.util.Collections; +import java.util.ArrayDeque; import java.util.List; import java.util.Random; import java.util.UUID; @@ -154,6 +147,7 @@ public final class Worker extends LoggingMain { private static BuildfarmConfigs configs = BuildfarmConfigs.getInstance(); + private boolean inGracefulShutdown = false; private boolean isPaused = false; private WorkerInstance instance; @@ -163,15 +157,10 @@ public final class Worker extends LoggingMain { private Server server; private Path root; private ExecFileSystem execFileSystem; - private @Nullable PersistentWorkerAwareExecOwnerPool execOwnerIndexResource; private Pipeline pipeline; - private PipelineStage matchStage; - private ShardWorkerContext context; private Backplane backplane; private LoadingCache workerStubs; private AtomicBoolean released = new AtomicBoolean(true); - private AtomicBoolean shutdownInitiated = new AtomicBoolean(false); - private boolean startWritable = true; /** * The method will prepare the worker for graceful shutdown when the worker is ready. Note on @@ -184,10 +173,11 @@ public void prepareWorkerForGracefulShutdown() { "Graceful Shutdown is not enabled. Worker is shutting down without finishing executions" + " in progress."); } else { + inGracefulShutdown = true; log.info( "Graceful Shutdown - The current worker will not be registered again and should be" + " shutdown gracefully!"); - context.prepareForGracefulShutdown(); + pipeline.stopMatchingOperations(); int scanRate = 30; // check every 30 seconds int timeWaited = 0; int timeOut = configs.getWorker().getGracefulShutdownSeconds(); @@ -241,15 +231,51 @@ public void put(ExecutionContext executionContext) throws InterruptedException { private Server createServer( ServerBuilder serverBuilder, + @Nullable CASFileCache storage, Instance instance, - WorkerProfileService workerProfileService) { + Pipeline pipeline, + ShardWorkerContext context) { serverBuilder.addService(healthStatusManager.getHealthService()); serverBuilder.addService(new ContentAddressableStorageService(instance)); serverBuilder.addService(new ByteStreamService(instance)); - serverBuilder.addService(new WorkerControl(this)); - serverBuilder.addService(ProtoReflectionServiceV1.newInstance()); - serverBuilder.addService(workerProfileService); + serverBuilder.addService(new ShutDownWorkerGracefully(this)); + serverBuilder.addService(ProtoReflectionService.newInstance()); + // We will build a worker's server based on it's capabilities. + // A worker that is capable of execution will construct an execution pipeline. + // It will use various execution phases for it's profile service. + // On the other hand, a worker that is only capable of CAS storage does not need a pipeline. + if (configs.getWorker().getCapabilities().isExecution()) { + // all claims should be released upon pipeline completion, but it is safe to ensure that + // they are whether empty or not, so do so. + PutOperationStage completeStage = + new ReleaseClaimStage(operation -> context.deactivate(operation.getName())); + PipelineStage errorStage = completeStage; /* new ErrorStage(); */ + SuperscalarPipelineStage reportResultStage = + new ReportResultStage(context, completeStage, errorStage); + SuperscalarPipelineStage executeActionStage = + new ExecuteActionStage(context, reportResultStage, errorStage); + PipelineStage releaseClaimAndRequeueStage = new ReleaseClaimStage(context::requeue); + SuperscalarPipelineStage inputFetchStage = + new InputFetchStage(context, executeActionStage, releaseClaimAndRequeueStage); + PipelineStage matchStage = + new MatchStage(context, inputFetchStage, releaseClaimAndRequeueStage); + + pipeline.add(matchStage, 4); + pipeline.add(inputFetchStage, 3); + pipeline.add(executeActionStage, 2); + pipeline.add(reportResultStage, 1); + + serverBuilder.addService( + new WorkerProfileService( + storage, + matchStage, + inputFetchStage, + executeActionStage, + reportResultStage, + completeStage, + backplane)); + } GrpcMetrics.handleGrpcMetricIntercepts(serverBuilder, configs.getWorker().getGrpcMetrics()); serverBuilder.intercept(new ServerHeadersInterceptor(meta -> {})); if (configs.getServer().getMaxInboundMessageSizeBytes() != 0) { @@ -314,7 +340,7 @@ private ExecFileSystem createFuseExecFileSystem( } } - private void initializeExecFileSystem( + private ExecFileSystem createExecFileSystem( InputStreamFactory remoteInputStreamFactory, ExecutorService removeDirectoryService, ExecutorService accessRecorder, @@ -327,6 +353,7 @@ private void initializeExecFileSystem( checkState(storage != null, "no exec fs cas specified"); if (storage instanceof CASFileCache cfc) { FileSystem fileSystem = cfc.getRoot().getFileSystem(); + PoolResource execOwnerIndexResource = null; ImmutableMap owners = ImmutableMap.of(); // there's some sense that ownerNames might be specifiable as 1, but not 2, and 3 being the // minimum passing the config validation @@ -334,10 +361,7 @@ private void initializeExecFileSystem( if (!Strings.isNullOrEmpty(ownerName)) { owners = ImmutableMap.of(ownerName, getOwner(ownerName, fileSystem)); execOwnerIndexResource = - new PersistentWorkerAwareExecOwnerPool( - PersistentExecutor.workerIndex, - Collections.singleton(ownerName), - REPORT_RESULT_STAGE); + new PoolResource(new Dispenser<>(ownerName), REPORT_RESULT_STAGE); } } else { ImmutableMap.Builder builder = ImmutableMap.builder(); @@ -350,20 +374,18 @@ private void initializeExecFileSystem( } owners = builder.build(); execOwnerIndexResource = - new PersistentWorkerAwareExecOwnerPool( - PersistentExecutor.workerIndex, owners.keySet(), REPORT_RESULT_STAGE); + new PoolResource(new ArrayDeque<>(owners.keySet()), REPORT_RESULT_STAGE); } if (execOwnerIndexResource != null) { - resourceSet.resources.put( - LocalResourceSet.EXEC_OWNER_RESOURCE_NAME, execOwnerIndexResource); + resourceSet.poolResources.put( + ShardWorkerContext.EXEC_OWNER_RESOURCE_NAME, execOwnerIndexResource); } - execFileSystem = - createCFCExecFileSystem( - removeDirectoryService, accessRecorder, fetchService, cfc, owners); + return createCFCExecFileSystem( + removeDirectoryService, accessRecorder, fetchService, cfc, owners); } else { // FIXME not the only fuse backing capacity... - execFileSystem = createFuseExecFileSystem(remoteInputStreamFactory, storage); + return createFuseExecFileSystem(remoteInputStreamFactory, storage); } } @@ -596,7 +618,7 @@ boolean isWorkerPausedFromNewWork() { if (pausedFile.exists() && !isPaused) { isPaused = true; log.log(Level.INFO, "The current worker is paused from taking on new work!"); - context.prepareForGracefulShutdown(); + pipeline.stopMatchingOperations(); workerPausedMetric.inc(); } } catch (Exception e) { @@ -608,7 +630,7 @@ boolean isWorkerPausedFromNewWork() { void registerIfExpired() { long now = System.currentTimeMillis(); if (now >= workerRegistrationExpiresAt - && !context.inGracefulShutdown() + && !inGracefulShutdown && !isWorkerPausedFromNewWork()) { // worker must be registered to match addWorker(nextRegistration(now)); @@ -714,15 +736,16 @@ public double get() { zstdBufferPool, configs.getWorker().getStorages()); // may modify resourceSet to provide additional resources - initializeExecFileSystem( - remoteInputStreamFactory, - removeDirectoryService, - accessRecorder, - fetchService, - storage, - resourceSet, - configs.getWorker().getExecOwner(), - execOwners); + execFileSystem = + createExecFileSystem( + remoteInputStreamFactory, + removeDirectoryService, + accessRecorder, + fetchService, + storage, + resourceSet, + configs.getWorker().getExecOwner(), + execOwners); instance = new WorkerInstance(configs.getWorker().getPublicName(), backplane, storage); @@ -735,7 +758,7 @@ public double get() { writer = new LocalCasWriter(execFileSystem); } - context = + ShardWorkerContext context = new ShardWorkerContext( configs.getWorker().getPublicName(), Duration.newBuilder().setSeconds(configs.getWorker().getOperationPollPeriod()).build(), @@ -763,72 +786,18 @@ public double get() { writer); pipeline = new Pipeline(); - SuperscalarPipelineStage inputFetchStage = null; - SuperscalarPipelineStage executeActionStage = null; - SuperscalarPipelineStage reportResultStage = null; - PutOperationStage completeStage = null; - if (configs.getWorker().getCapabilities().isExecution()) { - // all claims should be released upon pipeline completion, but it is safe to ensure that - // they are whether empty or not, so do so. - completeStage = new ReleaseClaimStage(operation -> context.deactivate(operation.getName())); - PipelineStage errorStage = completeStage; /* new ErrorStage(); */ - reportResultStage = new ReportResultStage(context, completeStage, errorStage); - executeActionStage = - new ExecuteActionStage( - context, reportResultStage, errorStage, execFileSystem, execOwnerIndexResource); - // FIXME this implies and requires that context::requeue performs the context::deactivate - // function - PipelineStage releaseClaimAndRequeueStage = new ReleaseClaimStage(context::requeue); - inputFetchStage = - new InputFetchStage(context, executeActionStage, releaseClaimAndRequeueStage); - matchStage = new MatchStage(context, inputFetchStage, releaseClaimAndRequeueStage); - - pipeline.add(matchStage, 4); - pipeline.add(inputFetchStage, 3); - pipeline.add(executeActionStage, 2); - pipeline.add(reportResultStage, 1); - } - - String workerName = InetAddress.getLocalHost().getHostName(); - - WorkerProfileService workerProfileService = - new WorkerProfileService( - workerName, - configs.getWorker().getPublicName(), - (CASFileCache) storage, - matchStage, - inputFetchStage, - executeActionStage, - reportResultStage, - completeStage); - server = createServer(serverBuilder, instance, workerProfileService); + server = createServer(serverBuilder, (CASFileCache) storage, instance, pipeline, context); removeWorker(configs.getWorker().getPublicName()); boolean skipLoad = configs.getWorker().getStorages().getFirst().isSkipLoad(); - ListenableFuture fileSystemStarted = - execFileSystem.start( - (digests) -> addBlobsLocation(digests, configs.getWorker().getPublicName()), - skipLoad, - startWritable); + execFileSystem.start( + (digests) -> addBlobsLocation(digests, configs.getWorker().getPublicName()), skipLoad); server.start(); - Futures.addCallback( - fileSystemStarted, - new FutureCallback<>() { - @Override - public void onSuccess(Void result) { - healthStatusManager.setStatus( - HealthStatusManager.SERVICE_NAME_ALL_SERVICES, ServingStatus.SERVING); - PrometheusPublisher.startHttpServer(configs.getPrometheusPort()); - } - - @Override - public void onFailure(Throwable t) { - log.log(SEVERE, "execFileSystem start failure", t); - } - }, - directExecutor()); + healthStatusManager.setStatus( + HealthStatusManager.SERVICE_NAME_ALL_SERVICES, ServingStatus.SERVING); + PrometheusPublisher.startHttpServer(configs.getPrometheusPort()); startFailsafeRegistration(); pipeline.start(); @@ -848,58 +817,16 @@ protected void onShutdown() throws InterruptedException { private void awaitTermination() throws InterruptedException { pipeline.join(); - if (server != null && !server.isTerminated()) { - int retries = 5; - while (retries > 0) { - if (shutdownInitiated.get()) { - server.shutdown(); - retries--; - } - if (server.awaitTermination(shutdownWaitTimeInSeconds, SECONDS)) { - retries = 1; - break; - } - } - if (retries == 0) { - server.shutdownNow(); - server.awaitTermination(); - } - } - } - - public Iterable pipelineChange(Iterable changes) { - for (PipelineChange change : changes) { - for (PipelineStage stage : pipeline) { - if (change.getStage().equals(stage.getName())) { - stage.setPaused(change.getPaused()); - if (change.getWidth() > 0) { - stage.setWidth(change.getWidth()); - } - } - } - } - - return Iterables.transform( - pipeline, - stage -> - PipelineChange.newBuilder() - .setStage(stage.getName()) - .setPaused(stage.isPaused()) - .setWidth(stage.getWidth()) - .build()); + server.awaitTermination(); } public void initiateShutdown() { - if (context != null) { - context.prepareForGracefulShutdown(); - } if (pipeline != null) { - pipeline.interrupt(matchStage); + pipeline.stopMatchingOperations(); } if (server != null) { server.shutdown(); } - shutdownInitiated.set(true); } private synchronized void awaitRelease() throws InterruptedException { @@ -940,11 +867,7 @@ private void shutdown() throws InterruptedException { inputFetchSlotsTotal.set(0); if (execFileSystem != null) { log.info("Stopping exec filesystem"); - try { - execFileSystem.stop(); - } catch (IOException e) { - log.log(SEVERE, "error shutting down exec filesystem", e); - } + execFileSystem.stop(); execFileSystem = null; } if (server != null) { diff --git a/src/main/java/build/buildfarm/worker/shard/WorkerInstance.java b/src/main/java/build/buildfarm/worker/shard/WorkerInstance.java index 4446c4fa8b..156e25e097 100644 --- a/src/main/java/build/buildfarm/worker/shard/WorkerInstance.java +++ b/src/main/java/build/buildfarm/worker/shard/WorkerInstance.java @@ -90,11 +90,6 @@ public String readResourceName(Compressor.Value compressor, Digest blobDigest) { throw new UnsupportedOperationException(); } - @Override - public boolean isReadOnly() { - return contentAddressableStorage.isReadOnly(); - } - @Override public void getBlob( Compressor.Value compressor, diff --git a/src/main/java/build/buildfarm/worker/shard/WorkerProfileService.java b/src/main/java/build/buildfarm/worker/shard/WorkerProfileService.java index f9565680ec..b93ed92a66 100644 --- a/src/main/java/build/buildfarm/worker/shard/WorkerProfileService.java +++ b/src/main/java/build/buildfarm/worker/shard/WorkerProfileService.java @@ -14,8 +14,11 @@ package build.buildfarm.worker.shard; +import build.buildfarm.backplane.Backplane; import build.buildfarm.cas.cfc.CASFileCache; import build.buildfarm.v1test.StageInformation; +import build.buildfarm.v1test.WorkerListMessage; +import build.buildfarm.v1test.WorkerListRequest; import build.buildfarm.v1test.WorkerProfileGrpc; import build.buildfarm.v1test.WorkerProfileMessage; import build.buildfarm.v1test.WorkerProfileRequest; @@ -25,35 +28,34 @@ import build.buildfarm.worker.SuperscalarPipelineStage; import com.google.common.base.Strings; import io.grpc.stub.StreamObserver; +import java.io.IOException; import javax.annotation.Nullable; public class WorkerProfileService extends WorkerProfileGrpc.WorkerProfileImplBase { - private final String name; - private final String endpoint; private final @Nullable CASFileCache storage; - private final @Nullable PipelineStage matchStage; - private final @Nullable SuperscalarPipelineStage inputFetchStage; - private final @Nullable SuperscalarPipelineStage executeActionStage; - private final @Nullable SuperscalarPipelineStage reportResultStage; - private final @Nullable PutOperationStage completeStage; + private final PipelineStage matchStage; + private final SuperscalarPipelineStage inputFetchStage; + private final SuperscalarPipelineStage executeActionStage; + private final SuperscalarPipelineStage reportResultStage; + private final PutOperationStage completeStage; + private final Backplane backplane; public WorkerProfileService( - String name, - String endpoint, @Nullable CASFileCache storage, - @Nullable PipelineStage matchStage, - @Nullable SuperscalarPipelineStage inputFetchStage, - @Nullable SuperscalarPipelineStage executeActionStage, - @Nullable SuperscalarPipelineStage reportResultStage, - @Nullable PutOperationStage completeStage) { - this.name = name; - this.endpoint = endpoint; + PipelineStage matchStage, + SuperscalarPipelineStage inputFetchStage, + SuperscalarPipelineStage executeActionStage, + SuperscalarPipelineStage reportResultStage, + PutOperationStage completeStage, + Backplane backplane) { + super(); this.storage = storage; this.matchStage = matchStage; this.inputFetchStage = inputFetchStage; this.executeActionStage = executeActionStage; this.reportResultStage = reportResultStage; this.completeStage = (PutOperationStage) completeStage; + this.backplane = backplane; } private StageInformation unaryStageInformation(String name, @Nullable String operationName) { @@ -78,8 +80,7 @@ private StageInformation superscalarStageInformation(SuperscalarPipelineStage st public void getWorkerProfile( WorkerProfileRequest request, StreamObserver responseObserver) { // get usage of CASFileCache - WorkerProfileMessage.Builder replyBuilder = - WorkerProfileMessage.newBuilder().setName(name).setEndpoint(endpoint); + WorkerProfileMessage.Builder replyBuilder = WorkerProfileMessage.newBuilder(); // FIXME deliver full local storage chain if (storage != null) { @@ -100,34 +101,40 @@ public void getWorkerProfile( // produce: slots that are not consistent with operations, operations // in multiple stages even in reverse due to claim progress // in short: this is for monitoring, not for guaranteed consistency checks - if (matchStage != null - && inputFetchStage != null - && executeActionStage != null - && reportResultStage != null) { - String matchOperation = matchStage.getOperationName(); + String matchOperation = matchStage.getOperationName(); + replyBuilder + .addStages(superscalarStageInformation(reportResultStage)) + .addStages(superscalarStageInformation(executeActionStage)) + .addStages(superscalarStageInformation(inputFetchStage)) + .addStages(unaryStageInformation(matchStage.getName(), matchOperation)); + + // get average time costs on each stage + OperationStageDurations[] durations = completeStage.getAverageTimeCostPerStage(); + for (OperationStageDurations duration : durations) { replyBuilder - .addStages(superscalarStageInformation(reportResultStage)) - .addStages(superscalarStageInformation(executeActionStage)) - .addStages(superscalarStageInformation(inputFetchStage)) - .addStages(unaryStageInformation(matchStage.getName(), matchOperation)); + .addTimesBuilder() + .setQueuedToMatch(duration.queuedToMatch) + .setMatchToInputFetchStart(duration.matchToInputFetchStart) + .setInputFetchStartToComplete(duration.inputFetchStartToComplete) + .setInputFetchCompleteToExecutionStart(duration.inputFetchCompleteToExecutionStart) + .setExecutionStartToComplete(duration.executionStartToComplete) + .setExecutionCompleteToOutputUploadStart(duration.executionCompleteToOutputUploadStart) + .setOutputUploadStartToComplete(duration.outputUploadStartToComplete) + .setOperationCount(duration.operationCount) + .setPeriod(duration.period); } + responseObserver.onNext(replyBuilder.build()); + responseObserver.onCompleted(); + } - // get average time costs on each stage - if (completeStage != null) { - OperationStageDurations[] durations = completeStage.getAverageTimeCostPerStage(); - for (OperationStageDurations duration : durations) { - replyBuilder - .addTimesBuilder() - .setQueuedToMatch(duration.queuedToMatch) - .setMatchToInputFetchStart(duration.matchToInputFetchStart) - .setInputFetchStartToComplete(duration.inputFetchStartToComplete) - .setInputFetchCompleteToExecutionStart(duration.inputFetchCompleteToExecutionStart) - .setExecutionStartToComplete(duration.executionStartToComplete) - .setExecutionCompleteToOutputUploadStart(duration.executionCompleteToOutputUploadStart) - .setOutputUploadStartToComplete(duration.outputUploadStartToComplete) - .setOperationCount(duration.operationCount) - .setPeriod(duration.period); - } + @Override + public void getWorkerList( + WorkerListRequest request, StreamObserver responseObserver) { + WorkerListMessage.Builder replyBuilder = WorkerListMessage.newBuilder(); + try { + replyBuilder.addAllWorkers(backplane.getStorageWorkers()); + } catch (IOException e) { + responseObserver.onError(e); } responseObserver.onNext(replyBuilder.build()); responseObserver.onCompleted(); diff --git a/src/main/java/build/buildfarm/worker/util/BUILD b/src/main/java/build/buildfarm/worker/util/BUILD index 957d5cc37f..d2713a3d10 100644 --- a/src/main/java/build/buildfarm/worker/util/BUILD +++ b/src/main/java/build/buildfarm/worker/util/BUILD @@ -7,17 +7,20 @@ java_library( deps = [ "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/instance", + "//src/main/java/build/buildfarm/instance/stub", + "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", - "@buildfarm_maven//:com_google_code_gson_gson", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", + "@maven//:com_google_code_gson_gson", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", ], ) diff --git a/src/main/protobuf/build/buildfarm/v1test/BUILD b/src/main/protobuf/build/buildfarm/v1test/BUILD index eb720e1ce0..e440b65b92 100644 --- a/src/main/protobuf/build/buildfarm/v1test/BUILD +++ b/src/main/protobuf/build/buildfarm/v1test/BUILD @@ -1,6 +1,6 @@ -load("@com_google_protobuf//bazel:java_proto_library.bzl", "java_proto_library") -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@grpc-java//:java_grpc_library.bzl", "java_grpc_library") +load("@protobuf//bazel:java_proto_library.bzl", "java_proto_library") +load("@protobuf//bazel:proto_library.bzl", "proto_library") package(default_visibility = ["//visibility:public"]) @@ -8,12 +8,12 @@ proto_library( name = "buildfarm_proto", srcs = ["buildfarm.proto"], deps = [ - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:timestamp_proto", "@googleapis//google/api:annotations_proto", "@googleapis//google/api:client_proto", "@googleapis//google/longrunning:operations_proto", "@googleapis//google/rpc:status_proto", + "@protobuf//:duration_proto", + "@protobuf//:timestamp_proto", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_proto", ], ) diff --git a/src/main/protobuf/build/buildfarm/v1test/buildfarm.proto b/src/main/protobuf/build/buildfarm/v1test/buildfarm.proto index 623f7a9226..5649b263c2 100644 --- a/src/main/protobuf/build/buildfarm/v1test/buildfarm.proto +++ b/src/main/protobuf/build/buildfarm/v1test/buildfarm.proto @@ -593,25 +593,7 @@ message StageInformation { repeated string operation_names = 4; } -message BatchWorkerProfilesResponse { - message Response { - string worker_name = 1; - - WorkerProfileMessage profile = 2; - - google.rpc.Status status = 3; - } - - repeated Response responses = 1; -} - message WorkerProfileMessage { - // useful name for worker - string name = 15; - - // preferred way to contact this worker - string endpoint = 16; - // number of Entry in CAS int64 cas_entry_count = 1; @@ -644,22 +626,18 @@ message WorkerProfileMessage { repeated OperationTimesBetweenStages times = 10; } -message WorkerProfileRequest { - string instance_name = 1; +message WorkerProfileRequest {} - string worker_name = 2; -} - -message BatchWorkerProfilesRequest { - string instance_name = 1; +message WorkerListRequest {} - repeated string worker_names = 2; +message WorkerListMessage { + repeated string workers = 1; } service WorkerProfile { rpc GetWorkerProfile(WorkerProfileRequest) returns (WorkerProfileMessage) {} - rpc BatchWorkerProfiles(BatchWorkerProfilesRequest) returns (BatchWorkerProfilesResponse) {} + rpc GetWorkerList(WorkerListRequest) returns (WorkerListMessage) {} } message PrepareWorkerForGracefulShutDownRequest { @@ -670,42 +648,9 @@ message PrepareWorkerForGracefulShutDownRequest { message PrepareWorkerForGracefulShutDownRequestResults {} -service WorkerControl { +service ShutDownWorker { rpc PrepareWorkerForGracefulShutdown(PrepareWorkerForGracefulShutDownRequest) - returns (PrepareWorkerForGracefulShutDownRequestResults) { - option (google.api.http) = { - post: "/v1test/{instance_name=**}/pipeline:change" - body: "*" - }; - } - - rpc PipelineChange(WorkerPipelineChangeRequest) - returns (WorkerPipelineChangeResponse) { - option (google.api.http) = { - post: "/v1test/{instance_name=**}/pipeline:change" - body: "*" - }; - } -} - -message PipelineChange { - string stage = 1; - - bool paused = 2; - - int32 width = 3; -} - -message WorkerPipelineChangeRequest { - string instance_name = 1; - - string worker_name = 2; - - repeated PipelineChange changes = 3; -} - -message WorkerPipelineChangeResponse { - repeated PipelineChange changes = 1; + returns (PrepareWorkerForGracefulShutDownRequestResults) {} } message FetchQualifiers { @@ -723,9 +668,3 @@ message Digest { build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 3; } - -message WorkerExecutedMetadata { - int64 fetched_bytes = 1; - - repeated string linked_input_directories = 2; -} diff --git a/src/test/java/build/buildfarm/BUILD b/src/test/java/build/buildfarm/BUILD index dea77b8670..720389ee29 100644 --- a/src/test/java/build/buildfarm/BUILD +++ b/src/test/java/build/buildfarm/BUILD @@ -17,7 +17,7 @@ java_library( ], visibility = ["//src/test/java/build/buildfarm:__subpackages__"], deps = [ - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:junit_junit", + "@maven//:com_google_guava_guava", + "@maven//:junit_junit", ], ) diff --git a/src/test/java/build/buildfarm/cas/BUILD b/src/test/java/build/buildfarm/cas/BUILD index 766c3e570a..4d25277bf4 100644 --- a/src/test/java/build/buildfarm/cas/BUILD +++ b/src/test/java/build/buildfarm/cas/BUILD @@ -15,19 +15,19 @@ java_test( "//src/test/java/build/buildfarm:test_runner", "//src/test/java/build/buildfarm/common/grpc", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_inprocess", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_util", - "@buildfarm_maven//:org_mockito_mockito_core", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_inprocess", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_util", + "@maven//:org_mockito_mockito_core", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/cas/cfc/CASFileCacheTest.java b/src/test/java/build/buildfarm/cas/cfc/CASFileCacheTest.java index 71dce1c566..3be0b30a65 100644 --- a/src/test/java/build/buildfarm/cas/cfc/CASFileCacheTest.java +++ b/src/test/java/build/buildfarm/cas/cfc/CASFileCacheTest.java @@ -47,6 +47,7 @@ import build.buildfarm.cas.DigestMismatchException; import build.buildfarm.cas.cfc.CASFileCache.CancellableOutputStream; import build.buildfarm.cas.cfc.CASFileCache.Entry; +import build.buildfarm.cas.cfc.CASFileCache.StartupCacheResults; import build.buildfarm.common.DigestUtil; import build.buildfarm.common.DigestUtil.HashFunction; import build.buildfarm.common.InputStreamFactory; @@ -196,7 +197,7 @@ public void putCreatesFile() throws IOException, InterruptedException { ByteString blob = ByteString.copyFromUtf8("Hello, World"); Digest blobDigest = DIGEST_UTIL.compute(blob); blobs.put(blobDigest, blob); - Path path = fileCache.put(blobDigest, false).path(); + Path path = fileCache.put(blobDigest, false); assertThat(Files.exists(path)).isTrue(); } @@ -220,7 +221,7 @@ public void putCreatesExecutable() throws IOException, InterruptedException { ByteString blob = ByteString.copyFromUtf8("executable"); Digest blobDigest = DIGEST_UTIL.compute(blob); blobs.put(blobDigest, blob); - Path path = fileCache.put(blobDigest, true).path(); + Path path = fileCache.put(blobDigest, true); assertThat(Files.isExecutable(path)).isTrue(); } @@ -300,7 +301,7 @@ public void expireUnreferencedEntryRemovesBlobFile() throws IOException, Interru ByteString bigBlob = ByteString.copyFrom(bigData); Digest bigDigest = DIGEST_UTIL.compute(bigBlob); blobs.put(bigDigest, bigBlob); - Path bigPath = fileCache.put(bigDigest, false).path(); + Path bigPath = fileCache.put(bigDigest, false); decrementReference(bigPath); @@ -308,23 +309,28 @@ public void expireUnreferencedEntryRemovesBlobFile() throws IOException, Interru ByteString strawBlob = ByteString.copyFrom(strawData); Digest strawDigest = DIGEST_UTIL.compute(strawBlob); blobs.put(strawDigest, strawBlob); - Path strawPath = fileCache.put(strawDigest, false).path(); + Path strawPath = fileCache.put(strawDigest, false); assertThat(Files.exists(bigPath)).isFalse(); assertThat(Files.exists(strawPath)).isTrue(); } @Test - public void startEmptyCas() throws Exception { + public void startEmptyCas() throws IOException, InterruptedException { // start the file cache with no files. // the cache should start without any initial files in the cache. - fileCache.start(false).get(); - - assertThat(storage).isEmpty(); + StartupCacheResults results = fileCache.start(false); + + // check the startuo results to ensure no files were processed + assertThat(results.load().loadSkipped()).isFalse(); + assertThat(results.load().scan().computeDirs().size()).isEqualTo(0); + assertThat(results.load().scan().deleteFiles().size()).isEqualTo(0); + assertThat(results.load().scan().fileKeys().size()).isEqualTo(0); + assertThat(results.load().invalidDirectories().size()).isEqualTo(0); } @Test - public void startCasAssumeDirectory() throws Exception { + public void startCasAssumeDirectory() throws IOException, InterruptedException { // create a "_dir" file on the root Path path = root.resolve("foobar_dir"); ByteString blob = ByteString.copyFromUtf8("content"); @@ -332,15 +338,18 @@ public void startCasAssumeDirectory() throws Exception { // start the CAS with a file whose name indicates its a directory // the cache should start and consider it a compute directory - fileCache.start(false).get(); - - // check the current state to ensure no files were processed - assertThat(Files.exists(path)).isFalse(); - assertThat(storage.isEmpty()); + StartupCacheResults results = fileCache.start(false); + + // check the startup results to ensure no files were processed + assertThat(results.load().loadSkipped()).isFalse(); + assertThat(results.load().scan().computeDirs().size()).isEqualTo(0); + assertThat(results.load().scan().deleteFiles().size()).isEqualTo(1); + assertThat(results.load().scan().fileKeys().size()).isEqualTo(0); + assertThat(results.load().invalidDirectories().size()).isEqualTo(0); } @Test - public void startLoadsExistingBlob() throws Exception { + public void startLoadsExistingBlob() throws IOException, InterruptedException { FileStore fileStore = Files.getFileStore(root); ByteString blob = ByteString.copyFromUtf8("blob"); Digest blobDigest = DIGEST_UTIL.compute(blob); @@ -351,19 +360,24 @@ public void startLoadsExistingBlob() throws Exception { Files.write(execPath, blob.toByteArray()); EvenMoreFiles.setReadOnlyPerms(execPath, true, fileStore); - fileCache.start(false).get(); + StartupCacheResults results = fileCache.start(false); - assertThat(storage.size()).isEqualTo(2); + // check the startup results to ensure our two files were processed + assertThat(results.load().loadSkipped()).isFalse(); + assertThat(results.load().scan().computeDirs().size()).isEqualTo(0); + assertThat(results.load().scan().deleteFiles().size()).isEqualTo(0); + assertThat(results.load().scan().fileKeys().size()).isEqualTo(2); + assertThat(results.load().invalidDirectories().size()).isEqualTo(0); // explicitly not providing blob via blobs, this would throw if fetched from factory // // FIXME https://github.com/google/truth/issues/285 assertThat(Path) is ambiguous - assertThat(fileCache.put(blobDigest, false).path().equals(path)).isTrue(); - assertThat(fileCache.put(blobDigest, true).path().equals(execPath)).isTrue(); + assertThat(fileCache.put(blobDigest, false).equals(path)).isTrue(); + assertThat(fileCache.put(blobDigest, true).equals(execPath)).isTrue(); } @Test - public void startSkipsLoadingExistingBlob() throws Exception { + public void startSkipsLoadingExistingBlob() throws IOException, InterruptedException { FileStore fileStore = Files.getFileStore(root); ByteString blob = ByteString.copyFromUtf8("blob"); Digest blobDigest = DIGEST_UTIL.compute(blob); @@ -374,16 +388,18 @@ public void startSkipsLoadingExistingBlob() throws Exception { Files.write(execPath, blob.toByteArray()); EvenMoreFiles.setReadOnlyPerms(execPath, true, fileStore); - fileCache.start(/* skipLoad= */ true).get(); + StartupCacheResults results = fileCache.start(/* skipLoad= */ true); - // check the current state to ensure our two files were processed - assertThat(storage).isEmpty(); - assertThat(Files.exists(path)); - assertThat(Files.exists(execPath)); + // check the startup results to ensure our two files were processed + assertThat(results.load().loadSkipped()).isTrue(); + assertThat(results.load().scan().computeDirs().size()).isEqualTo(0); + assertThat(results.load().scan().deleteFiles().size()).isEqualTo(0); + assertThat(results.load().scan().fileKeys().size()).isEqualTo(0); + assertThat(results.load().invalidDirectories().size()).isEqualTo(0); } @Test - public void startRemovesInvalidEntries() throws Exception { + public void startRemovesInvalidEntries() throws IOException, InterruptedException { Path tooFewComponents = root.resolve("00").resolve("toofewcomponents"); Path tooManyComponents = root.resolve("00").resolve("too_many_components_here"); Path invalidDigest = root.resolve("00").resolve("digest"); @@ -397,7 +413,7 @@ public void startRemovesInvalidEntries() throws Exception { Files.write( invalidExec, validBlob.toByteArray()); // content would match but for invalid exec field - fileCache.start(/* skipLoad= */ false).get(); + fileCache.start(/* skipLoad= */ false); assertThat(Files.exists(tooFewComponents)).isFalse(); assertThat(Files.exists(tooManyComponents)).isFalse(); @@ -433,7 +449,7 @@ public void expireEntryWaitsForUnreferencedEntry() ByteString bigContent = ByteString.copyFrom(bigData); Digest bigDigest = DIGEST_UTIL.compute(bigContent); blobs.put(bigDigest, bigContent); - Path bigPath = fileCache.put(bigDigest, /* isExecutable= */ false).path(); + Path bigPath = fileCache.put(bigDigest, /* isExecutable= */ false); AtomicBoolean started = new AtomicBoolean(false); ExecutorService service = newSingleThreadExecutor(); @@ -463,9 +479,7 @@ public void expireEntryWaitsForUnreferencedEntry() } @Test - public void containsRecordsAccess() throws Exception { - fileCache.start(false).get(); - + public void containsRecordsAccess() throws IOException, InterruptedException { ByteString contentOne = ByteString.copyFromUtf8("one"); Digest digestOne = DIGEST_UTIL.compute(contentOne); blobs.put(digestOne, contentOne); @@ -476,21 +490,19 @@ public void containsRecordsAccess() throws Exception { Digest digestThree = DIGEST_UTIL.compute(contentThree); blobs.put(digestThree, contentThree); - String pathOne = - fileCache.put(digestOne, /* isExecutable= */ false).path().getFileName().toString(); - String pathTwo = - fileCache.put(digestTwo, /* isExecutable= */ false).path().getFileName().toString(); + String pathOne = fileCache.put(digestOne, /* isExecutable= */ false).getFileName().toString(); + String pathTwo = fileCache.put(digestTwo, /* isExecutable= */ false).getFileName().toString(); String pathThree = - fileCache.put(digestThree, /* isExecutable= */ false).path().getFileName().toString(); + fileCache.put(digestThree, /* isExecutable= */ false).getFileName().toString(); fileCache.decrementReferences( ImmutableList.of(pathOne, pathTwo, pathThree), ImmutableList.of(), DIGEST_UTIL.getDigestFunction()); - /* sentinel <- three <- two <- one <- sentinel */ + /* three -> two -> one */ assertThat(storage.get(pathOne).after).isEqualTo(storage.get(pathTwo)); assertThat(storage.get(pathTwo).after).isEqualTo(storage.get(pathThree)); - /* sentinel <- one <- three <- two <- sentinel */ + /* one -> three -> two */ assertThat( fileCache.findMissingBlobs( ImmutableList.of(DigestUtil.toDigest(digestOne)), digestOne.getDigestFunction())) @@ -888,14 +900,12 @@ public void duplicateExpiredEntrySuppressesDigestExpiration() } blobs.put(expiringBlob.getDigest(), expiringBlob.getData()); decrementReference( - fileCache - .put(expiringBlob.getDigest(), /* isExecutable= */ false) - .path()); // expected eviction + fileCache.put(expiringBlob.getDigest(), /* isExecutable= */ false)); // expected eviction blobs.clear(); decrementReference( - fileCache - .put(expiringBlob.getDigest(), /* isExecutable= */ true) - .path()); // should be fed from storage directly, not through delegate + fileCache.put( + expiringBlob.getDigest(), + /* isExecutable= */ true)); // should be fed from storage directly, not through delegate fileCache.put(new Blob(ByteString.copyFromUtf8("Hello, World"), DIGEST_UTIL)); @@ -1195,7 +1205,7 @@ public int read(byte[] buf, int offset, int len) throws IOException { ByteString blob = ByteString.copyFromUtf8("Flaky Entry"); Digest blobDigest = DIGEST_UTIL.compute(blob); blobs.put(blobDigest, blob); - Path path = flakyExternalCAS.put(blobDigest, false).path(); + Path path = flakyExternalCAS.put(blobDigest, false); assertThat(Files.exists(path)).isTrue(); // would not have been created if not valid } diff --git a/src/test/java/build/buildfarm/common/BUILD b/src/test/java/build/buildfarm/common/BUILD index c291b776da..520871ed9f 100644 --- a/src/test/java/build/buildfarm/common/BUILD +++ b/src/test/java/build/buildfarm/common/BUILD @@ -11,19 +11,19 @@ java_test( "//src/main/java/build/buildfarm/common", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_luben_zstd_jni", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_luben_zstd_jni", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_testing", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", ], ) diff --git a/src/test/java/build/buildfarm/common/DigestUtilTest.java b/src/test/java/build/buildfarm/common/DigestUtilTest.java index 7b909f6c2d..2ad7a15ed8 100644 --- a/src/test/java/build/buildfarm/common/DigestUtilTest.java +++ b/src/test/java/build/buildfarm/common/DigestUtilTest.java @@ -107,7 +107,7 @@ public void computesSha512Hash() { @Test public void computeEmptyIsCachedEmpty() { DigestUtil digestUtil = new DigestUtil(HashFunction.BLAKE3); - Digest digest = digestUtil.compute(ByteString.EMPTY); + Digest digest = digestUtil.compute(ByteString.empty()); // reference comparison assertThat(digest == digestUtil.empty()).isTrue(); } diff --git a/src/test/java/build/buildfarm/common/config/BUILD b/src/test/java/build/buildfarm/common/config/BUILD index 4af5dd3915..c15a7c80af 100644 --- a/src/test/java/build/buildfarm/common/config/BUILD +++ b/src/test/java/build/buildfarm/common/config/BUILD @@ -7,10 +7,10 @@ java_test( deps = [ "//src/main/java/build/buildfarm/common/config", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:me_dinowernli_java_grpc_prometheus", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_testing", + "@maven//:me_dinowernli_java_grpc_prometheus", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/common/grpc/BUILD b/src/test/java/build/buildfarm/common/grpc/BUILD index ebb6c7f70f..cd19ebd8c9 100644 --- a/src/test/java/build/buildfarm/common/grpc/BUILD +++ b/src/test/java/build/buildfarm/common/grpc/BUILD @@ -7,13 +7,13 @@ java_library( visibility = ["//src/test/java/build/buildfarm:__subpackages__"], deps = [ "//src/main/java/build/buildfarm/common/grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_stub", ], ) @@ -24,17 +24,17 @@ java_test( deps = [ "//src/main/java/build/buildfarm/common/grpc", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_inprocess", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:io_grpc_grpc_util", - "@buildfarm_maven//:org_mockito_mockito_core", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_inprocess", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_testing", + "@maven//:io_grpc_grpc_util", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/common/io/BUILD b/src/test/java/build/buildfarm/common/io/BUILD index 243d063bca..8905be91e3 100644 --- a/src/test/java/build/buildfarm/common/io/BUILD +++ b/src/test/java/build/buildfarm/common/io/BUILD @@ -7,18 +7,20 @@ java_test( deps = [ "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/base", + "//src/main/java/build/buildfarm/common/grpc", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:commons_io_commons_io", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:org_apache_commons_commons_compress", - "@buildfarm_maven//:org_mockito_mockito_core", + "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:commons_io_commons_io", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_testing", + "@maven//:org_apache_commons_commons_compress", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/common/redis/BUILD b/src/test/java/build/buildfarm/common/redis/BUILD index 7eea917eb6..62c9d81355 100644 --- a/src/test/java/build/buildfarm/common/redis/BUILD +++ b/src/test/java/build/buildfarm/common/redis/BUILD @@ -7,12 +7,12 @@ COMMON_DEPS = [ "//src/main/java/build/buildfarm/common/redis", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_fppt_jedis_mock", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", + "@maven//:com_github_fppt_jedis_mock", + "@maven//:com_google_guava_guava", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", ] NATIVE_REDIS_TESTS = [ diff --git a/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueMockTest.java b/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueMockTest.java index 3b306a695d..8e59d2b422 100644 --- a/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueMockTest.java +++ b/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueMockTest.java @@ -122,7 +122,7 @@ public void takeElementDequeuedOnNonBlock() throws Exception { BalancedQueueEntry entry = queue.take(redis, service); // ASSERT - assertThat(entry.value()).isEqualTo("foo"); + assertThat(entry.getValue()).isEqualTo("foo"); service.shutdown(); assertThat(service.awaitTermination(1, SECONDS)).isTrue(); } @@ -145,7 +145,7 @@ public void dequeueElementDequeuedOnBlock() throws Exception { BalancedQueueEntry entry = queue.take(redis, service); // ASSERT - assertThat(entry.value()).isEqualTo("foo"); + assertThat(entry.getValue()).isEqualTo("foo"); service.shutdown(); assertThat(service.awaitTermination(1, SECONDS)).isTrue(); } @@ -266,7 +266,7 @@ public void visitCheckVisitOfEachElement() throws Exception { Visitor visitor = new Visitor<>() { public void visit(BalancedQueueEntry entry) { - visited.add(entry.value()); + visited.add(entry.getValue()); } }; queue.visit(redis, visitor); @@ -310,7 +310,7 @@ public void visitDequeueCheckVisitOfEachElement() throws Exception { Visitor visitor = new Visitor<>() { public void visit(BalancedQueueEntry entry) { - visited.add(entry.value()); + visited.add(entry.getValue()); } }; queue.visitDequeue(redis, visitor); diff --git a/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueTest.java b/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueTest.java index a8551ee636..8e9f6bcf57 100644 --- a/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueTest.java +++ b/src/test/java/build/buildfarm/common/redis/BalancedRedisQueueTest.java @@ -461,7 +461,7 @@ public void visitCheckVisitOfEachElement() throws Exception { Visitor visitor = new Visitor<>() { public void visit(BalancedQueueEntry entry) { - visited.add(entry.value()); + visited.add(entry.getValue()); } }; queue.visit(jedis, visitor); @@ -501,7 +501,7 @@ public void visitCheckVisitOfEachElementPriority() throws Exception { Visitor visitor = new Visitor<>() { public void visit(BalancedQueueEntry entry) { - visited.add(entry.value()); + visited.add(entry.getValue()); } }; queue.visit(jedis, visitor); diff --git a/src/test/java/build/buildfarm/common/redis/RedisMapMockTest.java b/src/test/java/build/buildfarm/common/redis/RedisMapMockTest.java index dfc12a2efe..3fef137f61 100644 --- a/src/test/java/build/buildfarm/common/redis/RedisMapMockTest.java +++ b/src/test/java/build/buildfarm/common/redis/RedisMapMockTest.java @@ -24,7 +24,6 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.params.GetExParams; /** * @class RedisMapMockTest @@ -96,24 +95,4 @@ public void getGet() throws Exception { verify(redis, times(1)).get("test:key"); assertThat(value).isEqualTo("value"); } - - // Function under test: getex - // Reason for testing: test how an element is looked up in a map - // Failure explanation: jedis was not called as expected - @Test - public void getGetEx() throws Exception { - // ARRANGE - JedisCluster redis = mock(JedisCluster.class); - GetExParams params = GetExParams.getExParams().ex(60); - when(redis.getEx("test:key", params)).thenReturn("value"); - RedisMap map = new RedisMap("test"); - - // ACT - map.insert(redis, "key", "value", 60); - String value = map.getex(redis, "key", 60); - - // ASSERT - verify(redis, times(1)).getEx("test:key", params); - assertThat(value).isEqualTo("value"); - } } diff --git a/src/test/java/build/buildfarm/common/resources/BUILD b/src/test/java/build/buildfarm/common/resources/BUILD index f94146bdf8..48da4449b4 100644 --- a/src/test/java/build/buildfarm/common/resources/BUILD +++ b/src/test/java/build/buildfarm/common/resources/BUILD @@ -11,19 +11,19 @@ java_test( "//src/main/java/build/buildfarm/common/resources:resource_java_proto", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_testing", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/common/s3/BUILD b/src/test/java/build/buildfarm/common/s3/BUILD index 9f1c28065e..72f2721173 100644 --- a/src/test/java/build/buildfarm/common/s3/BUILD +++ b/src/test/java/build/buildfarm/common/s3/BUILD @@ -8,9 +8,9 @@ java_test( deps = [ "//src/main/java/build/buildfarm/common/s3:S3Bucket", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_amazonaws_aws_java_sdk_s3", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_amazonaws_aws_java_sdk_s3", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/common/services/BUILD b/src/test/java/build/buildfarm/common/services/BUILD index 3cb742c9ce..30c74956bd 100644 --- a/src/test/java/build/buildfarm/common/services/BUILD +++ b/src/test/java/build/buildfarm/common/services/BUILD @@ -17,17 +17,17 @@ java_test( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_inprocess", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_inprocess", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/examples/BUILD b/src/test/java/build/buildfarm/examples/BUILD index dc259e2474..8c595628b7 100644 --- a/src/test/java/build/buildfarm/examples/BUILD +++ b/src/test/java/build/buildfarm/examples/BUILD @@ -11,18 +11,18 @@ java_test( "//src/main/java/build/buildfarm/common/config", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_testing", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", ], ) diff --git a/src/test/java/build/buildfarm/instance/BUILD b/src/test/java/build/buildfarm/instance/BUILD index 36c1895b5c..f8df9fa4e1 100644 --- a/src/test/java/build/buildfarm/instance/BUILD +++ b/src/test/java/build/buildfarm/instance/BUILD @@ -10,12 +10,12 @@ java_test( "//src/main/java/build/buildfarm/instance", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", "@googleapis//google/longrunning:longrunning_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/instance/DummyInstanceBase.java b/src/test/java/build/buildfarm/instance/DummyInstanceBase.java index f62d8fce4f..d4b2fe0e75 100644 --- a/src/test/java/build/buildfarm/instance/DummyInstanceBase.java +++ b/src/test/java/build/buildfarm/instance/DummyInstanceBase.java @@ -16,11 +16,11 @@ import build.buildfarm.common.Watcher; import build.buildfarm.common.Write; import build.buildfarm.v1test.BackplaneStatus; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.GetClientStartTimeRequest; import build.buildfarm.v1test.GetClientStartTimeResult; import build.buildfarm.v1test.PrepareWorkerForGracefulShutDownRequestResults; import build.buildfarm.v1test.Tree; +import build.buildfarm.v1test.WorkerListMessage; import build.buildfarm.v1test.WorkerProfileMessage; import com.google.common.util.concurrent.ListenableFuture; import com.google.longrunning.Operation; @@ -114,11 +114,6 @@ public String getTree( throw new UnsupportedOperationException(); } - @Override - public boolean isReadOnly() { - throw new UnsupportedOperationException(); - } - @Override public Write getBlobWrite( Compressor.Value compressor, @@ -214,12 +209,12 @@ public ServerCapabilities getCapabilities() { } @Override - public ListenableFuture getWorkerProfile(String name) { + public WorkerProfileMessage getWorkerProfile() { throw new UnsupportedOperationException(); } @Override - public ListenableFuture batchWorkerProfiles(Iterable names) { + public WorkerListMessage getWorkerList() { throw new UnsupportedOperationException(); } diff --git a/src/test/java/build/buildfarm/instance/server/BUILD b/src/test/java/build/buildfarm/instance/server/BUILD index 513be19e46..262999757b 100644 --- a/src/test/java/build/buildfarm/instance/server/BUILD +++ b/src/test/java/build/buildfarm/instance/server/BUILD @@ -14,15 +14,15 @@ java_test( "//src/main/java/build/buildfarm/instance/server", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/instance/server/NodeInstanceTest.java b/src/test/java/build/buildfarm/instance/server/NodeInstanceTest.java index cb8fe29354..759da525ce 100644 --- a/src/test/java/build/buildfarm/instance/server/NodeInstanceTest.java +++ b/src/test/java/build/buildfarm/instance/server/NodeInstanceTest.java @@ -61,11 +61,11 @@ import build.buildfarm.common.io.FeedbackOutputStream; import build.buildfarm.common.net.URL; import build.buildfarm.v1test.BackplaneStatus; -import build.buildfarm.v1test.BatchWorkerProfilesResponse; import build.buildfarm.v1test.Digest; import build.buildfarm.v1test.GetClientStartTimeRequest; import build.buildfarm.v1test.GetClientStartTimeResult; import build.buildfarm.v1test.PrepareWorkerForGracefulShutDownRequestResults; +import build.buildfarm.v1test.WorkerListMessage; import build.buildfarm.v1test.WorkerProfileMessage; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -205,13 +205,12 @@ public void deregisterWorker(String workerName) { } @Override - public ListenableFuture getWorkerProfile(String name) { + public WorkerProfileMessage getWorkerProfile() { throw new UnsupportedOperationException(); } @Override - public ListenableFuture batchWorkerProfiles( - Iterable names) { + public WorkerListMessage getWorkerList() { throw new UnsupportedOperationException(); } diff --git a/src/test/java/build/buildfarm/instance/shard/BUILD b/src/test/java/build/buildfarm/instance/shard/BUILD index fcb6519d81..ee314b9732 100644 --- a/src/test/java/build/buildfarm/instance/shard/BUILD +++ b/src/test/java/build/buildfarm/instance/shard/BUILD @@ -19,19 +19,19 @@ java_test( "//src/main/java/build/buildfarm/instance/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -57,19 +57,19 @@ java_test( "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -93,19 +93,19 @@ java_test( "//src/main/java/build/buildfarm/instance/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -118,11 +118,11 @@ java_test( "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/redis", "//src/main/java/build/buildfarm/instance/shard", - "@buildfarm_maven//:com_github_fppt_jedis_mock", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", + "@maven//:com_github_fppt_jedis_mock", + "@maven//:com_google_guava_guava", + "@maven//:com_google_truth_truth", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", ], ) @@ -146,19 +146,19 @@ java_test( "//src/main/java/build/buildfarm/instance/stub", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -182,19 +182,19 @@ java_test( "//src/main/java/build/buildfarm/instance/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -218,19 +218,19 @@ java_test( "//src/main/java/build/buildfarm/instance/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_ben_manes_caffeine_caffeine", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:redis_clients_jedis", "@googleapis//google/longrunning:longrunning_java_proto", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_ben_manes_caffeine_caffeine", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", + "@maven//:redis_clients_jedis", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -248,8 +248,8 @@ java_test( "//src/main/java/build/buildfarm/instance/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_fppt_jedis_mock", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:redis_clients_jedis", + "@maven//:com_github_fppt_jedis_mock", + "@maven//:com_google_truth_truth", + "@maven//:redis_clients_jedis", ], ) diff --git a/src/test/java/build/buildfarm/instance/shard/RedisShardBackplaneTest.java b/src/test/java/build/buildfarm/instance/shard/RedisShardBackplaneTest.java index 6d03d3a7ad..a5a5c35f07 100644 --- a/src/test/java/build/buildfarm/instance/shard/RedisShardBackplaneTest.java +++ b/src/test/java/build/buildfarm/instance/shard/RedisShardBackplaneTest.java @@ -25,12 +25,8 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; -import build.bazel.remote.execution.v2.Action; -import build.bazel.remote.execution.v2.ActionResult; -import build.bazel.remote.execution.v2.ExecuteOperationMetadata; import build.bazel.remote.execution.v2.RequestMetadata; import build.buildfarm.common.DigestUtil; -import build.buildfarm.common.DigestUtil.ActionKey; import build.buildfarm.common.DigestUtil.HashFunction; import build.buildfarm.common.config.BuildfarmConfigs; import build.buildfarm.common.config.Queue; @@ -48,13 +44,11 @@ import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.ShardWorker; import build.buildfarm.v1test.WorkerChange; -import build.buildfarm.v1test.WorkerExecutedMetadata; import build.buildfarm.worker.resources.LocalResourceSet; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.longrunning.Operation; -import com.google.protobuf.Any; import com.google.protobuf.ByteString; import com.google.protobuf.util.JsonFormat; import java.io.IOException; @@ -160,7 +154,7 @@ public void prequeueUpdatesOperationPrequeuesAndPublishes() throws IOException { eq(jedis), eq(DigestUtil.asActionKey(actionDigest).toString()), eq(opName), - eq(RedisShardBackplane.executionPrinter.print(op)))) + eq(RedisShardBackplane.operationPrinter.print(op)))) .thenReturn(true); assertThat(backplane.prequeue(executeEntry, op, /* ignoreMerge= */ false)).isTrue(); @@ -170,7 +164,7 @@ public void prequeueUpdatesOperationPrequeuesAndPublishes() throws IOException { eq(jedis), eq(DigestUtil.asActionKey(actionDigest).toString()), eq(opName), - eq(RedisShardBackplane.executionPrinter.print(op))); + eq(RedisShardBackplane.operationPrinter.print(op))); verifyNoMoreInteractions(state.executions); OperationChange opChange = verifyChangePublished(backplane.executionChannel(opName), jedis); assertThat(opChange.hasReset()).isTrue(); @@ -508,34 +502,4 @@ public void testAddWorker() throws IOException { JsonFormat.printer().print(shardWorker)); verify(jedis, times(1)).publish(anyString(), anyString()); } - - @Test - public void putActionResultPurgesUnknownAuxiliaryMetadatas() throws Exception { - UnifiedJedis jedis = mock(UnifiedJedis.class); - RedisClient client = new RedisClient(jedis); - DistributedState state = new DistributedState(); - state.actionCache = mock(RedisMap.class); - RedisShardBackplane backplane = createBackplane("put-action-result-purges-test"); - backplane.start(client, state, "putActionResultPurges/test:0000", name -> {}); - DigestUtil digestUtil = new DigestUtil(HashFunction.SHA256); - ActionKey actionKey = digestUtil.computeActionKey(Action.getDefaultInstance()); - - ActionResult.Builder actionResult = ActionResult.newBuilder(); - // action results cannot currently parse ExecuteOperationMetadata, ensure this continues to be - // true. - actionResult - .getExecutionMetadataBuilder() - .addAuxiliaryMetadata(Any.pack(ExecuteOperationMetadata.getDefaultInstance())) - .addAuxiliaryMetadata(Any.pack(WorkerExecutedMetadata.getDefaultInstance())); - - backplane.putActionResult(actionKey, actionResult.build()); - - ArgumentCaptor resultCaptor = ArgumentCaptor.forClass(String.class); - verify(state.actionCache, times(1)) - .insert(eq(jedis), eq(actionKey.toString()), resultCaptor.capture(), any(Integer.class)); - verifyNoMoreInteractions(state.actionCache); - String json = resultCaptor.getValue(); - assertThat( - backplane.parseActionResult(json).getExecutionMetadata().getAuxiliaryMetadataCount() == 1); - } } diff --git a/src/test/java/build/buildfarm/instance/stub/BUILD b/src/test/java/build/buildfarm/instance/stub/BUILD index c60f48ffbc..ff5722484b 100644 --- a/src/test/java/build/buildfarm/instance/stub/BUILD +++ b/src/test/java/build/buildfarm/instance/stub/BUILD @@ -5,7 +5,7 @@ java_test( size = "small", srcs = glob(["*.java"]), data = [ - "//examples:example_properties", + "//config:base_properties", ], test_class = "build.buildfarm.AllTests", deps = [ @@ -16,17 +16,17 @@ java_test( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_inprocess", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_util", - "@buildfarm_maven//:org_mockito_mockito_core", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_inprocess", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_util", + "@maven//:org_mockito_mockito_core", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/metrics/BUILD b/src/test/java/build/buildfarm/metrics/BUILD index 5587831914..e7b819ca95 100644 --- a/src/test/java/build/buildfarm/metrics/BUILD +++ b/src/test/java/build/buildfarm/metrics/BUILD @@ -13,16 +13,16 @@ java_test( "//src/main/java/build/buildfarm/metrics/log", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:io_grpc_grpc_testing", - "@buildfarm_maven//:org_mockito_mockito_core", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_stub", + "@maven//:io_grpc_grpc_testing", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/proxy/http/BUILD b/src/test/java/build/buildfarm/proxy/http/BUILD index 8b0cd689eb..d604abcf78 100644 --- a/src/test/java/build/buildfarm/proxy/http/BUILD +++ b/src/test/java/build/buildfarm/proxy/http/BUILD @@ -10,16 +10,16 @@ java_test( "//src/main/java/build/buildfarm/proxy/http", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_inprocess", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_inprocess", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/server/BUILD b/src/test/java/build/buildfarm/server/BUILD index e178b81163..a1029ad34d 100644 --- a/src/test/java/build/buildfarm/server/BUILD +++ b/src/test/java/build/buildfarm/server/BUILD @@ -21,24 +21,24 @@ java_test( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", "//third_party/remote-apis:build_bazel_remote_execution_v2_remote_execution_java_grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_protobuf_protobuf_java_util", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_inprocess", - "@buildfarm_maven//:io_grpc_grpc_netty", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:me_dinowernli_java_grpc_prometheus", - "@buildfarm_maven//:org_mockito_mockito_core", - "@com_google_protobuf//:protobuf_java", "@googleapis//google/bytestream:bytestream_java_grpc", "@googleapis//google/bytestream:bytestream_java_proto", "@googleapis//google/longrunning:longrunning_java_grpc", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_protobuf_protobuf_java_util", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_inprocess", + "@maven//:io_grpc_grpc_netty", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:io_grpc_grpc_stub", + "@maven//:me_dinowernli_java_grpc_prometheus", + "@maven//:org_mockito_mockito_core", + "@protobuf//:protobuf_java", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/server/services/BUILD b/src/test/java/build/buildfarm/server/services/BUILD index 0adbbd2ed5..8b482cf885 100644 --- a/src/test/java/build/buildfarm/server/services/BUILD +++ b/src/test/java/build/buildfarm/server/services/BUILD @@ -14,11 +14,11 @@ java_test( "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", "//third_party/remote-apis:build_bazel_remote_asset_v1_remote_asset_java_grpc", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_stub", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_guava_guava", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_stub", + "@maven//:org_mockito_mockito_core", "@remoteapis//build/bazel/remote/asset/v1:remote_asset_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/worker/BUILD b/src/test/java/build/buildfarm/worker/BUILD index b9d8cf2ab5..16646cfe55 100644 --- a/src/test/java/build/buildfarm/worker/BUILD +++ b/src/test/java/build/buildfarm/worker/BUILD @@ -1,9 +1,20 @@ load("@rules_java//java:java_test.bzl", "java_test") +config_setting( + name = "fuse", + values = {"define": "fuse=true"}, +) + java_test( name = "tests", size = "small", - srcs = glob(["*.java"]), + srcs = glob( + ["*.java"], + exclude = ["FuseCASTest.java"], + ) + select({ + ":fuse": ["FuseCASTest.java"], + "//conditions:default": [], + }), data = ["//examples:example_configs"], plugins = ["//src/main/java/build/buildfarm/common:lombok"], test_class = "build.buildfarm.AllTests", @@ -13,24 +24,23 @@ java_test( "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/worker", - "//src/main/java/build/buildfarm/worker/filesystem", "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_serceman_jnr_fuse", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:org_mockito_mockito_core", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/worker/filesystem/CFCExecFileSystemTest.java b/src/test/java/build/buildfarm/worker/CFCExecFileSystemTest.java similarity index 97% rename from src/test/java/build/buildfarm/worker/filesystem/CFCExecFileSystemTest.java rename to src/test/java/build/buildfarm/worker/CFCExecFileSystemTest.java index 30b8f89400..c79788efaf 100644 --- a/src/test/java/build/buildfarm/worker/filesystem/CFCExecFileSystemTest.java +++ b/src/test/java/build/buildfarm/worker/CFCExecFileSystemTest.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.truth.Truth.assertThat; diff --git a/src/test/java/build/buildfarm/worker/DequeueMatchEvaluatorTest.java b/src/test/java/build/buildfarm/worker/DequeueMatchEvaluatorTest.java index 8c23a3751e..48b59ab93e 100644 --- a/src/test/java/build/buildfarm/worker/DequeueMatchEvaluatorTest.java +++ b/src/test/java/build/buildfarm/worker/DequeueMatchEvaluatorTest.java @@ -229,7 +229,7 @@ public void shouldKeepOperationClaimsResource() throws Exception { .build(); // PRE-ASSERT - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(1); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(1); // ACT Claim claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -237,7 +237,7 @@ public void shouldKeepOperationClaimsResource() throws Exception { // ASSERT // the worker accepts because the resource is available. assertThat(claim).isNotNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(0); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(0); // ACT claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -245,7 +245,7 @@ public void shouldKeepOperationClaimsResource() throws Exception { // ASSERT // the worker rejects because there are no resources left. assertThat(claim).isNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(0); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(0); } // Function under test: acquireClaim @@ -266,7 +266,7 @@ public void rejectOperationIgnoresResource() throws Exception { .build(); // PRE-ASSERT - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(1); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(1); // ACT Claim claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -274,7 +274,7 @@ public void rejectOperationIgnoresResource() throws Exception { // ASSERT // the worker rejects because the os is not satisfied assertThat(claim).isNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(1); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(1); } // Function under test: acquireClaim @@ -296,8 +296,8 @@ public void shouldKeepOperationClaimsMultipleResource() throws Exception { .build(); // PRE-ASSERT - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(2); - assertThat(resourceSet.resources.get("BAR").available()).isEqualTo(4); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(2); + assertThat(resourceSet.resources.get("BAR").semaphore().availablePermits()).isEqualTo(4); // ACT Claim claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -305,8 +305,8 @@ public void shouldKeepOperationClaimsMultipleResource() throws Exception { // ASSERT // the worker accepts because the resource is available. assertThat(claim).isNotNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(1); - assertThat(resourceSet.resources.get("BAR").available()).isEqualTo(2); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(1); + assertThat(resourceSet.resources.get("BAR").semaphore().availablePermits()).isEqualTo(2); // ACT claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -314,8 +314,8 @@ public void shouldKeepOperationClaimsMultipleResource() throws Exception { // ASSERT // the worker accepts because the resource is available. assertThat(claim).isNotNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(0); - assertThat(resourceSet.resources.get("BAR").available()).isEqualTo(0); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(0); + assertThat(resourceSet.resources.get("BAR").semaphore().availablePermits()).isEqualTo(0); // ACT claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -323,8 +323,8 @@ public void shouldKeepOperationClaimsMultipleResource() throws Exception { // ASSERT // the worker rejects because there are no resources left. assertThat(claim).isNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(0); - assertThat(resourceSet.resources.get("BAR").available()).isEqualTo(0); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(0); + assertThat(resourceSet.resources.get("BAR").semaphore().availablePermits()).isEqualTo(0); } // Function under test: acquireClaim @@ -352,9 +352,9 @@ public void shouldKeepOperationFailsToClaimSameAmountRemains() throws Exception .build(); // PRE-ASSERT - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(50); - assertThat(resourceSet.resources.get("BAR").available()).isEqualTo(100); - assertThat(resourceSet.resources.get("BAZ").available()).isEqualTo(200); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(50); + assertThat(resourceSet.resources.get("BAR").semaphore().availablePermits()).isEqualTo(100); + assertThat(resourceSet.resources.get("BAZ").semaphore().availablePermits()).isEqualTo(200); // ACT Claim claim = acquireClaim(workerProvisions, resourceSet, platform); @@ -363,9 +363,9 @@ public void shouldKeepOperationFailsToClaimSameAmountRemains() throws Exception // the worker rejects because there are no resources left. // The same amount are returned. assertThat(claim).isNull(); - assertThat(resourceSet.resources.get("FOO").available()).isEqualTo(50); - assertThat(resourceSet.resources.get("BAR").available()).isEqualTo(100); - assertThat(resourceSet.resources.get("BAZ").available()).isEqualTo(200); + assertThat(resourceSet.resources.get("FOO").semaphore().availablePermits()).isEqualTo(50); + assertThat(resourceSet.resources.get("BAR").semaphore().availablePermits()).isEqualTo(100); + assertThat(resourceSet.resources.get("BAZ").semaphore().availablePermits()).isEqualTo(200); } @Test diff --git a/src/test/java/build/buildfarm/worker/ExecuteActionStageTest.java b/src/test/java/build/buildfarm/worker/ExecuteActionStageTest.java index a8be937731..1f16717efc 100644 --- a/src/test/java/build/buildfarm/worker/ExecuteActionStageTest.java +++ b/src/test/java/build/buildfarm/worker/ExecuteActionStageTest.java @@ -21,7 +21,6 @@ import build.buildfarm.v1test.ExecuteEntry; import build.buildfarm.v1test.QueueEntry; -import build.buildfarm.worker.filesystem.ExecFileSystem; import java.nio.file.Path; import org.junit.Test; import org.junit.runner.RunWith; @@ -45,9 +44,7 @@ public void errorPathDestroysExecDir() throws Exception { .setExecDir(Path.of("error-operation-path")) .build(); - ExecFileSystem execFileSystem = mock(ExecFileSystem.class); - PipelineStage executeActionStage = - new ExecuteActionStage(context, /* output= */ null, error, execFileSystem, null); + PipelineStage executeActionStage = new ExecuteActionStage(context, /* output= */ null, error); executeActionStage.error().put(errorContext); verify(context, times(1)).destroyExecDir(errorContext.execDir); verify(error, times(1)).put(errorContext); diff --git a/src/test/java/build/buildfarm/worker/filesystem/FuseCASTest.java b/src/test/java/build/buildfarm/worker/FuseCASTest.java similarity index 99% rename from src/test/java/build/buildfarm/worker/filesystem/FuseCASTest.java rename to src/test/java/build/buildfarm/worker/FuseCASTest.java index 1532f8489e..283b54ea4b 100644 --- a/src/test/java/build/buildfarm/worker/filesystem/FuseCASTest.java +++ b/src/test/java/build/buildfarm/worker/FuseCASTest.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; diff --git a/src/test/java/build/buildfarm/worker/InputFetchStageTest.java b/src/test/java/build/buildfarm/worker/InputFetchStageTest.java index 103a9281fd..58db40d57c 100644 --- a/src/test/java/build/buildfarm/worker/InputFetchStageTest.java +++ b/src/test/java/build/buildfarm/worker/InputFetchStageTest.java @@ -123,7 +123,6 @@ public void put(ExecutionContext executionContext) { any(Runnable.class), any(Deadline.class), any(Executor.class)); - verify(workerContext, times(2)).inGracefulShutdown(); verifyNoMoreInteractions(workerContext); ExecutionContext executionContext = error.getExecutionContexts().getFirst(); assertThat(executionContext).isEqualTo(badContext); diff --git a/src/test/java/build/buildfarm/worker/InputFetcherTest.java b/src/test/java/build/buildfarm/worker/InputFetcherTest.java index 66b302dcfa..145baf9a49 100644 --- a/src/test/java/build/buildfarm/worker/InputFetcherTest.java +++ b/src/test/java/build/buildfarm/worker/InputFetcherTest.java @@ -34,9 +34,7 @@ import build.buildfarm.v1test.ExecuteEntry; import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.QueuedOperation; -import build.buildfarm.v1test.WorkerExecutedMetadata; -import build.buildfarm.worker.filesystem.ExecDirException; -import build.buildfarm.worker.filesystem.ExecDirException.ViolationException; +import build.buildfarm.worker.ExecDirException.ViolationException; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.longrunning.Operation; @@ -100,8 +98,7 @@ public Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + UserPrincipal owner) throws IOException { Path root = Path.of(operationName); throw new ExecDirException( diff --git a/src/test/java/build/buildfarm/worker/filesystem/OutputDirectoryTest.java b/src/test/java/build/buildfarm/worker/OutputDirectoryTest.java similarity index 99% rename from src/test/java/build/buildfarm/worker/filesystem/OutputDirectoryTest.java rename to src/test/java/build/buildfarm/worker/OutputDirectoryTest.java index 72caffbe80..b6d82e4b24 100644 --- a/src/test/java/build/buildfarm/worker/filesystem/OutputDirectoryTest.java +++ b/src/test/java/build/buildfarm/worker/OutputDirectoryTest.java @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package build.buildfarm.worker.filesystem; +package build.buildfarm.worker; import static com.google.common.truth.Truth.assertThat; diff --git a/src/test/java/build/buildfarm/worker/StubWorkerContext.java b/src/test/java/build/buildfarm/worker/StubWorkerContext.java index b7cf6e5d32..f04821a7e5 100644 --- a/src/test/java/build/buildfarm/worker/StubWorkerContext.java +++ b/src/test/java/build/buildfarm/worker/StubWorkerContext.java @@ -27,7 +27,6 @@ import build.buildfarm.common.config.ExecutionPolicy; import build.buildfarm.v1test.QueueEntry; import build.buildfarm.v1test.QueuedOperation; -import build.buildfarm.v1test.WorkerExecutedMetadata; import build.buildfarm.worker.resources.ResourceLimits; import com.google.common.collect.ImmutableList; import com.google.longrunning.Operation; @@ -64,16 +63,6 @@ public void resumePoller( throw new UnsupportedOperationException(); } - @Override - public boolean inGracefulShutdown() { - throw new UnsupportedOperationException(); - } - - @Override - public void prepareForGracefulShutdown() { - throw new UnsupportedOperationException(); - } - @Override public void match(MatchListener listener) throws InterruptedException { throw new UnsupportedOperationException(); @@ -146,8 +135,7 @@ public Path createExecDir( DigestFunction.Value digestFunction, Action action, Command command, - UserPrincipal owner, - WorkerExecutedMetadata.Builder workerExecutedMetadata) + UserPrincipal owner) throws IOException, InterruptedException { throw new UnsupportedOperationException(); } diff --git a/src/test/java/build/buildfarm/worker/SuperscalarPipelineStageTest.java b/src/test/java/build/buildfarm/worker/SuperscalarPipelineStageTest.java index d431cabd8a..37639c2146 100644 --- a/src/test/java/build/buildfarm/worker/SuperscalarPipelineStageTest.java +++ b/src/test/java/build/buildfarm/worker/SuperscalarPipelineStageTest.java @@ -52,8 +52,8 @@ protected int claimsRequired(ExecutionContext executionContext) { throw new UnsupportedOperationException(); } - boolean hasOpenSlots() { - return !claims.isEmpty(); + boolean isFull() { + return claims.size() == width; } @Override @@ -78,7 +78,7 @@ protected int claimsRequired(ExecutionContext executionContext) { Thread interruptor = new Thread( () -> { - while (stage.hasOpenSlots()) { + while (!stage.isFull()) { try { MICROSECONDS.sleep(1); } catch (InterruptedException e) { diff --git a/src/test/java/build/buildfarm/worker/cgroup/BUILD b/src/test/java/build/buildfarm/worker/cgroup/BUILD index 7007fde70f..186a2f4beb 100644 --- a/src/test/java/build/buildfarm/worker/cgroup/BUILD +++ b/src/test/java/build/buildfarm/worker/cgroup/BUILD @@ -9,9 +9,9 @@ java_test( "//src/main/java/build/buildfarm/common/base", "//src/main/java/build/buildfarm/worker/cgroup", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:org_projectlombok_lombok", + "@maven//:com_google_guava_guava", + "@maven//:com_google_truth_truth", + "@maven//:org_mockito_mockito_core", + "@maven//:org_projectlombok_lombok", ], ) diff --git a/src/test/java/build/buildfarm/worker/cgroup/CGroupVersionProviderTest.java b/src/test/java/build/buildfarm/worker/cgroup/CGroupVersionProviderTest.java index 49ddb2a425..57553c80aa 100644 --- a/src/test/java/build/buildfarm/worker/cgroup/CGroupVersionProviderTest.java +++ b/src/test/java/build/buildfarm/worker/cgroup/CGroupVersionProviderTest.java @@ -36,6 +36,7 @@ @RunWith(JUnit4.class) public class CGroupVersionProviderTest { + @Before public void setup() throws IOException { Assume.assumeFalse(isWindows()); @@ -48,6 +49,7 @@ private void testDetectCGroups(String type, CGroupVersion detectedVersion) throw try (MockedStatic mockedFiles = Mockito.mockStatic(Files.class); MockedStatic mockedPath = Mockito.mockStatic(Path.class)) { + // Mock Path.of to return a path we can control Path mockPath = mock(Path.class); mockedPath.when(() -> Path.of("/sys/fs/cgroup")).thenReturn(mockPath); @@ -101,6 +103,7 @@ public void testDetectCGroupsV2() throws IOException { public void testNoCGroupsWhenDirectoryDoesntExist() { try (MockedStatic mockedFiles = Mockito.mockStatic(Files.class); MockedStatic mockedPath = Mockito.mockStatic(Path.class)) { + // Mock Path.of to return a path we can control Path mockPath = mock(Path.class); mockedPath.when(() -> Path.of("/sys/fs/cgroup")).thenReturn(mockPath); @@ -137,6 +140,7 @@ public void testNoCGroupsWhenDirectoryDoesntExist() { public void testIOExceptionHandling() { try (MockedStatic mockedFiles = Mockito.mockStatic(Files.class); MockedStatic mockedPath = Mockito.mockStatic(Path.class)) { + // Mock Path.of to return a path we can control Path mockPath = mock(Path.class); mockedPath.when(() -> Path.of("/sys/fs/cgroup")).thenReturn(mockPath); diff --git a/src/test/java/build/buildfarm/worker/cgroup/GroupTest.java b/src/test/java/build/buildfarm/worker/cgroup/GroupTest.java index 8c5aa192b7..a9776aa240 100644 --- a/src/test/java/build/buildfarm/worker/cgroup/GroupTest.java +++ b/src/test/java/build/buildfarm/worker/cgroup/GroupTest.java @@ -33,6 +33,7 @@ /** Tests for {@link Group#getSelfCgroup} method. */ @RunWith(JUnit4.class) public class GroupTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); private Path procSelfCgroupPath; diff --git a/src/test/java/build/buildfarm/worker/cgroup/GroupV1Test.java b/src/test/java/build/buildfarm/worker/cgroup/GroupV1Test.java index 659503cfe3..22e8d430c6 100644 --- a/src/test/java/build/buildfarm/worker/cgroup/GroupV1Test.java +++ b/src/test/java/build/buildfarm/worker/cgroup/GroupV1Test.java @@ -31,6 +31,7 @@ /** These test CGroups v1 behavior. When CGroups v1 is no longer supported, delete these tests. */ @RunWith(JUnit4.class) public class GroupV1Test { + @Before public void setup() { Group.VERSION = CGroupVersion.CGROUPS_V1; diff --git a/src/test/java/build/buildfarm/worker/cgroup/GroupV2Test.java b/src/test/java/build/buildfarm/worker/cgroup/GroupV2Test.java index c43deaa804..7d9e060f5d 100644 --- a/src/test/java/build/buildfarm/worker/cgroup/GroupV2Test.java +++ b/src/test/java/build/buildfarm/worker/cgroup/GroupV2Test.java @@ -31,6 +31,7 @@ /** These test CGroups v2 behavior exclusively. */ @RunWith(JUnit4.class) public class GroupV2Test { + @Before public void setup() { Group.VERSION = CGroupVersion.CGROUPS_V2; diff --git a/src/test/java/build/buildfarm/worker/filesystem/BUILD.bazel b/src/test/java/build/buildfarm/worker/filesystem/BUILD.bazel deleted file mode 100644 index 991f57e09a..0000000000 --- a/src/test/java/build/buildfarm/worker/filesystem/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_java//java:java_test.bzl", "java_test") - -config_setting( - name = "fuse", - values = {"define": "fuse=true"}, -) - -java_test( - name = "tests", - size = "small", - srcs = glob( - ["*.java"], - exclude = ["FuseCASTest.java"], - ) + select({ - ":fuse": ["FuseCASTest.java"], - "//conditions:default": [], - }), - data = ["//examples:example_configs"], - plugins = ["//src/main/java/build/buildfarm/common:lombok"], - test_class = "build.buildfarm.AllTests", - deps = [ - "//src/main/java/build/buildfarm/worker/filesystem", - "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", - ], -) diff --git a/src/test/java/build/buildfarm/worker/persistent/BUILD b/src/test/java/build/buildfarm/worker/persistent/BUILD index 539e2e855a..9f76bbe20a 100644 --- a/src/test/java/build/buildfarm/worker/persistent/BUILD +++ b/src/test/java/build/buildfarm/worker/persistent/BUILD @@ -8,28 +8,31 @@ java_test( deps = [ "//persistentworkers/src/main/java/persistent/bazel:bazel-persistent-workers", "//persistentworkers/src/main/java/persistent/common:persistent-common", + "//persistentworkers/src/main/java/persistent/common/util", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", "//src/main/java/build/buildfarm/common", "//src/main/java/build/buildfarm/common/config", + "//src/main/java/build/buildfarm/instance", + "//src/main/java/build/buildfarm/worker", "//src/main/java/build/buildfarm/worker/persistent", "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", "//src/test/java/build/buildfarm/worker/util:worker_test_utils", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_serceman_jnr_fuse", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:org_mockito_mockito_core", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/src/test/java/build/buildfarm/worker/persistent/ProtoCoordinatorTest.java b/src/test/java/build/buildfarm/worker/persistent/ProtoCoordinatorTest.java index 62b7cdcb86..6d8a7a2812 100644 --- a/src/test/java/build/buildfarm/worker/persistent/ProtoCoordinatorTest.java +++ b/src/test/java/build/buildfarm/worker/persistent/ProtoCoordinatorTest.java @@ -34,21 +34,19 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import persistent.bazel.client.WorkerIndex; import persistent.bazel.client.WorkerKey; @RunWith(JUnit4.class) public class ProtoCoordinatorTest { - private WorkerKey makeWorkerKey(WorkerInputs workerFiles, Path workRootsDir) { - return Keymaker.makeKey( - Keymaker.makeBasicKey( - ImmutableList.of("workerExecCmd"), - ImmutableList.of("workerInitArgs"), - ImmutableMap.of(), - "executionName"), - null, - ImmutableList.of(), + private WorkerKey makeWorkerKey( + WorkFilesContext ctx, WorkerInputs workerFiles, Path workRootsDir) { + return Keymaker.make( + ctx.opRoot, workRootsDir, + ImmutableList.of("workerExecCmd"), + ImmutableList.of("workerInitArgs"), + ImmutableMap.of(), + "executionName", workerFiles); } @@ -70,8 +68,7 @@ public Path jimFsRoot() { @Test public void testProtoCoordinator() throws Exception { - WorkerIndex workerIndex = new WorkerIndex(); - ProtoCoordinator pc = ProtoCoordinator.ofCommonsPool(workerIndex, 4); + ProtoCoordinator pc = ProtoCoordinator.ofCommonsPool(4); Path fsRoot = jimFsRoot(); Path opRoot = fsRoot.resolve("opRoot"); @@ -101,7 +98,7 @@ public void testProtoCoordinator() throws Exception { Files.createFile(file); } - WorkerKey key = makeWorkerKey(workerFiles, fsRoot.resolve("workRootsDir")); + WorkerKey key = makeWorkerKey(ctx, workerFiles, fsRoot.resolve("workRootsDir")); Path workRoot = key.getExecRoot(); Path toolsRoot = key.getToolRoot(); @@ -109,7 +106,7 @@ public void testProtoCoordinator() throws Exception { // Assert: all Tools are copied into "/workRootsDir/*/" assertThat(toolsRoot.toString()).startsWith(workRoot.toString()); assertThat(toolsRoot.toString()).endsWith(key.getWorkerFilesCombinedHash().toString()); - pc.copyToolInputsIntoWorkerToolRoot(key, workerFiles, key.getOwner()); + pc.copyToolInputsIntoWorkerToolRoot(key, workerFiles); assertThat(Files.exists(workRoot)).isTrue(); assertThat(Files.exists(toolsRoot)).isTrue(); diff --git a/src/test/java/build/buildfarm/worker/resources/BUILD b/src/test/java/build/buildfarm/worker/resources/BUILD index 324a22f09c..ed32ed5d32 100644 --- a/src/test/java/build/buildfarm/worker/resources/BUILD +++ b/src/test/java/build/buildfarm/worker/resources/BUILD @@ -5,7 +5,7 @@ java_test( size = "small", srcs = glob(["*.java"]), data = [ - "//examples:example_properties", + "//config:base_properties", ], test_class = "build.buildfarm.AllTests", deps = [ @@ -14,12 +14,12 @@ java_test( "//src/main/java/build/buildfarm/worker/resources", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/worker/shard/BUILD b/src/test/java/build/buildfarm/worker/shard/BUILD index 3c58bf570a..03901d8d62 100644 --- a/src/test/java/build/buildfarm/worker/shard/BUILD +++ b/src/test/java/build/buildfarm/worker/shard/BUILD @@ -13,18 +13,16 @@ java_test( "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/instance", "//src/main/java/build/buildfarm/worker", - "//src/main/java/build/buildfarm/worker/filesystem", - "//src/main/java/build/buildfarm/worker/persistent", "//src/main/java/build/buildfarm/worker/resources", "//src/main/java/build/buildfarm/worker/shard", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:org_mockito_mockito_core", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_core", + "@maven//:org_mockito_mockito_core", ], ) diff --git a/src/test/java/build/buildfarm/worker/shard/ShardWorkerContextTest.java b/src/test/java/build/buildfarm/worker/shard/ShardWorkerContextTest.java index 5fa4b5b2ea..10ce7cd133 100644 --- a/src/test/java/build/buildfarm/worker/shard/ShardWorkerContextTest.java +++ b/src/test/java/build/buildfarm/worker/shard/ShardWorkerContextTest.java @@ -20,6 +20,7 @@ import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -34,6 +35,7 @@ import build.buildfarm.backplane.Backplane; import build.buildfarm.cas.ContentAddressableStorage; import build.buildfarm.common.Claim; +import build.buildfarm.common.Dispenser; import build.buildfarm.common.InputStreamFactory; import build.buildfarm.common.config.BuildfarmConfigs; import build.buildfarm.common.config.ExecutionPolicy; @@ -41,12 +43,11 @@ import build.buildfarm.instance.Instance; import build.buildfarm.v1test.Digest; import build.buildfarm.v1test.QueueEntry; +import build.buildfarm.worker.ExecFileSystem; import build.buildfarm.worker.MatchListener; import build.buildfarm.worker.WorkerContext; -import build.buildfarm.worker.filesystem.ExecFileSystem; -import build.buildfarm.worker.persistent.PersistentExecutor; -import build.buildfarm.worker.persistent.PersistentWorkerAwareExecOwnerPool; import build.buildfarm.worker.resources.LocalResourceSet; +import build.buildfarm.worker.resources.LocalResourceSet.PoolResource; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.jimfs.Jimfs; @@ -54,7 +55,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import org.junit.Before; import org.junit.Test; @@ -143,16 +143,9 @@ public void queueEntryWithExecutionPolicyPlatformMatches() throws Exception { when(backplane.dispatchOperation(any(List.class), any(LocalResourceSet.class))) .thenReturn(queueEntry) .thenReturn(null); // provide a match completion in failure case - ContentAddressableStorage storage = mock(ContentAddressableStorage.class); - when(execFileSystem.getStorage()).thenReturn(storage); - MatchListener listener = mock(MatchListener.class); when(listener.onWaitStart()).thenReturn(true); context.match(listener); - - // one readonly check for each iteration - verify(storage, times(2)).isReadOnly(); - verifyNoMoreInteractions(storage); verify(listener, times(1)).onEntry(eq(queueEntry), any(Claim.class)); verify(listener, times(1)).onWaitStart(); } @@ -170,18 +163,9 @@ public void dequeueMatchSettingsPlatformRejectsInvalidQueueEntry() throws Except when(backplane.dispatchOperation(any(List.class), any(LocalResourceSet.class))) .thenReturn(queueEntry) .thenReturn(null); // provide a match completion in failure case - ContentAddressableStorage storage = mock(ContentAddressableStorage.class); - when(execFileSystem.getStorage()).thenReturn(storage); - MatchListener listener = mock(MatchListener.class); context.match(listener); - - // one readonly check for each iteration - verify(storage, times(2)).isReadOnly(); - verifyNoMoreInteractions(storage); - verify(listener, times(1)).onWaitStart(); - verify(listener, times(1)).onEntry(null, null); - verifyNoMoreInteractions(listener); + verify(listener, never()).onEntry(eq(queueEntry), any(Claim.class)); } @Test @@ -198,15 +182,9 @@ public void dequeueMatchSettingsPlatformAcceptsValidQueueEntry() throws Exceptio when(backplane.dispatchOperation(any(List.class), any(LocalResourceSet.class))) .thenReturn(queueEntry) .thenReturn(null); // provide a match completion in failure case - ContentAddressableStorage storage = mock(ContentAddressableStorage.class); - when(execFileSystem.getStorage()).thenReturn(storage); MatchListener listener = mock(MatchListener.class); when(listener.onWaitStart()).thenReturn(true); context.match(listener); - - // one readonly check for each iteration - verify(storage, times(2)).isReadOnly(); - verifyNoMoreInteractions(storage); verify(listener, times(1)).onEntry(eq(queueEntry), any(Claim.class)); verify(listener, times(1)).onWaitStart(); } @@ -234,12 +212,9 @@ public void uploadOutputsWorkingDirectoryRelative() throws Exception { @Test public void resourceExhaustedIgnoresEntryWithExecOwner() throws Exception { LocalResourceSet resourceSet = new LocalResourceSet(); - resourceSet.resources.put( - LocalResourceSet.EXEC_OWNER_RESOURCE_NAME, - new PersistentWorkerAwareExecOwnerPool( - PersistentExecutor.workerIndex, - Collections.singleton("exec-user-name"), - REPORT_RESULT_STAGE)); + resourceSet.poolResources.put( + ShardWorkerContext.EXEC_OWNER_RESOURCE_NAME, + new PoolResource(new Dispenser<>("exec-user-name"), REPORT_RESULT_STAGE)); WorkerContext context = createTestContext(/* policies= */ ImmutableList.of(), resourceSet); Platform platform = @@ -250,16 +225,10 @@ public void resourceExhaustedIgnoresEntryWithExecOwner() throws Exception { when(backplane.dispatchOperation(any(List.class), any(LocalResourceSet.class))) .thenReturn(queueEntry) .thenReturn(null); // provide a match completion in failure case - ContentAddressableStorage storage = mock(ContentAddressableStorage.class); - when(execFileSystem.getStorage()).thenReturn(storage); - MatchListener listener = mock(MatchListener.class); + when(listener.onWaitStart()).thenReturn(true); context.match(listener); - - // one readonly check for each iteration, 3 here - verify(storage, times(3)).isReadOnly(); - verifyNoMoreInteractions(storage); verify(listener, times(1)).onEntry(null, null); // twice because there were 2 dequeues to complete queueEntry verify(listener, times(2)).onWaitStart(); diff --git a/src/test/java/build/buildfarm/worker/util/BUILD b/src/test/java/build/buildfarm/worker/util/BUILD index 52cb324553..a86ad3c234 100644 --- a/src/test/java/build/buildfarm/worker/util/BUILD +++ b/src/test/java/build/buildfarm/worker/util/BUILD @@ -8,24 +8,26 @@ java_library( visibility = ["//src/test/java:__subpackages__"], deps = [ "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", + "//src/main/java/build/buildfarm/cas", "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/worker/util", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_serceman_jnr_fuse", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:org_mockito_mockito_core", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) @@ -36,25 +38,28 @@ java_test( srcs = glob(["*Test.java"]), test_class = "build.buildfarm.AllTests", deps = [ + ":worker_test_utils", "//persistentworkers/src/main/protobuf:worker_protocol_java_proto", + "//src/main/java/build/buildfarm/cas", "//src/main/java/build/buildfarm/common", + "//src/main/java/build/buildfarm/common/config", "//src/main/java/build/buildfarm/worker/util", "//src/main/protobuf/build/buildfarm/v1test:buildfarm_java_proto", "//src/test/java/build/buildfarm:test_runner", - "@buildfarm_maven//:com_github_jnr_jnr_constants", - "@buildfarm_maven//:com_github_jnr_jnr_ffi", - "@buildfarm_maven//:com_github_serceman_jnr_fuse", - "@buildfarm_maven//:com_google_guava_guava", - "@buildfarm_maven//:com_google_jimfs_jimfs", - "@buildfarm_maven//:com_google_protobuf_protobuf_java", - "@buildfarm_maven//:com_google_truth_truth", - "@buildfarm_maven//:io_grpc_grpc_api", - "@buildfarm_maven//:io_grpc_grpc_context", - "@buildfarm_maven//:io_grpc_grpc_core", - "@buildfarm_maven//:io_grpc_grpc_protobuf", - "@buildfarm_maven//:org_mockito_mockito_core", - "@buildfarm_maven//:org_projectlombok_lombok", "@googleapis//google/rpc:rpc_java_proto", + "@maven//:com_github_jnr_jnr_constants", + "@maven//:com_github_jnr_jnr_ffi", + "@maven//:com_github_serceman_jnr_fuse", + "@maven//:com_google_guava_guava", + "@maven//:com_google_jimfs_jimfs", + "@maven//:com_google_protobuf_protobuf_java", + "@maven//:com_google_truth_truth", + "@maven//:io_grpc_grpc_api", + "@maven//:io_grpc_grpc_context", + "@maven//:io_grpc_grpc_core", + "@maven//:io_grpc_grpc_protobuf", + "@maven//:org_mockito_mockito_core", + "@maven//:org_projectlombok_lombok", "@remoteapis//build/bazel/remote/execution/v2:remote_execution_java_proto", ], ) diff --git a/third_party/grpc-java-12207.patch b/third_party/grpc-java-12207.patch deleted file mode 100644 index d63fc0e999..0000000000 --- a/third_party/grpc-java-12207.patch +++ /dev/null @@ -1,66 +0,0 @@ -commit a37d3eb349e048b953633027ed011cda8b68c603 -Author: George Gensure -Date: Thu Jul 10 09:49:54 2025 -0400 - - Guarantee missing stream promise delivery - - In observed cases, whether RST_STREAM or another failure from netty or - the server, listeners can fail to be notified when a connection yields a - null stream for the selected streamId. This causes hangs in clients, - despite deadlines, with no obvious resolution. - - Tests which relied upon this promise succeeding must now change. - -diff --git a/netty/src/main/java/io/grpc/netty/NettyClientHandler.java b/netty/src/main/java/io/grpc/netty/NettyClientHandler.java -index a5fa0f800..276fa623c 100644 ---- a/netty/src/main/java/io/grpc/netty/NettyClientHandler.java -+++ b/netty/src/main/java/io/grpc/netty/NettyClientHandler.java -@@ -738,14 +738,19 @@ class NettyClientHandler extends AbstractNettyHandler { - - // Attach the client stream to the HTTP/2 stream object as user data. - stream.setHttp2Stream(http2Stream); -+ promise.setSuccess(); -+ } else { -+ // Otherwise, the stream has been cancelled and Netty is sending a -+ // RST_STREAM frame which causes it to purge pending writes from the -+ // flow-controller and delete the http2Stream. The stream listener has already -+ // been notified of cancellation so there is nothing to do. -+ // -+ // This process has been observed to fail in some circumstances, leaving listeners -+ // unanswered. Ensure that some exception has been delivered consistent with the -+ // implied RST_STREAM result above. -+ Status status = Status.INTERNAL.withDescription("unknown stream for connection"); -+ promise.setFailure(status.asRuntimeException()); - } -- // Otherwise, the stream has been cancelled and Netty is sending a -- // RST_STREAM frame which causes it to purge pending writes from the -- // flow-controller and delete the http2Stream. The stream listener has already -- // been notified of cancellation so there is nothing to do. -- -- // Just forward on the success status to the original promise. -- promise.setSuccess(); - } else { - Throwable cause = future.cause(); - if (cause instanceof StreamBufferingEncoder.Http2GoAwayException) { -diff --git a/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java -index f8fbeea9b..dd4fcb4ea 100644 ---- a/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java -+++ b/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java -@@ -268,7 +268,7 @@ public class NettyClientHandlerTest extends NettyHandlerTestBase -Date: Tue Jul 15 15:00:24 2025 -0700 - - netty: Associate netty stream eagerly to avoid client hang - - In #12185, RPCs were randomly hanging. In #12207 this was tracked down - to the headers promise completing successfully, but the netty stream - was null. This was because the headers write hadn't completed but - stream.close() had been called by goingAway(). - -diff --git a/netty/src/main/java/io/grpc/netty/NettyClientHandler.java b/netty/src/main/java/io/grpc/netty/NettyClientHandler.java -index 276fa623c..d6bb37904 100644 ---- a/netty/src/main/java/io/grpc/netty/NettyClientHandler.java -+++ b/netty/src/main/java/io/grpc/netty/NettyClientHandler.java -@@ -773,6 +773,19 @@ class NettyClientHandler extends AbstractNettyHandler { - } - } - }); -+ // When the HEADERS are not buffered because of MAX_CONCURRENT_STREAMS in -+ // StreamBufferingEncoder, the stream is created immediately even if the bytes of the HEADERS -+ // are delayed because the OS may have too much buffered and isn't accepting the write. The -+ // write promise is also delayed until flush(). However, we need to associate the netty stream -+ // with the transport state so that goingAway() and forcefulClose() and able to notify the -+ // stream of failures. -+ // -+ // This leaves a hole when MAX_CONCURRENT_STREAMS is reached, as http2Stream will be null, but -+ // it is better than nothing. -+ Http2Stream http2Stream = connection().stream(streamId); -+ if (http2Stream != null) { -+ http2Stream.setProperty(streamKey, stream); -+ } - } - - /** -diff --git a/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java b/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java -index dd4fcb4ea..5a2605eea 100644 ---- a/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java -+++ b/netty/src/test/java/io/grpc/netty/NettyClientHandlerTest.java -@@ -453,6 +453,26 @@ public class NettyClientHandlerTest extends NettyHandlerTestBase captor = ArgumentCaptor.forClass(Status.class); -+ verify(streamListener).closed(captor.capture(), same(REFUSED), -+ ArgumentMatchers.notNull()); -+ assertEquals(Status.UNAVAILABLE.getCode(), captor.getValue().getCode()); -+ assertEquals( -+ "Abrupt GOAWAY closed sent stream. HTTP/2 error code: NO_ERROR, " -+ + "debug data: this is a test", -+ captor.getValue().getDescription()); -+ assertTrue(future.isDone()); -+ } -+ - @Test - public void receivedGoAway_shouldFailBufferedStreamsExceedingMaxConcurrentStreams() - throws Exception { diff --git a/third_party/remote_apis_grpc.patch b/third_party/remote_apis_grpc.patch deleted file mode 100644 index b4eb096c71..0000000000 --- a/third_party/remote_apis_grpc.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- MODULE.bazel -+++ MODULE.bazel -@@ -3,7 +3,7 @@ - version = "", - ) - --bazel_dep(name = "grpc", version = "1.66.0.bcr.2") -+bazel_dep(name = "grpc", version = "1.74.1") - bazel_dep(name = "rules_proto", version = "6.0.2") - bazel_dep(name = "protobuf", version = "28.1") - bazel_dep(name = "googleapis", version = "0.0.0-20240819-fe8ba054a") From ed1223a9251f865dd5876747f125da4b1fc5d306 Mon Sep 17 00:00:00 2001 From: Timothy Blamires Date: Thu, 12 Mar 2026 13:37:43 -0600 Subject: [PATCH 2/2] Add config to enable as-nobody wrapper for non-persistent worker actions We want to enable as-nobody for normal actions to prevent them from writing to read-only input files that are shared via symlinks between actions. However, buildfarm does not set file ownership on persistent worker exec roots, so running them as nobody would break output writes. Since persistent workers copy their inputs rather than symlinking them, this is not a concern. Add alwaysUseAsNobodyExceptPersistentWorkers config option to enable as-nobody for all actions except those running on persistent workers. --- _site/docs/configuration/configuration.md | 1 + .../buildfarm/common/config/SandboxSettings.java | 7 +++++++ src/main/java/build/buildfarm/worker/Executor.java | 3 ++- .../java/build/buildfarm/worker/WorkerContext.java | 3 ++- .../buildfarm/worker/shard/ShardWorkerContext.java | 11 +++++++++-- .../build/buildfarm/worker/StubWorkerContext.java | 3 ++- 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/_site/docs/configuration/configuration.md b/_site/docs/configuration/configuration.md index 3f67519043..e0236dc0b3 100644 --- a/_site/docs/configuration/configuration.md +++ b/_site/docs/configuration/configuration.md @@ -345,6 +345,7 @@ Using the sandbox can be configurable by the client via `exec_properties`. Howev |-----------------------|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | alwaysUseSandbox | boolean, _false_ | Whether or not to always use the sandbox when running actions. It may be preferred to enforce sandbox usage than rely on client selection. | | alwaysUseAsNobody | boolean, _false_ | Whether or not to always use the as-nobody wrapper when running actions. It may be preferred to enforce this wrapper instead of relying on client selection. | +| alwaysUseAsNobodyExceptPersistentWorkers | boolean, _false_ | Same as alwaysUseAsNobody but does not apply to actions running on persistent workers. Ignored if alwaysUseAsNobody is set to true. | | alwaysUseCgroups | boolean, _true_ | Whether or not to use cgroups when sandboxing actions. It may be preferred to enforce cgroup usage. | | alwaysUseTmpFs | boolean, _false_ | Whether or not to always use tmpfs when using the sandbox. It may be preferred to enforce sandbox usage than rely on client selection. | | additionalWritePaths | List of Strings, _[]_ | Additional paths the sandbox is allowed to write to. Suggestions may include: /tmp, /dev/shm | diff --git a/src/main/java/build/buildfarm/common/config/SandboxSettings.java b/src/main/java/build/buildfarm/common/config/SandboxSettings.java index e9d1a48341..436181cc7e 100644 --- a/src/main/java/build/buildfarm/common/config/SandboxSettings.java +++ b/src/main/java/build/buildfarm/common/config/SandboxSettings.java @@ -41,6 +41,13 @@ public class SandboxSettings { */ private boolean alwaysUseAsNobody = false; + /** + * @field alwaysUseAsNobodyExceptPersistentWorkers + * @brief Always use the as-nobody wrapper, but does not apply to actions running on persistent + * workers. + */ + private boolean alwaysUseAsNobodyExceptPersistentWorkers = false; + /** * @field alwaysUseCgroups * @brief Whether or not to use cgroups when sandboxing actions. diff --git a/src/main/java/build/buildfarm/worker/Executor.java b/src/main/java/build/buildfarm/worker/Executor.java index 2b0647104a..1e2f0cd364 100644 --- a/src/main/java/build/buildfarm/worker/Executor.java +++ b/src/main/java/build/buildfarm/worker/Executor.java @@ -310,7 +310,8 @@ private long executePolled( executionContext.claim.owner(), arguments, executionContext.command, - workingDirectory)) { + workingDirectory, + shouldRunOnPersistentWorker(limits))) { // Windows requires that relative command programs are absolutized Iterator argumentItr = command.getArgumentsList().iterator(); boolean absolutizeExe = diff --git a/src/main/java/build/buildfarm/worker/WorkerContext.java b/src/main/java/build/buildfarm/worker/WorkerContext.java index d36852cad8..8ed8bca164 100644 --- a/src/main/java/build/buildfarm/worker/WorkerContext.java +++ b/src/main/java/build/buildfarm/worker/WorkerContext.java @@ -134,7 +134,8 @@ IOResource limitExecution( @Nullable UserPrincipal owner, ImmutableList.Builder arguments, Command command, - Path workingDirectory); + Path workingDirectory, + boolean runsOnPersistentWorker); int commandExecutionClaims(Command command); diff --git a/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java b/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java index 9deb30f122..4b1aa47624 100644 --- a/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java +++ b/src/main/java/build/buildfarm/worker/shard/ShardWorkerContext.java @@ -887,7 +887,8 @@ public IOResource limitExecution( @Nullable UserPrincipal owner, ImmutableList.Builder arguments, Command command, - Path workingDirectory) { + Path workingDirectory, + boolean runsOnPersistentWorker) { ResourceLimits limits = commandExecutionSettings(command); IOResource resource; if (shouldLimitCoreUsage()) { @@ -920,7 +921,13 @@ public boolean isReferenced() { addLinuxSandboxCli(arguments, options); } - if (configs.getWorker().getSandboxSettings().isAlwaysUseAsNobody() || limits.fakeUsername) { + if (configs.getWorker().getSandboxSettings().isAlwaysUseAsNobody() + || (!runsOnPersistentWorker + && configs + .getWorker() + .getSandboxSettings() + .isAlwaysUseAsNobodyExceptPersistentWorkers()) + || limits.fakeUsername) { arguments.add(configs.getExecutionWrappers().getAsNobody()); } diff --git a/src/test/java/build/buildfarm/worker/StubWorkerContext.java b/src/test/java/build/buildfarm/worker/StubWorkerContext.java index f04821a7e5..28f591bcaa 100644 --- a/src/test/java/build/buildfarm/worker/StubWorkerContext.java +++ b/src/test/java/build/buildfarm/worker/StubWorkerContext.java @@ -205,7 +205,8 @@ public IOResource limitExecution( UserPrincipal owner, ImmutableList.Builder arguments, Command command, - Path workingDirectory) { + Path workingDirectory, + boolean runsOnPersistentWorker) { throw new UnsupportedOperationException(); }