diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..2855941f405d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @basetenlabs/runtime-fabric diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..77c630665285 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: CI + +on: + pull_request: + branches: + - v0.22-base + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + GO_VERSION: "1.24" + +jobs: + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - uses: golangci/golangci-lint-action@v7 + with: + version: latest + args: --modules-download-mode=vendor + only-new-issues: true + + unit-tests: + name: Unit Tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + - name: Run unit tests + run: | + go test -mod=vendor -count=1 -timeout 10m \ + $(go list -mod=vendor ./... | grep -v -E '(worker/(containerd|runc)|cache$|cache/contenthash|snapshot$|source/(git|http)|util/overlay)$') + env: + SKIP_INTEGRATION_TESTS: "1" diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml new file mode 100644 index 000000000000..9ff5ad527a08 --- /dev/null +++ b/.github/workflows/post-merge.yml @@ -0,0 +1,94 @@ +name: Post-Merge + +on: + push: + branches: + - v0.22-base + workflow_dispatch: + inputs: + skip_integration_tests: + description: 'Skip integration tests' + required: false + type: boolean + default: false + skip_build: + description: 'Skip image build & push' + required: false + type: boolean + default: false + +permissions: + contents: read + +env: + GO_VERSION: "1.24" + SETUP_BUILDX_VERSION: "edge" + SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" + IMAGE_NAME: "baseten/buildkit" + +jobs: + integration-tests: + name: Integration Tests + if: ${{ !inputs.skip_integration_tests }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: crazy-max/ghaction-github-runtime@v3 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + with: + version: ${{ env.SETUP_BUILDX_VERSION }} + driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} + buildkitd-flags: --debug + - name: Build integration test image + uses: docker/bake-action@v6 + with: + targets: integration-tests + set: | + *.cache-from=type=gha,scope=integration-tests + *.cache-to=type=gha,scope=integration-tests,repository=${{ github.repository }},ghtoken=${{ secrets.GITHUB_TOKEN }} + - name: Run integration tests + run: ./hack/test integration + env: + TESTPKGS: ./client ./cmd/buildctl ./solver ./frontend + TESTFLAGS: "-v --parallel=6 --timeout=30m" + GOTESTSUM_FORMAT: standard-verbose + TEST_IMAGE_BUILD: "0" + CACHE_FROM: type=gha,scope=integration-tests + + build-and-push: + name: Build & Push Image + if: ${{ !inputs.skip_build }} + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + with: + version: ${{ env.SETUP_BUILDX_VERSION }} + driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} + buildkitd-flags: --debug + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Generate tag + id: tag + run: | + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + TIMESTAMP=$(date -u +%Y%m%d%H%M%S) + echo "tag=v0.22.0-${SHORT_SHA}-${TIMESTAMP}" >> $GITHUB_OUTPUT + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + target: buildkit + platforms: linux/amd64 + tags: ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} + cache-from: type=gha,scope=build-image + cache-to: type=gha,scope=build-image,mode=max + - name: Print image + run: echo "Pushed ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}" diff --git a/.golangci.yml b/.golangci.yml index 2efd20f86d9f..71de6e6abb13 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -63,10 +63,14 @@ linters: gosec: excludes: - G101 + - G115 + - G118 + - G120 + - G122 - G402 - G504 - G601 - - G115 + - G703 config: G306: "0644" govet: @@ -119,6 +123,7 @@ linters: text: if-return paths: - .*\.pb\.go$ + - .*_test\.go$ - examples formatters: enable: diff --git a/Dockerfile b/Dockerfile index 7324853f914f..c5635a513def 100644 --- a/Dockerfile +++ b/Dockerfile @@ -201,7 +201,7 @@ FROM scratch AS release COPY --link --from=releaser /out/ / FROM alpine:${ALPINE_VERSION} AS buildkit-export-alpine -RUN apk add --no-cache fuse3 git openssh pigz xz iptables ip6tables \ +RUN apk add --no-cache fuse3 git openssh pigz isa-l xz iptables ip6tables \ && ln -s fusermount3 /usr/bin/fusermount COPY --link examples/buildctl-daemonless/buildctl-daemonless.sh /usr/bin/ VOLUME /var/lib/buildkit @@ -213,6 +213,7 @@ RUN apt-get update \ git \ openssh-client \ pigz \ + isal \ xz-utils \ iptables \ ca-certificates \ @@ -458,7 +459,7 @@ VOLUME /var/lib/buildkit # rootless builds a rootless variant of buildkitd image FROM alpine:${ALPINE_VERSION} AS rootless -RUN apk add --no-cache fuse3 fuse-overlayfs git openssh pigz shadow-uidmap xz +RUN apk add --no-cache fuse3 fuse-overlayfs git openssh pigz isa-l shadow-uidmap xz RUN adduser -D -u 1000 user \ && mkdir -p /run/user/1000 /home/user/.local/tmp /home/user/.local/share/buildkit \ && chown -R user /run/user/1000 /home/user \ diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go index efed916c8f72..d76537782cee 100644 --- a/api/services/control/control.pb.go +++ b/api/services/control/control.pb.go @@ -164,7 +164,8 @@ type DiskUsageRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Filter []string `protobuf:"bytes,1,rep,name=filter,proto3" json:"filter,omitempty"` + Filter []string `protobuf:"bytes,1,rep,name=filter,proto3" json:"filter,omitempty"` + AgeLimit int64 `protobuf:"varint,2,opt,name=ageLimit,proto3" json:"ageLimit,omitempty"` } func (x *DiskUsageRequest) Reset() { @@ -204,6 +205,13 @@ func (x *DiskUsageRequest) GetFilter() []string { return nil } +func (x *DiskUsageRequest) GetAgeLimit() int64 { + if x != nil { + return x.AgeLimit + } + return 0 +} + type DiskUsageResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2063,452 +2071,454 @@ var file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = [ 0x70, 0x61, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x55, 0x73, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, - 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x22, 0x2a, 0x0a, 0x10, + 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x22, 0x46, 0x0a, 0x10, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, 0x4a, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x6b, - 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, - 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x22, 0x87, 0x03, 0x0a, 0x0b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x49, 0x6e, 0x55, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x49, - 0x6e, 0x55, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1a, 0x0a, 0x06, 0x50, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, - 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, - 0x0a, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, - 0x4c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, - 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, - 0x55, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, - 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x53, 0x68, - 0x61, 0x72, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xf4, - 0x07, 0x0a, 0x0c, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, - 0x66, 0x12, 0x2e, 0x0a, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x2e, 0x0a, 0x12, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, - 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x12, 0x75, 0x0a, 0x17, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, - 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, - 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, - 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x61, 0x67, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x22, 0x4a, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x06, 0x72, 0x65, 0x63, + 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x73, 0x61, + 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x22, 0x87, 0x03, 0x0a, 0x0b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, + 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x49, 0x6e, + 0x55, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x49, 0x6e, 0x55, 0x73, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, + 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1a, 0x0a, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, + 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x4c, 0x61, + 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x4c, 0x61, 0x73, 0x74, + 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, 0x61, 0x67, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x6f, + 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x68, 0x61, 0x72, + 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, + 0x12, 0x18, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xf4, 0x07, 0x0a, 0x0c, 0x53, + 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x52, + 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x2e, 0x0a, + 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, + 0x12, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x75, 0x0a, 0x17, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, - 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x57, - 0x0a, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, - 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, + 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, + 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x17, 0x45, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, + 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x57, 0x0a, 0x0d, 0x46, 0x72, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, + 0x74, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, + 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x5a, 0x0a, + 0x0e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, - 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, - 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x22, 0x0a, - 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x5a, 0x0a, 0x0e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x46, - 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x1a, 0x0a, - 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x49, 0x0a, 0x0c, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2e, - 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x6f, - 0x6c, 0x69, 0x63, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, - 0x74, 0x65, 0x72, 0x52, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x34, - 0x0a, 0x15, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x72, 0x1a, 0x4a, 0x0a, 0x1c, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x1a, 0x40, 0x0a, 0x12, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x51, 0x0a, 0x13, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, - 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xad, 0x03, 0x0a, 0x0c, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, - 0x52, 0x65, 0x66, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x44, 0x65, - 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x14, 0x49, 0x6d, 0x70, 0x6f, - 0x72, 0x74, 0x52, 0x65, 0x66, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, - 0x66, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x6f, 0x0a, 0x15, - 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x45, 0x78, 0x70, 0x6f, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x46, 0x72, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x49, 0x0a, 0x0c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, + 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2e, 0x50, 0x6f, 0x6c, 0x69, + 0x63, 0x79, 0x52, 0x0c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, + 0x12, 0x38, 0x0a, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, + 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x45, 0x6e, 0x61, 0x62, 0x6c, + 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, + 0x1a, 0x4a, 0x0a, 0x1c, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, + 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x40, 0x0a, 0x12, + 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x51, + 0x0a, 0x13, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x66, 0x69, + 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xad, 0x03, 0x0a, 0x0c, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x44, + 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x14, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, + 0x66, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x14, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x73, 0x44, 0x65, + 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x6f, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, - 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x3d, 0x0a, - 0x07, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x07, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x07, - 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x07, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x1a, 0x48, 0x0a, 0x1a, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, + 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, + 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x07, 0x45, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x07, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x07, 0x49, 0x6d, 0x70, 0x6f, + 0x72, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, + 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x1a, 0x48, 0x0a, 0x1a, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xa7, 0x01, 0x0a, 0x11, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x44, 0x0a, 0x05, 0x41, + 0x74, 0x74, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x41, 0x74, 0x74, 0x72, + 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb7, 0x01, 0x0a, 0x0d, + 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, + 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, + 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x1a, 0x43, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa7, 0x01, 0x0a, 0x11, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x54, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x44, 0x0a, 0x05, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, - 0x41, 0x74, 0x74, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0xb7, 0x01, 0x0a, 0x0d, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x61, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x70, - 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x43, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x21, 0x0a, 0x0d, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, - 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x22, 0xf0, 0x01, 0x0a, - 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x34, 0x0a, 0x08, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, - 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x52, 0x08, 0x76, 0x65, 0x72, - 0x74, 0x65, 0x78, 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, - 0x78, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, - 0x73, 0x12, 0x2f, 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, - 0x67, 0x73, 0x12, 0x3b, 0x0a, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x57, 0x61, - 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x22, - 0xa3, 0x02, 0x0a, 0x06, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, - 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x38, 0x0a, 0x09, - 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x0d, - 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xa4, 0x02, 0x0a, 0x0c, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, - 0x61, 0x6c, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x34, 0x0a, 0x07, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x65, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x87, 0x01, 0x0a, - 0x09, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, - 0x72, 0x74, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, - 0x65, 0x78, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xc4, 0x01, 0x0a, 0x0d, 0x56, 0x65, 0x72, 0x74, 0x65, - 0x78, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, - 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, - 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x22, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x0a, 0x06, 0x72, 0x61, - 0x6e, 0x67, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, - 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x22, 0x0a, - 0x0c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x22, 0x2c, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, - 0x53, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, - 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x22, 0x0d, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x61, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x21, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x22, 0xf0, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x08, 0x76, + 0x65, 0x72, 0x74, 0x65, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, + 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x52, 0x08, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x65, + 0x73, 0x12, 0x3a, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x12, 0x2f, 0x0a, + 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, + 0x65, 0x72, 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x3b, + 0x0a, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, + 0x67, 0x52, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x22, 0xa3, 0x02, 0x0a, 0x06, + 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x61, + 0x63, 0x68, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x64, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x22, 0xa4, 0x02, 0x0a, 0x0c, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x38, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x38, + 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x87, 0x01, 0x0a, 0x09, 0x56, 0x65, 0x72, + 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x38, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6d, + 0x73, 0x67, 0x22, 0xc4, 0x01, 0x0a, 0x0d, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x57, 0x61, 0x72, + 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x14, 0x0a, 0x05, + 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x72, 0x6c, 0x12, 0x22, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x22, 0x0a, 0x0c, 0x42, 0x79, 0x74, + 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x2c, 0x0a, + 0x12, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, 0x53, 0x0a, 0x13, 0x4c, + 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, + 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x22, 0x0d, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, + 0x61, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x51, 0x0a, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x93, 0x01, 0x0a, 0x13, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, + 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x41, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, + 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x1c, 0x0a, 0x09, + 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x46, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x46, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x11, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x93, 0x01, 0x0a, 0x13, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, - 0x0a, 0x0a, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0a, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, - 0x12, 0x1c, 0x0a, 0x09, 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x09, 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, - 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x8e, 0x01, 0x0a, - 0x11, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x27, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, - 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, - 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xd3, 0x09, - 0x0a, 0x12, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x12, 0x5d, 0x0a, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, - 0x74, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, - 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, - 0x73, 0x12, 0x38, 0x0a, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x52, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x28, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, - 0x41, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, - 0x3c, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, + 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xd3, 0x09, 0x0a, 0x12, 0x42, 0x75, + 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, + 0x65, 0x66, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x5d, + 0x0a, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x46, 0x72, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, + 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x12, 0x38, 0x0a, + 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x09, 0x45, 0x78, + 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x28, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, - 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, - 0x66, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, - 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x4b, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x0b, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, - 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, - 0x1e, 0x0a, 0x0a, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0a, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x32, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x74, 0x72, - 0x61, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x6e, - 0x75, 0x6d, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x0f, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, - 0x65, 0x70, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6e, 0x75, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x53, - 0x74, 0x65, 0x70, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x54, - 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x6e, 0x75, 0x6d, - 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x11, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x6e, 0x75, 0x6d, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, - 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, 0x42, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0d, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x6e, - 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x40, 0x0a, - 0x12, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x43, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5d, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, + 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3c, 0x0a, 0x0b, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, 0x04, 0x6c, 0x6f, 0x67, + 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x66, 0x0a, 0x10, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, + 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x45, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x4b, + 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, + 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x47, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x6e, 0x75, 0x6d, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0e, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, + 0x24, 0x0a, 0x0d, 0x6e, 0x75, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x74, 0x65, 0x70, 0x73, + 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, + 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x43, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x11, 0x6e, 0x75, 0x6d, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, + 0x65, 0x70, 0x73, 0x12, 0x42, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, + 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, 0x75, + 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x46, 0x72, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x43, 0x0a, 0x15, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x79, 0x0a, 0x19, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, - 0x65, 0x66, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x06, 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, 0x1c, - 0x0a, 0x1a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, - 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xe8, 0x01, 0x0a, - 0x0a, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, - 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x4f, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc1, 0x02, 0x0a, 0x0f, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x10, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x40, 0x0a, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x1a, 0x58, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, - 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x01, 0x0a, 0x08, - 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, - 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x05, 0x41, 0x74, 0x74, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, - 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0x5d, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x79, 0x0a, 0x19, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x16, + 0x0a, 0x06, 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, + 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1a, + 0x0a, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, 0x1c, 0x0a, 0x1a, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xe8, 0x01, 0x0a, 0x0a, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, + 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x12, 0x4f, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x2a, 0x3f, 0x0a, 0x15, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, - 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, - 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x4d, - 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, - 0x45, 0x44, 0x10, 0x02, 0x32, 0x89, 0x06, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x12, 0x54, 0x0a, 0x09, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x22, 0x2e, + 0x02, 0x38, 0x01, 0x22, 0xc1, 0x02, 0x0a, 0x0f, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, + 0x10, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x12, 0x40, 0x0a, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x58, 0x0a, + 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, + 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x01, 0x0a, 0x08, 0x45, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x41, 0x74, 0x74, 0x72, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x72, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, + 0x41, 0x74, 0x74, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, + 0x3f, 0x0a, 0x15, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x41, 0x52, + 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, + 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, + 0x32, 0x89, 0x06, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x54, 0x0a, 0x09, + 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x73, + 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, - 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, - 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x30, 0x01, - 0x12, 0x48, 0x0a, 0x05, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x06, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x07, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, - 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x48, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x72, 0x75, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, + 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x30, 0x01, 0x12, 0x48, 0x0a, 0x05, + 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x28, 0x01, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x73, 0x12, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x45, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x62, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x2e, 0x6d, - 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x62, 0x0a, 0x12, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, - 0x79, 0x12, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, - 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, - 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x30, 0x01, 0x12, - 0x6f, 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, - 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x2b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, - 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, - 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, - 0x6f, 0x62, 0x79, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2f, 0x61, 0x70, 0x69, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x3b, 0x6d, 0x6f, 0x62, 0x79, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x5f, - 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x30, 0x01, 0x12, 0x6f, 0x0a, 0x12, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, + 0x79, 0x12, 0x2b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, + 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x6f, 0x62, 0x79, 0x2f, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x3b, 0x6d, 0x6f, + 0x62, 0x79, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x5f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/services/control/control.proto b/api/services/control/control.proto index 24816d05cd8d..3e47752ff2f9 100644 --- a/api/services/control/control.proto +++ b/api/services/control/control.proto @@ -36,6 +36,7 @@ message PruneRequest { message DiskUsageRequest { repeated string filter = 1; + int64 ageLimit = 2; } message DiskUsageResponse { diff --git a/api/services/control/control_vtproto.pb.go b/api/services/control/control_vtproto.pb.go index e220f1bd5557..1d72ca07e501 100644 --- a/api/services/control/control_vtproto.pb.go +++ b/api/services/control/control_vtproto.pb.go @@ -56,6 +56,7 @@ func (m *DiskUsageRequest) CloneVT() *DiskUsageRequest { return (*DiskUsageRequest)(nil) } r := new(DiskUsageRequest) + r.AgeLimit = m.AgeLimit if rhs := m.Filter; rhs != nil { tmpContainer := make([]string, len(rhs)) copy(tmpContainer, rhs) @@ -831,6 +832,9 @@ func (this *DiskUsageRequest) EqualVT(that *DiskUsageRequest) bool { return false } } + if this.AgeLimit != that.AgeLimit { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -2019,6 +2023,11 @@ func (m *DiskUsageRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.AgeLimit != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AgeLimit)) + i-- + dAtA[i] = 0x10 + } if len(m.Filter) > 0 { for iNdEx := len(m.Filter) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Filter[iNdEx]) @@ -3991,6 +4000,9 @@ func (m *DiskUsageRequest) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.AgeLimit != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AgeLimit)) + } n += len(m.unknownFields) return n } @@ -5006,6 +5018,25 @@ func (m *DiskUsageRequest) UnmarshalVT(dAtA []byte) error { } m.Filter = append(m.Filter, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AgeLimit", wireType) + } + m.AgeLimit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AgeLimit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/cache/blobs.go b/cache/blobs.go index 09836772d08a..9bafb8f1c2e2 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -100,7 +100,11 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool return nil, nil } if !createIfNeeded { - return nil, errors.WithStack(ErrNoBlobs) + bklog.G(ctx).Warnf( + "computeBlobChain: ErrNoBlobs (createIfNeeded=false) ref=%s kind=%s blobOnly=%t snapshotID=%s compression=%s", + sr.ID(), sr.kind(), sr.getBlobOnly(), sr.getSnapshotID(), comp.Type, + ) + return nil, errors.Wrapf(ErrNoBlobs, "ref %s (kind=%s, snapshotID=%s, blobOnly=%t)", sr.ID(), sr.kind(), sr.getSnapshotID(), sr.getBlobOnly()) } l, ctx, err := leaseutil.NewLease(ctx, sr.cm.LeaseManager, leaseutil.MakeTemporary) diff --git a/cache/manager.go b/cache/manager.go index 63c676a19540..d8bd58e88503 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "maps" + "os" + "path/filepath" "slices" "strings" "sync" @@ -125,6 +127,8 @@ func NewManager(opt ManagerOpt) (Manager, error) { return nil, err } + cm.cleanupParallelExtractDirs() + p, err := newSharableMountPool(opt.MountPoolRoot) if err != nil { return nil, err @@ -207,9 +211,14 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, if p != nil { releaseParent = true } + existingImageRefs := ref.getImageRefs() if err := setImageRefMetadata(ref.cacheMetadata, opts...); err != nil { return nil, errors.Wrapf(err, "failed to append image ref metadata to ref %s", ref.ID()) } + bklog.G(ctx).Infof( + "DEDUP-HIT blobchain=%s reusingRef=%s blobDigest=%s existingImageRefs=%v newImageRefs=%v", + blobChainID, ref.ID(), ref.getBlob(), existingImageRefs, ref.getImageRefs(), + ) return ref, nil } @@ -340,6 +349,25 @@ func (cm *cacheManager) init(ctx context.Context) error { return nil } +// cleanupParallelExtractDirs removes any leftover temp directories from +// parallel layer extraction that weren't cleaned up (e.g. due to a crash). +func (cm *cacheManager) cleanupParallelExtractDirs() { + entries, err := os.ReadDir(cm.root) + if err != nil { + return + } + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), "buildkit-parallel-extract-") { + p := filepath.Join(cm.root, e.Name()) + if err := os.RemoveAll(p); err != nil { + bklog.G(context.TODO()).Warnf("failed to clean up parallel extract temp dir %s: %v", p, err) + } else { + bklog.G(context.TODO()).Infof("cleaned up leftover parallel extract temp dir %s", p) + } + } + } +} + // IdentityMapping returns the userns remapping used for refs func (cm *cacheManager) IdentityMapping() *user.IdentityMapping { return cm.Snapshotter.IdentityMapping() @@ -1452,6 +1480,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) if err := cm.markShared(m); err != nil { return nil, err } + cutOff := time.Now().Add(-opt.AgeLimit) var du []*client.UsageInfo for id, cr := range m { @@ -1468,9 +1497,15 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) RecordType: cr.recordType, Shared: cr.shared, } - if filter.Match(adaptUsageInfo(c)) { - du = append(du, c) + if !filter.Match(adaptUsageInfo(c)) { + continue + } + if opt.AgeLimit > 0 { + if c.LastUsedAt != nil && c.LastUsedAt.After(cutOff) { + continue + } } + du = append(du, c) } eg, ctx := errgroup.WithContext(ctx) diff --git a/cache/refs.go b/cache/refs.go index 3bd534854ae0..7a6fff9c998c 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "time" @@ -48,6 +49,51 @@ import ( var additionalAnnotations = append(append(compression.EStargzAnnotations, obdlabel.OverlayBDAnnotations...), labels.LabelUncompressed) +func parallelExtractEnabled() bool { + v, _ := strconv.ParseBool(os.Getenv("BUILDKIT_PARALLEL_EXTRACT")) + return v +} + +// extractFSPath extracts the writable fs directory path from snapshot mounts. +// For overlay, this is the upperdir. For bind mounts (base layer), it's the Source. +func extractFSPath(mounts []mount.Mount) string { + if len(mounts) == 0 { + return "" + } + m := mounts[0] + if m.Type == "bind" { + return m.Source + } + for _, opt := range m.Options { + if strings.HasPrefix(opt, "upperdir=") { + return strings.TrimPrefix(opt, "upperdir=") + } + } + return "" +} + +// isLinearLayerChain reports whether chain is a pure linear Layer/BaseLayer +// stack with no Merge or Diff refs. Parallel extract is only safe for such +// chains because Phase 2's parentID tracking assumes each entry's overlay +// parent is exactly the previous entry in the chain. +func isLinearLayerChain(chain []*immutableRef) bool { + for i, ref := range chain { + switch ref.kind() { + case BaseLayer: + if i != 0 { + return false + } + case Layer: + if i == 0 || ref.layerParent != chain[i-1] { + return false + } + default: + return false + } + } + return true +} + // Ref is a reference to cacheable objects. type Ref interface { Mountable @@ -221,6 +267,21 @@ const ( Diff ) +func (k refKind) String() string { + switch k { + case BaseLayer: + return "BaseLayer" + case Layer: + return "Layer" + case Merge: + return "Merge" + case Diff: + return "Diff" + default: + return fmt.Sprintf("refKind(%d)", int(k)) + } +} + func (cr *cacheRecord) kind() refKind { if len(cr.mergeParents) > 0 { return Merge @@ -1010,6 +1071,7 @@ func (sr *immutableRef) ensureLocalContentBlob(ctx context.Context, s session.Gr func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr error) { if (sr.kind() == Layer || sr.kind() == BaseLayer) && !sr.getBlobOnly() { + bklog.G(ctx).Infof("Extract: skipping extract for ref %s", sr.ID()) return nil } @@ -1029,6 +1091,23 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } } + if parallelExtractEnabled() && sr.cm.Snapshotter.Name() == "overlayfs" { + chain := sr.layerChain() + if isLinearLayerChain(chain) { + var needsExtract []*immutableRef + for _, ref := range chain { + if ref.getBlobOnly() { + needsExtract = append(needsExtract, ref) + } + } + if len(needsExtract) > 0 { + bklog.G(ctx).Infof("parallel extract: extracting %d/%d layers in parallel", len(needsExtract), len(chain)) + return sr.parallelExtractLayers(ctx, chain, needsExtract, s) + } + return nil + } + } + return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) } @@ -1343,6 +1422,15 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr } dh := dhs[desc.Digest] + dhRef := "" + if dh != nil { + dhRef = dh.Ref + } + bklog.G(ctx).Infof( + "UNLAZY-LAYER ref=%s digest=%s ensureContentStore=%t dhFound=%t dh.Ref=%q dhsLen=%d imageRefs=%v", + sr.ID(), desc.Digest, ensureContentStore, dh != nil, dhRef, len(dhs), sr.getImageRefs(), + ) + eg.Go(func() error { // unlazies if needed, otherwise a no-op return lazyRefProvider{ @@ -1414,6 +1502,172 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr return nil } +// parallelExtractLayers extracts multiple layers in parallel by decompressing +// into temp directories first, then doing the sequential Prepare→Commit chain +// with pre-populated content. This avoids the sequential Apply bottleneck where +// each layer must wait for its parent to be fully extracted before starting. +// +// Only used for the overlay snapshotter when BUILDKIT_PARALLEL_EXTRACT=1. +func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immutableRef, needsExtract []*immutableRef, s session.Group) (rerr error) { + if sr.cm.Applier == nil { + return errors.New("parallel extract requires an applier") + } + + if _, ok := leases.FromContext(ctx); !ok { + leaseCtx, done, err := leaseutil.WithLease(ctx, sr.cm.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return err + } + defer done(leaseCtx) + ctx = leaseCtx + } + + sp, ctx := tracing.StartSpan(ctx, "parallel-extract", trace.WithAttributes( + attribute.Int("layers.total", len(chain)), + attribute.Int("layers.extract", len(needsExtract)), + )) + defer sp.End() + + // Phase 1: parallel download + decompress into temp dirs. + // A parent temp dir is created under cm.root to ensure it's on the same + // filesystem as the snapshotter (avoiding EXDEV from os.Rename in Phase 2) + // and to simplify cleanup. + parentTmpDir, err := os.MkdirTemp(sr.cm.root, "buildkit-parallel-extract-") + if err != nil { + return errors.Wrap(err, "parallel extract: failed to create temp dir") + } + defer os.RemoveAll(parentTmpDir) + + type extractResult struct { + tmpDir string + desc ocispecs.Descriptor + } + extractResults := make(map[string]*extractResult, len(needsExtract)) + var mu sync.Mutex + + eg, egctx := errgroup.WithContext(ctx) + for _, ref := range needsExtract { + ref := ref + eg.Go(func() (rerr error) { + dhs := ref.descHandlers + desc, err := ref.ociDesc(egctx, dhs, true) + if err != nil { + return err + } + dh := dhs[desc.Digest] + + if err := (lazyRefProvider{ref: ref, desc: desc, dh: dh, session: s}).Unlazy(egctx); err != nil { + return err + } + + pg := ref.progress + if pg == nil && dh != nil { + pg = dh.Progress + } + if pg != nil { + _, stopProgress := pg.Start(egctx) + defer stopProgress(rerr) + statusDone := pg.Status("extracting "+desc.Digest.String(), "extracting") + defer statusDone() + } + + tmpDir, err := os.MkdirTemp(parentTmpDir, ref.ID()+"-") + if err != nil { + return err + } + fsDir := filepath.Join(tmpDir, "fs") + if err := os.Mkdir(fsDir, 0755); err != nil { + return err + } + + mounts := []mount.Mount{{ + Source: fsDir, + Type: "bind", + Options: []string{"rw", "rbind"}, + }} + if _, err := ref.cm.Applier.Apply(egctx, desc, mounts); err != nil { + return err + } + + mu.Lock() + extractResults[ref.ID()] = &extractResult{tmpDir: tmpDir, desc: desc} + mu.Unlock() + return nil + }) + } + + if err := eg.Wait(); err != nil { + return err + } + + // Phase 2: sequential Prepare → swap content → Commit. + // Walk the full chain in order. Layers already extracted are skipped + // (just advance parentID). Layers that were parallel-extracted get their + // temp dir content swapped into the snapshotter path. + parentID := "" + for _, ref := range chain { + if !ref.getBlobOnly() { + parentID = ref.getSnapshotID() + continue + } + + result := extractResults[ref.ID()] + if result == nil { + return errors.Errorf("parallel extract: missing result for layer %s", ref.ID()) + } + + key := fmt.Sprintf("extract-%s %s", identity.NewID(), ref.getChainID()) + if err := ref.cm.Snapshotter.Prepare(ctx, key, parentID); err != nil { + return err + } + + mountable, err := ref.cm.Snapshotter.Mounts(ctx, key) + if err != nil { + return err + } + mounts, unmount, err := mountable.Mount() + if err != nil { + return err + } + + fsPath := extractFSPath(mounts) + if err := unmount(); err != nil { + return err + } + + if fsPath == "" { + return errors.Errorf("parallel extract: could not determine fs path from mounts for layer %s", ref.ID()) + } + + entries, err := os.ReadDir(fsPath) + if err != nil { + return errors.Wrapf(err, "parallel extract: failed to read fs dir") + } + if len(entries) > 0 { + return errors.Errorf("parallel extract: expected empty fs dir but found %d entries at %s", len(entries), fsPath) + } + if err := os.Remove(fsPath); err != nil { + return errors.Wrapf(err, "parallel extract: failed to remove empty fs dir") + } + if err := os.Rename(filepath.Join(result.tmpDir, "fs"), fsPath); err != nil { + return errors.Wrapf(err, "parallel extract: failed to rename extracted content into snapshot path") + } + + if err := ref.cm.Snapshotter.Commit(ctx, ref.getSnapshotID(), key); err != nil { + if !errors.Is(err, cerrdefs.ErrAlreadyExists) { + return err + } + } + ref.queueBlobOnly(false) + ref.queueSize(sizeUnknown) + if err := ref.commitMetadata(); err != nil { + return err + } + parentID = ref.getSnapshotID() + } + return nil +} + func (sr *immutableRef) Release(ctx context.Context) error { sr.cm.mu.Lock() defer sr.cm.mu.Unlock() diff --git a/cache/refs_test.go b/cache/refs_test.go new file mode 100644 index 000000000000..b1195644e0a6 --- /dev/null +++ b/cache/refs_test.go @@ -0,0 +1,88 @@ +package cache + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestImmutableRef(kind refKind, parent *immutableRef) *immutableRef { + ref := &immutableRef{ + cacheRecord: &cacheRecord{ + mu: &sync.Mutex{}, + }, + } + switch kind { + case Layer: + ref.layerParent = parent + case Merge: + ref.mergeParents = []*immutableRef{parent} + case Diff: + ref.diffParents = &diffParents{lower: parent} + } + return ref +} + +func TestIsLinearLayerChain(t *testing.T) { + base := newTestImmutableRef(BaseLayer, nil) + layer1 := newTestImmutableRef(Layer, base) + layer2 := newTestImmutableRef(Layer, layer1) + merge := newTestImmutableRef(Merge, base) + diff := newTestImmutableRef(Diff, base) + + tests := []struct { + name string + chain []*immutableRef + want bool + }{ + {"empty chain", nil, true}, + {"single base layer", []*immutableRef{base}, true}, + {"base + layer", []*immutableRef{base, layer1}, true}, + {"base + layer + layer", []*immutableRef{base, layer1, layer2}, true}, + {"single layer (no base)", []*immutableRef{layer1}, false}, + {"base in wrong position", []*immutableRef{layer1, base}, false}, + {"merge ref", []*immutableRef{base, merge}, false}, + {"diff ref", []*immutableRef{base, diff}, false}, + {"broken parent link", []*immutableRef{base, layer2}, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isLinearLayerChain(tt.chain) + require.Equal(t, tt.want, got) + }) + } +} + +func TestCleanupParallelExtractDirs(t *testing.T) { + root := t.TempDir() + + // Create dirs that should be cleaned up + require.NoError(t, os.Mkdir(filepath.Join(root, "buildkit-parallel-extract-abc123"), 0755)) + require.NoError(t, os.Mkdir(filepath.Join(root, "buildkit-parallel-extract-xyz789"), 0755)) + // Create a file inside one to verify RemoveAll works + require.NoError(t, os.WriteFile(filepath.Join(root, "buildkit-parallel-extract-abc123", "data"), []byte("test"), 0644)) + + // Create dirs that should NOT be cleaned up + require.NoError(t, os.Mkdir(filepath.Join(root, "snapshots"), 0755)) + require.NoError(t, os.Mkdir(filepath.Join(root, "other-dir"), 0755)) + + cm := &cacheManager{root: root} + cm.cleanupParallelExtractDirs() + + entries, err := os.ReadDir(root) + require.NoError(t, err) + + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + + require.NotContains(t, names, "buildkit-parallel-extract-abc123") + require.NotContains(t, names, "buildkit-parallel-extract-xyz789") + require.Contains(t, names, "snapshots") + require.Contains(t, names, "other-dir") +} diff --git a/cache/remote.go b/cache/remote.go index c0e3cc6b48c2..ff58869de0ed 100644 --- a/cache/remote.go +++ b/cache/remote.go @@ -338,6 +338,10 @@ func (p lazyRefProvider) Unlazy(ctx context.Context) error { if p.dh == nil { // shouldn't happen, if you have a lazy immutable ref it already should be validated // that descriptor handlers exist for it + bklog.G(ctx).Warnf( + "UNLAZY-NIL-DH ref=%s digest=%s imageRefs=%v", + p.ref.ID(), p.desc.Digest, p.ref.getImageRefs(), + ) return struct{}{}, errors.New("unexpected nil descriptor handler") } @@ -350,11 +354,19 @@ func (p lazyRefProvider) Unlazy(ctx context.Context) error { // For now, just pull down the whole content and then return a ReaderAt from the local content // store. If efficient partial reads are desired in the future, something more like a "tee" // that caches remote partial reads to a local store may need to replace this. + bklog.G(ctx).Infof( + "UNLAZY-FETCH ref=%s digest=%s mediaType=%s size=%d dh.Ref=%q imageRefs=%v", + p.ref.ID(), p.desc.Digest, p.desc.MediaType, p.desc.Size, p.dh.Ref, p.ref.getImageRefs(), + ) err := contentutil.Copy(ctx, p.ref.cm.ContentStore, &pullprogress.ProviderWithProgress{ Provider: p.dh.Provider(p.session), Manager: p.ref.cm.ContentStore, }, p.desc, p.dh.Ref, logs.LoggerFromContext(ctx)) if err != nil { + bklog.G(ctx).Warnf( + "UNLAZY-FETCH-ERR ref=%s digest=%s dh.Ref=%q imageRefs=%v err=%v", + p.ref.ID(), p.desc.Digest, p.dh.Ref, p.ref.getImageRefs(), err, + ) return struct{}{}, err } diff --git a/client/client_test.go b/client/client_test.go index 83cc70dac822..f6c67d5eddae 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -113,6 +113,8 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testBuildHTTPSourceAuthHeaderSecret, testBuildHTTPSourceHeader, testBuildPushAndValidate, + testEagerExportCompress, + testEagerExportPush, testBuildExportWithUncompressed, testBuildExportScratch, testResolveAndHosts, @@ -5080,6 +5082,135 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) { require.False(t, ok) } +func testEagerExportCompress(t *testing.T, sb integration.Sandbox) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + st = busybox.Run(llb.Shlex(`sh -c "echo layer1 > /file1"`), llb.Dir("/wd")).AddMount("/wd", st) + st = busybox.Run(llb.Shlex(`sh -c "echo layer2 > /file2"`), llb.Dir("/wd")).AddMount("/wd", st) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testeagercompress:latest" + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + "eager-export": "compress", + }, + }, + }, + }, nil) + require.NoError(t, err) + + // Verify the pushed image is pullable and has the right content. + pullSt := llb.Image(target) + def, err = pullSt.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "file1")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer1") + + dt, err = os.ReadFile(filepath.Join(destDir, "file2")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer2") +} + +func testEagerExportPush(t *testing.T, sb integration.Sandbox) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + st = busybox.Run(llb.Shlex(`sh -c "echo layer1 > /file1"`), llb.Dir("/wd")).AddMount("/wd", st) + st = busybox.Run(llb.Shlex(`sh -c "echo layer2 > /file2"`), llb.Dir("/wd")).AddMount("/wd", st) + st = busybox.Run(llb.Shlex(`sh -c "echo layer3 > /file3"`), llb.Dir("/wd")).AddMount("/wd", st) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testeagerpush:latest" + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + "eager-export": "push", + }, + }, + }, + }, nil) + require.NoError(t, err) + + // Verify the pushed image is pullable and has the right content. + pullSt := llb.Image(target) + def, err = pullSt.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "file1")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer1") + + dt, err = os.ReadFile(filepath.Join(destDir, "file2")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer2") + + dt, err = os.ReadFile(filepath.Join(destDir, "file3")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer3") +} + func testStargzLazyRegistryCacheImportExport(t *testing.T, sb integration.Sandbox) { workers.CheckFeatureCompat(t, sb, workers.FeatureCacheExport, diff --git a/client/diskusage.go b/client/diskusage.go index a690f4818168..2471ac297cff 100644 --- a/client/diskusage.go +++ b/client/diskusage.go @@ -31,7 +31,7 @@ func (c *Client) DiskUsage(ctx context.Context, opts ...DiskUsageOption) ([]*Usa o.SetDiskUsageOption(info) } - req := &controlapi.DiskUsageRequest{Filter: info.Filter} + req := &controlapi.DiskUsageRequest{Filter: info.Filter, AgeLimit: int64(info.AgeLimit)} resp, err := c.ControlClient().DiskUsage(ctx, req) if err != nil { return nil, errors.Wrap(err, "failed to call diskusage") @@ -72,7 +72,8 @@ type DiskUsageOption interface { } type DiskUsageInfo struct { - Filter []string + Filter []string + AgeLimit time.Duration } type UsageRecordType string @@ -85,3 +86,15 @@ const ( UsageRecordTypeCacheMount UsageRecordType = "exec.cachemount" UsageRecordTypeRegular UsageRecordType = "regular" ) + +type diskUsageOptionFunc func(*DiskUsageInfo) + +func (f diskUsageOptionFunc) SetDiskUsageOption(info *DiskUsageInfo) { + f(info) +} + +func WithAgeLimit(age time.Duration) DiskUsageOption { + return diskUsageOptionFunc(func(info *DiskUsageInfo) { + info.AgeLimit = age + }) +} diff --git a/client/llb/source.go b/client/llb/source.go index cb1f47206de3..d6eba0758981 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -322,6 +322,12 @@ func Git(url, ref string, opts ...GitOption) State { addCap(&gi.Constraints, pb.CapSourceGitMountSSHSock) } + checksum := gi.Checksum + if checksum != "" { + attrs[pb.AttrGitChecksum] = checksum + addCap(&gi.Constraints, pb.CapSourceGitChecksum) + } + addCap(&gi.Constraints, pb.CapSourceGit) source := NewSource("git://"+id, attrs, gi.Constraints) @@ -345,6 +351,7 @@ type GitInfo struct { addAuthCap bool KnownSSHHosts string MountSSHSock string + Checksum string } func KeepGitDir() GitOption { @@ -373,6 +380,12 @@ func MountSSHSock(sshID string) GitOption { }) } +func GitChecksum(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.Checksum = v + }) +} + // AuthOption can be used with either HTTP or Git sources. type AuthOption interface { GitOption diff --git a/cmd/buildctl/common/trace.go b/cmd/buildctl/common/trace.go index 47e6ad5e9682..3e40ccb78cbb 100644 --- a/cmd/buildctl/common/trace.go +++ b/cmd/buildctl/common/trace.go @@ -2,7 +2,9 @@ package common import ( "context" + "errors" "os" + "time" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/tracing/delegated" @@ -69,7 +71,12 @@ func AttachAppContext(app *cli.App) error { if span != nil { span.End() } - return tp.Shutdown(context.TODO()) + ctx, cancel := context.WithTimeoutCause(context.Background(), 5*time.Second, errors.New("tracer shutdown timeout")) + defer cancel() + // Trace export is best-effort; don't fail the build if the + // collector is unreachable or slow. + _ = tp.Shutdown(ctx) + return nil } return nil } diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index d3bf198d8fbf..7b94462c0492 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -13,6 +13,7 @@ import ( "slices" "strconv" "strings" + "time" "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/containerd/containerd/v2/defaults" @@ -20,7 +21,6 @@ import ( "github.com/containerd/platforms" sddaemon "github.com/coreos/go-systemd/v22/daemon" "github.com/gofrs/flock" - "github.com/hashicorp/go-multierror" "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/cache/remotecache/azblob" "github.com/moby/buildkit/cache/remotecache/gha" @@ -399,12 +399,15 @@ func main() { } app.After = func(_ *cli.Context) (err error) { + ctx, cancel := context.WithTimeoutCause(context.Background(), 5*time.Second, errors.New("telemetry shutdown timeout")) + defer cancel() for _, c := range closers { - if e := c(context.TODO()); e != nil { - err = multierror.Append(err, e) - } + // Closers here are telemetry providers (TracerProvider, + // MeterProvider). Failures are non-fatal — an unreachable + // collector should not prevent clean daemon shutdown. + _ = c(ctx) } - return err + return nil } profiler.Attach(app) diff --git a/control/control.go b/control/control.go index dc73be3da02b..39abe468f3cb 100644 --- a/control/control.go +++ b/control/control.go @@ -51,9 +51,7 @@ import ( tracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -167,7 +165,8 @@ func (c *Controller) DiskUsage(ctx context.Context, r *controlapi.DiskUsageReque } for _, w := range workers { du, err := w.DiskUsage(ctx, client.DiskUsageInfo{ - Filter: r.Filter, + Filter: r.Filter, + AgeLimit: time.Duration(r.AgeLimit), }) if err != nil { return nil, err @@ -288,11 +287,12 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr func (c *Controller) Export(ctx context.Context, req *tracev1.ExportTraceServiceRequest) (*tracev1.ExportTraceServiceResponse, error) { if c.opt.TraceCollector == nil { - return nil, status.Errorf(codes.Unavailable, "trace collector not configured") + return &tracev1.ExportTraceServiceResponse{}, nil } - err := c.opt.TraceCollector.ExportSpans(ctx, transform.Spans(req.GetResourceSpans())) - if err != nil { - return nil, err + ctx, cancel := context.WithTimeoutCause(ctx, 5*time.Second, errors.New("trace export timeout")) + defer cancel() + if err := c.opt.TraceCollector.ExportSpans(ctx, transform.Spans(req.GetResourceSpans())); err != nil { + bklog.G(ctx).WithError(err).Debug("trace export to collector failed") } return &tracev1.ExportTraceServiceResponse{}, nil } @@ -510,6 +510,16 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* procs = append(procs, proc.ProvenanceProcessor(attrs)) } + eagerExport, eagerPushCfg, err := resolveEagerExport(req.Exporters, expis) + if err != nil { + return nil, err + } + + pushRegCfg, err := resolvePushRegistryConfig(req.Exporters, expis) + if err != nil { + return nil, err + } + resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{ Frontend: req.Frontend, Definition: req.Definition, @@ -520,6 +530,9 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* Exporters: expis, CacheExporters: cacheExporters, EnableSessionExporter: req.EnableSessionExporter, + EagerExport: eagerExport, + EagerPushConfig: eagerPushCfg, + PushRegistryConfig: pushRegCfg, }, entitlementsFromPB(req.Entitlements), procs, req.Internal, req.SourcePolicy) if err != nil { return nil, err @@ -529,6 +542,127 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* }, nil } +// resolveEagerExport checks whether the exporter requests eager export and +// validates the configuration. Returns EagerExportNone if the flag is not set. +func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.ExporterInstance) (llbsolver.EagerExportMode, *exporter.EagerPushConfig, error) { + if len(rawExporters) != 1 { + for _, ex := range rawExporters { + if _, ok := ex.Attrs[string(exptypes.OptKeyEagerExport)]; ok { + return 0, nil, errors.Errorf("eager-export requires exactly one exporter, got %d", len(rawExporters)) + } + } + return llbsolver.EagerExportNone, nil, nil + } + + ex := rawExporters[0] + v, ok := ex.Attrs[string(exptypes.OptKeyEagerExport)] + if !ok { + return llbsolver.EagerExportNone, nil, nil + } + + var mode llbsolver.EagerExportMode + switch v { + case exptypes.OptValEagerExportCompress: + mode = llbsolver.EagerExportCompress + case exptypes.OptValEagerExportPush: + mode = llbsolver.EagerExportPush + default: + return 0, nil, errors.Errorf("invalid eager-export value %q", v) + } + + if expis[0].Type() != client.ExporterImage { + return 0, nil, errors.Errorf("eager-export requires image exporter, got %q", expis[0].Type()) + } + + var pushCfg *exporter.EagerPushConfig + if mode == llbsolver.EagerExportPush { + push := ex.Attrs[string(exptypes.OptKeyPush)] + if push != "true" { + return 0, nil, errors.New("eager-export=push requires push=true") + } + provider, ok := expis[0].(exporter.EagerExportProvider) + if !ok { + return 0, nil, errors.New("eager-export=push: exporter does not support eager push") + } + pushCfg = provider.EagerPushConfig() + if pushCfg == nil { + return 0, nil, errors.New("eager-export=push requires a single image name (not empty, wildcard, or comma-separated)") + } + } + return mode, pushCfg, nil +} + +// resolvePushRegistryConfig scans all exporters for a single image exporter +// with push=true and prefer-push-registry=true, and returns its push registry +// configuration. Other exporter types (e.g. oci tarball, local) are ignored. +// Returns an error if multiple image exporters have the flag set (ambiguous +// destination). +func resolvePushRegistryConfig(rawExporters []*controlapi.Exporter, expis []exporter.ExporterInstance) (*exporter.EagerPushConfig, error) { + var found *exporter.EagerPushConfig + for i, ex := range rawExporters { + cfg, err := exporterPushRegistryConfig(ex, expis[i]) + if err != nil { + return nil, err + } + if cfg == nil { + continue + } + if found != nil { + return nil, errors.New("prefer-push-registry set on multiple image exporters; destination is ambiguous") + } + found = cfg + } + return found, nil +} + +// exporterPushRegistryConfig returns the push registry config for a single +// exporter if it has prefer-push-registry=true, or nil if the flag isn't set +// (or the exporter doesn't support eager push config). Validation errors are +// returned when the flag is set but the exporter is misconfigured. +func exporterPushRegistryConfig(ex *controlapi.Exporter, expi exporter.ExporterInstance) (*exporter.EagerPushConfig, error) { + enabled, err := parseBoolAttr(ex.Attrs, string(exptypes.OptKeyPreferPushRegistry)) + if err != nil || !enabled { + return nil, err + } + if expi.Type() != client.ExporterImage { + return nil, errors.Errorf("prefer-push-registry requires image exporter, got %q", expi.Type()) + } + push, err := parseBoolAttr(ex.Attrs, string(exptypes.OptKeyPush)) + if err != nil { + return nil, err + } + if !push { + return nil, errors.New("prefer-push-registry requires push=true") + } + provider, ok := expi.(exporter.EagerExportProvider) + if !ok { + return nil, nil + } + cfg := provider.EagerPushConfig() + if cfg == nil { + return nil, errors.New("prefer-push-registry requires a single image name") + } + return cfg, nil +} + +// parseBoolAttr returns true if the attr is present and evaluates to a truthy +// value. Follows the same convention as the image exporter: empty value means +// true (for --output type=image,flag shorthand). +func parseBoolAttr(attrs map[string]string, key string) (bool, error) { + v, ok := attrs[key] + if !ok { + return false, nil + } + if v == "" { + return true, nil + } + b, err := strconv.ParseBool(v) + if err != nil { + return false, errors.Wrapf(err, "non-bool value specified for %s", key) + } + return b, nil +} + func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Control_StatusServer) error { if err := sendTimestampHeader(stream); err != nil { return err @@ -645,7 +779,7 @@ func (c *Controller) gc() { } <-done if size > 0 { - bklog.G(ctx).Debugf("gc cleaned up %d bytes", size) + bklog.G(ctx).Infof("gc cleaned up %d bytes", size) go c.throttledReleaseUnreferenced() } } diff --git a/control/control_test.go b/control/control_test.go index 504c94f61602..1d001da42739 100644 --- a/control/control_test.go +++ b/control/control_test.go @@ -1,12 +1,43 @@ package control import ( + "context" "testing" controlapi "github.com/moby/buildkit/api/services/control" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/solver/llbsolver" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// mockExporterInstance is a minimal ExporterInstance for testing resolveEagerExport. +type mockExporterInstance struct { + typ string + attrs map[string]string +} + +func (m *mockExporterInstance) ID() int { return 0 } +func (m *mockExporterInstance) Name() string { return "mock" } +func (m *mockExporterInstance) Config() *exporter.Config { return exporter.NewConfig() } +func (m *mockExporterInstance) Type() string { return m.typ } +func (m *mockExporterInstance) Attrs() map[string]string { return m.attrs } +func (m *mockExporterInstance) Export(context.Context, *exporter.Source, exptypes.InlineCache, string) (map[string]string, exporter.DescriptorReference, error) { + return nil, nil, nil +} + +// mockEagerExporterInstance also implements EagerExportProvider. +type mockEagerExporterInstance struct { + mockExporterInstance + pushCfg *exporter.EagerPushConfig +} + +func (m *mockEagerExporterInstance) EagerPushConfig() *exporter.EagerPushConfig { + return m.pushCfg +} + func TestDuplicateCacheOptions(t *testing.T) { var testCases = []struct { name string @@ -146,3 +177,150 @@ func TestParseCacheExportIgnoreError(t *testing.T) { }) } } + +func TestResolveEagerExport(t *testing.T) { + imageExporter := &mockEagerExporterInstance{ + mockExporterInstance: mockExporterInstance{typ: client.ExporterImage}, + pushCfg: &exporter.EagerPushConfig{ + TargetName: "docker.io/library/test:latest", + }, + } + + tests := []struct { + name string + exporters []*controlapi.Exporter + instances []exporter.ExporterInstance + wantMode llbsolver.EagerExportMode + wantPushCfg bool + wantErr string + }{ + { + name: "no eager-export attr returns none", + exporters: []*controlapi.Exporter{{Attrs: map[string]string{}}}, + instances: []exporter.ExporterInstance{imageExporter}, + wantMode: llbsolver.EagerExportNone, + }, + { + name: "compress mode", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportCompress, + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantMode: llbsolver.EagerExportCompress, + }, + { + name: "push mode with push=true", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + string(exptypes.OptKeyPush): "true", + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantMode: llbsolver.EagerExportPush, + wantPushCfg: true, + }, + { + name: "push mode without push=true errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantErr: "eager-export=push requires push=true", + }, + { + name: "invalid eager-export value errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): "bogus", + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantErr: "invalid eager-export value", + }, + { + name: "non-image exporter errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportCompress, + }, + }}, + instances: []exporter.ExporterInstance{ + &mockExporterInstance{typ: "local"}, + }, + wantErr: "eager-export requires image exporter", + }, + { + name: "multiple exporters with eager-export errors", + exporters: []*controlapi.Exporter{ + {Attrs: map[string]string{string(exptypes.OptKeyEagerExport): "compress"}}, + {Attrs: map[string]string{}}, + }, + instances: []exporter.ExporterInstance{imageExporter, imageExporter}, + wantErr: "eager-export requires exactly one exporter", + }, + { + name: "multiple exporters without eager-export is fine", + exporters: []*controlapi.Exporter{{Attrs: map[string]string{}}, {Attrs: map[string]string{}}}, + instances: []exporter.ExporterInstance{imageExporter, imageExporter}, + wantMode: llbsolver.EagerExportNone, + }, + { + name: "zero exporters returns none", + exporters: nil, + instances: nil, + wantMode: llbsolver.EagerExportNone, + }, + { + name: "push mode with nil EagerPushConfig errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + string(exptypes.OptKeyPush): "true", + }, + }}, + instances: []exporter.ExporterInstance{ + &mockEagerExporterInstance{ + mockExporterInstance: mockExporterInstance{typ: client.ExporterImage}, + pushCfg: nil, + }, + }, + wantErr: "eager-export=push requires a single image name", + }, + { + name: "push mode with non-EagerExportProvider exporter errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + string(exptypes.OptKeyPush): "true", + }, + }}, + instances: []exporter.ExporterInstance{ + &mockExporterInstance{typ: client.ExporterImage}, + }, + wantErr: "exporter does not support eager push", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, pushCfg, err := resolveEagerExport(tt.exporters, tt.instances) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantMode, mode) + if tt.wantPushCfg { + assert.NotNil(t, pushCfg) + } else { + assert.Nil(t, pushCfg) + } + }) + } +} diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 0dd8255e237c..e1678e17d145 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -27,6 +27,7 @@ import ( "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/leaseutil" @@ -170,6 +171,23 @@ func (e *imageExporter) Resolve(ctx context.Context, id int, opt map[string]stri return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } i.nameCanonical = b + case exptypes.OptKeyEagerExport: + switch v { + case exptypes.OptValEagerExportCompress, exptypes.OptValEagerExportPush: + i.eagerExport = v + default: + return nil, errors.Errorf("invalid value %q for %s, must be \"compress\" or \"push\"", v, k) + } + case exptypes.OptKeyPreferPushRegistry: + if v == "" { + i.preferPushRegistry = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.preferPushRegistry = b default: if i.meta == nil { i.meta = make(map[string][]byte) @@ -195,6 +213,8 @@ type imageExporterInstance struct { nameCanonical bool danglingPrefix string danglingEmptyOnly bool + eagerExport string // "", "compress", or "push" + preferPushRegistry bool meta map[string][]byte } @@ -214,6 +234,20 @@ func (e *imageExporterInstance) Type() string { return client.ExporterImage } +func (e *imageExporterInstance) EagerPushConfig() *exporter.EagerPushConfig { + name := e.opts.ImageName + if name == "" || name == "*" || strings.Contains(name, ",") { + return nil + } + return &exporter.EagerPushConfig{ + TargetName: name, + RegistryHosts: e.opt.RegistryHosts, + Insecure: e.insecure, + ContentStore: e.opt.ImageWriter.ContentStore(), + PreferPushRegistry: e.preferPushRegistry, + } +} + func (e *imageExporterInstance) Attrs() map[string]string { return e.attrs } @@ -226,6 +260,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source maps.Copy(src.Metadata, e.meta) opts := e.opts + opts.EagerExport = e.eagerExport as, _, err := ParseAnnotations(src.Metadata) if err != nil { return nil, nil, err @@ -337,7 +372,10 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source eg.Go(func() error { remotes, err := ref.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) if err != nil { - return err + if errors.Is(err, cache.ErrNoBlobs) { + bklog.G(ctx).Warnf("imageExporter unlazy: ErrNoBlobs for top-level ref=%s", ref.ID()) + } + return errors.Wrapf(err, "imageExporter unlazy: top-level ref %s", ref.ID()) } remote := remotes[0] if unlazier, ok := remote.Provider.(cache.Unlazier); ok { @@ -402,7 +440,10 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou for _, ref := range refs { remotes, err := ref.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) if err != nil { - return err + if errors.Is(err, cache.ErrNoBlobs) { + bklog.G(ctx).Warnf("pushImage: ErrNoBlobs for top-level ref=%s targetName=%s", ref.ID(), targetName) + } + return errors.Wrapf(err, "pushImage: top-level ref %s", ref.ID()) } remote := remotes[0] for _, desc := range remote.Descriptors { @@ -410,7 +451,7 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou addAnnotations(annotations, desc) } } - return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations) + return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, e.eagerExport == exptypes.OptValEagerExportPush, annotations) } func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, s session.Group) (err0 error) { diff --git a/exporter/containerimage/exptypes/keys.go b/exporter/containerimage/exptypes/keys.go index 7cfa8dc024d9..043692304d19 100644 --- a/exporter/containerimage/exptypes/keys.go +++ b/exporter/containerimage/exptypes/keys.go @@ -85,4 +85,25 @@ var ( // Rewrite timestamps in layers to match SOURCE_DATE_EPOCH // Value: bool OptKeyRewriteTimestamp ImageExporterOptKey = "rewrite-timestamp" + + // Eagerly compress (and optionally push) layer blobs during the build, + // rather than waiting for all vertices to complete first. + // Requires a single image exporter with push enabled. + // Value: string + // compress — compress layers as vertices complete; push everything at finalize + // push — compress AND push layer blobs as vertices complete; only push manifest at finalize + OptKeyEagerExport ImageExporterOptKey = "eager-export" + + // When pulling base image layers, check the push (destination) registry + // first and pull from there if the layer exists. Falls back to the origin + // registry transparently. Useful when the push registry is closer than + // the origin (e.g., ECR in the same region vs Docker Hub). + // Requires push=true so the destination registry is known. + // Value: bool + OptKeyPreferPushRegistry ImageExporterOptKey = "prefer-push-registry" +) + +const ( + OptValEagerExportCompress = "compress" + OptValEagerExportPush = "push" ) diff --git a/exporter/containerimage/opts.go b/exporter/containerimage/opts.go index 1ccadbd946d8..0cf4d1862d1d 100644 --- a/exporter/containerimage/opts.go +++ b/exporter/containerimage/opts.go @@ -21,8 +21,9 @@ type ImageCommitOpts struct { Annotations AnnotationsGroup Epoch *time.Time - ForceInlineAttestations bool // force inline attestations to be attached - RewriteTimestamp bool // rewrite timestamps in layers to match the epoch + ForceInlineAttestations bool // force inline attestations to be attached + RewriteTimestamp bool // rewrite timestamps in layers to match the epoch + EagerExport string // "compress" or "push" — layers already compressed by eager pipeline } func (c *ImageCommitOpts) Load(ctx context.Context, opt map[string]string) (map[string]string, error) { diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 63f390a85b49..c40b8917aac8 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -138,7 +138,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session baseImg = &baseImgX } - remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), ref) + remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), opts.EagerExport, ref) if err != nil { return nil, err } @@ -200,7 +200,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session refs = append(refs, r) } - remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), refs...) + remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), opts.EagerExport, refs...) if err != nil { return nil, err } @@ -350,7 +350,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return &idxDesc, nil } -func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefConfig, s session.Group, refs ...cache.ImmutableRef) ([]solver.Remote, error) { +func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefConfig, s session.Group, eagerExport string, refs ...cache.ImmutableRef) ([]solver.Remote, error) { attr := []attribute.KeyValue{ attribute.String("exportLayers.compressionType", refCfg.Compression.Type.String()), attribute.Bool("exportLayers.forceCompression", refCfg.Compression.Force), @@ -360,8 +360,15 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC } span, ctx := tracing.StartSpan(ctx, "export layers", trace.WithAttributes(attr...)) + // When eager export is active, blobs have already been compressed during + // the build. Use createIfNeeded=false to read existing blobs without + // re-compressing. If eager compression somehow missed a layer, this will + // return ErrNoBlobs — that is intentional; the build should fail rather + // than silently fall back. + createIfNeeded := eagerExport == "" + eg, ctx := errgroup.WithContext(ctx) - layersDone := progress.OneOff(ctx, "exporting layers") + layersDone := progress.OneOff(ctx, exportLayersProgressID(eagerExport)) out := make([]solver.Remote, len(refs)) @@ -371,9 +378,12 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC return } eg.Go(func() error { - remotes, err := ref.GetRemotes(ctx, true, refCfg, false, s) + remotes, err := ref.GetRemotes(ctx, createIfNeeded, refCfg, false, s) if err != nil { - return err + if errors.Is(err, cache.ErrNoBlobs) { + bklog.G(ctx).Warnf("exportLayers: ErrNoBlobs for top-level ref=%s eagerExport=%q createIfNeeded=%t", ref.ID(), eagerExport, createIfNeeded) + } + return errors.Wrapf(err, "exportLayers: top-level ref %s", ref.ID()) } remote := remotes[0] out[i] = *remote @@ -387,6 +397,13 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC return out, err } +func exportLayersProgressID(eagerExport string) string { + if eagerExport != "" { + return "loading eagerly compressed layers" + } + return "exporting layers" +} + // rewriteImageLayerWithEpoch rewrites the file timestamps in the layer blob to match the epoch, and returns a new descriptor that points to // the new blob. // diff --git a/exporter/exporter.go b/exporter/exporter.go index c16f174558ba..a4c611742186 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -3,6 +3,8 @@ package exporter import ( "context" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/solver/result" @@ -54,3 +56,19 @@ func NewConfigWithCompression(comp compression.Config) *Config { func (c *Config) Compression() compression.Config { return c.compression } + +// EagerPushConfig holds the registry details needed to push individual layer +// blobs during the build, before the final manifest is assembled. +type EagerPushConfig struct { + TargetName string + RegistryHosts docker.RegistryHosts + Insecure bool + ContentStore content.Store + PreferPushRegistry bool +} + +// EagerExportProvider is an optional interface that ExporterInstances can +// implement to supply configuration for the eager export pipeline. +type EagerExportProvider interface { + EagerPushConfig() *EagerPushConfig +} diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 71ef28a3a2b0..b52c03155baf 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -937,27 +937,21 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { case *instructions.WorkdirCommand: err = dispatchWorkdir(d, c, true, &opt) case *instructions.AddCommand: - var checksum digest.Digest - if c.Checksum != "" { - checksum, err = digest.Parse(c.Checksum) - } - if err == nil { - err = dispatchCopy(d, copyConfig{ - params: c.SourcesAndDest, - excludePatterns: c.ExcludePatterns, - source: opt.buildContext, - isAddCommand: true, - cmdToPrint: c, - chown: c.Chown, - chmod: c.Chmod, - link: c.Link, - keepGitDir: c.KeepGitDir, - checksum: checksum, - location: c.Location(), - ignoreMatcher: opt.dockerIgnoreMatcher, - opt: opt, - }) - } + err = dispatchCopy(d, copyConfig{ + params: c.SourcesAndDest, + excludePatterns: c.ExcludePatterns, + source: opt.buildContext, + isAddCommand: true, + cmdToPrint: c, + chown: c.Chown, + chmod: c.Chmod, + link: c.Link, + keepGitDir: c.KeepGitDir, + checksum: c.Checksum, + location: c.Location(), + ignoreMatcher: opt.dockerIgnoreMatcher, + opt: opt, + }) if err == nil { for _, src := range c.SourcePaths { if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { @@ -1470,8 +1464,8 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if len(cfg.params.SourcePaths) != 1 { return errors.New("checksum can't be specified for multiple sources") } - if !isHTTPSource(cfg.params.SourcePaths[0]) { - return errors.New("checksum can't be specified for non-HTTP(S) sources") + if !isHTTPSource(cfg.params.SourcePaths[0]) && !isGitSource(cfg.params.SourcePaths[0]) { + return errors.New("checksum requires HTTP(S) or Git sources") } } @@ -1519,6 +1513,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if cfg.keepGitDir { gitOptions = append(gitOptions, llb.KeepGitDir()) } + if cfg.checksum != "" { + gitOptions = append(gitOptions, llb.GitChecksum(cfg.checksum)) + } st := llb.Git(gitRef.Remote, commit, gitOptions...) opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: chopt, @@ -1547,7 +1544,15 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { } } - st := llb.HTTP(src, llb.Filename(f), llb.WithCustomName(pgName), llb.Checksum(cfg.checksum), dfCmd(cfg.params)) + var checksum digest.Digest + if cfg.checksum != "" { + checksum, err = digest.Parse(cfg.checksum) + if err != nil { + return err + } + } + + st := llb.HTTP(src, llb.Filename(f), llb.WithCustomName(pgName), llb.Checksum(checksum), dfCmd(cfg.params)) opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: chopt, @@ -1674,7 +1679,7 @@ type copyConfig struct { chmod string link bool keepGitDir bool - checksum digest.Digest + checksum string parents bool location []parser.Range ignoreMatcher *patternmatcher.PatternMatcher @@ -2265,11 +2270,15 @@ func isHTTPSource(src string) bool { if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { return false } + return !isGitSource(src) +} + +func isGitSource(src string) bool { // https://github.com/ORG/REPO.git is a git source, not an http source if gitRef, gitErr := gitutil.ParseGitRef(src); gitRef != nil && gitErr == nil { - return false + return true } - return true + return false } func isEnabledForStage(stage string, value string) bool { diff --git a/frontend/dockerfile/dockerfile_addchecksum_test.go b/frontend/dockerfile/dockerfile_addchecksum_test.go index 050291845168..c2b77ff1f4bd 100644 --- a/frontend/dockerfile/dockerfile_addchecksum_test.go +++ b/frontend/dockerfile/dockerfile_addchecksum_test.go @@ -162,6 +162,6 @@ ADD --checksum=%s foo /tmp/foo dockerui.DefaultLocalNameContext: dir, }, }, nil) - require.Error(t, err, "checksum can't be specified for non-HTTP(S) sources") + require.Error(t, err, "checksum requires HTTP(S) or Git sources") }) } diff --git a/frontend/dockerfile/dockerfile_addgit_test.go b/frontend/dockerfile/dockerfile_addgit_test.go index db8d0d9649fa..60b56de630a7 100644 --- a/frontend/dockerfile/dockerfile_addgit_test.go +++ b/frontend/dockerfile/dockerfile_addgit_test.go @@ -5,7 +5,9 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" + "strings" "testing" "text/template" @@ -19,6 +21,7 @@ import ( var addGitTests = integration.TestFuncs( testAddGit, + testAddGitChecksumCache, ) func init() { @@ -52,6 +55,18 @@ func testAddGit(t *testing.T, sb integration.Sandbox) { err = runShell(gitDir, gitCommands...) require.NoError(t, err) + revParseCmd := exec.Command("git", "rev-parse", "v0.0.2") + revParseCmd.Dir = gitDir + commitHashB, err := revParseCmd.Output() + require.NoError(t, err) + commitHashV2 := strings.TrimSpace(string(commitHashB)) + + revParseCmd = exec.Command("git", "rev-parse", "v0.0.3") + revParseCmd.Dir = gitDir + commitHashB, err = revParseCmd.Output() + require.NoError(t, err) + commitHashV3 := strings.TrimSpace(string(commitHashB)) + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir)))) defer server.Close() serverURL := server.URL @@ -68,7 +83,7 @@ RUN cd /x && \ # Complicated case ARG REPO="{{.ServerURL}}/.git" ARG TAG="v0.0.2" -ADD --keep-git-dir=true --chown=4242:8484 ${REPO}#${TAG} /buildkit-chowned +ADD --keep-git-dir=true --chown=4242:8484 --checksum={{.Checksum}} ${REPO}#${TAG} /buildkit-chowned RUN apk add git USER 4242 RUN cd /buildkit-chowned && \ @@ -78,9 +93,9 @@ RUN cd /buildkit-chowned && \ [ -z "$(git status -s)" ] `, map[string]string{ "ServerURL": serverURL, + "Checksum": commitHashV2, }) require.NoError(t, err) - t.Logf("dockerfile=%s", dockerfile) dir := integration.Tmpdir(t, fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600), @@ -97,6 +112,242 @@ RUN cd /buildkit-chowned && \ }, }, nil) require.NoError(t, err) + + // Additional test: ADD from Git URL with checksum but without keep-git-dir flag + dockerfile2, err := applyTemplate(` +FROM alpine +ARG REPO="{{.ServerURL}}/.git" +ARG TAG="v0.0.3" +ADD --checksum={{.Checksum}} ${REPO}#${TAG} /nogitdir +RUN [ -f /nogitdir/foo ] +RUN [ "$(cat /nogitdir/foo)" = "foo of v0.0.3" ] +RUN [ ! -d /nogitdir/.git ] +`, map[string]string{ + "ServerURL": serverURL, + "Checksum": commitHashV3, + }) + require.NoError(t, err) + + dir2 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir2, + dockerui.DefaultLocalNameContext: dir2, + }, + }, nil) + require.NoError(t, err) + + // access initial ref again that was already pulled + dockerfile3, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG TAG="v0.0.2" + ADD --keep-git-dir --checksum={{.Checksum}} ${REPO}#${TAG} /nogitdir + RUN [ -f /nogitdir/foo ] + RUN [ "$(cat /nogitdir/foo)" = "foo of v0.0.2" ] + RUN [ -d /nogitdir/.git ] + `, map[string]string{ + "ServerURL": serverURL, + "Checksum": commitHashV2, + }) + require.NoError(t, err) + + dir3 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile3), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir3, + dockerui.DefaultLocalNameContext: dir3, + }, + }, nil) + require.NoError(t, err) + + // Additional test: ADD from Git URL using commitHashV3 for both checksum and ref + dockerfile4, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG COMMIT="{{.Checksum}}" + ADD --keep-git-dir=true --checksum={{.Checksum}} ${REPO}#${COMMIT} /commitdir + RUN [ -f /commitdir/foo ] + RUN [ "$(cat /commitdir/foo)" = "foo of v0.0.3" ] + RUN [ -d /commitdir/.git ] + `, map[string]string{ + "ServerURL": serverURL, + "Checksum": commitHashV3, + }) + require.NoError(t, err) + + dir4 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile4), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir4, + dockerui.DefaultLocalNameContext: dir4, + }, + }, nil) + require.NoError(t, err) + + // checksum does not match + dockerfile5, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG TAG="v0.0.3" + ADD --checksum={{.WrongChecksum}} ${REPO}#${TAG} /faildir + `, map[string]string{ + "ServerURL": serverURL, + "WrongChecksum": commitHashV2, // v0.0.2 hash, but ref is v0.0.3 + }) + require.NoError(t, err) + + dir5 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile5), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir5, + dockerui.DefaultLocalNameContext: dir5, + }, + }, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "expected checksum to match") + + // checksum is garbage + dockerfile6, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG TAG="v0.0.3" + ADD --checksum=foobar ${REPO}#${TAG} /faildir + `, map[string]string{ + "ServerURL": serverURL, + }) + require.NoError(t, err) + + dir6 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile6), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir6, + dockerui.DefaultLocalNameContext: dir6, + }, + }, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid checksum") + require.Contains(t, err.Error(), "expected hex commit hash") +} + +func testAddGitChecksumCache(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + f := getFrontend(t, sb) + + gitDir, err := os.MkdirTemp("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(gitDir) + gitCommands := []string{ + "git init", + "git config --local user.email test", + "git config --local user.name test", + } + makeCommit := func(tag string) []string { + return []string{ + "echo foo of " + tag + " >foo", + "git add foo", + "git commit -m " + tag, + "git tag " + tag, + } + } + gitCommands = append(gitCommands, makeCommit("v0.0.1")...) + gitCommands = append(gitCommands, makeCommit("v0.0.2")...) + gitCommands = append(gitCommands, "git update-server-info") + err = runShell(gitDir, gitCommands...) + require.NoError(t, err) + + revParseCmd := exec.Command("git", "rev-parse", "v0.0.2") + revParseCmd.Dir = gitDir + commitHashB, err := revParseCmd.Output() + require.NoError(t, err) + commitHash := strings.TrimSpace(string(commitHashB)) + + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir)))) + defer server.Close() + serverURL := server.URL + + // First build: without checksum, from tag, generate unique.txt from /dev/urandom and copy to scratch + dockerfile1 := ` +FROM alpine AS src +ADD --keep-git-dir ` + serverURL + `/.git#v0.0.2 /repo +RUN head -c 16 /dev/urandom | base64 > /repo/unique.txt + +FROM scratch +COPY --from=src /repo/unique.txt / +` + dir1 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile1), 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + destDir1 := t.TempDir() + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir1, + }, + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir1, + dockerui.DefaultLocalNameContext: dir1, + }, + }, nil) + require.NoError(t, err) + + unique1, err := os.ReadFile(filepath.Join(destDir1, "unique.txt")) + require.NoError(t, err) + + // Second build: with checksum, should match cache even though this one sets commitHash and get same unique.txt + dockerfile2 := ` +FROM alpine AS src +ADD --keep-git-dir --checksum=` + commitHash + ` ` + serverURL + `/.git#v0.0.2 /repo +RUN head -c 16 /dev/urandom | base64 > /repo/unique.txt + +FROM scratch +COPY --from=src /repo/unique.txt / +` + dir2 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600), + ) + + destDir2 := t.TempDir() + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir2, + }, + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir2, + dockerui.DefaultLocalNameContext: dir2, + }, + }, nil) + require.NoError(t, err) + + unique2, err := os.ReadFile(filepath.Join(destDir2, "unique.txt")) + require.NoError(t, err) + + require.Equal(t, string(unique1), string(unique2), "cache should be matched and unique file content should be the same") } func applyTemplate(tmpl string, x any) (string, error) { diff --git a/solver/jobs.go b/solver/jobs.go index 2e4a95475b34..2ce721bd7b57 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -296,6 +296,11 @@ func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(any) er return nil } +// OnVertexCompleteFunc is called after a vertex produces results, either from +// execution (cache miss) or cache load (cache hit). The callback receives the +// vertex and its output results. Implementations must be safe for concurrent use. +type OnVertexCompleteFunc func(vtx Vertex, results []Result) + type Job struct { list *Solver pr *progress.MultiReader @@ -306,9 +311,14 @@ type Job struct { startedTime time.Time completedTime time.Time - progressCloser func(error) - SessionID string - uniqueID string // unique ID is used for provenance. We use a different field that client can't control + progressCloser func(error) + SessionID string + uniqueID string // unique ID is used for provenance. We use a different field that client can't control + onVertexComplete OnVertexCompleteFunc +} + +func (j *Job) SetOnVertexComplete(f OnVertexCompleteFunc) { + j.onVertexComplete = f } type SolverOpt struct { @@ -907,6 +917,9 @@ func (s *sharedOp) LoadCache(ctx context.Context, rec *CacheRecord) (Result, err res, err := s.Cache().Load(withAncestorCacheOpts(ctx, s.st), rec) tracing.FinishWithError(span, err) notifyCompleted(err, true) + if err == nil && res != nil { + s.fireOnVertexComplete([]Result{res}) + } return res, err } @@ -1127,6 +1140,8 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, s.subBuilder.mu.Unlock() s.execRes = &execRes{execRes: wrapShared(res), execExporters: subExporters} + + s.fireOnVertexComplete(res) } s.execErr = err } @@ -1141,6 +1156,22 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, return unwrapShared(res.execRes), res.execExporters, nil } +func (s *sharedOp) fireOnVertexComplete(results []Result) { + s.st.mu.Lock() + var callbacks []OnVertexCompleteFunc + for j := range s.st.jobs { + if j.onVertexComplete != nil { + callbacks = append(callbacks, j.onVertexComplete) + } + } + vtx := s.st.vtx + s.st.mu.Unlock() + + for _, cb := range callbacks { + cb(vtx, results) + } +} + func (s *sharedOp) getOp() (Op, error) { s.opOnce.Do(func() { s.subBuilder = s.st.builder() diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go new file mode 100644 index 000000000000..fe2b1f93e10d --- /dev/null +++ b/solver/llbsolver/eager.go @@ -0,0 +1,786 @@ +package llbsolver + +import ( + "context" + "os" + "runtime" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/remotes" + "github.com/moby/buildkit/cache" + cacheconfig "github.com/moby/buildkit/cache/config" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/frontend" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/compression" + pushutil "github.com/moby/buildkit/util/push" + "github.com/moby/buildkit/util/resolver/limited" + "github.com/moby/buildkit/util/resolver/retryhandler" + "github.com/moby/buildkit/util/tracing" + "github.com/moby/buildkit/worker" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// Keep the pools large enough for a ref's descriptor chain to fan out without +// serializing pushes. +const ( + defaultEagerWorkers = 128 + defaultEagerPushWorkers = 128 +) + +type eagerWorkItem struct { + ref cache.ImmutableRef +} + +// eagerPushItem is a single unique push descriptor handed to the push pool. +// Duplicate descriptor requests are tracked per digest and do not enqueue +// additional work. +type eagerPushItem struct { + refID string + desc ocispecs.Descriptor + handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error) + attempt uint64 +} + +// eagerPipeline manages background compression and pushing of layer blobs as +// build vertices complete, rather than deferring all work to finalize. +// +// It is split into two worker pools: +// - Compress pool: receives refs from the solver vertex callback, runs +// GetRemotes (which performs blob compression for the full parent chain), +// and dispatches each pushable descriptor onto the push pool. +// - Push pool: receives individual descriptors and uploads them to the +// registry. Decoupling lets a single ref fan out parallel pushes across +// its descriptor chain (instead of serializing them in one goroutine), +// which matches the parallelism that vanilla buildkit gets from +// images.Dispatch. +type eagerPipeline struct { + mode EagerExportMode + refCfg cacheconfig.RefConfig + sessionID string + pushCfg *exporter.EagerPushConfig + + compressWork chan eagerWorkItem + pushWork chan eagerPushItem + done chan struct{} + + compressWG sync.WaitGroup + pushWG sync.WaitGroup + + // closeMu gates new senders from entering shutdown. Senders increment + // senderWg while holding the mutex so wait() can stop admission before + // waiting for all in-flight send attempts to finish. + closeMu sync.Mutex + closing bool + senderWg sync.WaitGroup + waitOnce sync.Once + + // ctx carries the lease so compressed blobs are GC-protected. + ctx context.Context + cancelCause context.CancelCauseFunc + + // pusher is created at pipeline init when mode is EagerExportPush. + pusher remotes.Pusher + + // keepRefIDs is the set of ImmutableRef.ID()s whose blobs are part of + // the final exported image manifest. It is populated once, by wait(), + // after the frontend has fully resolved its result. While nil, every + // ref's blobs are eligible for compression+push. Once non-nil, processRef + // and requestPushDescriptor skip refs + // whose ID is not in the set, and wait() cancels in-flight + // pushes whose only requesters are non-kept refs. + keepRefIDs atomic.Pointer[map[string]struct{}] + // inflight tracks per-digest cancel funcs and the set of refIDs that + // requested each digest. Used by wait() to cancel uploads + // whose every requester turns out to be non-kept. + inflight sync.Map // map[string]*pushTracker + + mu sync.Mutex + firstErr error +} + +// pushTracker holds bookkeeping for a single digest's eager push. A digest is +// enqueued at most once at a time; duplicate requesters are recorded here but do +// not occupy push worker slots. +type pushTracker struct { + mu sync.Mutex + refIDs map[string]struct{} + enqueued bool + pushing bool + pushed bool + cancel context.CancelFunc + cancelled bool + attempt uint64 +} + +func eagerWorkerCount() int { + return envWorkerCount("BUILDKIT_EAGER_EXPORT_WORKERS", defaultEagerWorkers) +} + +func eagerPushWorkerCount() int { + return envWorkerCount("BUILDKIT_EAGER_PUSH_WORKERS", defaultEagerPushWorkers) +} + +func envWorkerCount(env string, fallback int) int { + if s := os.Getenv(env); s != "" { + if n, err := strconv.Atoi(s); err == nil && n > 0 { + return n + } + } + return max(fallback, runtime.NumCPU()) +} + +func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compression.Config, sessionID string, sm *session.Manager, pushCfg *exporter.EagerPushConfig) (*eagerPipeline, error) { + if mode == EagerExportPush && pushCfg == nil { + return nil, errors.New("eager-export=push requires push config") + } + + var pusher remotes.Pusher + if mode == EagerExportPush { + var err error + pusher, err = pushutil.NewPusher(ctx, sm, sessionID, pushCfg.TargetName, pushCfg.Insecure, pushCfg.RegistryHosts) + if err != nil { + return nil, errors.Wrap(err, "eager-export=push: failed to create pusher") + } + } + + pipelineCtx, cancel := context.WithCancelCause(ctx) + ep := &eagerPipeline{ + mode: mode, + refCfg: cacheconfig.RefConfig{ + Compression: comp, + }, + sessionID: sessionID, + pushCfg: pushCfg, + pusher: pusher, + ctx: pipelineCtx, + cancelCause: cancel, + compressWork: make(chan eagerWorkItem, 256), + pushWork: make(chan eagerPushItem, 1024), + done: make(chan struct{}), + } + + numCompress := eagerWorkerCount() + ep.compressWG.Add(numCompress) + for range numCompress { + go ep.compressWorker() + } + + if mode == EagerExportPush { + numPush := eagerPushWorkerCount() + ep.pushWG.Add(numPush) + for range numPush { + go ep.pushWorker() + } + bklog.G(ctx).Infof("eager pipeline started compress_workers=%d push_workers=%d", numCompress, numPush) + } else { + bklog.G(ctx).Infof("eager pipeline started compress_workers=%d (no push)", numCompress) + } + + return ep, nil +} + +func (ep *eagerPipeline) cancel(err error) { + if ep.cancelCause != nil { + ep.cancelCause(err) + } +} + +func (ep *eagerPipeline) compressWorker() { + defer ep.compressWG.Done() + for { + select { + case <-ep.ctx.Done(): + return + case item, ok := <-ep.compressWork: + if !ok { + return + } + if err := ep.processRef(item.ref); err != nil { + ep.recordErr(err) + } + item.ref.Release(context.TODO()) + } + } +} + +func (ep *eagerPipeline) pushWorker() { + defer ep.pushWG.Done() + for { + // Don't bail out on ctx.Done here: wait() closes pushWork only after + // compression has finished enqueueing, and the worker should drain any + // remaining items so cancellation bookkeeping can settle. + item, ok := <-ep.pushWork + if !ok { + return + } + if err := ep.pushDescriptor(ep.ctx, item); err != nil { + ep.recordErr(err) + } + } +} + +func (ep *eagerPipeline) recordErr(err error) { + ep.mu.Lock() + if ep.firstErr == nil { + ep.firstErr = err + } + ep.mu.Unlock() +} + +// onVertexComplete is the callback registered on the solver Job. It extracts +// ImmutableRefs from vertex results, clones them for safe async use, and +// sends them to the compress pool. Fires that arrive after shutdown starts +// are rejected before enqueue and release their clones. +func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { + for _, res := range results { + if res == nil { + continue + } + workerRef, ok := res.Sys().(*worker.WorkerRef) + if !ok || workerRef.ImmutableRef == nil { + continue + } + + ep.closeMu.Lock() + if ep.closing { + ep.closeMu.Unlock() + continue + } + ep.senderWg.Add(1) + ep.closeMu.Unlock() + + cloned := workerRef.ImmutableRef.Clone() + select { + case ep.compressWork <- eagerWorkItem{ref: cloned}: + ep.senderWg.Done() + case <-ep.done: + cloned.Release(context.TODO()) + ep.senderWg.Done() + case <-ep.ctx.Done(): + cloned.Release(context.TODO()) + ep.senderWg.Done() + return + } + } +} + +// computeEagerKeepSet walks the resolved frontend Result and returns the +// set of ImmutableRef.ID()s that belong to the final exported image — +// including every layer in each output ref's parent chain. +// +// Why the chain matters: each layer is its own ImmutableRef with its own +// ID, *equal to* the ID of the intermediate vertex that produced it. The +// solver's onVertexComplete fires once per vertex during the build, so +// the eager pipeline pushes layer L_n via vertex V_n's processRef long +// before V_final completes. If we only kept V_final.ID() we'd filter +// every intermediate vertex's processRef, and all the real layer pushes +// would get deferred to wait() — defeating the entire point of eager +// export. +// +// res must already be fully resolved: every ResultProxy.Result(ctx) call +// must have completed without error. The caller in solver.go ensures +// this via eg.Wait() right before invoking us. +// +// If res is nil or contains zero refs, returns an empty map (== filter +// everything). Errors resolving individual refs are logged but not +// returned: a partial keep-set is safer than failing the build, since a +// missing entry just costs some wasted bandwidth, not correctness. +func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]struct{} { + keep := make(map[string]struct{}) + if res == nil { + return keep + } + res.EachRef(func(rp solver.ResultProxy) error { + if rp == nil { + return nil + } + cached, err := rp.Result(ctx) + if err != nil { + bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to resolve ref") + return nil + } + workerRef, ok := cached.Sys().(*worker.WorkerRef) + if !ok || workerRef.ImmutableRef == nil { + return nil + } + // LayerChain walks BaseLayer → ... → tip, including the tip + // itself. Each entry is a *clone* of the underlying ref — we + // only need its ID, then must release. + chain := workerRef.ImmutableRef.LayerChain() + for _, layer := range chain { + if layer == nil { + continue + } + keep[layer.ID()] = struct{}{} + } + if err := chain.Release(context.WithoutCancel(ctx)); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to release chain clones") + } + return nil + }) + return keep +} + +func computeEagerExportRefs(ctx context.Context, res *frontend.Result) (map[string]struct{}, error) { + return computeEagerKeepSet(ctx, res), nil +} + +// keepSet returns the current keep-set, or nil if none has been set yet. +// While nil, no filtering is applied (every ref is eligible). +func (ep *eagerPipeline) keepSet() map[string]struct{} { + if p := ep.keepRefIDs.Load(); p != nil { + return *p + } + return nil +} + +// isKept reports whether refID should be retained. If no keep-set has been +// installed yet, every ref is considered kept. +func (ep *eagerPipeline) isKept(refID string) bool { + keep := ep.keepSet() + if keep == nil { + return true + } + _, ok := keep[refID] + return ok +} + +func (ep *eagerPipeline) hasKeptRequester(refIDs map[string]struct{}) bool { + keep := ep.keepSet() + if keep == nil { + return len(refIDs) > 0 + } + for refID := range refIDs { + if _, ok := keep[refID]; ok { + return true + } + } + return false +} + +// trackerFor returns (creating if necessary) the pushTracker for digest. +func (ep *eagerPipeline) trackerFor(digest string) *pushTracker { + if v, ok := ep.inflight.Load(digest); ok { + return v.(*pushTracker) + } + nt := &pushTracker{refIDs: make(map[string]struct{})} + actual, _ := ep.inflight.LoadOrStore(digest, nt) + return actual.(*pushTracker) +} + +// processRef compresses a single ref's blob (and its parent chain) and, in +// push mode, dispatches each pushable descriptor onto the push pool. Parent +// compression is deduplicated by flightcontrol inside computeBlobChain, so +// overlapping parent chains across workers are only compressed once. +func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { + ctx := ep.ctx + s := session.NewGroup(ep.sessionID) + refID := ref.ID() + + // If the keep-set is already installed (e.g. queued items processed + // during shutdown drain), skip non-kept refs without paying for + // compression. + if !ep.isKept(refID) { + bklog.G(ctx).Debugf("eager compress skipped (filtered) ref=%s", refID) + return nil + } + + bklog.G(ctx).Infof("eager compress starting ref=%s", refID) + compressStart := time.Now() + compressSpan, compressCtx := tracing.StartSpan(ctx, "eager compress ref", trace.WithAttributes( + attribute.String("ref.id", refID), + )) + rems, err := ref.GetRemotes(compressCtx, true, ep.refCfg, false, s) + descCount, totalBytes := summarizeDescriptors(rems) + compressSpan.SetAttributes( + attribute.Int("descriptors", descCount), + attribute.Int64("bytes", totalBytes), + ) + tracing.FinishWithError(compressSpan, err) + if err != nil { + bklog.G(ctx).WithError(err).Warnf("eager compress failed ref=%s", refID) + return err + } + bklog.G(ctx).Infof("eager compress done ref=%s descriptors=%d bytes=%d duration=%s", + refID, descCount, totalBytes, time.Since(compressStart).Round(time.Millisecond)) + + if ep.mode != EagerExportPush { + return nil + } + + // Re-check after the (potentially long) compress: the keep-set may + // have been installed while we were inside GetRemotes. + if !ep.isKept(refID) { + bklog.G(ctx).Debugf("eager push skipped after compress (filtered) ref=%s", refID) + return nil + } + + pushStart := time.Now() + if err := ep.dispatchPushes(ctx, refID, rems); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager push failed ref=%s", refID) + return err + } + bklog.G(ctx).Infof("eager push dispatched ref=%s descriptors=%d bytes=%d duration=%s", + refID, descCount, totalBytes, time.Since(pushStart).Round(time.Millisecond)) + return nil +} + +func summarizeDescriptors(rems []*solver.Remote) (count int, totalBytes int64) { + if len(rems) == 0 { + return 0, 0 + } + for _, desc := range rems[0].Descriptors { + if !shouldEagerPushDesc(desc) { + continue + } + count++ + totalBytes += desc.Size + } + return +} + +// dispatchPushes sends every not-yet-enqueued pushable descriptor in a ref's +// chain to the push pool. It deliberately does not wait for completion; final +// export waits for the push pool after all compression workers have finished +// enqueueing. +func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems []*solver.Remote) error { + if len(rems) == 0 { + return nil + } + + remote := rems[0] + handler := retryhandler.New( + limited.PushHandler(ep.pusher, remote.Provider, ep.pushCfg.TargetName), + nil, + ) + + enqueued := 0 + for _, desc := range remote.Descriptors { + if !shouldEagerPushDesc(desc) { + continue + } + attempt, shouldEnqueue := ep.requestPushDescriptor(refID, desc) + if !shouldEnqueue { + continue + } + item := eagerPushItem{ + refID: refID, + desc: desc, + handler: handler, + attempt: attempt, + } + select { + case ep.pushWork <- item: + enqueued++ + case <-ctx.Done(): + ep.unmarkQueuedDescriptor(refID, desc, attempt) + return context.Cause(ctx) + } + } + + bklog.G(ctx).Debugf("eager push dispatched ref=%s enqueued=%d", refID, enqueued) + return nil +} + +func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { + return !images.IsNonDistributable(desc.MediaType) +} + +// requestPushDescriptor records that refID needs desc and returns true only +// for the first requester that should enqueue actual push work. Duplicate +// requesters are bookkeeping only: they do not occupy push worker slots. +func (ep *eagerPipeline) requestPushDescriptor(refID string, desc ocispecs.Descriptor) (uint64, bool) { + digest := desc.Digest.String() + tracker := ep.trackerFor(digest) + + tracker.mu.Lock() + defer tracker.mu.Unlock() + + tracker.refIDs[refID] = struct{}{} + + if tracker.pushed { + bklog.G(ep.ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", refID, digest, desc.Size) + return 0, false + } + if !ep.hasKeptRequester(tracker.refIDs) { + bklog.G(ep.ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", refID, digest, desc.Size) + return 0, false + } + if tracker.enqueued || tracker.pushing { + bklog.G(ep.ctx).Debugf("eager push already in progress ref=%s digest=%s size=%d", refID, digest, desc.Size) + return 0, false + } + + tracker.attempt++ + tracker.enqueued = true + tracker.cancelled = false + return tracker.attempt, true +} + +func (ep *eagerPipeline) unmarkQueuedDescriptor(refID string, desc ocispecs.Descriptor, attempt uint64) { + digest := desc.Digest.String() + if v, ok := ep.inflight.Load(digest); ok { + tracker := v.(*pushTracker) + tracker.mu.Lock() + if tracker.attempt == attempt && !tracker.pushing && !tracker.pushed { + tracker.enqueued = false + } + delete(tracker.refIDs, refID) + tracker.mu.Unlock() + } +} + +// pushDescriptor uploads a single descriptor. Deduplication has already +// happened at enqueue time, so push workers only handle real push attempts. If +// a kept requester arrives after a non-kept upload was cancelled, it can enqueue +// a fresh item once the cancelled attempt clears the tracker state. +func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) error { + digest := item.desc.Digest.String() + parentCtx := ctx + span, spanCtx := tracing.StartSpan(ctx, "eager push descriptor", trace.WithAttributes( + attribute.String("ref.id", item.refID), + attribute.String("digest", digest), + attribute.Int64("size", item.desc.Size), + )) + ctx = spanCtx + tracker := ep.trackerFor(digest) + + tracker.mu.Lock() + if item.attempt != 0 && tracker.attempt != item.attempt { + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("deduped", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Debugf("eager push skipped stale attempt ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } + tracker.refIDs[item.refID] = struct{}{} + if tracker.pushed { + tracker.enqueued = false + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("deduped", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } + if !ep.hasKeptRequester(tracker.refIDs) { + tracker.enqueued = false + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("filtered", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } + + pushCtx, cancel := context.WithCancel(ctx) + defer cancel() + + tracker.enqueued = false + tracker.pushing = true + tracker.cancel = cancel + tracker.cancelled = false + tracker.mu.Unlock() + defer func() { + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.cancel = nil + tracker.pushing = false + } + tracker.mu.Unlock() + }() + + start := time.Now() + bklog.G(ctx).Infof("eager push starting ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + + _, err := item.handler(pushCtx, item.desc) + if err == nil && pushCtx.Err() != nil && parentCtx.Err() == nil { + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.pushing = false + tracker.enqueued = false + tracker.cancel = nil + tracker.cancelled = true + } + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("cancelled", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil + } + if err != nil { + // Distinguish a deliberate keep-set cancellation from a real + // failure: only the former has pushCtx done while the parent + // ctx is still alive. + if pushCtx.Err() != nil && parentCtx.Err() == nil { + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.pushing = false + tracker.enqueued = false + tracker.cancel = nil + tracker.cancelled = true + } + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("cancelled", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil + } + tracing.FinishWithError(span, err) + return err + } + + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.pushed = true + tracker.pushing = false + tracker.enqueued = false + tracker.cancel = nil + tracker.cancelled = false + } + tracker.mu.Unlock() + tracing.FinishWithError(span, nil) + bklog.G(ctx).Infof("eager push done ref=%s digest=%s size=%d duration=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil +} + +// wait shuts down both pools in order: stop new compress senders, drain +// compressWork, then drain pushWork. Compress workers enqueue push work without +// waiting for it; once compressWG.Wait returns, every reachable descriptor has +// either been enqueued or deduped, so closing pushWork is safe. +// +// The optional keep-set is the canonical set of ImmutableRef.ID()s that +// belong to the final image manifest, computed by the caller from the +// resolved frontend Result. When non-nil: +// - any in-flight push whose every requester is non-kept is cancelled +// immediately, freeing its registry-channel slot; +// - any items still queued in pushWork or compressWork for non-kept refs +// are skipped on dequeue; +// - kept refs continue to compress and push as normal. +// +// Pass nil to keep every ref (back-compat for non-gateway frontends or +// callers that haven't computed a keep-set). +func (ep *eagerPipeline) wait(keepOpt ...map[string]struct{}) error { + ep.waitOnce.Do(func() { + var keep map[string]struct{} + if len(keepOpt) > 0 { + keep = keepOpt[0] + } + if keep != nil { + cp := keep + ep.keepRefIDs.Store(&cp) + n := ep.cancelNonKeptInflight(keep) + bklog.G(ep.ctx).Infof("eager wait keep_set_size=%d cancelled_inflight=%d", len(keep), n) + } + + ep.closeMu.Lock() + if ep.done == nil { + ep.done = make(chan struct{}) + } + ep.closing = true + ep.closeMu.Unlock() + + close(ep.done) + ep.senderWg.Wait() + close(ep.compressWork) + ep.compressWG.Wait() + ep.drainCompress() + + if ep.pushWork != nil { + close(ep.pushWork) + ep.pushWG.Wait() + ep.drainPushWork() + } + }) + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.firstErr +} + +// cancelNonKeptInflight walks every digest currently being pushed and +// cancels the upload if no requester is in the keep-set. Returns the +// number of digests for which at least one cancel was issued. +// +// A digest with at least one kept requester is left entirely alone. A digest +// whose requesters are all non-kept can have its single active push cancelled; +// queued items will observe the keep-set when dequeued and skip without pushing. +func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { + var digestsCancelled int + ep.inflight.Range(func(key, val any) bool { + digest := key.(string) + tracker := val.(*pushTracker) + + tracker.mu.Lock() + defer tracker.mu.Unlock() + + for refID := range tracker.refIDs { + if _, ok := keep[refID]; ok { + return true + } + } + if tracker.cancel == nil { + // Nothing in flight — queued items will be filtered by + // isKept() at dequeue. + return true + } + tracker.cancel() + tracker.cancel = nil + tracker.pushing = false + tracker.enqueued = false + tracker.cancelled = true + tracker.attempt++ + digestsCancelled++ + bklog.G(ep.ctx).Infof("eager push cancel digest=%s requesters=%d cancelled_waiters=%d", + digest, len(tracker.refIDs), 1) + return true + }) + return digestsCancelled +} + +func (ep *eagerPipeline) applyExportRefs(exportRefs map[string]struct{}) int { + cp := exportRefs + ep.keepRefIDs.Store(&cp) + return ep.cancelNonKeptInflight(exportRefs) +} + +// drainCompress releases any refs left in compressWork after workers have +// exited (e.g. via ctx cancellation before the channel was drained). +func (ep *eagerPipeline) drainCompress() { + for { + select { + case item, ok := <-ep.compressWork: + if !ok { + return + } + item.ref.Release(context.TODO()) + default: + return + } + } +} + +// drainPushWork clears any leftover queued push items after push workers exit. +func (ep *eagerPipeline) drainPushWork() { + for { + select { + case item, ok := <-ep.pushWork: + if !ok { + return + } + ep.unmarkQueuedDescriptor(item.refID, item.desc, item.attempt) + default: + return + } + } +} diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go new file mode 100644 index 000000000000..38fd2a1a6765 --- /dev/null +++ b/solver/llbsolver/eager_test.go @@ -0,0 +1,499 @@ +package llbsolver + +import ( + "context" + "os" + "runtime" + "sync" + "sync/atomic" + "testing" + + "github.com/containerd/containerd/v2/core/images" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/worker" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEagerWorkerCount_Default(t *testing.T) { + os.Unsetenv("BUILDKIT_EAGER_EXPORT_WORKERS") + n := eagerWorkerCount() + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), n) +} + +func TestEagerWorkerCount_EnvOverride(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "2") + assert.Equal(t, 2, eagerWorkerCount()) +} + +func TestEagerWorkerCount_EnvInvalid(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "not-a-number") + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) +} + +func TestEagerWorkerCount_EnvZero(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "0") + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) +} + +func TestEagerWorkerCount_EnvNegative(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "-1") + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) +} + +func TestEagerPushWorkerCount_Default(t *testing.T) { + os.Unsetenv("BUILDKIT_EAGER_PUSH_WORKERS") + assert.Equal(t, max(defaultEagerPushWorkers, runtime.NumCPU()), eagerPushWorkerCount()) +} + +func TestEagerPushWorkerCount_EnvOverride(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_PUSH_WORKERS", "5") + assert.Equal(t, 5, eagerPushWorkerCount()) +} + +func TestNewEagerPipeline_PushRequiresConfig(t *testing.T) { + _, err := newEagerPipeline(context.Background(), EagerExportPush, compression.Config{}, "", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "push config") +} + +func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { + ep := &eagerPipeline{ + compressWork: make(chan eagerWorkItem), + } + ep.firstErr = assert.AnError + + err := ep.wait(nil) + assert.Equal(t, assert.AnError, err) +} + +func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { + ep := &eagerPipeline{ + compressWork: make(chan eagerWorkItem), + } + + err := ep.wait(nil) + assert.NoError(t, err) +} + +func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { + var released atomic.Int32 + ep := &eagerPipeline{ + compressWork: make(chan eagerWorkItem, 10), + } + + ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} + ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} + + err := ep.wait(nil) + require.NoError(t, err) + assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") +} + +func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { + ep := &eagerPipeline{ + compressWork: make(chan eagerWorkItem), + } + + require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait(nil), "second wait must not panic") +} + +func TestEagerPipeline_CompressWorkerExitsOnContextCancel(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + ep := &eagerPipeline{ + mode: EagerExportCompress, + ctx: ctx, + compressWork: make(chan eagerWorkItem, 10), + } + + cancel(nil) + + ep.compressWG.Add(1) + go ep.compressWorker() + ep.compressWG.Wait() +} + +func TestEagerPipeline_CompressWorkerExitsOnChannelClose(t *testing.T) { + ep := &eagerPipeline{ + mode: EagerExportCompress, + ctx: context.Background(), + compressWork: make(chan eagerWorkItem), + } + + ep.compressWG.Add(1) + go ep.compressWorker() + + close(ep.compressWork) + ep.compressWG.Wait() +} + +func TestEagerPipeline_PushWorkerExitsOnChannelClose(t *testing.T) { + ep := &eagerPipeline{ + ctx: context.Background(), + pushWork: make(chan eagerPushItem), + } + + ep.pushWG.Add(1) + go ep.pushWorker() + + close(ep.pushWork) + ep.pushWG.Wait() +} + +// Late fires of onVertexComplete must release the clone instead of sending +// into the (now closed) work channel. +func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { + var cloned atomic.Int32 + ep := &eagerPipeline{ + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 10), + done: make(chan struct{}), + } + require.NoError(t, ep.wait(nil)) + + var released atomic.Int32 + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) + + require.NotPanics(t, func() { + ep.onVertexComplete(nil, []solver.Result{res}) + }) + assert.Zero(t, cloned.Load()) + assert.Zero(t, released.Load()) +} + +// Many concurrent late fires after wait() must be rejected before cloning. +func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { + var cloned atomic.Int32 + ep := &eagerPipeline{ + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 10), + done: make(chan struct{}), + } + require.NoError(t, ep.wait(nil)) + + var released atomic.Int32 + const fires = 100 + + var wg sync.WaitGroup + wg.Add(fires) + for range fires { + go func() { + defer wg.Done() + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) + ep.onVertexComplete(nil, []solver.Result{res}) + }() + } + wg.Wait() + + assert.Zero(t, cloned.Load()) + assert.Zero(t, released.Load()) +} + +// A sender admitted before wait() but blocked on a full queue must take the +// done path, release its clone, and exit without panic. +func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) { + var cloned atomic.Int32 + var released atomic.Int32 + ep := &eagerPipeline{ + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 1), + done: make(chan struct{}), + } + ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} + + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) + + var senderWg sync.WaitGroup + senderWg.Add(1) + go func() { + defer senderWg.Done() + ep.onVertexComplete(nil, []solver.Result{res}) + }() + + for range 1000 { + if cloned.Load() == 1 { + break + } + runtime.Gosched() + } + require.Equal(t, int32(1), cloned.Load()) + + require.NotPanics(t, func() { + require.NoError(t, ep.wait(nil)) + }) + senderWg.Wait() + + assert.Equal(t, int32(2), released.Load()) +} + +// Fires racing concurrently with wait() must not panic and must not leak: +// every admitted clone is either drained by wait() or released by the sender. +func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { + var cloned atomic.Int32 + ep := &eagerPipeline{ + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 256), + done: make(chan struct{}), + } + + var released atomic.Int32 + const fires = 200 + + var wg sync.WaitGroup + wg.Add(fires) + for range fires { + go func() { + defer wg.Done() + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) + ep.onVertexComplete(nil, []solver.Result{res}) + }() + } + + require.NotPanics(t, func() { + require.NoError(t, ep.wait(nil)) + }) + wg.Wait() + + assert.Equal(t, cloned.Load(), released.Load()) +} + +func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { + descs := []ocispecs.Descriptor{ + { + Digest: digest.FromString("push me"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + }, + { + Digest: digest.FromString("skip me"), + MediaType: ocispecs.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // deprecated but still supported + }, + { + Digest: digest.FromString("push me too"), + MediaType: images.MediaTypeDockerSchema2Layer, + }, + } + + var pushed []digest.Digest + ep := &eagerPipeline{} + handler := func(_ context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + pushed = append(pushed, desc.Digest) + return nil, nil + } + + for _, desc := range descs { + if !shouldEagerPushDesc(desc) { + continue + } + err := ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "test-ref", + desc: desc, + handler: handler, + }) + require.NoError(t, err) + } + + assert.Equal(t, []digest.Digest{ + digest.FromString("push me"), + digest.FromString("push me too"), + }, pushed) +} + +// pushDescriptor must short-circuit on the second call for the same digest, +// without invoking the handler again. This is the fix for the noisy +// "eager pushing blob" log entries that appeared once per shared parent +// blob per ref. +func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("shared-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 1234, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + for range 5 { + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "ref-test", + desc: desc, + handler: handler, + })) + } + assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once across repeated calls for the same digest") +} + +// pushDescriptor must skip — without invoking the handler — when the +// keep-set is installed and the requesting refID is not in it. +func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("intermediate-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 4096, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) + + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "intermediate-ref", + desc: desc, + handler: handler, + })) + assert.Zero(t, calls.Load(), "non-kept ref must not invoke the push handler") + + // And the digest must still appear in the inflight tracker so + // cancelNonKeptInflight can reason about it. + v, ok := ep.inflight.Load(desc.Digest.String()) + require.True(t, ok) + tracker := v.(*pushTracker) + tracker.mu.Lock() + _, hasRef := tracker.refIDs["intermediate-ref"] + tracker.mu.Unlock() + assert.True(t, hasRef, "tracker must record requester even when filtered") +} + +// pushDescriptor must still push when at least one kept ref requests the +// digest, even if a non-kept ref also asked for it. +func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("shared-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 4096, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) + + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "intermediate-ref", + desc: desc, + handler: handler, + })) + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "final-ref", + desc: desc, + handler: handler, + })) + assert.Equal(t, int32(1), calls.Load(), "kept-ref call must drive exactly one push") +} + +// cancelNonKeptInflight must cancel the active push for a digest whose +// requesters are all non-kept, and must leave digests with at least one +// kept requester completely alone. +func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { + ep := &eagerPipeline{ctx: context.Background()} + + keep := map[string]struct{}{"final-ref": {}} + + // Digest A: two non-kept requesters share one active push. + var cancelA atomic.Bool + ep.inflight.Store("digest-a", &pushTracker{ + refIDs: map[string]struct{}{"int-ref-1": {}, "int-ref-2": {}}, + cancel: func() { cancelA.Store(true) }, + pushing: true, + }) + // Digest B: requested by both kept and non-kept — must be spared + // even though it has an active push. + var cancelB atomic.Bool + ep.inflight.Store("digest-b", &pushTracker{ + refIDs: map[string]struct{}{"int-ref-3": {}, "final-ref": {}}, + cancel: func() { cancelB.Store(true) }, + pushing: true, + }) + // Digest C: non-kept requester but no cancel yet (still queued). + // Must not be counted and must not panic. + ep.inflight.Store("digest-c", &pushTracker{ + refIDs: map[string]struct{}{"int-ref-4": {}}, + }) + + n := ep.cancelNonKeptInflight(keep) + assert.Equal(t, 1, n, "exactly one digest's active push should be cancelled") + assert.True(t, cancelA.Load(), "digest A active push must be cancelled") + assert.False(t, cancelB.Load(), "digest B must be spared (kept requester)") + + trackerA, _ := ep.inflight.Load("digest-a") + assert.Nil(t, trackerA.(*pushTracker).cancel, "cancel should be cleared after cancellation") +} + +// After cancelNonKeptInflight has fired, a kept ref that arrives later +// must not be skipped — it still needs to push the blob (the previous +// closure was cancelled before completion). +func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("retry-after-cancel"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 4096, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) + + // Simulate a prior cancel pass: tracker exists with the requester + // recorded and no active push. + ep.inflight.Store(desc.Digest.String(), &pushTracker{ + refIDs: map[string]struct{}{"int-ref": {}}, + }) + + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "final-ref", + desc: desc, + handler: handler, + })) + assert.Equal(t, int32(1), calls.Load(), "kept ref must drive a fresh push after a prior cancellation") +} + +// releaseTracker is a minimal cache.ImmutableRef stub that counts +// Release calls. Clones share the counter. +type releaseTracker struct { + cache.ImmutableRef + released *atomic.Int32 + cloned *atomic.Int32 +} + +func (r *releaseTracker) Release(context.Context) error { + r.released.Add(1) + return nil +} + +func (r *releaseTracker) Clone() cache.ImmutableRef { + if r.cloned != nil { + r.cloned.Add(1) + } + return &releaseTracker{released: r.released} +} + +func (r *releaseTracker) ID() string { return "release-tracker" } + +type fakeWorker struct{ worker.Worker } + +func (fakeWorker) ID() string { return "fake-worker" } + +func newWorkerRefResult(ref cache.ImmutableRef) solver.Result { + return worker.NewWorkerRefResult(ref, fakeWorker{}) +} diff --git a/solver/llbsolver/ops/source.go b/solver/llbsolver/ops/source.go index d6d1712a442b..388094550ab9 100644 --- a/solver/llbsolver/ops/source.go +++ b/solver/llbsolver/ops/source.go @@ -5,6 +5,7 @@ import ( "strings" "sync" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" @@ -18,17 +19,18 @@ import ( const sourceCacheType = "buildkit.source.v0" type SourceOp struct { - mu sync.Mutex - op *pb.Op_Source - platform *pb.Platform - sm *source.Manager - src source.SourceInstance - sessM *session.Manager - w worker.Worker - vtx solver.Vertex - parallelism *semaphore.Weighted - pin string - id source.Identifier + mu sync.Mutex + op *pb.Op_Source + platform *pb.Platform + sm *source.Manager + src source.SourceInstance + sessM *session.Manager + w worker.Worker + vtx solver.Vertex + parallelism *semaphore.Weighted + pin string + id source.Identifier + eagerPushCfg *exporter.EagerPushConfig } var _ solver.Op = &SourceOp{} @@ -48,6 +50,10 @@ func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm }, nil } +func (s *SourceOp) SetEagerPushConfig(cfg *exporter.EagerPushConfig) { + s.eagerPushCfg = cfg +} + func (s *SourceOp) IsProvenanceProvider() {} func (s *SourceOp) Pin() (source.Identifier, string) { @@ -68,6 +74,11 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error) if err != nil { return nil, err } + if s.eagerPushCfg != nil { + if setter, ok := src.(pushRegistryConfigSetter); ok { + setter.SetEagerPushConfig(s.eagerPushCfg) + } + } s.src = src s.id = id return s.src, nil @@ -112,6 +123,10 @@ func (s *SourceOp) Exec(ctx context.Context, g session.Group, _ []solver.Result) return []solver.Result{worker.NewWorkerRefResult(ref, s.w)}, nil } +type pushRegistryConfigSetter interface { + SetEagerPushConfig(*exporter.EagerPushConfig) +} + func (s *SourceOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { if s.parallelism == nil { return func() {}, nil diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 36fc7fd08502..49d34e42a063 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -51,14 +51,27 @@ import ( ) const ( - keyEntitlements = "llb.entitlements" - keySourcePolicy = "llb.sourcepolicy" + keyEntitlements = "llb.entitlements" + keySourcePolicy = "llb.sourcepolicy" + keyEagerPushConfig = "llb.eagerpushconfig" +) + +// EagerExportMode controls whether layers are handled during the build. +type EagerExportMode int + +const ( + EagerExportNone EagerExportMode = iota + EagerExportCompress // compress layers as vertices complete + EagerExportPush // compress AND push layer blobs as vertices complete ) type ExporterRequest struct { Exporters []exporter.ExporterInstance CacheExporters []RemoteCacheExporter EnableSessionExporter bool + EagerExport EagerExportMode + EagerPushConfig *exporter.EagerPushConfig // non-nil when EagerExport == EagerExportPush + PushRegistryConfig *exporter.EagerPushConfig // non-nil when prefer-push-registry=true } type RemoteCacheExporter struct { @@ -141,10 +154,28 @@ func (s *Solver) resolver() solver.ResolveOpFunc { if err != nil { return nil, err } - return w.ResolveOp(v, s.Bridge(b), s.sm) + op, err := w.ResolveOp(v, s.Bridge(b), s.sm) + if err != nil { + return nil, err + } + if setter, ok := op.(eagerPushConfigSetter); ok { + var pushCfg *exporter.EagerPushConfig + b.EachValue(context.TODO(), keyEagerPushConfig, func(v any) error { + pushCfg, _ = v.(*exporter.EagerPushConfig) + return nil + }) + if pushCfg != nil { + setter.SetEagerPushConfig(pushCfg) + } + } + return op, nil } } +type eagerPushConfigSetter interface { + SetEagerPushConfig(*exporter.EagerPushConfig) +} + func (s *Solver) bridge(b solver.Builder) *provenanceBridge { return &provenanceBridge{llbBridge: &llbBridge{ builder: b, @@ -515,6 +546,52 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SessionID = sessionID + createLease := func(ctx context.Context) (context.Context, error) { + lm, err := s.leaseManager() + if err != nil { + return ctx, err + } + var done func(context.Context) error + ctx, done, err = leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) + if err != nil { + return ctx, err + } + releasers = append(releasers, func() { + done(context.WithoutCancel(ctx)) + }) + return ctx, nil + } + + // Start eager export before the build so completed vertices can enqueue work. + // The lease is needed early to protect compressed blobs during the build. + var eager *eagerPipeline + eagerWaited := false + if exp.EagerExport != EagerExportNone && len(exp.Exporters) > 0 { + ctx, err = createLease(ctx) + if err != nil { + return nil, err + } + comp := exp.Exporters[0].Config().Compression() + eager, err = newEagerPipeline(ctx, exp.EagerExport, comp, sessionID, s.sm, exp.EagerPushConfig) + if err != nil { + return nil, err + } + j.SetOnVertexComplete(eager.onVertexComplete) + defer func() { + if eagerWaited { + return + } + eager.cancel(errors.WithStack(context.Canceled)) + if err := eager.wait(); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager export cleanup failed") + } + }() + } + + if exp.PushRegistryConfig != nil { + j.SetValue(keyEagerPushConfig, exp.PushRegistryConfig) + } + br := s.bridge(j) var fwd gateway.LLBBridgeForwarder if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" { @@ -586,6 +663,27 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } + // Exporters need the final layer digests, so wait for eager work first. + if eager != nil { + // Filter out completed vertices that are not part of the final result. + exportRefs, err := computeEagerExportRefs(ctx, res) + if err != nil { + bklog.G(ctx).WithError(err).Warnf("failed to compute eager export refs; skipping eager export filtering") + } else { + cancelled := eager.applyExportRefs(exportRefs) + bklog.G(ctx).Infof("eager export_refs=%d cancelled_inflight=%d", len(exportRefs), cancelled) + } + if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { + span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) + err := eager.wait() + eagerWaited = true + tracing.FinishWithError(span, err) + return err + }); err != nil { + return nil, errors.Wrap(err, "eager export pipeline failed") + } + } + resProv, err = addProvenanceToResult(res, br) if err != nil { return nil, err @@ -617,21 +715,13 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - // Functions that create new objects in containerd (eg. content blobs) need to have a lease to ensure - // that the object is not garbage collected immediately. This is protected by the indivual components, - // but because creating a lease is not cheap and requires a disk write, we create a single lease here - // early and let all the exporters, cache export and provenance creation use the same one. - lm, err := s.leaseManager() - if err != nil { - return nil, err - } - ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) - if err != nil { - return nil, err + // Without eager export, keep the original lease timing. + if eager == nil { + ctx, err = createLease(ctx) + if err != nil { + return nil, err + } } - releasers = append(releasers, func() { - done(context.WithoutCancel(ctx)) - }) cacheExporters, inlineCacheExporter := splitCacheExporters(exp.CacheExporters) @@ -1090,6 +1180,17 @@ func withDescHandlerCacheOpts(ctx context.Context, ref cache.ImmutableRef) conte }) } +func eagerWaitProgressID(mode EagerExportMode) string { + switch mode { + case EagerExportPush: + return "waiting for eager compression and push to finish" + case EagerExportCompress: + return "waiting for eager compression to finish" + default: + return "waiting for eager export to finish" + } +} + func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client.SolveStatus) error { if err := s.history.Status(ctx, id, statusChan); err != nil { if !errors.Is(err, os.ErrNotExist) { diff --git a/solver/pb/attr.go b/solver/pb/attr.go index b18223dcdc6a..1157d987750e 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -6,6 +6,8 @@ const AttrAuthHeaderSecret = "git.authheadersecret" const AttrAuthTokenSecret = "git.authtokensecret" const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" +const AttrGitChecksum = "git.checksum" + const AttrLocalSessionID = "local.session" const AttrLocalUniqueID = "local.unique" const AttrIncludePatterns = "local.includepattern" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index ce5b0d4ea9d2..75c298ae0d5c 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -30,6 +30,7 @@ const ( CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts" CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock" CapSourceGitSubdir apicaps.CapID = "source.git.subdir" + CapSourceGitChecksum apicaps.CapID = "source.git.checksum" CapSourceHTTP apicaps.CapID = "source.http" CapSourceHTTPAuth apicaps.CapID = "source.http.auth" @@ -222,6 +223,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitChecksum, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceHTTP, Enabled: true, diff --git a/solver/vertex_callback_test.go b/solver/vertex_callback_test.go new file mode 100644 index 000000000000..6c4b26a2bf8e --- /dev/null +++ b/solver/vertex_callback_test.go @@ -0,0 +1,138 @@ +package solver + +import ( + "context" + "sync" + "sync/atomic" + "testing" + + digest "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" +) + +type mockVertex struct { + dgst digest.Digest + name string +} + +func (v *mockVertex) Digest() digest.Digest { return v.dgst } +func (v *mockVertex) Sys() any { return nil } +func (v *mockVertex) Options() VertexOptions { return VertexOptions{} } +func (v *mockVertex) Inputs() []Edge { return nil } +func (v *mockVertex) Name() string { return v.name } + +type mockResult struct { + id string +} + +func (r *mockResult) ID() string { return r.id } +func (r *mockResult) Release(context.Context) error { return nil } +func (r *mockResult) Sys() any { return nil } +func (r *mockResult) Clone() Result { return r } + +func TestFireOnVertexComplete_CallsRegisteredCallbacks(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:abc", name: "test-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + var called atomic.Int32 + var receivedVtx Vertex + var receivedResults []Result + var mu sync.Mutex + + j := &Job{} + j.SetOnVertexComplete(func(v Vertex, results []Result) { + called.Add(1) + mu.Lock() + receivedVtx = v + receivedResults = results + mu.Unlock() + }) + st.jobs[j] = struct{}{} + + results := []Result{&mockResult{id: "res1"}} + so.fireOnVertexComplete(results) + + assert.Equal(t, int32(1), called.Load()) + mu.Lock() + assert.Equal(t, vtx, receivedVtx) + assert.Len(t, receivedResults, 1) + assert.Equal(t, "res1", receivedResults[0].ID()) + mu.Unlock() +} + +func TestFireOnVertexComplete_MultipleJobs(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:def", name: "multi-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + var called atomic.Int32 + + for range 3 { + j := &Job{} + j.SetOnVertexComplete(func(v Vertex, results []Result) { + called.Add(1) + }) + st.jobs[j] = struct{}{} + } + + so.fireOnVertexComplete([]Result{&mockResult{id: "r"}}) + + assert.Equal(t, int32(3), called.Load()) +} + +func TestFireOnVertexComplete_SkipsJobsWithoutCallback(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:ghi", name: "skip-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + var called atomic.Int32 + + jobWithCallback := &Job{} + jobWithCallback.SetOnVertexComplete(func(v Vertex, results []Result) { + called.Add(1) + }) + jobWithoutCallback := &Job{} + + st.jobs[jobWithCallback] = struct{}{} + st.jobs[jobWithoutCallback] = struct{}{} + + so.fireOnVertexComplete([]Result{&mockResult{id: "r"}}) + + assert.Equal(t, int32(1), called.Load()) +} + +func TestFireOnVertexComplete_NoJobs(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:jkl", name: "noop-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + // Should not panic with no jobs + so.fireOnVertexComplete([]Result{&mockResult{id: "r"}}) +} + +func TestSetOnVertexComplete(t *testing.T) { + j := &Job{} + assert.Nil(t, j.onVertexComplete) + + called := false + j.SetOnVertexComplete(func(v Vertex, results []Result) { + called = true + }) + assert.NotNil(t, j.onVertexComplete) + + j.onVertexComplete(nil, nil) + assert.True(t, called) +} diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index dde923c3b860..ac7cb6766b36 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -3,7 +3,9 @@ package containerimage import ( "context" "encoding/json" + "io" "maps" + "os" "runtime" "time" @@ -18,9 +20,12 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb/sourceresolver" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/estargz" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/imageutil" @@ -47,7 +52,8 @@ type puller struct { layerLimit *int vtx solver.Vertex ResolverType - store sourceresolver.ResolveImageConfigOptStore + store sourceresolver.ResolveImageConfigOptStore + eagerPushCfg *exporter.EagerPushConfig g flightcontrol.Group[struct{}] cacheKeyErr error @@ -60,6 +66,10 @@ type puller struct { *pull.Puller } +func (p *puller) SetEagerPushConfig(cfg *exporter.EagerPushConfig) { + p.eagerPushCfg = cfg +} + func mainManifestKey(desc ocispecs.Descriptor, platform ocispecs.Platform, layerLimit *int) (digest.Digest, error) { dt, err := json.Marshal(struct { Digest digest.Digest @@ -136,6 +146,10 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach p.manifest.Descriptors = p.manifest.Descriptors[:*ll] } + if p.eagerPushCfg != nil && p.eagerPushCfg.PreferPushRegistry && p.ResolverType == ResolverTypeRegistry && len(p.manifest.Descriptors) > 0 { + p.wrapProviderWithPushFallback(g) + } + if len(p.manifest.Descriptors) > 0 { progressController := &controller.Controller{ WriterFactory: progressFactory, @@ -283,6 +297,105 @@ func (p *puller) Snapshot(ctx context.Context, g session.Group) (ir cache.Immuta return current, nil } +// wrapProviderWithPushFallback wraps the manifest's Provider so that when +// a layer blob is lazily fetched, it first tries the push (destination) +// registry. If the layer already exists there, it is pulled from the closer +// registry instead of the origin. On any failure, the original provider is +// used transparently. +func (p *puller) wrapProviderWithPushFallback(g session.Group) { + cfg := p.eagerPushCfg + pushResolver := resolver.DefaultPool.GetResolver(cfg.RegistryHosts, cfg.TargetName, "pull", p.SessionManager, g) + origProvider := p.manifest.Provider + pushRef := cfg.TargetName + + bklog.L.Infof("prefer-push-registry: wrapping provider for %s with fallback to push registry %s", p.Ref, pushRef) + + p.manifest.Provider = func(g session.Group) content.Provider { + return &pushFallbackProvider{ + pushResolver: pushResolver.WithSession(g), + pushRef: pushRef, + origin: origProvider(g), + } + } +} + +// pushFallbackProvider tries fetching a blob from the push registry first, +// falling back to the origin provider on any error (404, auth, network, etc.). +type pushFallbackProvider struct { + pushResolver remotes.Resolver + pushRef string + origin content.Provider +} + +const ( + pushRegistryProbeTimeoutEnv = "BUILDKIT_PREFER_PUSH_REGISTRY_PROBE_TIMEOUT" + defaultPushRegistryProbeTimeout = 500 * time.Millisecond +) + +// pushRegistryProbeTimeout bounds how long we'll wait for the push registry +// to respond to the existence probe before falling back to origin. Configurable +// via BUILDKIT_PREFER_PUSH_REGISTRY_PROBE_TIMEOUT (Go duration format, e.g. "750ms"). +var pushRegistryProbeTimeout = func() time.Duration { + v := os.Getenv(pushRegistryProbeTimeoutEnv) + if v == "" { + return defaultPushRegistryProbeTimeout + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + bklog.L.Warnf("prefer-push-registry: invalid %s=%q, using default %s", pushRegistryProbeTimeoutEnv, v, defaultPushRegistryProbeTimeout) + return defaultPushRegistryProbeTimeout + } + bklog.L.Infof("prefer-push-registry: probe timeout overridden to %s via %s", d, pushRegistryProbeTimeoutEnv) + return d +}() + +func (p *pushFallbackProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { + if err := p.probe(ctx, desc); err != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe against %s failed (%v), falling back to origin", desc.Digest, p.pushRef, err) + return p.origin.ReaderAt(ctx, desc) + } + + fetcher, err := p.pushResolver.Fetcher(ctx, p.pushRef) + if err != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe succeeded but fetcher failed for %s (%v), falling back to origin", desc.Digest, p.pushRef, err) + return p.origin.ReaderAt(ctx, desc) + } + ra, err := contentutil.FromFetcher(fetcher).ReaderAt(ctx, desc) + if err != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe succeeded but ReaderAt failed for %s (%v), falling back to origin", desc.Digest, p.pushRef, err) + return p.origin.ReaderAt(ctx, desc) + } + + bklog.G(ctx).Infof("prefer-push-registry: layer %s pulled from push registry %s", desc.Digest, p.pushRef) + return ra, nil +} + +// probe forces an actual HTTP request against the push registry to confirm +// the blob exists. Bounded by pushRegistryProbeTimeout. We can't reuse the +// resulting ReaderAt because its context (with the short timeout) is captured +// inside containerd's httpReadSeeker; the caller must construct a fresh one +// with the original context for the real read. +func (p *pushFallbackProvider) probe(ctx context.Context, desc ocispecs.Descriptor) error { + probeCtx, cancel := context.WithTimeoutCause(ctx, pushRegistryProbeTimeout, errors.WithStack(context.DeadlineExceeded)) + defer cancel() + + fetcher, err := p.pushResolver.Fetcher(probeCtx, p.pushRef) + if err != nil { + return err + } + ra, err := contentutil.FromFetcher(fetcher).ReaderAt(probeCtx, desc) + if err != nil { + return err + } + defer ra.Close() + + probe := make([]byte, 1) + if _, err := ra.ReadAt(probe, 0); err != nil && !errors.Is(err, io.EOF) { + return err + } + return nil +} + // cacheKeyFromConfig returns a stable digest from image config. If image config // is a known oci image we will use chainID of layers. func cacheKeyFromConfig(dt []byte, layerLimit *int) (digest.Digest, error) { diff --git a/source/containerimage/pull_test.go b/source/containerimage/pull_test.go new file mode 100644 index 000000000000..20aa6902ac98 --- /dev/null +++ b/source/containerimage/pull_test.go @@ -0,0 +1,271 @@ +package containerimage + +import ( + "bytes" + "context" + "errors" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/remotes" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" +) + +const testPushRef = "push.example.com/test:latest" + +func testDesc(payload []byte) ocispecs.Descriptor { + return ocispecs.Descriptor{ + Digest: digest.FromBytes(payload), + Size: int64(len(payload)), + } +} + +// fakeResolver implements just enough of remotes.Resolver to back +// pushFallbackProvider in tests. Only Fetcher is exercised. +type fakeResolver struct { + fetcherFn func(ctx context.Context, ref string) (remotes.Fetcher, error) +} + +func (r *fakeResolver) Resolve(ctx context.Context, ref string) (string, ocispecs.Descriptor, error) { + return ref, ocispecs.Descriptor{}, errors.New("not implemented") +} +func (r *fakeResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + return r.fetcherFn(ctx, ref) +} +func (r *fakeResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + return nil, errors.New("not implemented") +} + +// fetcherFunc adapts a function to remotes.Fetcher for terse test setup. +type fetcherFunc func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) + +func (f fetcherFunc) Fetch(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return f(ctx, desc) +} + +// originProvider serves a fixed payload and counts ReaderAt invocations so +// tests can assert whether the fallback was exercised. +type originProvider struct { + payload []byte + calls int32 +} + +func (o *originProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { + atomic.AddInt32(&o.calls, 1) + return &bytesReaderAt{data: o.payload}, nil +} + +type bytesReaderAt struct { + data []byte +} + +func (b *bytesReaderAt) ReadAt(p []byte, off int64) (int, error) { + if off >= int64(len(b.data)) { + return 0, io.EOF + } + n := copy(p, b.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil +} +func (b *bytesReaderAt) Close() error { return nil } +func (b *bytesReaderAt) Size() int64 { return int64(len(b.data)) } + +// seekableBuffer wraps bytes.Reader as an io.ReadCloser+Seeker so it works +// with the FromFetcher readerAt wrapper which seeks for offset reads. +type seekableBuffer struct { + *bytes.Reader +} + +func (s *seekableBuffer) Close() error { return nil } + +func newSeekableBuffer(data []byte) io.ReadCloser { + return &seekableBuffer{Reader: bytes.NewReader(data)} +} + +// blockingReader blocks Read until ctx is done, simulating a slow registry. +type blockingReader struct { + ctx context.Context +} + +func (b *blockingReader) Read(p []byte) (int, error) { + <-b.ctx.Done() + return 0, b.ctx.Err() +} +func (b *blockingReader) Close() error { return nil } + +func TestPushFallbackProvider_Hit(t *testing.T) { + t.Parallel() + + pushPayload := []byte("from-push-registry") + originPayload := []byte("from-origin") + + var fetcherCalls int32 + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + atomic.AddInt32(&fetcherCalls, 1) + return newSeekableBuffer(pushPayload), nil + }), nil + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(pushPayload)) + require.NoError(t, err) + defer ra.Close() + + require.Equal(t, int32(0), atomic.LoadInt32(&origin.calls), "origin should not be called on hit") + // Probe + real read both call Fetcher exactly once. + require.Equal(t, int32(2), atomic.LoadInt32(&fetcherCalls)) + + got := make([]byte, len(pushPayload)) + n, err := ra.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, pushPayload, got[:n]) +} + +func TestPushFallbackProvider_NotFound(t *testing.T) { + t.Parallel() + + originPayload := []byte("from-origin") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return nil, errors.New("blob not found") + }), nil + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(originPayload)) + require.NoError(t, err) + defer ra.Close() + + require.Equal(t, int32(1), atomic.LoadInt32(&origin.calls), "origin should be called on miss") + + got := make([]byte, len(originPayload)) + n, err := ra.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, originPayload, got[:n]) +} + +func TestPushFallbackProvider_FetcherError(t *testing.T) { + t.Parallel() + + originPayload := []byte("from-origin") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return nil, errors.New("auth failed") + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(originPayload)) + require.NoError(t, err) + defer ra.Close() + + require.Equal(t, int32(1), atomic.LoadInt32(&origin.calls), "origin should be called when fetcher errors") +} + +func TestPushFallbackProvider_EmptyBlobIsHit(t *testing.T) { + t.Parallel() + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return newSeekableBuffer(nil), nil + }), nil + }, + } + origin := &originProvider{payload: []byte("origin-fallback")} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(nil)) + require.NoError(t, err) + defer ra.Close() + + // EOF on the 1-byte probe of an empty blob is not a "missing" signal — + // the blob exists, it's just empty. We should commit to the push reader. + require.Equal(t, int32(0), atomic.LoadInt32(&origin.calls)) +} + +func TestPushFallbackProvider_ProbeTimesOut(t *testing.T) { + t.Parallel() + + prevTimeout := pushRegistryProbeTimeout + pushRegistryProbeTimeout = 50 * time.Millisecond + defer func() { pushRegistryProbeTimeout = prevTimeout }() + + originPayload := []byte("from-origin") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return &blockingReader{ctx: ctx}, nil + }), nil + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + start := time.Now() + ra, err := p.ReaderAt(context.Background(), testDesc(originPayload)) + elapsed := time.Since(start) + + require.NoError(t, err) + defer ra.Close() + require.Equal(t, int32(1), atomic.LoadInt32(&origin.calls), "origin should be called when probe times out") + require.Less(t, elapsed, 1*time.Second, "should fall back well before the parent ctx deadline") + require.GreaterOrEqual(t, elapsed, 50*time.Millisecond, "should wait for the probe timeout") +} + +func TestPushFallbackProvider_ReturnedReaderIsolatedFromProbeTimeout(t *testing.T) { + t.Parallel() + + prevTimeout := pushRegistryProbeTimeout + pushRegistryProbeTimeout = 50 * time.Millisecond + defer func() { pushRegistryProbeTimeout = prevTimeout }() + + pushPayload := []byte("from-push-registry") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return newSeekableBuffer(pushPayload), nil + }), nil + }, + } + origin := &originProvider{payload: []byte("origin-fallback")} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(pushPayload)) + require.NoError(t, err) + defer ra.Close() + + // Sleep past the probe timeout. If the returned ReaderAt's underlying + // context were the (now-canceled) probeCtx, the read below would fail. + time.Sleep(2 * pushRegistryProbeTimeout) + + got := make([]byte, len(pushPayload)) + n, err := ra.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, pushPayload, got[:n]) +} diff --git a/source/git/identifier.go b/source/git/identifier.go index 77951399b08a..ac2b0dbe6c04 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -13,6 +13,7 @@ import ( type GitIdentifier struct { Remote string Ref string + Checksum string Subdir string KeepGitDir bool AuthTokenSecret string diff --git a/source/git/source.go b/source/git/source.go index acba38551ea1..a1671796ecb4 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -92,6 +92,8 @@ func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, pla id.KnownSSHHosts = v case pb.AttrMountSSHSock: id.MountSSHSock = v + case pb.AttrGitChecksum: + id.Checksum = v } } @@ -349,10 +351,26 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index gs.locker.Lock(remote) defer gs.locker.Unlock(remote) - if ref := gs.src.Ref; ref != "" && gitutil.IsCommitSHA(ref) { - cacheKey := gs.shaToCacheKey(ref, "") + if gs.src.Checksum != "" { + matched, err := regexp.MatchString("^[a-fA-F0-9]+$", gs.src.Checksum) + if err != nil || !matched { + return "", "", nil, false, errors.Errorf("invalid checksum %s for Git URL, expected hex commit hash", gs.src.Checksum) + } + } + + var refCommitFullHash, ref2 string + if gitutil.IsCommitSHA(gs.src.Checksum) && !gs.src.KeepGitDir { + refCommitFullHash = gs.src.Checksum + ref2 = gs.src.Ref + } + if refCommitFullHash == "" && gitutil.IsCommitSHA(gs.src.Ref) { + refCommitFullHash = gs.src.Ref + } + if refCommitFullHash != "" { + cacheKey := gs.shaToCacheKey(refCommitFullHash, ref2) gs.cacheKey = cacheKey - return cacheKey, ref, nil, true, nil + // gs.src.Checksum is verified when checking out the commit + return cacheKey, refCommitFullHash, nil, true, nil } gs.getAuthToken(ctx, g) @@ -415,7 +433,9 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index if !gitutil.IsCommitSHA(sha) { return "", "", nil, false, errors.Errorf("invalid commit sha %q", sha) } - + if gs.src.Checksum != "" && !strings.HasPrefix(sha, gs.src.Checksum) { + return "", "", nil, false, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, sha) + } cacheKey := gs.shaToCacheKey(sha, usedRef) gs.cacheKey = cacheKey return cacheKey, sha, nil, true, nil @@ -536,6 +556,17 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out subdir = "." } + if gs.src.Checksum != "" { + actualHashBuf, err := git.Run(ctx, "rev-parse", ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to rev-parse %s for %s", ref, urlutil.RedactCredentials(gs.src.Remote)) + } + actualHash := strings.TrimSpace(string(actualHashBuf)) + if !strings.HasPrefix(actualHash, gs.src.Checksum) { + return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, actualHash) + } + } + if gs.src.KeepGitDir && subdir == "." { checkoutDirGit := filepath.Join(checkoutDir, ".git") if err := os.MkdirAll(checkoutDir, 0711); err != nil { diff --git a/source/git/source_test.go b/source/git/source_test.go index 5962814af1ff..9e9f568f15dd 100644 --- a/source/git/source_test.go +++ b/source/git/source_test.go @@ -148,6 +148,16 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { require.NoError(t, err) require.Equal(t, "subcontents\n", string(dt)) + + // The key should not change regardless to the existence of Checksum + // https://github.com/moby/buildkit/pull/5975#discussion_r2092206059 + id.Checksum = pin3 + g, err = gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + key4, pin4, _, _, err := g.CacheKey(ctx, nil, 0) + require.NoError(t, err) + require.Equal(t, key3, key4) + require.Equal(t, pin3, pin4) } func TestFetchBySHA(t *testing.T) { @@ -304,54 +314,75 @@ func testFetchUnreferencedRefSha(t *testing.T, ref string, keepGitDir bool) { } func TestFetchByTag(t *testing.T) { - testFetchByTag(t, "lightweight-tag", "third", false, true, false) + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeNone) } func TestFetchByTagKeepGitDir(t *testing.T) { - testFetchByTag(t, "lightweight-tag", "third", false, true, true) + testFetchByTag(t, "lightweight-tag", "third", false, true, true, testChecksumModeNone) } func TestFetchByTagFull(t *testing.T) { - testFetchByTag(t, "refs/tags/lightweight-tag", "third", false, true, true) + testFetchByTag(t, "refs/tags/lightweight-tag", "third", false, true, true, testChecksumModeNone) } func TestFetchByAnnotatedTag(t *testing.T) { - testFetchByTag(t, "v1.2.3", "second", true, false, false) + testFetchByTag(t, "v1.2.3", "second", true, false, false, testChecksumModeNone) } func TestFetchByAnnotatedTagKeepGitDir(t *testing.T) { - testFetchByTag(t, "v1.2.3", "second", true, false, true) + testFetchByTag(t, "v1.2.3", "second", true, false, true, testChecksumModeNone) } func TestFetchByAnnotatedTagFull(t *testing.T) { - testFetchByTag(t, "refs/tags/v1.2.3", "second", true, false, true) + testFetchByTag(t, "refs/tags/v1.2.3", "second", true, false, true, testChecksumModeNone) } func TestFetchByBranch(t *testing.T) { - testFetchByTag(t, "feature", "withsub", false, true, false) + testFetchByTag(t, "feature", "withsub", false, true, false, testChecksumModeNone) } func TestFetchByBranchKeepGitDir(t *testing.T) { - testFetchByTag(t, "feature", "withsub", false, true, true) + testFetchByTag(t, "feature", "withsub", false, true, true, testChecksumModeNone) } func TestFetchByBranchFull(t *testing.T) { - testFetchByTag(t, "refs/heads/feature", "withsub", false, true, true) + testFetchByTag(t, "refs/heads/feature", "withsub", false, true, true, testChecksumModeNone) } func TestFetchByRef(t *testing.T) { - testFetchByTag(t, "test", "feature", false, true, false) + testFetchByTag(t, "test", "feature", false, true, false, testChecksumModeNone) } func TestFetchByRefKeepGitDir(t *testing.T) { - testFetchByTag(t, "test", "feature", false, true, true) + testFetchByTag(t, "test", "feature", false, true, true, testChecksumModeNone) } func TestFetchByRefFull(t *testing.T) { - testFetchByTag(t, "refs/test", "feature", false, true, true) + testFetchByTag(t, "refs/test", "feature", false, true, true, testChecksumModeNone) +} + +func TestFetchByTagWithChecksum(t *testing.T) { + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeValid) } -func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotatedTag, hasFoo13File, keepGitDir bool) { +func TestFetchByTagWithChecksumPartial(t *testing.T) { + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeValidPartial) +} + +func TestFetchByTagWithChecksumInvalid(t *testing.T) { + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeInvalid) +} + +type testChecksumMode int + +const ( + testChecksumModeNone testChecksumMode = iota + testChecksumModeValid + testChecksumModeValidPartial + testChecksumModeInvalid +) + +func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotatedTag, hasFoo13File, keepGitDir bool, checksumMode testChecksumMode) { if runtime.GOOS == "windows" { t.Skip("Depends on unimplemented containerd bind-mount support on Windows") } @@ -366,6 +397,28 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated id := &GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir} + if checksumMode != testChecksumModeNone { + cmd := exec.Command("git", "rev-parse", tag) + cmd.Dir = repo.mainPath + + out, err := cmd.Output() + require.NoError(t, err) + + sha := strings.TrimSpace(string(out)) + require.Equal(t, 40, len(sha)) + + switch checksumMode { + case testChecksumModeValid: + id.Checksum = sha + case testChecksumModeValidPartial: + id.Checksum = sha[:8] + case testChecksumModeInvalid: + id.Checksum = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + default: + // NOTREACHED + } + } + g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -383,6 +436,10 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated require.Equal(t, 40, len(pin1)) ref1, err := g.Snapshot(ctx, nil) + if checksumMode == testChecksumModeInvalid { + require.ErrorContains(t, err, "expected checksum to match "+id.Checksum) + return + } require.NoError(t, err) defer ref1.Release(context.TODO()) diff --git a/util/push/push.go b/util/push/push.go index 5c6ec95e013b..fa89f9e170ee 100644 --- a/util/push/push.go +++ b/util/push/push.go @@ -46,7 +46,33 @@ func Pusher(ctx context.Context, resolver remotes.Resolver, ref string) (remotes return &pusher{Pusher: p}, nil } -func Push(ctx context.Context, sm *session.Manager, sid string, provider content.Provider, manager content.Manager, dgst digest.Digest, ref string, insecure bool, hosts docker.RegistryHosts, byDigest bool, annotations map[digest.Digest]map[string]string) error { +// NewPusher creates a registry pusher for the given target reference. It +// handles reference parsing, insecure registry overrides, resolver creation, +// and pusher wrapping. +func NewPusher(ctx context.Context, sm *session.Manager, sid string, ref string, insecure bool, hosts docker.RegistryHosts) (remotes.Pusher, error) { + parsed, err := reference.ParseNormalizedNamed(ref) + if err != nil { + return nil, err + } + + scope := "push" + if insecure { + insecureTrue := true + httpTrue := true + hosts = resolver.NewRegistryConfig(map[string]resolverconfig.RegistryConfig{ + reference.Domain(parsed): { + Insecure: &insecureTrue, + PlainHTTP: &httpTrue, + }, + }) + scope += ":insecure" + } + + r := resolver.DefaultPool.GetResolver(hosts, ref, scope, sm, session.NewGroup(sid)) + return Pusher(ctx, r, ref) +} + +func Push(ctx context.Context, sm *session.Manager, sid string, provider content.Provider, manager content.Manager, dgst digest.Digest, ref string, insecure bool, hosts docker.RegistryHosts, byDigest bool, eager bool, annotations map[digest.Digest]map[string]string) error { ctx = contentutil.RegisterContentPayloadTypes(ctx) desc := ocispecs.Descriptor{ Digest: dgst, @@ -70,22 +96,7 @@ func Push(ctx context.Context, sm *session.Manager, sid string, provider content ref = r.String() } - scope := "push" - if insecure { - insecureTrue := true - httpTrue := true - hosts = resolver.NewRegistryConfig(map[string]resolverconfig.RegistryConfig{ - reference.Domain(parsed): { - Insecure: &insecureTrue, - PlainHTTP: &httpTrue, - }, - }) - scope += ":insecure" - } - - resolver := resolver.DefaultPool.GetResolver(hosts, ref, scope, sm, session.NewGroup(sid)) - - pusher, err := Pusher(ctx, resolver, ref) + pusher, err := NewPusher(ctx, sm, sid, ref, insecure, hosts) if err != nil { return err } @@ -128,7 +139,7 @@ func Push(ctx context.Context, sm *session.Manager, sid string, provider content return err } - layersDone := progress.OneOff(ctx, "pushing layers") + layersDone := progress.OneOff(ctx, pushLayersProgressID(eager)) err = images.Dispatch(ctx, skipNonDistributableBlobs(images.Handlers(handlers...)), nil, ocispecs.Descriptor{ Digest: dgst, Size: ra.Size(), @@ -147,6 +158,13 @@ func Push(ctx context.Context, sm *session.Manager, sid string, provider content return mfstDone(nil) } +func pushLayersProgressID(eager bool) string { + if eager { + return "pushing any remaining layers" + } + return "pushing layers" +} + // TODO: the containerd function for this is filtering too much, that needs to be fixed. // For now we just carry this. func skipNonDistributableBlobs(f images.HandlerFunc) images.HandlerFunc { diff --git a/util/tracing/detect/shutdown_test.go b/util/tracing/detect/shutdown_test.go new file mode 100644 index 000000000000..3f98d54f0f5d --- /dev/null +++ b/util/tracing/detect/shutdown_test.go @@ -0,0 +1,76 @@ +package detect_test + +import ( + "context" + "testing" + "time" + + "github.com/moby/buildkit/util/tracing/detect" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func newTestProvider(t *testing.T) *sdktrace.TracerProvider { + t.Helper() + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://192.0.2.1:4317") // RFC 5737 TEST-NET, guaranteed unreachable + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "grpc") + + exp, err := detect.NewSpanExporter(context.Background()) + if err != nil { + t.Fatalf("NewSpanExporter: %v", err) + } + if detect.IsNoneSpanExporter(exp) { + t.Fatal("expected OTLP exporter, got none") + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(detect.Resource()), + sdktrace.WithBatcher(exp), + ) + + tracer := tp.Tracer("test") + _, span := tracer.Start(context.Background(), "test-op") + span.End() + + return tp +} + +// TestShutdownStallsWithUnboundedContext reproduces +// https://github.com/moby/buildkit/issues/4616. +// Without a deadline on the context, Shutdown blocks until the BSP export +// timeout expires (default 30 s; reduced here via env for test speed). +func TestShutdownStallsWithUnboundedContext(t *testing.T) { + t.Setenv("OTEL_BSP_EXPORT_TIMEOUT", "2000") // 2 s instead of default 30 s + + tp := newTestProvider(t) + + start := time.Now() + _ = tp.Shutdown(context.TODO()) // the buggy call pattern + elapsed := time.Since(start) + + // Even with the reduced timeout, shutdown still blocks for the full + // export timeout (2 s here, 30 s in production). + if elapsed < 1500*time.Millisecond { + t.Fatalf("expected stall of ~2s, got %v", elapsed) + } + t.Logf("Unbounded shutdown stalled for %v (BSP export timeout)", elapsed) +} + +// TestShutdownRespectsDeadline verifies the fix: passing a bounded context +// makes Shutdown return promptly when the deadline is shorter than the export +// timeout. +func TestShutdownRespectsDeadline(t *testing.T) { + tp := newTestProvider(t) + + deadline := 500 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + + start := time.Now() + err := tp.Shutdown(ctx) + elapsed := time.Since(start) + + if elapsed > deadline+200*time.Millisecond { + t.Fatalf("Shutdown took %v, expected <= %v", elapsed, deadline+200*time.Millisecond) + } + t.Logf("Bounded shutdown completed in %v (err=%v)", elapsed, err) +} diff --git a/worker/base/worker.go b/worker/base/worker.go index 1922b9dc67c6..f0ffdabe2eca 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -317,8 +317,21 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm } } } + recoveredRefs := make(map[string]string, len(descHandlers)) + for d, h := range descHandlers { + recoveredRefs[string(d)] = h.Ref + } + bklog.G(ctx).Infof( + "LOADREF-RECOVER id=%s missingDigests=%v recoveredFromSolver=%v", + id, []digest.Digest(needsRemoteProviders), recoveredRefs, + ) opts = append(opts, descHandlers) ref, err = w.CacheMgr.Get(ctx, id, pg, opts...) + } else { + bklog.G(ctx).Warnf( + "LOADREF-RECOVER-NOGETTER id=%s missingDigests=%v (no CacheOptGetter in ctx)", + id, []digest.Digest(needsRemoteProviders), + ) } } if err != nil {