From 8e1bbedb004fa7580d64ba32f7d4162cd79141ea Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 10 Apr 2025 18:28:25 +0900 Subject: [PATCH 01/55] git source: add AttrGitChecksum Not integrated to util/giturl, as PR 5974 is not merged yet. Signed-off-by: Akihiro Suda (cherry picked from commit 6cbf02ae5b7daa3824654a8758300c8670d06758) --- client/llb/source.go | 13 ++++++ solver/pb/attr.go | 2 + solver/pb/caps.go | 7 ++++ source/git/identifier.go | 1 + source/git/source.go | 33 ++++++++++++++-- source/git/source_test.go | 83 +++++++++++++++++++++++++++++++++------ 6 files changed, 122 insertions(+), 17 deletions(-) diff --git a/client/llb/source.go b/client/llb/source.go index cb1f47206de3..d6eba0758981 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -322,6 +322,12 @@ func Git(url, ref string, opts ...GitOption) State { addCap(&gi.Constraints, pb.CapSourceGitMountSSHSock) } + checksum := gi.Checksum + if checksum != "" { + attrs[pb.AttrGitChecksum] = checksum + addCap(&gi.Constraints, pb.CapSourceGitChecksum) + } + addCap(&gi.Constraints, pb.CapSourceGit) source := NewSource("git://"+id, attrs, gi.Constraints) @@ -345,6 +351,7 @@ type GitInfo struct { addAuthCap bool KnownSSHHosts string MountSSHSock string + Checksum string } func KeepGitDir() GitOption { @@ -373,6 +380,12 @@ func MountSSHSock(sshID string) GitOption { }) } +func GitChecksum(v string) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.Checksum = v + }) +} + // AuthOption can be used with either HTTP or Git sources. type AuthOption interface { GitOption diff --git a/solver/pb/attr.go b/solver/pb/attr.go index b18223dcdc6a..1157d987750e 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -6,6 +6,8 @@ const AttrAuthHeaderSecret = "git.authheadersecret" const AttrAuthTokenSecret = "git.authtokensecret" const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" +const AttrGitChecksum = "git.checksum" + const AttrLocalSessionID = "local.session" const AttrLocalUniqueID = "local.unique" const AttrIncludePatterns = "local.includepattern" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index ce5b0d4ea9d2..75c298ae0d5c 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -30,6 +30,7 @@ const ( CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts" CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock" CapSourceGitSubdir apicaps.CapID = "source.git.subdir" + CapSourceGitChecksum apicaps.CapID = "source.git.checksum" CapSourceHTTP apicaps.CapID = "source.http" CapSourceHTTPAuth apicaps.CapID = "source.http.auth" @@ -222,6 +223,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitChecksum, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceHTTP, Enabled: true, diff --git a/source/git/identifier.go b/source/git/identifier.go index 77951399b08a..ac2b0dbe6c04 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -13,6 +13,7 @@ import ( type GitIdentifier struct { Remote string Ref string + Checksum string Subdir string KeepGitDir bool AuthTokenSecret string diff --git a/source/git/source.go b/source/git/source.go index acba38551ea1..d392b72b3b19 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -92,6 +92,8 @@ func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, pla id.KnownSSHHosts = v case pb.AttrMountSSHSock: id.MountSSHSock = v + case pb.AttrGitChecksum: + id.Checksum = v } } @@ -349,10 +351,19 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index gs.locker.Lock(remote) defer gs.locker.Unlock(remote) - if ref := gs.src.Ref; ref != "" && gitutil.IsCommitSHA(ref) { - cacheKey := gs.shaToCacheKey(ref, "") + var refCommitFullHash, ref2 string + if gitutil.IsCommitSHA(gs.src.Checksum) && !gs.src.KeepGitDir { + refCommitFullHash = gs.src.Checksum + ref2 = gs.src.Ref + } + if refCommitFullHash == "" && gitutil.IsCommitSHA(gs.src.Ref) { + refCommitFullHash = gs.src.Ref + } + if refCommitFullHash != "" { + cacheKey := gs.shaToCacheKey(refCommitFullHash, ref2) gs.cacheKey = cacheKey - return cacheKey, ref, nil, true, nil + // gs.src.Checksum is verified when checking out the commit + return cacheKey, refCommitFullHash, nil, true, nil } gs.getAuthToken(ctx, g) @@ -415,7 +426,9 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index if !gitutil.IsCommitSHA(sha) { return "", "", nil, false, errors.Errorf("invalid commit sha %q", sha) } - + if gs.src.Checksum != "" && !strings.HasPrefix(sha, gs.src.Checksum) { + return "", "", nil, false, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, sha) + } cacheKey := gs.shaToCacheKey(sha, usedRef) gs.cacheKey = cacheKey return cacheKey, sha, nil, true, nil @@ -536,6 +549,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out subdir = "." } + checkedoutRef := "HEAD" if gs.src.KeepGitDir && subdir == "." { checkoutDirGit := filepath.Join(checkoutDir, ".git") if err := os.MkdirAll(checkoutDir, 0711); err != nil { @@ -605,6 +619,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) } + checkedoutRef = ref // HEAD may not exist if subdir != "." { d, err := os.Open(filepath.Join(cd, subdir)) if err != nil { @@ -635,6 +650,16 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out } git = git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(gitDir)) + if gs.src.Checksum != "" { + actualHashBuf, err := git.Run(ctx, "rev-parse", checkedoutRef) + if err != nil { + return nil, errors.Wrapf(err, "failed to rev-parse %s for %s", checkedoutRef, urlutil.RedactCredentials(gs.src.Remote)) + } + actualHash := strings.TrimSpace(string(actualHashBuf)) + if !strings.HasPrefix(actualHash, gs.src.Checksum) { + return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, actualHash) + } + } _, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1") if err != nil { return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote)) diff --git a/source/git/source_test.go b/source/git/source_test.go index 5962814af1ff..9e9f568f15dd 100644 --- a/source/git/source_test.go +++ b/source/git/source_test.go @@ -148,6 +148,16 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { require.NoError(t, err) require.Equal(t, "subcontents\n", string(dt)) + + // The key should not change regardless to the existence of Checksum + // https://github.com/moby/buildkit/pull/5975#discussion_r2092206059 + id.Checksum = pin3 + g, err = gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + key4, pin4, _, _, err := g.CacheKey(ctx, nil, 0) + require.NoError(t, err) + require.Equal(t, key3, key4) + require.Equal(t, pin3, pin4) } func TestFetchBySHA(t *testing.T) { @@ -304,54 +314,75 @@ func testFetchUnreferencedRefSha(t *testing.T, ref string, keepGitDir bool) { } func TestFetchByTag(t *testing.T) { - testFetchByTag(t, "lightweight-tag", "third", false, true, false) + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeNone) } func TestFetchByTagKeepGitDir(t *testing.T) { - testFetchByTag(t, "lightweight-tag", "third", false, true, true) + testFetchByTag(t, "lightweight-tag", "third", false, true, true, testChecksumModeNone) } func TestFetchByTagFull(t *testing.T) { - testFetchByTag(t, "refs/tags/lightweight-tag", "third", false, true, true) + testFetchByTag(t, "refs/tags/lightweight-tag", "third", false, true, true, testChecksumModeNone) } func TestFetchByAnnotatedTag(t *testing.T) { - testFetchByTag(t, "v1.2.3", "second", true, false, false) + testFetchByTag(t, "v1.2.3", "second", true, false, false, testChecksumModeNone) } func TestFetchByAnnotatedTagKeepGitDir(t *testing.T) { - testFetchByTag(t, "v1.2.3", "second", true, false, true) + testFetchByTag(t, "v1.2.3", "second", true, false, true, testChecksumModeNone) } func TestFetchByAnnotatedTagFull(t *testing.T) { - testFetchByTag(t, "refs/tags/v1.2.3", "second", true, false, true) + testFetchByTag(t, "refs/tags/v1.2.3", "second", true, false, true, testChecksumModeNone) } func TestFetchByBranch(t *testing.T) { - testFetchByTag(t, "feature", "withsub", false, true, false) + testFetchByTag(t, "feature", "withsub", false, true, false, testChecksumModeNone) } func TestFetchByBranchKeepGitDir(t *testing.T) { - testFetchByTag(t, "feature", "withsub", false, true, true) + testFetchByTag(t, "feature", "withsub", false, true, true, testChecksumModeNone) } func TestFetchByBranchFull(t *testing.T) { - testFetchByTag(t, "refs/heads/feature", "withsub", false, true, true) + testFetchByTag(t, "refs/heads/feature", "withsub", false, true, true, testChecksumModeNone) } func TestFetchByRef(t *testing.T) { - testFetchByTag(t, "test", "feature", false, true, false) + testFetchByTag(t, "test", "feature", false, true, false, testChecksumModeNone) } func TestFetchByRefKeepGitDir(t *testing.T) { - testFetchByTag(t, "test", "feature", false, true, true) + testFetchByTag(t, "test", "feature", false, true, true, testChecksumModeNone) } func TestFetchByRefFull(t *testing.T) { - testFetchByTag(t, "refs/test", "feature", false, true, true) + testFetchByTag(t, "refs/test", "feature", false, true, true, testChecksumModeNone) +} + +func TestFetchByTagWithChecksum(t *testing.T) { + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeValid) } -func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotatedTag, hasFoo13File, keepGitDir bool) { +func TestFetchByTagWithChecksumPartial(t *testing.T) { + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeValidPartial) +} + +func TestFetchByTagWithChecksumInvalid(t *testing.T) { + testFetchByTag(t, "lightweight-tag", "third", false, true, false, testChecksumModeInvalid) +} + +type testChecksumMode int + +const ( + testChecksumModeNone testChecksumMode = iota + testChecksumModeValid + testChecksumModeValidPartial + testChecksumModeInvalid +) + +func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotatedTag, hasFoo13File, keepGitDir bool, checksumMode testChecksumMode) { if runtime.GOOS == "windows" { t.Skip("Depends on unimplemented containerd bind-mount support on Windows") } @@ -366,6 +397,28 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated id := &GitIdentifier{Remote: repo.mainURL, Ref: tag, KeepGitDir: keepGitDir} + if checksumMode != testChecksumModeNone { + cmd := exec.Command("git", "rev-parse", tag) + cmd.Dir = repo.mainPath + + out, err := cmd.Output() + require.NoError(t, err) + + sha := strings.TrimSpace(string(out)) + require.Equal(t, 40, len(sha)) + + switch checksumMode { + case testChecksumModeValid: + id.Checksum = sha + case testChecksumModeValidPartial: + id.Checksum = sha[:8] + case testChecksumModeInvalid: + id.Checksum = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + default: + // NOTREACHED + } + } + g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) @@ -383,6 +436,10 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated require.Equal(t, 40, len(pin1)) ref1, err := g.Snapshot(ctx, nil) + if checksumMode == testChecksumModeInvalid { + require.ErrorContains(t, err, "expected checksum to match "+id.Checksum) + return + } require.NoError(t, err) defer ref1.Release(context.TODO()) From 13f36e64a7a80a988f685f61020c4b86eb9fe14e Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 16 May 2025 14:51:28 +0900 Subject: [PATCH 02/55] dockerfile: implement `ADD --checksum=COMMIT_HASH GIT_URL` Signed-off-by: Akihiro Suda (cherry picked from commit 8ee2cf5cab160389251a0b5b9f379f5932225f8b) --- frontend/dockerfile/dockerfile2llb/convert.go | 63 +++++++++++-------- .../dockerfile/dockerfile_addchecksum_test.go | 2 +- frontend/dockerfile/dockerfile_addgit_test.go | 11 +++- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 71ef28a3a2b0..b52c03155baf 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -937,27 +937,21 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { case *instructions.WorkdirCommand: err = dispatchWorkdir(d, c, true, &opt) case *instructions.AddCommand: - var checksum digest.Digest - if c.Checksum != "" { - checksum, err = digest.Parse(c.Checksum) - } - if err == nil { - err = dispatchCopy(d, copyConfig{ - params: c.SourcesAndDest, - excludePatterns: c.ExcludePatterns, - source: opt.buildContext, - isAddCommand: true, - cmdToPrint: c, - chown: c.Chown, - chmod: c.Chmod, - link: c.Link, - keepGitDir: c.KeepGitDir, - checksum: checksum, - location: c.Location(), - ignoreMatcher: opt.dockerIgnoreMatcher, - opt: opt, - }) - } + err = dispatchCopy(d, copyConfig{ + params: c.SourcesAndDest, + excludePatterns: c.ExcludePatterns, + source: opt.buildContext, + isAddCommand: true, + cmdToPrint: c, + chown: c.Chown, + chmod: c.Chmod, + link: c.Link, + keepGitDir: c.KeepGitDir, + checksum: c.Checksum, + location: c.Location(), + ignoreMatcher: opt.dockerIgnoreMatcher, + opt: opt, + }) if err == nil { for _, src := range c.SourcePaths { if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { @@ -1470,8 +1464,8 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if len(cfg.params.SourcePaths) != 1 { return errors.New("checksum can't be specified for multiple sources") } - if !isHTTPSource(cfg.params.SourcePaths[0]) { - return errors.New("checksum can't be specified for non-HTTP(S) sources") + if !isHTTPSource(cfg.params.SourcePaths[0]) && !isGitSource(cfg.params.SourcePaths[0]) { + return errors.New("checksum requires HTTP(S) or Git sources") } } @@ -1519,6 +1513,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if cfg.keepGitDir { gitOptions = append(gitOptions, llb.KeepGitDir()) } + if cfg.checksum != "" { + gitOptions = append(gitOptions, llb.GitChecksum(cfg.checksum)) + } st := llb.Git(gitRef.Remote, commit, gitOptions...) opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: chopt, @@ -1547,7 +1544,15 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { } } - st := llb.HTTP(src, llb.Filename(f), llb.WithCustomName(pgName), llb.Checksum(cfg.checksum), dfCmd(cfg.params)) + var checksum digest.Digest + if cfg.checksum != "" { + checksum, err = digest.Parse(cfg.checksum) + if err != nil { + return err + } + } + + st := llb.HTTP(src, llb.Filename(f), llb.WithCustomName(pgName), llb.Checksum(checksum), dfCmd(cfg.params)) opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: chopt, @@ -1674,7 +1679,7 @@ type copyConfig struct { chmod string link bool keepGitDir bool - checksum digest.Digest + checksum string parents bool location []parser.Range ignoreMatcher *patternmatcher.PatternMatcher @@ -2265,11 +2270,15 @@ func isHTTPSource(src string) bool { if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { return false } + return !isGitSource(src) +} + +func isGitSource(src string) bool { // https://github.com/ORG/REPO.git is a git source, not an http source if gitRef, gitErr := gitutil.ParseGitRef(src); gitRef != nil && gitErr == nil { - return false + return true } - return true + return false } func isEnabledForStage(stage string, value string) bool { diff --git a/frontend/dockerfile/dockerfile_addchecksum_test.go b/frontend/dockerfile/dockerfile_addchecksum_test.go index 050291845168..c2b77ff1f4bd 100644 --- a/frontend/dockerfile/dockerfile_addchecksum_test.go +++ b/frontend/dockerfile/dockerfile_addchecksum_test.go @@ -162,6 +162,6 @@ ADD --checksum=%s foo /tmp/foo dockerui.DefaultLocalNameContext: dir, }, }, nil) - require.Error(t, err, "checksum can't be specified for non-HTTP(S) sources") + require.Error(t, err, "checksum requires HTTP(S) or Git sources") }) } diff --git a/frontend/dockerfile/dockerfile_addgit_test.go b/frontend/dockerfile/dockerfile_addgit_test.go index db8d0d9649fa..60797a34885e 100644 --- a/frontend/dockerfile/dockerfile_addgit_test.go +++ b/frontend/dockerfile/dockerfile_addgit_test.go @@ -5,7 +5,9 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" + "strings" "testing" "text/template" @@ -52,6 +54,12 @@ func testAddGit(t *testing.T, sb integration.Sandbox) { err = runShell(gitDir, gitCommands...) require.NoError(t, err) + revParseCmd := exec.Command("git", "rev-parse", "v0.0.2") + revParseCmd.Dir = gitDir + commitHashB, err := revParseCmd.Output() + require.NoError(t, err) + commitHash := strings.TrimSpace(string(commitHashB)) + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir)))) defer server.Close() serverURL := server.URL @@ -68,7 +76,7 @@ RUN cd /x && \ # Complicated case ARG REPO="{{.ServerURL}}/.git" ARG TAG="v0.0.2" -ADD --keep-git-dir=true --chown=4242:8484 ${REPO}#${TAG} /buildkit-chowned +ADD --keep-git-dir=true --chown=4242:8484 --checksum={{.Checksum}} ${REPO}#${TAG} /buildkit-chowned RUN apk add git USER 4242 RUN cd /buildkit-chowned && \ @@ -78,6 +86,7 @@ RUN cd /buildkit-chowned && \ [ -z "$(git status -s)" ] `, map[string]string{ "ServerURL": serverURL, + "Checksum": commitHash, }) require.NoError(t, err) t.Logf("dockerfile=%s", dockerfile) From 17ae6d087c9718fb443864e1606720b27128025f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 16 May 2025 18:10:28 -0700 Subject: [PATCH 03/55] git: verify checksum early and more tests Signed-off-by: Tonis Tiigi (cherry picked from commit 58f956b8075d756063d616eea88b9540c08ed696) --- frontend/dockerfile/dockerfile_addgit_test.go | 142 +++++++++++++++++- source/git/source.go | 30 ++-- 2 files changed, 157 insertions(+), 15 deletions(-) diff --git a/frontend/dockerfile/dockerfile_addgit_test.go b/frontend/dockerfile/dockerfile_addgit_test.go index 60797a34885e..34557f249b87 100644 --- a/frontend/dockerfile/dockerfile_addgit_test.go +++ b/frontend/dockerfile/dockerfile_addgit_test.go @@ -58,7 +58,13 @@ func testAddGit(t *testing.T, sb integration.Sandbox) { revParseCmd.Dir = gitDir commitHashB, err := revParseCmd.Output() require.NoError(t, err) - commitHash := strings.TrimSpace(string(commitHashB)) + commitHashV2 := strings.TrimSpace(string(commitHashB)) + + revParseCmd = exec.Command("git", "rev-parse", "v0.0.3") + revParseCmd.Dir = gitDir + commitHashB, err = revParseCmd.Output() + require.NoError(t, err) + commitHashV3 := strings.TrimSpace(string(commitHashB)) server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir)))) defer server.Close() @@ -86,10 +92,9 @@ RUN cd /buildkit-chowned && \ [ -z "$(git status -s)" ] `, map[string]string{ "ServerURL": serverURL, - "Checksum": commitHash, + "Checksum": commitHashV2, }) require.NoError(t, err) - t.Logf("dockerfile=%s", dockerfile) dir := integration.Tmpdir(t, fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600), @@ -106,6 +111,137 @@ RUN cd /buildkit-chowned && \ }, }, nil) require.NoError(t, err) + + // Additional test: ADD from Git URL with checksum but without keep-git-dir flag + dockerfile2, err := applyTemplate(` +FROM alpine +ARG REPO="{{.ServerURL}}/.git" +ARG TAG="v0.0.3" +ADD --checksum={{.Checksum}} ${REPO}#${TAG} /nogitdir +RUN [ -f /nogitdir/foo ] +RUN [ "$(cat /nogitdir/foo)" = "foo of v0.0.3" ] +RUN [ ! -d /nogitdir/.git ] +`, map[string]string{ + "ServerURL": serverURL, + "Checksum": commitHashV3, + }) + require.NoError(t, err) + + dir2 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir2, + dockerui.DefaultLocalNameContext: dir2, + }, + }, nil) + require.NoError(t, err) + + // access initial ref again that was already pulled + dockerfile3, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG TAG="v0.0.2" + ADD --keep-git-dir --checksum={{.Checksum}} ${REPO}#${TAG} /nogitdir + RUN [ -f /nogitdir/foo ] + RUN [ "$(cat /nogitdir/foo)" = "foo of v0.0.2" ] + RUN [ -d /nogitdir/.git ] + `, map[string]string{ + "ServerURL": serverURL, + "Checksum": commitHashV2, + }) + require.NoError(t, err) + + dir3 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile3), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir3, + dockerui.DefaultLocalNameContext: dir3, + }, + }, nil) + require.NoError(t, err) + + // Additional test: ADD from Git URL using commitHashV3 for both checksum and ref + dockerfile4, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG COMMIT="{{.Checksum}}" + ADD --keep-git-dir=true --checksum={{.Checksum}} ${REPO}#${COMMIT} /commitdir + RUN [ -f /commitdir/foo ] + RUN [ "$(cat /commitdir/foo)" = "foo of v0.0.3" ] + RUN [ -d /commitdir/.git ] + `, map[string]string{ + "ServerURL": serverURL, + "Checksum": commitHashV3, + }) + require.NoError(t, err) + + dir4 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile4), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir4, + dockerui.DefaultLocalNameContext: dir4, + }, + }, nil) + require.NoError(t, err) + + // checksum does not match + dockerfile5, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG TAG="v0.0.3" + ADD --checksum={{.WrongChecksum}} ${REPO}#${TAG} /faildir + `, map[string]string{ + "ServerURL": serverURL, + "WrongChecksum": commitHashV2, // v0.0.2 hash, but ref is v0.0.3 + }) + require.NoError(t, err) + + dir5 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile5), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir5, + dockerui.DefaultLocalNameContext: dir5, + }, + }, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "expected checksum to match") + + // checksum is garbage + dockerfile6, err := applyTemplate(` + FROM alpine + ARG REPO="{{.ServerURL}}/.git" + ARG TAG="v0.0.3" + ADD --checksum=foobar ${REPO}#${TAG} /faildir + `, map[string]string{ + "ServerURL": serverURL, + }) + require.NoError(t, err) + + dir6 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile6), 0600), + ) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir6, + dockerui.DefaultLocalNameContext: dir6, + }, + }, nil) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid checksum") + require.Contains(t, err.Error(), "expected hex commit hash") } func applyTemplate(tmpl string, x any) (string, error) { diff --git a/source/git/source.go b/source/git/source.go index d392b72b3b19..a1671796ecb4 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -351,6 +351,13 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index gs.locker.Lock(remote) defer gs.locker.Unlock(remote) + if gs.src.Checksum != "" { + matched, err := regexp.MatchString("^[a-fA-F0-9]+$", gs.src.Checksum) + if err != nil || !matched { + return "", "", nil, false, errors.Errorf("invalid checksum %s for Git URL, expected hex commit hash", gs.src.Checksum) + } + } + var refCommitFullHash, ref2 string if gitutil.IsCommitSHA(gs.src.Checksum) && !gs.src.KeepGitDir { refCommitFullHash = gs.src.Checksum @@ -549,7 +556,17 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out subdir = "." } - checkedoutRef := "HEAD" + if gs.src.Checksum != "" { + actualHashBuf, err := git.Run(ctx, "rev-parse", ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to rev-parse %s for %s", ref, urlutil.RedactCredentials(gs.src.Remote)) + } + actualHash := strings.TrimSpace(string(actualHashBuf)) + if !strings.HasPrefix(actualHash, gs.src.Checksum) { + return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, actualHash) + } + } + if gs.src.KeepGitDir && subdir == "." { checkoutDirGit := filepath.Join(checkoutDir, ".git") if err := os.MkdirAll(checkoutDir, 0711); err != nil { @@ -619,7 +636,6 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) } - checkedoutRef = ref // HEAD may not exist if subdir != "." { d, err := os.Open(filepath.Join(cd, subdir)) if err != nil { @@ -650,16 +666,6 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out } git = git.New(gitutil.WithWorkTree(checkoutDir), gitutil.WithGitDir(gitDir)) - if gs.src.Checksum != "" { - actualHashBuf, err := git.Run(ctx, "rev-parse", checkedoutRef) - if err != nil { - return nil, errors.Wrapf(err, "failed to rev-parse %s for %s", checkedoutRef, urlutil.RedactCredentials(gs.src.Remote)) - } - actualHash := strings.TrimSpace(string(actualHashBuf)) - if !strings.HasPrefix(actualHash, gs.src.Checksum) { - return nil, errors.Errorf("expected checksum to match %s, got %s", gs.src.Checksum, actualHash) - } - } _, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1") if err != nil { return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote)) From 93b71cd2ac486fc5670d31b05ada7f87ab809935 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 16 May 2025 18:57:00 -0700 Subject: [PATCH 04/55] git: add testcase for checking that adding checksum doesn't break cache Signed-off-by: Tonis Tiigi (cherry picked from commit 6b0bf3b21aeb958dd60ffc5175f7286721aab5f5) --- frontend/dockerfile/dockerfile_addgit_test.go | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/frontend/dockerfile/dockerfile_addgit_test.go b/frontend/dockerfile/dockerfile_addgit_test.go index 34557f249b87..60b56de630a7 100644 --- a/frontend/dockerfile/dockerfile_addgit_test.go +++ b/frontend/dockerfile/dockerfile_addgit_test.go @@ -21,6 +21,7 @@ import ( var addGitTests = integration.TestFuncs( testAddGit, + testAddGitChecksumCache, ) func init() { @@ -244,6 +245,111 @@ RUN [ ! -d /nogitdir/.git ] require.Contains(t, err.Error(), "expected hex commit hash") } +func testAddGitChecksumCache(t *testing.T, sb integration.Sandbox) { + integration.SkipOnPlatform(t, "windows") + f := getFrontend(t, sb) + + gitDir, err := os.MkdirTemp("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(gitDir) + gitCommands := []string{ + "git init", + "git config --local user.email test", + "git config --local user.name test", + } + makeCommit := func(tag string) []string { + return []string{ + "echo foo of " + tag + " >foo", + "git add foo", + "git commit -m " + tag, + "git tag " + tag, + } + } + gitCommands = append(gitCommands, makeCommit("v0.0.1")...) + gitCommands = append(gitCommands, makeCommit("v0.0.2")...) + gitCommands = append(gitCommands, "git update-server-info") + err = runShell(gitDir, gitCommands...) + require.NoError(t, err) + + revParseCmd := exec.Command("git", "rev-parse", "v0.0.2") + revParseCmd.Dir = gitDir + commitHashB, err := revParseCmd.Output() + require.NoError(t, err) + commitHash := strings.TrimSpace(string(commitHashB)) + + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(gitDir)))) + defer server.Close() + serverURL := server.URL + + // First build: without checksum, from tag, generate unique.txt from /dev/urandom and copy to scratch + dockerfile1 := ` +FROM alpine AS src +ADD --keep-git-dir ` + serverURL + `/.git#v0.0.2 /repo +RUN head -c 16 /dev/urandom | base64 > /repo/unique.txt + +FROM scratch +COPY --from=src /repo/unique.txt / +` + dir1 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile1), 0600), + ) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + destDir1 := t.TempDir() + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir1, + }, + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir1, + dockerui.DefaultLocalNameContext: dir1, + }, + }, nil) + require.NoError(t, err) + + unique1, err := os.ReadFile(filepath.Join(destDir1, "unique.txt")) + require.NoError(t, err) + + // Second build: with checksum, should match cache even though this one sets commitHash and get same unique.txt + dockerfile2 := ` +FROM alpine AS src +ADD --keep-git-dir --checksum=` + commitHash + ` ` + serverURL + `/.git#v0.0.2 /repo +RUN head -c 16 /dev/urandom | base64 > /repo/unique.txt + +FROM scratch +COPY --from=src /repo/unique.txt / +` + dir2 := integration.Tmpdir(t, + fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600), + ) + + destDir2 := t.TempDir() + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir2, + }, + }, + LocalMounts: map[string]fsutil.FS{ + dockerui.DefaultLocalNameDockerfile: dir2, + dockerui.DefaultLocalNameContext: dir2, + }, + }, nil) + require.NoError(t, err) + + unique2, err := os.ReadFile(filepath.Join(destDir2, "unique.txt")) + require.NoError(t, err) + + require.Equal(t, string(unique1), string(unique2), "cache should be matched and unique file content should be the same") +} + func applyTemplate(tmpl string, x any) (string, error) { var buf bytes.Buffer parsed, err := template.New("").Parse(tmpl) From a3712e24c8b31109c74d95345c435ea9530cc166 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 23 Oct 2024 18:03:47 -0700 Subject: [PATCH 05/55] allow duration based filters on diskusage requests Allows similar time-based filter that is allowed for prune requests so that DiskUsage request can be used to check which records would be candidates for pruning. Signed-off-by: Tonis Tiigi (cherry picked from commit 2307fb7d123fbe436e9b3d5449b57cc98ac8db4d) --- api/services/control/control.pb.go | 874 +++++++++++---------- api/services/control/control.proto | 1 + api/services/control/control_vtproto.pb.go | 31 + cache/manager.go | 11 +- client/diskusage.go | 17 +- control/control.go | 3 +- 6 files changed, 500 insertions(+), 437 deletions(-) diff --git a/api/services/control/control.pb.go b/api/services/control/control.pb.go index efed916c8f72..d76537782cee 100644 --- a/api/services/control/control.pb.go +++ b/api/services/control/control.pb.go @@ -164,7 +164,8 @@ type DiskUsageRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Filter []string `protobuf:"bytes,1,rep,name=filter,proto3" json:"filter,omitempty"` + Filter []string `protobuf:"bytes,1,rep,name=filter,proto3" json:"filter,omitempty"` + AgeLimit int64 `protobuf:"varint,2,opt,name=ageLimit,proto3" json:"ageLimit,omitempty"` } func (x *DiskUsageRequest) Reset() { @@ -204,6 +205,13 @@ func (x *DiskUsageRequest) GetFilter() []string { return nil } +func (x *DiskUsageRequest) GetAgeLimit() int64 { + if x != nil { + return x.AgeLimit + } + return 0 +} + type DiskUsageResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2063,452 +2071,454 @@ var file_github_com_moby_buildkit_api_services_control_control_proto_rawDesc = [ 0x70, 0x61, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x55, 0x73, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, - 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x22, 0x2a, 0x0a, 0x10, + 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x53, 0x70, 0x61, 0x63, 0x65, 0x22, 0x46, 0x0a, 0x10, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, 0x4a, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x6b, - 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, - 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, - 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x22, 0x87, 0x03, 0x0a, 0x0b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x49, 0x44, 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, - 0x0a, 0x05, 0x49, 0x6e, 0x55, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x49, - 0x6e, 0x55, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1a, 0x0a, 0x06, 0x50, 0x61, 0x72, 0x65, - 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, - 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, - 0x0a, 0x0a, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, - 0x4c, 0x61, 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, - 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, - 0x55, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, - 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, - 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0a, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, - 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x53, 0x68, - 0x61, 0x72, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, - 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xf4, - 0x07, 0x0a, 0x0c, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, - 0x66, 0x12, 0x2e, 0x0a, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x66, 0x69, 0x6e, - 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x2e, 0x0a, 0x12, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, - 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x12, 0x75, 0x0a, 0x17, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, - 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, - 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, - 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x67, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x61, 0x67, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x22, 0x4a, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x06, 0x72, 0x65, 0x63, + 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x73, 0x61, + 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x22, 0x87, 0x03, 0x0a, 0x0b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, + 0x12, 0x18, 0x0a, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x4d, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x49, 0x6e, + 0x55, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x49, 0x6e, 0x55, 0x73, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, + 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1a, 0x0a, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, + 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x4c, 0x61, + 0x73, 0x74, 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0a, 0x4c, 0x61, 0x73, 0x74, + 0x55, 0x73, 0x65, 0x64, 0x41, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x55, 0x73, 0x61, 0x67, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x55, 0x73, 0x61, 0x67, + 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x0a, 0x0b, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x52, 0x65, 0x63, 0x6f, + 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x68, 0x61, 0x72, + 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x53, 0x68, 0x61, 0x72, 0x65, 0x64, + 0x12, 0x18, 0x0a, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x07, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xf4, 0x07, 0x0a, 0x0c, 0x53, + 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x52, + 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x2e, 0x0a, + 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x0a, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, 0x0a, + 0x12, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x72, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x75, 0x0a, 0x17, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, - 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x57, - 0x0a, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, - 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, + 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, + 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x17, 0x45, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1a, + 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x57, 0x0a, 0x0d, 0x46, 0x72, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, + 0x74, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, + 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x45, 0x6e, 0x74, + 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x5a, 0x0a, + 0x0e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, - 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, - 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x05, 0x43, 0x61, 0x63, 0x68, 0x65, 0x12, 0x22, 0x0a, - 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x09, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x5a, 0x0a, 0x0e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x32, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x46, - 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x1a, 0x0a, - 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x08, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x49, 0x0a, 0x0c, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2e, - 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x0c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x6f, - 0x6c, 0x69, 0x63, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, - 0x74, 0x65, 0x72, 0x52, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x34, - 0x0a, 0x15, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x70, 0x6f, - 0x72, 0x74, 0x65, 0x72, 0x1a, 0x4a, 0x0a, 0x1c, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x1a, 0x40, 0x0a, 0x12, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x51, 0x0a, 0x13, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, - 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, - 0x44, 0x65, 0x66, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xad, 0x03, 0x0a, 0x0c, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, - 0x52, 0x65, 0x66, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x44, 0x65, - 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x14, 0x49, 0x6d, 0x70, 0x6f, - 0x72, 0x74, 0x52, 0x65, 0x66, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, - 0x66, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x6f, 0x0a, 0x15, - 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, - 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x45, 0x78, 0x70, 0x6f, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, + 0x70, 0x75, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x46, 0x72, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x49, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x49, 0x0a, 0x0c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, + 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x2e, 0x50, 0x6f, 0x6c, 0x69, + 0x63, 0x79, 0x52, 0x0c, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, + 0x12, 0x38, 0x0a, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, + 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x34, 0x0a, 0x15, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x72, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x45, 0x6e, 0x61, 0x62, 0x6c, + 0x65, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, + 0x1a, 0x4a, 0x0a, 0x1c, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, + 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x40, 0x0a, 0x12, + 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x51, + 0x0a, 0x13, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x24, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x44, 0x65, 0x66, 0x69, + 0x6e, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xad, 0x03, 0x0a, 0x0c, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x44, + 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x13, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x14, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, + 0x66, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x14, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x73, 0x44, 0x65, + 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x6f, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, - 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, - 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x3d, 0x0a, - 0x07, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x07, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x07, - 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, - 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x07, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x1a, 0x48, 0x0a, 0x1a, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x64, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, + 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, + 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x07, 0x45, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x07, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x3d, 0x0a, 0x07, 0x49, 0x6d, 0x70, 0x6f, + 0x72, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, 0x63, + 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, + 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x73, 0x1a, 0x48, 0x0a, 0x1a, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x41, 0x74, 0x74, 0x72, 0x73, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0xa7, 0x01, 0x0a, 0x11, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x44, 0x0a, 0x05, 0x41, + 0x74, 0x74, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x41, 0x74, 0x74, 0x72, + 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xb7, 0x01, 0x0a, 0x0d, + 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, + 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, + 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x1a, 0x43, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa7, 0x01, 0x0a, 0x11, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x54, - 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x44, 0x0a, 0x05, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x43, 0x61, 0x63, 0x68, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, - 0x41, 0x74, 0x74, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0xb7, 0x01, 0x0a, 0x0d, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x61, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, - 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x70, - 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x43, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x21, 0x0a, 0x0d, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, - 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x22, 0xf0, 0x01, 0x0a, - 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x34, 0x0a, 0x08, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x18, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, - 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x52, 0x08, 0x76, 0x65, 0x72, - 0x74, 0x65, 0x78, 0x65, 0x73, 0x12, 0x3a, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, - 0x78, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, - 0x73, 0x12, 0x2f, 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, - 0x67, 0x73, 0x12, 0x3b, 0x0a, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x57, 0x61, - 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x52, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x22, - 0xa3, 0x02, 0x0a, 0x06, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x63, 0x61, 0x63, 0x68, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, - 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x38, 0x0a, 0x09, - 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x0d, - 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xa4, 0x02, 0x0a, 0x0c, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, - 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, - 0x61, 0x6c, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x34, 0x0a, 0x07, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x65, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x87, 0x01, 0x0a, - 0x09, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, - 0x72, 0x74, 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, - 0x65, 0x78, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x22, 0xc4, 0x01, 0x0a, 0x0d, 0x56, 0x65, 0x72, 0x74, 0x65, - 0x78, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, - 0x65, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, - 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x65, - 0x74, 0x61, 0x69, 0x6c, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x22, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x0a, 0x06, 0x72, 0x61, - 0x6e, 0x67, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, - 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x22, 0x0a, - 0x0c, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, - 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x22, 0x2c, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, - 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, - 0x53, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, - 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, - 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x22, 0x0d, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x22, 0x61, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x21, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x22, 0xf0, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x08, 0x76, + 0x65, 0x72, 0x74, 0x65, 0x78, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, + 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x52, 0x08, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x65, + 0x73, 0x12, 0x3a, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x12, 0x2f, 0x0a, + 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x56, + 0x65, 0x72, 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x3b, + 0x0a, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, + 0x67, 0x52, 0x08, 0x77, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x22, 0xa3, 0x02, 0x0a, 0x06, + 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x16, + 0x0a, 0x06, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x61, + 0x63, 0x68, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x63, 0x61, 0x63, 0x68, + 0x65, 0x64, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, + 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, + 0x65, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x37, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x67, + 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x70, 0x62, 0x2e, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x52, 0x0d, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x47, 0x72, 0x6f, 0x75, + 0x70, 0x22, 0xa4, 0x02, 0x0a, 0x0c, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x49, 0x44, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, + 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x07, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x74, 0x61, + 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x12, 0x38, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x34, 0x0a, 0x07, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x38, + 0x0a, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x87, 0x01, 0x0a, 0x09, 0x56, 0x65, 0x72, + 0x74, 0x65, 0x78, 0x4c, 0x6f, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x38, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6d, + 0x73, 0x67, 0x22, 0xc4, 0x01, 0x0a, 0x0d, 0x56, 0x65, 0x72, 0x74, 0x65, 0x78, 0x57, 0x61, 0x72, + 0x6e, 0x69, 0x6e, 0x67, 0x12, 0x16, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x76, 0x65, 0x72, 0x74, 0x65, 0x78, 0x12, 0x14, 0x0a, 0x05, + 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x6c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x05, 0x73, 0x68, 0x6f, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x06, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, + 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, + 0x72, 0x6c, 0x12, 0x22, 0x0a, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0e, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x21, 0x0a, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x22, 0x22, 0x0a, 0x0c, 0x42, 0x79, 0x74, + 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x2c, 0x0a, + 0x12, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x22, 0x53, 0x0a, 0x13, 0x4c, + 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, + 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x22, 0x0d, 0x0a, 0x0b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, + 0x61, 0x0a, 0x0c, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x51, 0x0a, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x22, 0x93, 0x01, 0x0a, 0x13, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, + 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x41, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, + 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x1c, 0x0a, 0x09, + 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x09, 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x46, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x46, 0x69, 0x6c, 0x74, + 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x8e, 0x01, 0x0a, 0x11, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3b, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x27, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, - 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x93, 0x01, 0x0a, 0x13, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, - 0x0a, 0x0a, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0a, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x4f, 0x6e, 0x6c, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, - 0x12, 0x1c, 0x0a, 0x09, 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x09, 0x45, 0x61, 0x72, 0x6c, 0x79, 0x45, 0x78, 0x69, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, - 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x8e, 0x01, 0x0a, - 0x11, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x27, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, - 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, - 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xd3, 0x09, - 0x0a, 0x12, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x64, 0x12, 0x5d, 0x0a, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, - 0x74, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, - 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, - 0x73, 0x12, 0x38, 0x0a, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, - 0x52, 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x28, 0x0a, 0x05, 0x65, - 0x72, 0x72, 0x6f, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, - 0x41, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, - 0x3c, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, + 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, + 0x64, 0x52, 0x06, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x22, 0xd3, 0x09, 0x0a, 0x12, 0x42, 0x75, + 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, + 0x65, 0x66, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x12, 0x5d, + 0x0a, 0x0d, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x46, 0x72, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, + 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x12, 0x38, 0x0a, + 0x09, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x09, 0x45, 0x78, + 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x73, 0x12, 0x28, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x12, 0x38, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, - 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, - 0x66, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, - 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x12, 0x4b, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x0b, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, - 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, - 0x1e, 0x0a, 0x0a, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0a, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x32, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x74, 0x72, - 0x61, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0e, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x6e, - 0x75, 0x6d, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x0f, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0e, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, - 0x65, 0x70, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6e, 0x75, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x53, - 0x74, 0x65, 0x70, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x54, - 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x6e, 0x75, 0x6d, - 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x11, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x6e, 0x75, 0x6d, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, - 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, 0x42, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, - 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0d, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x6e, - 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x40, 0x0a, - 0x12, 0x46, 0x72, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0x43, 0x0a, 0x15, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5d, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, + 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3c, 0x0a, 0x0b, 0x43, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x43, 0x6f, + 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, 0x04, 0x6c, 0x6f, 0x67, + 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x66, 0x0a, 0x10, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, + 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x45, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x10, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x4b, + 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x31, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, + 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x47, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x06, 0x70, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x26, 0x0a, 0x0e, 0x6e, 0x75, 0x6d, 0x43, 0x61, + 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0e, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, + 0x24, 0x0a, 0x0d, 0x6e, 0x75, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x74, 0x65, 0x70, 0x73, + 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6e, 0x75, 0x6d, 0x54, 0x6f, 0x74, 0x61, 0x6c, + 0x53, 0x74, 0x65, 0x70, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x43, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, 0x65, 0x70, 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x11, 0x6e, 0x75, 0x6d, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x53, 0x74, + 0x65, 0x70, 0x73, 0x12, 0x42, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, + 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x20, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x57, 0x61, + 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x18, 0x13, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, 0x75, + 0x6d, 0x57, 0x61, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x46, 0x72, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x64, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x43, 0x0a, 0x15, 0x45, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x79, 0x0a, 0x19, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, - 0x65, 0x66, 0x12, 0x16, 0x0a, 0x06, 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x06, 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, 0x1c, - 0x0a, 0x1a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, - 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xe8, 0x01, 0x0a, - 0x0a, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, - 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, - 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x4f, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc1, 0x02, 0x0a, 0x0f, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x10, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x52, 0x10, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x40, 0x0a, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x73, 0x1a, 0x58, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, - 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x01, 0x0a, 0x08, - 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, - 0x41, 0x74, 0x74, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x72, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x05, 0x41, 0x74, 0x74, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, - 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0x5d, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x37, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x79, 0x0a, 0x19, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, + 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x52, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x52, 0x65, 0x66, 0x12, 0x16, + 0x0a, 0x06, 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, + 0x50, 0x69, 0x6e, 0x6e, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1a, + 0x0a, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x08, 0x46, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, 0x1c, 0x0a, 0x1a, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xe8, 0x01, 0x0a, 0x0a, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, + 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, + 0x7a, 0x65, 0x12, 0x4f, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x2a, 0x3f, 0x0a, 0x15, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, - 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, - 0x53, 0x54, 0x41, 0x52, 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x4d, - 0x50, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, - 0x45, 0x44, 0x10, 0x02, 0x32, 0x89, 0x06, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, - 0x12, 0x54, 0x0a, 0x09, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x22, 0x2e, + 0x02, 0x38, 0x01, 0x22, 0xc1, 0x02, 0x0a, 0x0f, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x48, 0x0a, 0x10, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, + 0x10, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x12, 0x40, 0x0a, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x0c, 0x41, 0x74, 0x74, 0x65, 0x73, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x48, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x58, 0x0a, + 0x0c, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, + 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x95, 0x01, 0x0a, 0x08, 0x45, 0x78, 0x70, 0x6f, + 0x72, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x41, 0x74, 0x74, 0x72, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x65, 0x72, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, + 0x41, 0x74, 0x74, 0x72, 0x73, 0x1a, 0x38, 0x0a, 0x0a, 0x41, 0x74, 0x74, 0x72, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, + 0x3f, 0x0a, 0x15, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x54, 0x41, 0x52, + 0x54, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, + 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x02, + 0x32, 0x89, 0x06, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x12, 0x54, 0x0a, 0x09, + 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x73, + 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, - 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, - 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, - 0x76, 0x31, 0x2e, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x30, 0x01, - 0x12, 0x48, 0x0a, 0x05, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, - 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, - 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x06, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x07, 0x53, 0x65, 0x73, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, - 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, - 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, + 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x55, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x48, 0x0a, 0x05, 0x50, 0x72, 0x75, 0x6e, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x50, + 0x72, 0x75, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x6f, + 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, + 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x30, 0x01, 0x12, 0x48, 0x0a, 0x05, + 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, + 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x6c, 0x76, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x1f, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x4d, 0x0a, 0x07, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, + 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x28, 0x01, 0x30, 0x01, 0x12, 0x5a, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x73, 0x12, 0x24, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, + 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x45, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x62, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, 0x2e, 0x6d, - 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x6d, 0x6f, - 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x62, 0x0a, 0x12, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, - 0x79, 0x12, 0x25, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, - 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, - 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, - 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x30, 0x01, 0x12, - 0x6f, 0x0a, 0x12, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, - 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x2b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, - 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, - 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, - 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x40, 0x5a, 0x3e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, - 0x6f, 0x62, 0x79, 0x2f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2f, 0x61, 0x70, 0x69, - 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, - 0x6c, 0x3b, 0x6d, 0x6f, 0x62, 0x79, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x5f, - 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x30, 0x01, 0x12, 0x6f, 0x0a, 0x12, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, + 0x79, 0x12, 0x2b, 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, + 0x74, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x2e, 0x6d, 0x6f, 0x62, 0x79, 0x2e, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2e, 0x76, + 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x48, 0x69, 0x73, + 0x74, 0x6f, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x40, 0x5a, 0x3e, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x6f, 0x62, 0x79, 0x2f, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x3b, 0x6d, 0x6f, + 0x62, 0x79, 0x5f, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x5f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/api/services/control/control.proto b/api/services/control/control.proto index 24816d05cd8d..3e47752ff2f9 100644 --- a/api/services/control/control.proto +++ b/api/services/control/control.proto @@ -36,6 +36,7 @@ message PruneRequest { message DiskUsageRequest { repeated string filter = 1; + int64 ageLimit = 2; } message DiskUsageResponse { diff --git a/api/services/control/control_vtproto.pb.go b/api/services/control/control_vtproto.pb.go index e220f1bd5557..1d72ca07e501 100644 --- a/api/services/control/control_vtproto.pb.go +++ b/api/services/control/control_vtproto.pb.go @@ -56,6 +56,7 @@ func (m *DiskUsageRequest) CloneVT() *DiskUsageRequest { return (*DiskUsageRequest)(nil) } r := new(DiskUsageRequest) + r.AgeLimit = m.AgeLimit if rhs := m.Filter; rhs != nil { tmpContainer := make([]string, len(rhs)) copy(tmpContainer, rhs) @@ -831,6 +832,9 @@ func (this *DiskUsageRequest) EqualVT(that *DiskUsageRequest) bool { return false } } + if this.AgeLimit != that.AgeLimit { + return false + } return string(this.unknownFields) == string(that.unknownFields) } @@ -2019,6 +2023,11 @@ func (m *DiskUsageRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.AgeLimit != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AgeLimit)) + i-- + dAtA[i] = 0x10 + } if len(m.Filter) > 0 { for iNdEx := len(m.Filter) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Filter[iNdEx]) @@ -3991,6 +4000,9 @@ func (m *DiskUsageRequest) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.AgeLimit != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AgeLimit)) + } n += len(m.unknownFields) return n } @@ -5006,6 +5018,25 @@ func (m *DiskUsageRequest) UnmarshalVT(dAtA []byte) error { } m.Filter = append(m.Filter, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AgeLimit", wireType) + } + m.AgeLimit = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AgeLimit |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/cache/manager.go b/cache/manager.go index 63c676a19540..bd109c9ebafc 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -1452,6 +1452,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) if err := cm.markShared(m); err != nil { return nil, err } + cutOff := time.Now().Add(-opt.AgeLimit) var du []*client.UsageInfo for id, cr := range m { @@ -1468,9 +1469,15 @@ func (cm *cacheManager) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) RecordType: cr.recordType, Shared: cr.shared, } - if filter.Match(adaptUsageInfo(c)) { - du = append(du, c) + if !filter.Match(adaptUsageInfo(c)) { + continue + } + if opt.AgeLimit > 0 { + if c.LastUsedAt != nil && c.LastUsedAt.After(cutOff) { + continue + } } + du = append(du, c) } eg, ctx := errgroup.WithContext(ctx) diff --git a/client/diskusage.go b/client/diskusage.go index a690f4818168..2471ac297cff 100644 --- a/client/diskusage.go +++ b/client/diskusage.go @@ -31,7 +31,7 @@ func (c *Client) DiskUsage(ctx context.Context, opts ...DiskUsageOption) ([]*Usa o.SetDiskUsageOption(info) } - req := &controlapi.DiskUsageRequest{Filter: info.Filter} + req := &controlapi.DiskUsageRequest{Filter: info.Filter, AgeLimit: int64(info.AgeLimit)} resp, err := c.ControlClient().DiskUsage(ctx, req) if err != nil { return nil, errors.Wrap(err, "failed to call diskusage") @@ -72,7 +72,8 @@ type DiskUsageOption interface { } type DiskUsageInfo struct { - Filter []string + Filter []string + AgeLimit time.Duration } type UsageRecordType string @@ -85,3 +86,15 @@ const ( UsageRecordTypeCacheMount UsageRecordType = "exec.cachemount" UsageRecordTypeRegular UsageRecordType = "regular" ) + +type diskUsageOptionFunc func(*DiskUsageInfo) + +func (f diskUsageOptionFunc) SetDiskUsageOption(info *DiskUsageInfo) { + f(info) +} + +func WithAgeLimit(age time.Duration) DiskUsageOption { + return diskUsageOptionFunc(func(info *DiskUsageInfo) { + info.AgeLimit = age + }) +} diff --git a/control/control.go b/control/control.go index dc73be3da02b..b392c72ee7ce 100644 --- a/control/control.go +++ b/control/control.go @@ -167,7 +167,8 @@ func (c *Controller) DiskUsage(ctx context.Context, r *controlapi.DiskUsageReque } for _, w := range workers { du, err := w.DiskUsage(ctx, client.DiskUsageInfo{ - Filter: r.Filter, + Filter: r.Filter, + AgeLimit: time.Duration(r.AgeLimit), }) if err != nil { return nil, err From d976b7fa5c6051a82f263c3df437a4958c27281c Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 16:45:36 -0700 Subject: [PATCH 06/55] add eager compression --- cache/blobs.go | 37 +++--- cache/config/config.go | 4 + cache/remote.go | 2 +- control/control.go | 46 +++++++ exporter/containerimage/export.go | 9 ++ exporter/containerimage/exptypes/keys.go | 13 ++ exporter/containerimage/opts.go | 5 +- exporter/containerimage/writer.go | 15 ++- solver/jobs.go | 29 ++++- solver/llbsolver/eager.go | 154 +++++++++++++++++++++++ solver/llbsolver/solver.go | 60 ++++++--- 11 files changed, 330 insertions(+), 44 deletions(-) create mode 100644 solver/llbsolver/eager.go diff --git a/cache/blobs.go b/cache/blobs.go index 09836772d08a..07768ab48291 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -42,7 +42,7 @@ var ErrNoBlobs = errors.Errorf("no blobs for snapshot") // a blob is missing and createIfNeeded is true, then the blob will be created, otherwise ErrNoBlobs will // be returned. Caller must hold a lease when calling this function. // If forceCompression is specified but the blob of compressionType doesn't exist, this function creates it. -func (sr *immutableRef) computeBlobChain(ctx context.Context, createIfNeeded bool, comp compression.Config, s session.Group) error { +func (sr *immutableRef) computeBlobChain(ctx context.Context, createIfNeeded bool, skipParents bool, comp compression.Config, s session.Group) error { if _, ok := leases.FromContext(ctx); !ok { return errors.Errorf("missing lease requirement for computeBlobChain") } @@ -68,29 +68,30 @@ func (sr *immutableRef) computeBlobChain(ctx context.Context, createIfNeeded boo // refs rather than every single layer present among their ancestors. filter := sr.layerSet() - return computeBlobChain(ctx, sr, createIfNeeded, comp, s, filter) + return computeBlobChain(ctx, sr, createIfNeeded, skipParents, comp, s, filter) } -func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool, comp compression.Config, s session.Group, filter map[string]struct{}) error { +func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool, skipParents bool, comp compression.Config, s session.Group, filter map[string]struct{}) error { eg, ctx := errgroup.WithContext(ctx) - switch sr.kind() { - case Merge: - for _, parent := range sr.mergeParents { - eg.Go(func() error { - return computeBlobChain(ctx, parent, createIfNeeded, comp, s, filter) - }) - } - case Diff: - if _, ok := filter[sr.ID()]; !ok && sr.diffParents.upper != nil { - // This diff is just re-using the upper blob, compute that + if !skipParents { + switch sr.kind() { + case Merge: + for _, parent := range sr.mergeParents { + eg.Go(func() error { + return computeBlobChain(ctx, parent, createIfNeeded, false, comp, s, filter) + }) + } + case Diff: + if _, ok := filter[sr.ID()]; !ok && sr.diffParents.upper != nil { + eg.Go(func() error { + return computeBlobChain(ctx, sr.diffParents.upper, createIfNeeded, false, comp, s, filter) + }) + } + case Layer: eg.Go(func() error { - return computeBlobChain(ctx, sr.diffParents.upper, createIfNeeded, comp, s, filter) + return computeBlobChain(ctx, sr.layerParent, createIfNeeded, false, comp, s, filter) }) } - case Layer: - eg.Go(func() error { - return computeBlobChain(ctx, sr.layerParent, createIfNeeded, comp, s, filter) - }) } if _, ok := filter[sr.ID()]; ok { diff --git a/cache/config/config.go b/cache/config/config.go index 5fcae329c677..0253ed42e472 100644 --- a/cache/config/config.go +++ b/cache/config/config.go @@ -5,4 +5,8 @@ import "github.com/moby/buildkit/util/compression" type RefConfig struct { Compression compression.Config PreferNonDistributable bool + // SkipParents skips recursive compression of parent layers. Used by the + // eager export pipeline where each layer is compressed independently as + // its vertex completes. + SkipParents bool } diff --git a/cache/remote.go b/cache/remote.go index c0e3cc6b48c2..3d6ac9dc8f30 100644 --- a/cache/remote.go +++ b/cache/remote.go @@ -141,7 +141,7 @@ func getAvailableBlobs(ctx context.Context, cs content.Store, chain *solver.Remo } func (sr *immutableRef) getRemote(ctx context.Context, createIfNeeded bool, refCfg config.RefConfig, s session.Group) (*solver.Remote, error) { - err := sr.computeBlobChain(ctx, createIfNeeded, refCfg.Compression, s) + err := sr.computeBlobChain(ctx, createIfNeeded, refCfg.SkipParents, refCfg.Compression, s) if err != nil { return nil, err } diff --git a/control/control.go b/control/control.go index b392c72ee7ce..2b1dc8f5a5ba 100644 --- a/control/control.go +++ b/control/control.go @@ -511,6 +511,11 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* procs = append(procs, proc.ProvenanceProcessor(attrs)) } + eagerExport, err := resolveEagerExport(req.Exporters, expis) + if err != nil { + return nil, err + } + resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{ Frontend: req.Frontend, Definition: req.Definition, @@ -521,6 +526,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* Exporters: expis, CacheExporters: cacheExporters, EnableSessionExporter: req.EnableSessionExporter, + EagerExport: eagerExport, }, entitlementsFromPB(req.Entitlements), procs, req.Internal, req.SourcePolicy) if err != nil { return nil, err @@ -530,6 +536,46 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* }, nil } +// resolveEagerExport checks whether the exporter requests eager export and +// validates the configuration. Returns EagerExportNone if the flag is not set. +func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.ExporterInstance) (llbsolver.EagerExportMode, error) { + if len(rawExporters) != 1 { + for _, ex := range rawExporters { + if _, ok := ex.Attrs[string(exptypes.OptKeyEagerExport)]; ok { + return 0, errors.Errorf("eager-export requires exactly one exporter, got %d", len(rawExporters)) + } + } + return llbsolver.EagerExportNone, nil + } + + ex := rawExporters[0] + v, ok := ex.Attrs[string(exptypes.OptKeyEagerExport)] + if !ok { + return llbsolver.EagerExportNone, nil + } + + var mode llbsolver.EagerExportMode + switch v { + case exptypes.OptValEagerExportCompress: + mode = llbsolver.EagerExportCompress + case exptypes.OptValEagerExportPush: + mode = llbsolver.EagerExportPush + default: + return 0, errors.Errorf("invalid eager-export value %q", v) + } + + if expis[0].Type() != client.ExporterImage { + return 0, errors.Errorf("eager-export requires image exporter, got %q", expis[0].Type()) + } + if mode == llbsolver.EagerExportPush { + push, _ := ex.Attrs[string(exptypes.OptKeyPush)] + if push != "true" { + return 0, errors.New("eager-export=push requires push=true") + } + } + return mode, nil +} + func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Control_StatusServer) error { if err := sendTimestampHeader(stream); err != nil { return err diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 0dd8255e237c..d535c84f137e 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -170,6 +170,13 @@ func (e *imageExporter) Resolve(ctx context.Context, id int, opt map[string]stri return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } i.nameCanonical = b + case exptypes.OptKeyEagerExport: + switch v { + case exptypes.OptValEagerExportCompress, exptypes.OptValEagerExportPush: + i.eagerExport = v + default: + return nil, errors.Errorf("invalid value %q for %s, must be \"compress\" or \"push\"", v, k) + } default: if i.meta == nil { i.meta = make(map[string][]byte) @@ -195,6 +202,7 @@ type imageExporterInstance struct { nameCanonical bool danglingPrefix string danglingEmptyOnly bool + eagerExport string // "", "compress", or "push" meta map[string][]byte } @@ -226,6 +234,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source maps.Copy(src.Metadata, e.meta) opts := e.opts + opts.EagerExport = e.eagerExport as, _, err := ParseAnnotations(src.Metadata) if err != nil { return nil, nil, err diff --git a/exporter/containerimage/exptypes/keys.go b/exporter/containerimage/exptypes/keys.go index 7cfa8dc024d9..9d5b4d149f8b 100644 --- a/exporter/containerimage/exptypes/keys.go +++ b/exporter/containerimage/exptypes/keys.go @@ -85,4 +85,17 @@ var ( // Rewrite timestamps in layers to match SOURCE_DATE_EPOCH // Value: bool OptKeyRewriteTimestamp ImageExporterOptKey = "rewrite-timestamp" + + // Eagerly compress (and optionally push) layer blobs during the build, + // rather than waiting for all vertices to complete first. + // Requires a single image exporter with push enabled. + // Value: string + // compress — compress layers as vertices complete; push everything at finalize + // push — compress AND push layer blobs as vertices complete; only push manifest at finalize + OptKeyEagerExport ImageExporterOptKey = "eager-export" +) + +const ( + OptValEagerExportCompress = "compress" + OptValEagerExportPush = "push" ) diff --git a/exporter/containerimage/opts.go b/exporter/containerimage/opts.go index 1ccadbd946d8..0cf4d1862d1d 100644 --- a/exporter/containerimage/opts.go +++ b/exporter/containerimage/opts.go @@ -21,8 +21,9 @@ type ImageCommitOpts struct { Annotations AnnotationsGroup Epoch *time.Time - ForceInlineAttestations bool // force inline attestations to be attached - RewriteTimestamp bool // rewrite timestamps in layers to match the epoch + ForceInlineAttestations bool // force inline attestations to be attached + RewriteTimestamp bool // rewrite timestamps in layers to match the epoch + EagerExport string // "compress" or "push" — layers already compressed by eager pipeline } func (c *ImageCommitOpts) Load(ctx context.Context, opt map[string]string) (map[string]string, error) { diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 63f390a85b49..c410892d572a 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -138,7 +138,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session baseImg = &baseImgX } - remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), ref) + remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), opts.EagerExport, ref) if err != nil { return nil, err } @@ -200,7 +200,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session refs = append(refs, r) } - remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), refs...) + remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), opts.EagerExport, refs...) if err != nil { return nil, err } @@ -350,7 +350,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session return &idxDesc, nil } -func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefConfig, s session.Group, refs ...cache.ImmutableRef) ([]solver.Remote, error) { +func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefConfig, s session.Group, eagerExport string, refs ...cache.ImmutableRef) ([]solver.Remote, error) { attr := []attribute.KeyValue{ attribute.String("exportLayers.compressionType", refCfg.Compression.Type.String()), attribute.Bool("exportLayers.forceCompression", refCfg.Compression.Force), @@ -360,6 +360,13 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC } span, ctx := tracing.StartSpan(ctx, "export layers", trace.WithAttributes(attr...)) + // When eager export is active, blobs have already been compressed during + // the build. Use createIfNeeded=false to read existing blobs without + // re-compressing. If eager compression somehow missed a layer, this will + // return ErrNoBlobs — that is intentional; the build should fail rather + // than silently fall back. + createIfNeeded := eagerExport == "" + eg, ctx := errgroup.WithContext(ctx) layersDone := progress.OneOff(ctx, "exporting layers") @@ -371,7 +378,7 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC return } eg.Go(func() error { - remotes, err := ref.GetRemotes(ctx, true, refCfg, false, s) + remotes, err := ref.GetRemotes(ctx, createIfNeeded, refCfg, false, s) if err != nil { return err } diff --git a/solver/jobs.go b/solver/jobs.go index 2e4a95475b34..b22f0fbfb4ed 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -296,6 +296,11 @@ func (sb *subBuilder) EachValue(ctx context.Context, key string, fn func(any) er return nil } +// OnVertexCompleteFunc is called after a vertex produces results, either from +// execution (cache miss) or cache load (cache hit). The callback receives the +// vertex and its output results. Implementations must be safe for concurrent use. +type OnVertexCompleteFunc func(vtx Vertex, results []Result) + type Job struct { list *Solver pr *progress.MultiReader @@ -306,9 +311,14 @@ type Job struct { startedTime time.Time completedTime time.Time - progressCloser func(error) - SessionID string - uniqueID string // unique ID is used for provenance. We use a different field that client can't control + progressCloser func(error) + SessionID string + uniqueID string // unique ID is used for provenance. We use a different field that client can't control + onVertexComplete OnVertexCompleteFunc +} + +func (j *Job) SetOnVertexComplete(f OnVertexCompleteFunc) { + j.onVertexComplete = f } type SolverOpt struct { @@ -907,6 +917,9 @@ func (s *sharedOp) LoadCache(ctx context.Context, rec *CacheRecord) (Result, err res, err := s.Cache().Load(withAncestorCacheOpts(ctx, s.st), rec) tracing.FinishWithError(span, err) notifyCompleted(err, true) + if err == nil && res != nil { + s.fireOnVertexComplete([]Result{res}) + } return res, err } @@ -1127,6 +1140,8 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, s.subBuilder.mu.Unlock() s.execRes = &execRes{execRes: wrapShared(res), execExporters: subExporters} + + s.fireOnVertexComplete(res) } s.execErr = err } @@ -1141,6 +1156,14 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, return unwrapShared(res.execRes), res.execExporters, nil } +func (s *sharedOp) fireOnVertexComplete(results []Result) { + for j := range s.st.jobs { + if j.onVertexComplete != nil { + j.onVertexComplete(s.st.vtx, results) + } + } +} + func (s *sharedOp) getOp() (Op, error) { s.opOnce.Do(func() { s.subBuilder = s.st.builder() diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go new file mode 100644 index 000000000000..27de0e8a4f23 --- /dev/null +++ b/solver/llbsolver/eager.go @@ -0,0 +1,154 @@ +package llbsolver + +import ( + "context" + "os" + "runtime" + "strconv" + "sync" + + "github.com/moby/buildkit/cache" + cacheconfig "github.com/moby/buildkit/cache/config" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/worker" +) + +const defaultEagerWorkers = 4 + +type eagerWorkItem struct { + ref cache.ImmutableRef +} + +// eagerPipeline manages background compression (and optionally push) of layer +// blobs as build vertices complete, rather than deferring all work to finalize. +type eagerPipeline struct { + mode EagerExportMode + refCfg cacheconfig.RefConfig + sessionID string + sm *session.Manager + + work chan eagerWorkItem + wg sync.WaitGroup + + // ctx carries the lease so compressed blobs are GC-protected. + ctx context.Context + + mu sync.Mutex + firstErr error +} + +func eagerWorkerCount() int { + if s := os.Getenv("BUILDKIT_EAGER_EXPORT_WORKERS"); s != "" { + if n, err := strconv.Atoi(s); err == nil && n > 0 { + return n + } + } + return max(defaultEagerWorkers, runtime.NumCPU()) +} + +func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compression.Config, sessionID string, sm *session.Manager) *eagerPipeline { + ep := &eagerPipeline{ + mode: mode, + refCfg: cacheconfig.RefConfig{ + Compression: comp, + SkipParents: true, + }, + sessionID: sessionID, + sm: sm, + ctx: ctx, + work: make(chan eagerWorkItem, 256), + } + + numWorkers := eagerWorkerCount() + ep.wg.Add(numWorkers) + for range numWorkers { + go ep.worker() + } + + return ep +} + +func (ep *eagerPipeline) worker() { + defer ep.wg.Done() + for { + select { + case <-ep.ctx.Done(): + return + case item, ok := <-ep.work: + if !ok { + return + } + if err := ep.processRef(item.ref); err != nil { + ep.mu.Lock() + if ep.firstErr == nil { + ep.firstErr = err + } + ep.mu.Unlock() + } + item.ref.Release(context.TODO()) + } + } +} + +// onVertexComplete is the callback registered on the solver Job. It extracts +// ImmutableRefs from vertex results, clones them for safe async use, and +// sends them to the worker pool for compression. +func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { + for _, res := range results { + if res == nil { + continue + } + workerRef, ok := res.Sys().(*worker.WorkerRef) + if !ok || workerRef.ImmutableRef == nil { + continue + } + + cloned := workerRef.ImmutableRef.Clone() + select { + case ep.work <- eagerWorkItem{ref: cloned}: + case <-ep.ctx.Done(): + cloned.Release(context.TODO()) + return + } + } +} + +// processRef compresses a single ref's blob. SkipParents is set in the refCfg, +// so only this ref's own layer is compressed — parent layers get their own +// callbacks and worker items. +func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { + ctx := ep.ctx + s := session.NewGroup(ep.sessionID) + + bklog.G(ctx).Debugf("eager compress starting for ref %s", ref.ID()) + _, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) + if err != nil { + bklog.G(ctx).WithError(err).Warnf("eager compress failed for ref %s", ref.ID()) + return err + } + bklog.G(ctx).Debugf("eager compress done for ref %s", ref.ID()) + + if ep.mode == EagerExportPush { + // TODO: implement eager blob push + bklog.G(ctx).Debugf("eager push not yet implemented for ref %s", ref.ID()) + } + return nil +} + +// wait closes the work channel and blocks until all workers finish. +// Returns the first error encountered by any worker. +func (ep *eagerPipeline) wait() error { + close(ep.work) + ep.wg.Wait() + // Release any refs left in the channel (e.g. if workers exited early + // due to context cancellation). + for item := range ep.work { + item.ref.Release(context.TODO()) + } + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.firstErr +} diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 36fc7fd08502..4207848b9bd9 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -55,10 +55,21 @@ const ( keySourcePolicy = "llb.sourcepolicy" ) +// EagerExportMode controls whether layer compression and/or pushing +// happens concurrently with the build, rather than after all vertices complete. +type EagerExportMode int + +const ( + EagerExportNone EagerExportMode = iota + EagerExportCompress // compress layers as vertices complete + EagerExportPush // compress AND push layer blobs as vertices complete +) + type ExporterRequest struct { Exporters []exporter.ExporterInstance CacheExporters []RemoteCacheExporter EnableSessionExporter bool + EagerExport EagerExportMode } type RemoteCacheExporter struct { @@ -515,6 +526,29 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SessionID = sessionID + // Create lease early so it covers both the build phase (eager compression + // creates content blobs) and the export/finalize phase (push to registry). + lm, err := s.leaseManager() + if err != nil { + return nil, err + } + ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) + if err != nil { + return nil, err + } + releasers = append(releasers, func() { + done(context.WithoutCancel(ctx)) + }) + + // Set up eager export pipeline before the build starts so the vertex + // completion callback can kick off compression/push during the build. + var eager *eagerPipeline + if exp.EagerExport != EagerExportNone && len(exp.Exporters) > 0 { + comp := exp.Exporters[0].Config().Compression() + eager = newEagerPipeline(ctx, exp.EagerExport, comp, sessionID, s.sm) + j.SetOnVertexComplete(eager.onVertexComplete) + } + br := s.bridge(j) var fwd gateway.LLBBridgeForwarder if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" { @@ -586,6 +620,14 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } + // Wait for all eager compression/push jobs to finish before running + // exporters. The manifest needs final blob digests from every layer. + if eager != nil { + if err := eager.wait(); err != nil { + return nil, errors.Wrap(err, "eager export pipeline failed") + } + } + resProv, err = addProvenanceToResult(res, br) if err != nil { return nil, err @@ -617,22 +659,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - // Functions that create new objects in containerd (eg. content blobs) need to have a lease to ensure - // that the object is not garbage collected immediately. This is protected by the indivual components, - // but because creating a lease is not cheap and requires a disk write, we create a single lease here - // early and let all the exporters, cache export and provenance creation use the same one. - lm, err := s.leaseManager() - if err != nil { - return nil, err - } - ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) - if err != nil { - return nil, err - } - releasers = append(releasers, func() { - done(context.WithoutCancel(ctx)) - }) - + // Lease already created earlier in the function (unconditionally or + // conditionally for eager export) — see createLease / leaseutil.WithLease above. cacheExporters, inlineCacheExporter := splitCacheExporters(exp.CacheExporters) if exp.EnableSessionExporter { From a48b0d590b35b8b2ce6ba7b8cc9884dc3b3eaf9e Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 17:15:32 -0700 Subject: [PATCH 07/55] address comments --- exporter/containerimage/writer.go | 11 +++++--- solver/jobs.go | 10 +++++++- solver/llbsolver/solver.go | 42 ++++++++++++++++++++----------- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index c410892d572a..97fbe3e069ac 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -362,10 +362,15 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC // When eager export is active, blobs have already been compressed during // the build. Use createIfNeeded=false to read existing blobs without - // re-compressing. If eager compression somehow missed a layer, this will - // return ErrNoBlobs — that is intentional; the build should fail rather - // than silently fall back. + // re-compressing, and SkipParents=true to avoid walking the parent chain + // (each layer was compressed independently by the eager pipeline). + // If eager compression somehow missed a layer, this will return + // ErrNoBlobs — that is intentional; the build should fail rather than + // silently fall back. createIfNeeded := eagerExport == "" + if eagerExport != "" { + refCfg.SkipParents = true + } eg, ctx := errgroup.WithContext(ctx) layersDone := progress.OneOff(ctx, "exporting layers") diff --git a/solver/jobs.go b/solver/jobs.go index b22f0fbfb4ed..2ce721bd7b57 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -1157,11 +1157,19 @@ func (s *sharedOp) Exec(ctx context.Context, inputs []Result) (outputs []Result, } func (s *sharedOp) fireOnVertexComplete(results []Result) { + s.st.mu.Lock() + var callbacks []OnVertexCompleteFunc for j := range s.st.jobs { if j.onVertexComplete != nil { - j.onVertexComplete(s.st.vtx, results) + callbacks = append(callbacks, j.onVertexComplete) } } + vtx := s.st.vtx + s.st.mu.Unlock() + + for _, cb := range callbacks { + cb(vtx, results) + } } func (s *sharedOp) getOp() (Op, error) { diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 4207848b9bd9..bd215aa074f5 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -526,24 +526,31 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SessionID = sessionID - // Create lease early so it covers both the build phase (eager compression - // creates content blobs) and the export/finalize phase (push to registry). - lm, err := s.leaseManager() - if err != nil { - return nil, err - } - ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) - if err != nil { - return nil, err + createLease := func() error { + lm, err := s.leaseManager() + if err != nil { + return err + } + var done func(context.Context) error + ctx, done, err = leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) + if err != nil { + return err + } + releasers = append(releasers, func() { + done(context.WithoutCancel(ctx)) + }) + return nil } - releasers = append(releasers, func() { - done(context.WithoutCancel(ctx)) - }) // Set up eager export pipeline before the build starts so the vertex // completion callback can kick off compression/push during the build. + // When eager export is active, the lease must be created early so + // compressed blobs are GC-protected during the build phase. var eager *eagerPipeline if exp.EagerExport != EagerExportNone && len(exp.Exporters) > 0 { + if err := createLease(); err != nil { + return nil, err + } comp := exp.Exporters[0].Config().Compression() eager = newEagerPipeline(ctx, exp.EagerExport, comp, sessionID, s.sm) j.SetOnVertexComplete(eager.onVertexComplete) @@ -659,8 +666,15 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - // Lease already created earlier in the function (unconditionally or - // conditionally for eager export) — see createLease / leaseutil.WithLease above. + // When eager export is not active, the lease is created here (the original + // location) — after the build completes but before export. This avoids + // adding a disk write before gateway forwarder registration. + if eager == nil { + if err := createLease(); err != nil { + return nil, err + } + } + cacheExporters, inlineCacheExporter := splitCacheExporters(exp.CacheExporters) if exp.EnableSessionExporter { From ce339b534d10bd5d9cba41f2ab832660e7b0f619 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 17:59:33 -0700 Subject: [PATCH 08/55] add push --- control/control.go | 29 +++++++---- exporter/containerimage/export.go | 13 +++++ exporter/exporter.go | 17 +++++++ solver/llbsolver/eager.go | 84 +++++++++++++++++++++++++++---- solver/llbsolver/solver.go | 10 ++-- util/push/push.go | 43 ++++++++++------ 6 files changed, 158 insertions(+), 38 deletions(-) diff --git a/control/control.go b/control/control.go index 2b1dc8f5a5ba..eeaf49ab3349 100644 --- a/control/control.go +++ b/control/control.go @@ -511,7 +511,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* procs = append(procs, proc.ProvenanceProcessor(attrs)) } - eagerExport, err := resolveEagerExport(req.Exporters, expis) + eagerExport, eagerPushCfg, err := resolveEagerExport(req.Exporters, expis) if err != nil { return nil, err } @@ -527,6 +527,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* CacheExporters: cacheExporters, EnableSessionExporter: req.EnableSessionExporter, EagerExport: eagerExport, + EagerPushConfig: eagerPushCfg, }, entitlementsFromPB(req.Entitlements), procs, req.Internal, req.SourcePolicy) if err != nil { return nil, err @@ -538,20 +539,20 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* // resolveEagerExport checks whether the exporter requests eager export and // validates the configuration. Returns EagerExportNone if the flag is not set. -func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.ExporterInstance) (llbsolver.EagerExportMode, error) { +func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.ExporterInstance) (llbsolver.EagerExportMode, *exporter.EagerPushConfig, error) { if len(rawExporters) != 1 { for _, ex := range rawExporters { if _, ok := ex.Attrs[string(exptypes.OptKeyEagerExport)]; ok { - return 0, errors.Errorf("eager-export requires exactly one exporter, got %d", len(rawExporters)) + return 0, nil, errors.Errorf("eager-export requires exactly one exporter, got %d", len(rawExporters)) } } - return llbsolver.EagerExportNone, nil + return llbsolver.EagerExportNone, nil, nil } ex := rawExporters[0] v, ok := ex.Attrs[string(exptypes.OptKeyEagerExport)] if !ok { - return llbsolver.EagerExportNone, nil + return llbsolver.EagerExportNone, nil, nil } var mode llbsolver.EagerExportMode @@ -561,19 +562,29 @@ func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.Ex case exptypes.OptValEagerExportPush: mode = llbsolver.EagerExportPush default: - return 0, errors.Errorf("invalid eager-export value %q", v) + return 0, nil, errors.Errorf("invalid eager-export value %q", v) } if expis[0].Type() != client.ExporterImage { - return 0, errors.Errorf("eager-export requires image exporter, got %q", expis[0].Type()) + return 0, nil, errors.Errorf("eager-export requires image exporter, got %q", expis[0].Type()) } + + var pushCfg *exporter.EagerPushConfig if mode == llbsolver.EagerExportPush { push, _ := ex.Attrs[string(exptypes.OptKeyPush)] if push != "true" { - return 0, errors.New("eager-export=push requires push=true") + return 0, nil, errors.New("eager-export=push requires push=true") + } + provider, ok := expis[0].(exporter.EagerExportProvider) + if !ok { + return 0, nil, errors.New("eager-export=push: exporter does not support eager push") + } + pushCfg = provider.EagerPushConfig() + if pushCfg == nil { + return 0, nil, errors.New("eager-export=push requires a single image name (not empty, wildcard, or comma-separated)") } } - return mode, nil + return mode, pushCfg, nil } func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Control_StatusServer) error { diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index d535c84f137e..11a57ce2ce05 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -222,6 +222,19 @@ func (e *imageExporterInstance) Type() string { return client.ExporterImage } +func (e *imageExporterInstance) EagerPushConfig() *exporter.EagerPushConfig { + name := e.opts.ImageName + if name == "" || name == "*" || strings.Contains(name, ",") { + return nil + } + return &exporter.EagerPushConfig{ + TargetName: name, + RegistryHosts: e.opt.RegistryHosts, + Insecure: e.insecure, + ContentStore: e.opt.ImageWriter.ContentStore(), + } +} + func (e *imageExporterInstance) Attrs() map[string]string { return e.attrs } diff --git a/exporter/exporter.go b/exporter/exporter.go index c16f174558ba..1b884d22107b 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -3,6 +3,8 @@ package exporter import ( "context" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/solver/result" @@ -54,3 +56,18 @@ func NewConfigWithCompression(comp compression.Config) *Config { func (c *Config) Compression() compression.Config { return c.compression } + +// EagerPushConfig holds the registry details needed to push individual layer +// blobs during the build, before the final manifest is assembled. +type EagerPushConfig struct { + TargetName string + RegistryHosts docker.RegistryHosts + Insecure bool + ContentStore content.Store +} + +// EagerExportProvider is an optional interface that ExporterInstances can +// implement to supply configuration for the eager export pipeline. +type EagerExportProvider interface { + EagerPushConfig() *EagerPushConfig +} diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 27de0e8a4f23..5a8c29b17f05 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -7,13 +7,21 @@ import ( "strconv" "sync" + "github.com/containerd/containerd/v2/core/remotes" "github.com/moby/buildkit/cache" cacheconfig "github.com/moby/buildkit/cache/config" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/util/flightcontrol" + pushutil "github.com/moby/buildkit/util/push" + "github.com/moby/buildkit/util/resolver/limited" + "github.com/moby/buildkit/util/resolver/retryhandler" "github.com/moby/buildkit/worker" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" ) const defaultEagerWorkers = 4 @@ -28,7 +36,7 @@ type eagerPipeline struct { mode EagerExportMode refCfg cacheconfig.RefConfig sessionID string - sm *session.Manager + pushCfg *exporter.EagerPushConfig work chan eagerWorkItem wg sync.WaitGroup @@ -36,6 +44,12 @@ type eagerPipeline struct { // ctx carries the lease so compressed blobs are GC-protected. ctx context.Context + // pusher is created at pipeline init when mode is EagerExportPush. + pusher remotes.Pusher + + // pushDedup prevents concurrent pushes of the same digest. + pushDedup flightcontrol.Group[struct{}] + mu sync.Mutex firstErr error } @@ -49,7 +63,20 @@ func eagerWorkerCount() int { return max(defaultEagerWorkers, runtime.NumCPU()) } -func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compression.Config, sessionID string, sm *session.Manager) *eagerPipeline { +func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compression.Config, sessionID string, sm *session.Manager, pushCfg *exporter.EagerPushConfig) (*eagerPipeline, error) { + if mode == EagerExportPush && pushCfg == nil { + return nil, errors.New("eager-export=push requires push config") + } + + var pusher remotes.Pusher + if mode == EagerExportPush { + var err error + pusher, err = pushutil.NewPusher(ctx, sm, sessionID, pushCfg.TargetName, pushCfg.Insecure, pushCfg.RegistryHosts) + if err != nil { + return nil, errors.Wrap(err, "eager-export=push: failed to create pusher") + } + } + ep := &eagerPipeline{ mode: mode, refCfg: cacheconfig.RefConfig{ @@ -57,7 +84,8 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio SkipParents: true, }, sessionID: sessionID, - sm: sm, + pushCfg: pushCfg, + pusher: pusher, ctx: ctx, work: make(chan eagerWorkItem, 256), } @@ -68,7 +96,7 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio go ep.worker() } - return ep + return ep, nil } func (ep *eagerPipeline) worker() { @@ -116,15 +144,15 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } } -// processRef compresses a single ref's blob. SkipParents is set in the refCfg, -// so only this ref's own layer is compressed — parent layers get their own -// callbacks and worker items. +// processRef compresses a single ref's blob and optionally pushes it. +// SkipParents is set in the refCfg, so only this ref's own layer is +// compressed — parent layers get their own callbacks and worker items. func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { ctx := ep.ctx s := session.NewGroup(ep.sessionID) bklog.G(ctx).Debugf("eager compress starting for ref %s", ref.ID()) - _, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) + remotes, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) if err != nil { bklog.G(ctx).WithError(err).Warnf("eager compress failed for ref %s", ref.ID()) return err @@ -132,12 +160,48 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { bklog.G(ctx).Debugf("eager compress done for ref %s", ref.ID()) if ep.mode == EagerExportPush { - // TODO: implement eager blob push - bklog.G(ctx).Debugf("eager push not yet implemented for ref %s", ref.ID()) + if err := ep.pushBlobs(ctx, remotes); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager push failed for ref %s", ref.ID()) + return err + } + bklog.G(ctx).Debugf("eager push done for ref %s", ref.ID()) + } + return nil +} + +// pushBlobs pushes each layer descriptor from the GetRemotes result to the +// registry. Pushes are deduplicated by digest via flightcontrol — if two +// workers try to push the same blob concurrently, only one upload happens. +func (ep *eagerPipeline) pushBlobs(ctx context.Context, rems []*solver.Remote) error { + if len(rems) == 0 { + return nil + } + + remote := rems[0] + handler := retryhandler.New( + limited.PushHandler(ep.pusher, remote.Provider, ep.pushCfg.TargetName), + nil, + ) + + for _, desc := range remote.Descriptors { + if err := ep.pushBlob(ctx, handler, desc); err != nil { + return err + } } return nil } +// pushBlob pushes a single descriptor, deduplicated by digest across all +// concurrent workers via flightcontrol. +func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error), desc ocispecs.Descriptor) error { + _, err := ep.pushDedup.Do(ctx, desc.Digest.String(), func(ctx context.Context) (struct{}, error) { + bklog.G(ctx).Debugf("eager pushing blob %s (%d bytes)", desc.Digest, desc.Size) + _, err := handler(ctx, desc) + return struct{}{}, err + }) + return err +} + // wait closes the work channel and blocks until all workers finish. // Returns the first error encountered by any worker. func (ep *eagerPipeline) wait() error { diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index bd215aa074f5..63a423bcc31d 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -61,8 +61,8 @@ type EagerExportMode int const ( EagerExportNone EagerExportMode = iota - EagerExportCompress // compress layers as vertices complete - EagerExportPush // compress AND push layer blobs as vertices complete + EagerExportCompress // compress layers as vertices complete + EagerExportPush // compress AND push layer blobs as vertices complete ) type ExporterRequest struct { @@ -70,6 +70,7 @@ type ExporterRequest struct { CacheExporters []RemoteCacheExporter EnableSessionExporter bool EagerExport EagerExportMode + EagerPushConfig *exporter.EagerPushConfig // non-nil when EagerExport == EagerExportPush } type RemoteCacheExporter struct { @@ -552,7 +553,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } comp := exp.Exporters[0].Config().Compression() - eager = newEagerPipeline(ctx, exp.EagerExport, comp, sessionID, s.sm) + eager, err = newEagerPipeline(ctx, exp.EagerExport, comp, sessionID, s.sm, exp.EagerPushConfig) + if err != nil { + return nil, err + } j.SetOnVertexComplete(eager.onVertexComplete) } diff --git a/util/push/push.go b/util/push/push.go index 5c6ec95e013b..c859cce8eba1 100644 --- a/util/push/push.go +++ b/util/push/push.go @@ -46,6 +46,32 @@ func Pusher(ctx context.Context, resolver remotes.Resolver, ref string) (remotes return &pusher{Pusher: p}, nil } +// NewPusher creates a registry pusher for the given target reference. It +// handles reference parsing, insecure registry overrides, resolver creation, +// and pusher wrapping. +func NewPusher(ctx context.Context, sm *session.Manager, sid string, ref string, insecure bool, hosts docker.RegistryHosts) (remotes.Pusher, error) { + parsed, err := reference.ParseNormalizedNamed(ref) + if err != nil { + return nil, err + } + + scope := "push" + if insecure { + insecureTrue := true + httpTrue := true + hosts = resolver.NewRegistryConfig(map[string]resolverconfig.RegistryConfig{ + reference.Domain(parsed): { + Insecure: &insecureTrue, + PlainHTTP: &httpTrue, + }, + }) + scope += ":insecure" + } + + r := resolver.DefaultPool.GetResolver(hosts, ref, scope, sm, session.NewGroup(sid)) + return Pusher(ctx, r, ref) +} + func Push(ctx context.Context, sm *session.Manager, sid string, provider content.Provider, manager content.Manager, dgst digest.Digest, ref string, insecure bool, hosts docker.RegistryHosts, byDigest bool, annotations map[digest.Digest]map[string]string) error { ctx = contentutil.RegisterContentPayloadTypes(ctx) desc := ocispecs.Descriptor{ @@ -70,22 +96,7 @@ func Push(ctx context.Context, sm *session.Manager, sid string, provider content ref = r.String() } - scope := "push" - if insecure { - insecureTrue := true - httpTrue := true - hosts = resolver.NewRegistryConfig(map[string]resolverconfig.RegistryConfig{ - reference.Domain(parsed): { - Insecure: &insecureTrue, - PlainHTTP: &httpTrue, - }, - }) - scope += ":insecure" - } - - resolver := resolver.DefaultPool.GetResolver(hosts, ref, scope, sm, session.NewGroup(sid)) - - pusher, err := Pusher(ctx, resolver, ref) + pusher, err := NewPusher(ctx, sm, sid, ref, insecure, hosts) if err != nil { return err } From 1b7a3967ec541c585a7658998ff37b5c462150db Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 20:39:08 -0700 Subject: [PATCH 09/55] add info logging --- solver/llbsolver/eager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 5a8c29b17f05..e13d6a6e1c40 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -151,20 +151,20 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { ctx := ep.ctx s := session.NewGroup(ep.sessionID) - bklog.G(ctx).Debugf("eager compress starting for ref %s", ref.ID()) + bklog.G(ctx).Infof("eager compress starting for ref %s", ref.ID()) remotes, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) if err != nil { bklog.G(ctx).WithError(err).Warnf("eager compress failed for ref %s", ref.ID()) return err } - bklog.G(ctx).Debugf("eager compress done for ref %s", ref.ID()) + bklog.G(ctx).Infof("eager compress done for ref %s", ref.ID()) if ep.mode == EagerExportPush { if err := ep.pushBlobs(ctx, remotes); err != nil { bklog.G(ctx).WithError(err).Warnf("eager push failed for ref %s", ref.ID()) return err } - bklog.G(ctx).Debugf("eager push done for ref %s", ref.ID()) + bklog.G(ctx).Infof("eager push done for ref %s", ref.ID()) } return nil } @@ -195,7 +195,7 @@ func (ep *eagerPipeline) pushBlobs(ctx context.Context, rems []*solver.Remote) e // concurrent workers via flightcontrol. func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error), desc ocispecs.Descriptor) error { _, err := ep.pushDedup.Do(ctx, desc.Digest.String(), func(ctx context.Context) (struct{}, error) { - bklog.G(ctx).Debugf("eager pushing blob %s (%d bytes)", desc.Digest, desc.Size) + bklog.G(ctx).Infof("eager pushing blob %s (%d bytes)", desc.Digest, desc.Size) _, err := handler(ctx, desc) return struct{}{}, err }) From 8f3a20818d0b8afc37d42ee3a9b4a323b3e8de84 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 21:23:21 -0700 Subject: [PATCH 10/55] fix bug --- cache/blobs.go | 36 +++++++++++++++---------------- cache/config/config.go | 4 ---- cache/remote.go | 2 +- exporter/containerimage/writer.go | 11 +++------- solver/llbsolver/eager.go | 8 +++---- 5 files changed, 25 insertions(+), 36 deletions(-) diff --git a/cache/blobs.go b/cache/blobs.go index 07768ab48291..22c25cc2e1ff 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -42,7 +42,7 @@ var ErrNoBlobs = errors.Errorf("no blobs for snapshot") // a blob is missing and createIfNeeded is true, then the blob will be created, otherwise ErrNoBlobs will // be returned. Caller must hold a lease when calling this function. // If forceCompression is specified but the blob of compressionType doesn't exist, this function creates it. -func (sr *immutableRef) computeBlobChain(ctx context.Context, createIfNeeded bool, skipParents bool, comp compression.Config, s session.Group) error { +func (sr *immutableRef) computeBlobChain(ctx context.Context, createIfNeeded bool, comp compression.Config, s session.Group) error { if _, ok := leases.FromContext(ctx); !ok { return errors.Errorf("missing lease requirement for computeBlobChain") } @@ -68,30 +68,28 @@ func (sr *immutableRef) computeBlobChain(ctx context.Context, createIfNeeded boo // refs rather than every single layer present among their ancestors. filter := sr.layerSet() - return computeBlobChain(ctx, sr, createIfNeeded, skipParents, comp, s, filter) + return computeBlobChain(ctx, sr, createIfNeeded, comp, s, filter) } -func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool, skipParents bool, comp compression.Config, s session.Group, filter map[string]struct{}) error { +func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool, comp compression.Config, s session.Group, filter map[string]struct{}) error { eg, ctx := errgroup.WithContext(ctx) - if !skipParents { - switch sr.kind() { - case Merge: - for _, parent := range sr.mergeParents { - eg.Go(func() error { - return computeBlobChain(ctx, parent, createIfNeeded, false, comp, s, filter) - }) - } - case Diff: - if _, ok := filter[sr.ID()]; !ok && sr.diffParents.upper != nil { - eg.Go(func() error { - return computeBlobChain(ctx, sr.diffParents.upper, createIfNeeded, false, comp, s, filter) - }) - } - case Layer: + switch sr.kind() { + case Merge: + for _, parent := range sr.mergeParents { eg.Go(func() error { - return computeBlobChain(ctx, sr.layerParent, createIfNeeded, false, comp, s, filter) + return computeBlobChain(ctx, parent, createIfNeeded, comp, s, filter) }) } + case Diff: + if _, ok := filter[sr.ID()]; !ok && sr.diffParents.upper != nil { + eg.Go(func() error { + return computeBlobChain(ctx, sr.diffParents.upper, createIfNeeded, comp, s, filter) + }) + } + case Layer: + eg.Go(func() error { + return computeBlobChain(ctx, sr.layerParent, createIfNeeded, comp, s, filter) + }) } if _, ok := filter[sr.ID()]; ok { diff --git a/cache/config/config.go b/cache/config/config.go index 0253ed42e472..5fcae329c677 100644 --- a/cache/config/config.go +++ b/cache/config/config.go @@ -5,8 +5,4 @@ import "github.com/moby/buildkit/util/compression" type RefConfig struct { Compression compression.Config PreferNonDistributable bool - // SkipParents skips recursive compression of parent layers. Used by the - // eager export pipeline where each layer is compressed independently as - // its vertex completes. - SkipParents bool } diff --git a/cache/remote.go b/cache/remote.go index 3d6ac9dc8f30..c0e3cc6b48c2 100644 --- a/cache/remote.go +++ b/cache/remote.go @@ -141,7 +141,7 @@ func getAvailableBlobs(ctx context.Context, cs content.Store, chain *solver.Remo } func (sr *immutableRef) getRemote(ctx context.Context, createIfNeeded bool, refCfg config.RefConfig, s session.Group) (*solver.Remote, error) { - err := sr.computeBlobChain(ctx, createIfNeeded, refCfg.SkipParents, refCfg.Compression, s) + err := sr.computeBlobChain(ctx, createIfNeeded, refCfg.Compression, s) if err != nil { return nil, err } diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 97fbe3e069ac..c410892d572a 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -362,15 +362,10 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC // When eager export is active, blobs have already been compressed during // the build. Use createIfNeeded=false to read existing blobs without - // re-compressing, and SkipParents=true to avoid walking the parent chain - // (each layer was compressed independently by the eager pipeline). - // If eager compression somehow missed a layer, this will return - // ErrNoBlobs — that is intentional; the build should fail rather than - // silently fall back. + // re-compressing. If eager compression somehow missed a layer, this will + // return ErrNoBlobs — that is intentional; the build should fail rather + // than silently fall back. createIfNeeded := eagerExport == "" - if eagerExport != "" { - refCfg.SkipParents = true - } eg, ctx := errgroup.WithContext(ctx) layersDone := progress.OneOff(ctx, "exporting layers") diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index e13d6a6e1c40..6f28a5f29943 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -81,7 +81,6 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio mode: mode, refCfg: cacheconfig.RefConfig{ Compression: comp, - SkipParents: true, }, sessionID: sessionID, pushCfg: pushCfg, @@ -144,9 +143,10 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } } -// processRef compresses a single ref's blob and optionally pushes it. -// SkipParents is set in the refCfg, so only this ref's own layer is -// compressed — parent layers get their own callbacks and worker items. +// processRef compresses a single ref's blob (and its parent chain) and +// optionally pushes it. Parent compression is deduplicated by flightcontrol +// inside computeBlobChain, so overlapping parent chains across workers are +// only compressed once. func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { ctx := ep.ctx s := session.NewGroup(ep.sessionID) From 0e5dec4fa4f1cbcac7f709f22286715f80573c5c Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 21:37:22 -0700 Subject: [PATCH 11/55] Add parallel layer extraction support Introduces BUILDKIT_PARALLEL_EXTRACT=1 env var to enable parallel layer decompression during image pull. When enabled on the overlay snapshotter, all layers are downloaded and decompressed into independent temp directories concurrently, then the sequential Prepare/Commit metadata chain runs with pre-populated content. Co-Authored-By: Claude Opus 4.6 --- cache/refs.go | 191 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) diff --git a/cache/refs.go b/cache/refs.go index 3bd534854ae0..f55bfc72ce72 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "sync" "time" @@ -48,6 +49,29 @@ import ( var additionalAnnotations = append(append(compression.EStargzAnnotations, obdlabel.OverlayBDAnnotations...), labels.LabelUncompressed) +func parallelExtractEnabled() bool { + v, _ := strconv.ParseBool(os.Getenv("BUILDKIT_PARALLEL_EXTRACT")) + return v +} + +// extractFSPath extracts the writable fs directory path from snapshot mounts. +// For overlay, this is the upperdir. For bind mounts (base layer), it's the Source. +func extractFSPath(mounts []mount.Mount) string { + if len(mounts) == 0 { + return "" + } + m := mounts[0] + if m.Type == "bind" { + return m.Source + } + for _, opt := range m.Options { + if strings.HasPrefix(opt, "upperdir=") { + return strings.TrimPrefix(opt, "upperdir=") + } + } + return "" +} + // Ref is a reference to cacheable objects. type Ref interface { Mountable @@ -1029,6 +1053,20 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } } + if parallelExtractEnabled() && sr.cm.Snapshotter.Name() == "overlay" { + chain := sr.layerChain() + var needsExtract []*immutableRef + for _, ref := range chain { + if ref.getBlobOnly() { + needsExtract = append(needsExtract, ref) + } + } + if len(needsExtract) > 0 { + return sr.parallelExtractLayers(ctx, chain, needsExtract, s) + } + return nil + } + return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) } @@ -1414,6 +1452,159 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr return nil } +// parallelExtractLayers extracts multiple layers in parallel by decompressing +// into temp directories first, then doing the sequential Prepare→Commit chain +// with pre-populated content. This avoids the sequential Apply bottleneck where +// each layer must wait for its parent to be fully extracted before starting. +// +// Only used for the overlay snapshotter when BUILDKIT_PARALLEL_EXTRACT=1. +func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immutableRef, needsExtract []*immutableRef, s session.Group) (rerr error) { + if sr.cm.Applier == nil { + return errors.New("parallel extract requires an applier") + } + + if _, ok := leases.FromContext(ctx); !ok { + leaseCtx, done, err := leaseutil.WithLease(ctx, sr.cm.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return err + } + defer done(leaseCtx) + ctx = leaseCtx + } + + sp, ctx := tracing.StartSpan(ctx, "parallel-extract", trace.WithAttributes( + attribute.Int("layers.total", len(chain)), + attribute.Int("layers.extract", len(needsExtract)), + )) + defer sp.End() + + // Phase 1: parallel download + decompress into temp dirs. + // Each layer is independently decompressed into its own temp directory. + type extractResult struct { + tmpDir string + desc ocispecs.Descriptor + } + extractResults := make(map[string]*extractResult, len(needsExtract)) + var mu sync.Mutex + + eg, egctx := errgroup.WithContext(ctx) + for _, ref := range needsExtract { + ref := ref + eg.Go(func() error { + dhs := ref.descHandlers + desc, err := ref.ociDesc(egctx, dhs, true) + if err != nil { + return err + } + dh := dhs[desc.Digest] + + if err := (lazyRefProvider{ref: ref, desc: desc, dh: dh, session: s}).Unlazy(egctx); err != nil { + return err + } + + tmpDir, err := os.MkdirTemp("", "buildkit-parallel-extract-") + if err != nil { + return err + } + fsDir := filepath.Join(tmpDir, "fs") + if err := os.Mkdir(fsDir, 0755); err != nil { + os.RemoveAll(tmpDir) + return err + } + + mounts := []mount.Mount{{ + Source: fsDir, + Type: "bind", + Options: []string{"rw", "rbind"}, + }} + if _, err := ref.cm.Applier.Apply(egctx, desc, mounts); err != nil { + os.RemoveAll(tmpDir) + return err + } + + mu.Lock() + extractResults[ref.ID()] = &extractResult{tmpDir: tmpDir, desc: desc} + mu.Unlock() + return nil + }) + } + + if err := eg.Wait(); err != nil { + for _, r := range extractResults { + os.RemoveAll(r.tmpDir) + } + return err + } + + // Phase 2: sequential Prepare → swap content → Commit. + // Walk the full chain in order. Layers already extracted are skipped + // (just advance parentID). Layers that were parallel-extracted get their + // temp dir content swapped into the snapshotter path. + parentID := "" + for _, ref := range chain { + if !ref.getBlobOnly() { + parentID = ref.getSnapshotID() + continue + } + + result := extractResults[ref.ID()] + if result == nil { + return errors.Errorf("parallel extract: missing result for layer %s", ref.ID()) + } + + key := fmt.Sprintf("extract-%s %s", identity.NewID(), ref.getChainID()) + if err := ref.cm.Snapshotter.Prepare(ctx, key, parentID); err != nil { + os.RemoveAll(result.tmpDir) + return err + } + + mountable, err := ref.cm.Snapshotter.Mounts(ctx, key) + if err != nil { + os.RemoveAll(result.tmpDir) + return err + } + mounts, unmount, err := mountable.Mount() + if err != nil { + os.RemoveAll(result.tmpDir) + return err + } + + fsPath := extractFSPath(mounts) + if err := unmount(); err != nil { + os.RemoveAll(result.tmpDir) + return err + } + + if fsPath == "" { + os.RemoveAll(result.tmpDir) + return errors.Errorf("parallel extract: could not determine fs path from mounts for layer %s", ref.ID()) + } + + if err := os.RemoveAll(fsPath); err != nil { + os.RemoveAll(result.tmpDir) + return errors.Wrapf(err, "parallel extract: failed to remove empty fs dir") + } + if err := os.Rename(filepath.Join(result.tmpDir, "fs"), fsPath); err != nil { + os.RemoveAll(result.tmpDir) + return errors.Wrapf(err, "parallel extract: failed to rename extracted content into snapshot path") + } + os.RemoveAll(result.tmpDir) + + if err := ref.cm.Snapshotter.Commit(ctx, ref.getSnapshotID(), key); err != nil { + if !errors.Is(err, cerrdefs.ErrAlreadyExists) { + return err + } + } + ref.queueBlobOnly(false) + ref.queueSize(sizeUnknown) + if err := ref.commitMetadata(); err != nil { + return err + } + parentID = ref.getSnapshotID() + } + return nil +} + func (sr *immutableRef) Release(ctx context.Context) error { sr.cm.mu.Lock() defer sr.cm.mu.Unlock() From 548d483c93ed0a6151a170f7ff1a2dafd1460764 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 21:56:45 -0700 Subject: [PATCH 12/55] debug loggiing + fix --- cache/refs.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cache/refs.go b/cache/refs.go index f55bfc72ce72..285f46dc4969 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1034,10 +1034,12 @@ func (sr *immutableRef) ensureLocalContentBlob(ctx context.Context, s session.Gr func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr error) { if (sr.kind() == Layer || sr.kind() == BaseLayer) && !sr.getBlobOnly() { + bklog.G(ctx).Infof("Extract: skipping extract for ref %s", sr.ID()) return nil } if sr.cm.Snapshotter.Name() == "stargz" { + bklog.G(ctx).Infof("Extract: preparing remote snapshots for ref %s", sr.ID()) if err := sr.withRemoteSnapshotLabelsStargzMode(ctx, s, func() { if rerr = sr.prepareRemoteSnapshotsStargzMode(ctx, s); rerr != nil { return @@ -1048,12 +1050,16 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } return rerr } else if sr.cm.Snapshotter.Name() == "overlaybd" { + bklog.G(ctx).Infof("Extract: preparing remote snapshots for overlaybd mode for ref %s", sr.ID()) if rerr = sr.prepareRemoteSnapshotsOverlaybdMode(ctx); rerr == nil { return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) } } - if parallelExtractEnabled() && sr.cm.Snapshotter.Name() == "overlay" { + bklog.G(ctx).Infof("Extract: parallelExtractEnabled=%v, BUILDKIT_PARALLEL_EXTRACT=%q, snapshotter=%s", + parallelExtractEnabled(), os.Getenv("BUILDKIT_PARALLEL_EXTRACT"), sr.cm.Snapshotter.Name()) + + if parallelExtractEnabled() && sr.cm.Snapshotter.Name() == "overlayfs" { chain := sr.layerChain() var needsExtract []*immutableRef for _, ref := range chain { @@ -1062,10 +1068,12 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } } if len(needsExtract) > 0 { + bklog.G(ctx).Infof("parallel extract: extracting %d/%d layers in parallel", len(needsExtract), len(chain)) return sr.parallelExtractLayers(ctx, chain, needsExtract, s) } return nil } + bklog.G(ctx).Infof("Extract: unlazy for ref %s", sr.ID()) return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) } From d048126b51365e1c8faea098d05ae946bd2d1311 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 9 Apr 2026 22:00:30 -0700 Subject: [PATCH 13/55] hm --- cache/refs.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/cache/refs.go b/cache/refs.go index 285f46dc4969..ba1ce96d509d 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1486,6 +1486,31 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu )) defer sp.End() + // Discover the snapshotter's on-disk root so temp dirs are on the same + // filesystem, avoiding EXDEV from os.Rename in Phase 2. + discoverKey := fmt.Sprintf("discover-%s", identity.NewID()) + if err := sr.cm.Snapshotter.Prepare(ctx, discoverKey, ""); err != nil { + return errors.Wrap(err, "parallel extract: failed to discover snapshotter root") + } + discoverMountable, err := sr.cm.Snapshotter.Mounts(ctx, discoverKey) + if err != nil { + sr.cm.Snapshotter.Remove(ctx, discoverKey) + return errors.Wrap(err, "parallel extract: failed to get mounts for root discovery") + } + discoverMounts, discoverUnmount, err := discoverMountable.Mount() + if err != nil { + sr.cm.Snapshotter.Remove(ctx, discoverKey) + return errors.Wrap(err, "parallel extract: failed to mount for root discovery") + } + snapshotsDir := filepath.Dir(filepath.Dir(extractFSPath(discoverMounts))) + discoverUnmount() + sr.cm.Snapshotter.Remove(ctx, discoverKey) + + if snapshotsDir == "" || snapshotsDir == "." { + return errors.New("parallel extract: could not determine snapshotter root") + } + bklog.G(ctx).Infof("parallel extract: using snapshotter root %s for temp dirs", snapshotsDir) + // Phase 1: parallel download + decompress into temp dirs. // Each layer is independently decompressed into its own temp directory. type extractResult struct { @@ -1510,7 +1535,7 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu return err } - tmpDir, err := os.MkdirTemp("", "buildkit-parallel-extract-") + tmpDir, err := os.MkdirTemp(snapshotsDir, "buildkit-parallel-extract-") if err != nil { return err } From dc202e8458786dce288eff3c00752c2a12b8bfe5 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 09:40:24 -0700 Subject: [PATCH 14/55] small change --- cache/blobs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cache/blobs.go b/cache/blobs.go index 22c25cc2e1ff..09836772d08a 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -82,6 +82,7 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool } case Diff: if _, ok := filter[sr.ID()]; !ok && sr.diffParents.upper != nil { + // This diff is just re-using the upper blob, compute that eg.Go(func() error { return computeBlobChain(ctx, sr.diffParents.upper, createIfNeeded, comp, s, filter) }) From c585ef2096bdf82cd87966a40d52d8373b5c4096 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 11:04:36 -0700 Subject: [PATCH 15/55] cleanup --- cache/refs.go | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/cache/refs.go b/cache/refs.go index ba1ce96d509d..a43624eb731c 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1039,7 +1039,6 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } if sr.cm.Snapshotter.Name() == "stargz" { - bklog.G(ctx).Infof("Extract: preparing remote snapshots for ref %s", sr.ID()) if err := sr.withRemoteSnapshotLabelsStargzMode(ctx, s, func() { if rerr = sr.prepareRemoteSnapshotsStargzMode(ctx, s); rerr != nil { return @@ -1050,15 +1049,11 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } return rerr } else if sr.cm.Snapshotter.Name() == "overlaybd" { - bklog.G(ctx).Infof("Extract: preparing remote snapshots for overlaybd mode for ref %s", sr.ID()) if rerr = sr.prepareRemoteSnapshotsOverlaybdMode(ctx); rerr == nil { return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) } } - bklog.G(ctx).Infof("Extract: parallelExtractEnabled=%v, BUILDKIT_PARALLEL_EXTRACT=%q, snapshotter=%s", - parallelExtractEnabled(), os.Getenv("BUILDKIT_PARALLEL_EXTRACT"), sr.cm.Snapshotter.Name()) - if parallelExtractEnabled() && sr.cm.Snapshotter.Name() == "overlayfs" { chain := sr.layerChain() var needsExtract []*immutableRef @@ -1073,7 +1068,6 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro } return nil } - bklog.G(ctx).Infof("Extract: unlazy for ref %s", sr.ID()) return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) } @@ -1486,33 +1480,9 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu )) defer sp.End() - // Discover the snapshotter's on-disk root so temp dirs are on the same - // filesystem, avoiding EXDEV from os.Rename in Phase 2. - discoverKey := fmt.Sprintf("discover-%s", identity.NewID()) - if err := sr.cm.Snapshotter.Prepare(ctx, discoverKey, ""); err != nil { - return errors.Wrap(err, "parallel extract: failed to discover snapshotter root") - } - discoverMountable, err := sr.cm.Snapshotter.Mounts(ctx, discoverKey) - if err != nil { - sr.cm.Snapshotter.Remove(ctx, discoverKey) - return errors.Wrap(err, "parallel extract: failed to get mounts for root discovery") - } - discoverMounts, discoverUnmount, err := discoverMountable.Mount() - if err != nil { - sr.cm.Snapshotter.Remove(ctx, discoverKey) - return errors.Wrap(err, "parallel extract: failed to mount for root discovery") - } - snapshotsDir := filepath.Dir(filepath.Dir(extractFSPath(discoverMounts))) - discoverUnmount() - sr.cm.Snapshotter.Remove(ctx, discoverKey) - - if snapshotsDir == "" || snapshotsDir == "." { - return errors.New("parallel extract: could not determine snapshotter root") - } - bklog.G(ctx).Infof("parallel extract: using snapshotter root %s for temp dirs", snapshotsDir) - // Phase 1: parallel download + decompress into temp dirs. - // Each layer is independently decompressed into its own temp directory. + // Temp dirs are created under cm.root to ensure they're on the same + // filesystem as the snapshotter, avoiding EXDEV from os.Rename in Phase 2. type extractResult struct { tmpDir string desc ocispecs.Descriptor @@ -1535,7 +1505,7 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu return err } - tmpDir, err := os.MkdirTemp(snapshotsDir, "buildkit-parallel-extract-") + tmpDir, err := os.MkdirTemp(sr.cm.root, "buildkit-parallel-extract-") if err != nil { return err } From 02aa68f0600e033ccfe6fae4fd10e9585bcee84a Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 11:25:16 -0700 Subject: [PATCH 16/55] cleanup --- cache/refs.go | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/cache/refs.go b/cache/refs.go index a43624eb731c..1ebabd604094 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1481,8 +1481,15 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu defer sp.End() // Phase 1: parallel download + decompress into temp dirs. - // Temp dirs are created under cm.root to ensure they're on the same - // filesystem as the snapshotter, avoiding EXDEV from os.Rename in Phase 2. + // A parent temp dir is created under cm.root to ensure it's on the same + // filesystem as the snapshotter (avoiding EXDEV from os.Rename in Phase 2) + // and to simplify cleanup. + parentTmpDir, err := os.MkdirTemp(sr.cm.root, "buildkit-parallel-extract-") + if err != nil { + return errors.Wrap(err, "parallel extract: failed to create temp dir") + } + defer os.RemoveAll(parentTmpDir) + type extractResult struct { tmpDir string desc ocispecs.Descriptor @@ -1505,13 +1512,12 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu return err } - tmpDir, err := os.MkdirTemp(sr.cm.root, "buildkit-parallel-extract-") + tmpDir, err := os.MkdirTemp(parentTmpDir, ref.ID()+"-") if err != nil { return err } fsDir := filepath.Join(tmpDir, "fs") if err := os.Mkdir(fsDir, 0755); err != nil { - os.RemoveAll(tmpDir) return err } @@ -1521,7 +1527,6 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu Options: []string{"rw", "rbind"}, }} if _, err := ref.cm.Applier.Apply(egctx, desc, mounts); err != nil { - os.RemoveAll(tmpDir) return err } @@ -1533,9 +1538,6 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu } if err := eg.Wait(); err != nil { - for _, r := range extractResults { - os.RemoveAll(r.tmpDir) - } return err } @@ -1557,41 +1559,33 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu key := fmt.Sprintf("extract-%s %s", identity.NewID(), ref.getChainID()) if err := ref.cm.Snapshotter.Prepare(ctx, key, parentID); err != nil { - os.RemoveAll(result.tmpDir) return err } mountable, err := ref.cm.Snapshotter.Mounts(ctx, key) if err != nil { - os.RemoveAll(result.tmpDir) return err } mounts, unmount, err := mountable.Mount() if err != nil { - os.RemoveAll(result.tmpDir) return err } fsPath := extractFSPath(mounts) if err := unmount(); err != nil { - os.RemoveAll(result.tmpDir) return err } if fsPath == "" { - os.RemoveAll(result.tmpDir) return errors.Errorf("parallel extract: could not determine fs path from mounts for layer %s", ref.ID()) } if err := os.RemoveAll(fsPath); err != nil { - os.RemoveAll(result.tmpDir) return errors.Wrapf(err, "parallel extract: failed to remove empty fs dir") } if err := os.Rename(filepath.Join(result.tmpDir, "fs"), fsPath); err != nil { - os.RemoveAll(result.tmpDir) return errors.Wrapf(err, "parallel extract: failed to rename extracted content into snapshot path") } - os.RemoveAll(result.tmpDir) if err := ref.cm.Snapshotter.Commit(ctx, ref.getSnapshotID(), key); err != nil { if !errors.Is(err, cerrdefs.ErrAlreadyExists) { From b88d6f344da6a52995fb147e7846dea13c8161b1 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 11:31:03 -0700 Subject: [PATCH 17/55] safety check --- cache/refs.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cache/refs.go b/cache/refs.go index 1ebabd604094..10f95df806d8 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1580,7 +1580,14 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu return errors.Errorf("parallel extract: could not determine fs path from mounts for layer %s", ref.ID()) } - if err := os.RemoveAll(fsPath); err != nil { + entries, err := os.ReadDir(fsPath) + if err != nil { + return errors.Wrapf(err, "parallel extract: failed to read fs dir") + } + if len(entries) > 0 { + return errors.Errorf("parallel extract: expected empty fs dir but found %d entries at %s", len(entries), fsPath) + } + if err := os.Remove(fsPath); err != nil { return errors.Wrapf(err, "parallel extract: failed to remove empty fs dir") } if err := os.Rename(filepath.Join(result.tmpDir, "fs"), fsPath); err != nil { From 825205b6dcef73bd9cba6bac563f7278035c8ce8 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 12:37:28 -0700 Subject: [PATCH 18/55] add logs --- cache/refs.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/cache/refs.go b/cache/refs.go index 10f95df806d8..ed7ce2057fcc 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1500,7 +1500,7 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu eg, egctx := errgroup.WithContext(ctx) for _, ref := range needsExtract { ref := ref - eg.Go(func() error { + eg.Go(func() (rerr error) { dhs := ref.descHandlers desc, err := ref.ociDesc(egctx, dhs, true) if err != nil { @@ -1512,6 +1512,17 @@ func (sr *immutableRef) parallelExtractLayers(ctx context.Context, chain []*immu return err } + pg := ref.progress + if pg == nil && dh != nil { + pg = dh.Progress + } + if pg != nil { + _, stopProgress := pg.Start(egctx) + defer stopProgress(rerr) + statusDone := pg.Status("extracting "+desc.Digest.String(), "extracting") + defer statusDone() + } + tmpDir, err := os.MkdirTemp(parentTmpDir, ref.ID()+"-") if err != nil { return err From 4349a0977ba5d331351f95d84e2c913daf55f55d Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 13:57:25 -0700 Subject: [PATCH 19/55] get faster decompression speeds --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7324853f914f..c5635a513def 100644 --- a/Dockerfile +++ b/Dockerfile @@ -201,7 +201,7 @@ FROM scratch AS release COPY --link --from=releaser /out/ / FROM alpine:${ALPINE_VERSION} AS buildkit-export-alpine -RUN apk add --no-cache fuse3 git openssh pigz xz iptables ip6tables \ +RUN apk add --no-cache fuse3 git openssh pigz isa-l xz iptables ip6tables \ && ln -s fusermount3 /usr/bin/fusermount COPY --link examples/buildctl-daemonless/buildctl-daemonless.sh /usr/bin/ VOLUME /var/lib/buildkit @@ -213,6 +213,7 @@ RUN apt-get update \ git \ openssh-client \ pigz \ + isal \ xz-utils \ iptables \ ca-certificates \ @@ -458,7 +459,7 @@ VOLUME /var/lib/buildkit # rootless builds a rootless variant of buildkitd image FROM alpine:${ALPINE_VERSION} AS rootless -RUN apk add --no-cache fuse3 fuse-overlayfs git openssh pigz shadow-uidmap xz +RUN apk add --no-cache fuse3 fuse-overlayfs git openssh pigz isa-l shadow-uidmap xz RUN adduser -D -u 1000 user \ && mkdir -p /run/user/1000 /home/user/.local/tmp /home/user/.local/share/buildkit \ && chown -R user /run/user/1000 /home/user \ From bf6084e058c9425e59570674bbac8431d314dadf Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 14:34:28 -0700 Subject: [PATCH 20/55] add tests --- client/client_test.go | 131 ++++++++++++++++++++++++ control/control_test.go | 178 +++++++++++++++++++++++++++++++++ solver/llbsolver/eager_test.go | 120 ++++++++++++++++++++++ solver/vertex_callback_test.go | 138 +++++++++++++++++++++++++ 4 files changed, 567 insertions(+) create mode 100644 solver/llbsolver/eager_test.go create mode 100644 solver/vertex_callback_test.go diff --git a/client/client_test.go b/client/client_test.go index 83cc70dac822..f6c67d5eddae 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -113,6 +113,8 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testBuildHTTPSourceAuthHeaderSecret, testBuildHTTPSourceHeader, testBuildPushAndValidate, + testEagerExportCompress, + testEagerExportPush, testBuildExportWithUncompressed, testBuildExportScratch, testResolveAndHosts, @@ -5080,6 +5082,135 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) { require.False(t, ok) } +func testEagerExportCompress(t *testing.T, sb integration.Sandbox) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + st = busybox.Run(llb.Shlex(`sh -c "echo layer1 > /file1"`), llb.Dir("/wd")).AddMount("/wd", st) + st = busybox.Run(llb.Shlex(`sh -c "echo layer2 > /file2"`), llb.Dir("/wd")).AddMount("/wd", st) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testeagercompress:latest" + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + "eager-export": "compress", + }, + }, + }, + }, nil) + require.NoError(t, err) + + // Verify the pushed image is pullable and has the right content. + pullSt := llb.Image(target) + def, err = pullSt.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "file1")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer1") + + dt, err = os.ReadFile(filepath.Join(destDir, "file2")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer2") +} + +func testEagerExportPush(t *testing.T, sb integration.Sandbox) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + busybox := llb.Image("busybox:latest") + st := llb.Scratch() + st = busybox.Run(llb.Shlex(`sh -c "echo layer1 > /file1"`), llb.Dir("/wd")).AddMount("/wd", st) + st = busybox.Run(llb.Shlex(`sh -c "echo layer2 > /file2"`), llb.Dir("/wd")).AddMount("/wd", st) + st = busybox.Run(llb.Shlex(`sh -c "echo layer3 > /file3"`), llb.Dir("/wd")).AddMount("/wd", st) + + def, err := st.Marshal(sb.Context()) + require.NoError(t, err) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testeagerpush:latest" + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + "eager-export": "push", + }, + }, + }, + }, nil) + require.NoError(t, err) + + // Verify the pushed image is pullable and has the right content. + pullSt := llb.Image(target) + def, err = pullSt.Marshal(sb.Context()) + require.NoError(t, err) + + destDir := t.TempDir() + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Exports: []ExportEntry{ + { + Type: ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "file1")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer1") + + dt, err = os.ReadFile(filepath.Join(destDir, "file2")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer2") + + dt, err = os.ReadFile(filepath.Join(destDir, "file3")) + require.NoError(t, err) + require.Contains(t, string(dt), "layer3") +} + func testStargzLazyRegistryCacheImportExport(t *testing.T, sb integration.Sandbox) { workers.CheckFeatureCompat(t, sb, workers.FeatureCacheExport, diff --git a/control/control_test.go b/control/control_test.go index 504c94f61602..cfaeb06392b3 100644 --- a/control/control_test.go +++ b/control/control_test.go @@ -1,12 +1,43 @@ package control import ( + "context" "testing" controlapi "github.com/moby/buildkit/api/services/control" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/solver/llbsolver" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// mockExporterInstance is a minimal ExporterInstance for testing resolveEagerExport. +type mockExporterInstance struct { + typ string + attrs map[string]string +} + +func (m *mockExporterInstance) ID() int { return 0 } +func (m *mockExporterInstance) Name() string { return "mock" } +func (m *mockExporterInstance) Config() *exporter.Config { return exporter.NewConfig() } +func (m *mockExporterInstance) Type() string { return m.typ } +func (m *mockExporterInstance) Attrs() map[string]string { return m.attrs } +func (m *mockExporterInstance) Export(context.Context, *exporter.Source, exptypes.InlineCache, string) (map[string]string, exporter.DescriptorReference, error) { + return nil, nil, nil +} + +// mockEagerExporterInstance also implements EagerExportProvider. +type mockEagerExporterInstance struct { + mockExporterInstance + pushCfg *exporter.EagerPushConfig +} + +func (m *mockEagerExporterInstance) EagerPushConfig() *exporter.EagerPushConfig { + return m.pushCfg +} + func TestDuplicateCacheOptions(t *testing.T) { var testCases = []struct { name string @@ -146,3 +177,150 @@ func TestParseCacheExportIgnoreError(t *testing.T) { }) } } + +func TestResolveEagerExport(t *testing.T) { + imageExporter := &mockEagerExporterInstance{ + mockExporterInstance: mockExporterInstance{typ: client.ExporterImage}, + pushCfg: &exporter.EagerPushConfig{ + TargetName: "docker.io/library/test:latest", + }, + } + + tests := []struct { + name string + exporters []*controlapi.Exporter + instances []exporter.ExporterInstance + wantMode llbsolver.EagerExportMode + wantPushCfg bool + wantErr string + }{ + { + name: "no eager-export attr returns none", + exporters: []*controlapi.Exporter{{Attrs: map[string]string{}}}, + instances: []exporter.ExporterInstance{imageExporter}, + wantMode: llbsolver.EagerExportNone, + }, + { + name: "compress mode", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportCompress, + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantMode: llbsolver.EagerExportCompress, + }, + { + name: "push mode with push=true", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + string(exptypes.OptKeyPush): "true", + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantMode: llbsolver.EagerExportPush, + wantPushCfg: true, + }, + { + name: "push mode without push=true errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantErr: "eager-export=push requires push=true", + }, + { + name: "invalid eager-export value errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): "bogus", + }, + }}, + instances: []exporter.ExporterInstance{imageExporter}, + wantErr: "invalid eager-export value", + }, + { + name: "non-image exporter errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportCompress, + }, + }}, + instances: []exporter.ExporterInstance{ + &mockExporterInstance{typ: "local"}, + }, + wantErr: "eager-export requires image exporter", + }, + { + name: "multiple exporters with eager-export errors", + exporters: []*controlapi.Exporter{ + {Attrs: map[string]string{string(exptypes.OptKeyEagerExport): "compress"}}, + {Attrs: map[string]string{}}, + }, + instances: []exporter.ExporterInstance{imageExporter, imageExporter}, + wantErr: "eager-export requires exactly one exporter", + }, + { + name: "multiple exporters without eager-export is fine", + exporters: []*controlapi.Exporter{{Attrs: map[string]string{}}, {Attrs: map[string]string{}}}, + instances: []exporter.ExporterInstance{imageExporter, imageExporter}, + wantMode: llbsolver.EagerExportNone, + }, + { + name: "zero exporters returns none", + exporters: nil, + instances: nil, + wantMode: llbsolver.EagerExportNone, + }, + { + name: "push mode with nil EagerPushConfig errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + string(exptypes.OptKeyPush): "true", + }, + }}, + instances: []exporter.ExporterInstance{ + &mockEagerExporterInstance{ + mockExporterInstance: mockExporterInstance{typ: client.ExporterImage}, + pushCfg: nil, + }, + }, + wantErr: "eager-export=push requires a single image name", + }, + { + name: "push mode with non-EagerExportProvider exporter errors", + exporters: []*controlapi.Exporter{{ + Attrs: map[string]string{ + string(exptypes.OptKeyEagerExport): exptypes.OptValEagerExportPush, + string(exptypes.OptKeyPush): "true", + }, + }}, + instances: []exporter.ExporterInstance{ + &mockExporterInstance{typ: client.ExporterImage}, + }, + wantErr: "exporter does not support eager push", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mode, pushCfg, err := resolveEagerExport(tt.exporters, tt.instances) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantMode, mode) + if tt.wantPushCfg { + assert.NotNil(t, pushCfg) + } else { + assert.Nil(t, pushCfg) + } + }) + } +} diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go new file mode 100644 index 000000000000..c80f089649fe --- /dev/null +++ b/solver/llbsolver/eager_test.go @@ -0,0 +1,120 @@ +package llbsolver + +import ( + "context" + "os" + "runtime" + "sync/atomic" + "testing" + + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/util/compression" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEagerWorkerCount_Default(t *testing.T) { + os.Unsetenv("BUILDKIT_EAGER_EXPORT_WORKERS") + n := eagerWorkerCount() + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), n) +} + +func TestEagerWorkerCount_EnvOverride(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "2") + assert.Equal(t, 2, eagerWorkerCount()) +} + +func TestEagerWorkerCount_EnvInvalid(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "not-a-number") + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) +} + +func TestEagerWorkerCount_EnvZero(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "0") + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) +} + +func TestEagerWorkerCount_EnvNegative(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_EXPORT_WORKERS", "-1") + assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) +} + +func TestNewEagerPipeline_PushRequiresConfig(t *testing.T) { + _, err := newEagerPipeline(context.Background(), EagerExportPush, compression.Config{}, "", nil, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "push config") +} + +func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { + ep := &eagerPipeline{ + work: make(chan eagerWorkItem), + } + ep.firstErr = assert.AnError + + err := ep.wait() + assert.Equal(t, assert.AnError, err) +} + +func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { + ep := &eagerPipeline{ + work: make(chan eagerWorkItem), + } + + err := ep.wait() + assert.NoError(t, err) +} + +func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { + var released atomic.Int32 + ep := &eagerPipeline{ + work: make(chan eagerWorkItem, 10), + } + + ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} + ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} + + err := ep.wait() + assert.NoError(t, err) + assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") +} + +func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + ep := &eagerPipeline{ + mode: EagerExportCompress, + ctx: ctx, + work: make(chan eagerWorkItem, 10), + } + + cancel() + + ep.wg.Add(1) + go ep.worker() + ep.wg.Wait() +} + +func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { + ep := &eagerPipeline{ + mode: EagerExportCompress, + ctx: context.Background(), + work: make(chan eagerWorkItem), + } + + ep.wg.Add(1) + go ep.worker() + + close(ep.work) + ep.wg.Wait() +} + +// releaseTracker is a minimal stub that satisfies cache.ImmutableRef +// for testing ref lifecycle (Release calls). All other methods panic. +type releaseTracker struct { + cache.ImmutableRef + released *atomic.Int32 +} + +func (r *releaseTracker) Release(context.Context) error { + r.released.Add(1) + return nil +} diff --git a/solver/vertex_callback_test.go b/solver/vertex_callback_test.go new file mode 100644 index 000000000000..29bb6985f63c --- /dev/null +++ b/solver/vertex_callback_test.go @@ -0,0 +1,138 @@ +package solver + +import ( + "context" + "sync" + "sync/atomic" + "testing" + + digest "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/assert" +) + +type mockVertex struct { + dgst digest.Digest + name string +} + +func (v *mockVertex) Digest() digest.Digest { return v.dgst } +func (v *mockVertex) Sys() any { return nil } +func (v *mockVertex) Options() VertexOptions { return VertexOptions{} } +func (v *mockVertex) Inputs() []Edge { return nil } +func (v *mockVertex) Name() string { return v.name } + +type mockResult struct { + id string +} + +func (r *mockResult) ID() string { return r.id } +func (r *mockResult) Release(context.Context) error { return nil } +func (r *mockResult) Sys() any { return nil } +func (r *mockResult) Clone() Result { return r } + +func TestFireOnVertexComplete_CallsRegisteredCallbacks(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:abc", name: "test-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + var called atomic.Int32 + var receivedVtx Vertex + var receivedResults []Result + var mu sync.Mutex + + j := &Job{} + j.SetOnVertexComplete(func(v Vertex, results []Result) { + called.Add(1) + mu.Lock() + receivedVtx = v + receivedResults = results + mu.Unlock() + }) + st.jobs[j] = struct{}{} + + results := []Result{&mockResult{id: "res1"}} + so.fireOnVertexComplete(results) + + assert.Equal(t, int32(1), called.Load()) + mu.Lock() + assert.Equal(t, vtx, receivedVtx) + assert.Len(t, receivedResults, 1) + assert.Equal(t, "res1", receivedResults[0].ID()) + mu.Unlock() +} + +func TestFireOnVertexComplete_MultipleJobs(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:def", name: "multi-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + var called atomic.Int32 + + for range 3 { + j := &Job{} + j.SetOnVertexComplete(func(v Vertex, results []Result) { + called.Add(1) + }) + st.jobs[j] = struct{}{} + } + + so.fireOnVertexComplete([]Result{&mockResult{id: "r"}}) + + assert.Equal(t, int32(3), called.Load()) +} + +func TestFireOnVertexComplete_SkipsJobsWithoutCallback(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:ghi", name: "skip-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + var called atomic.Int32 + + jobWithCallback := &Job{} + jobWithCallback.SetOnVertexComplete(func(v Vertex, results []Result) { + called.Add(1) + }) + jobWithoutCallback := &Job{} + + st.jobs[jobWithCallback] = struct{}{} + st.jobs[jobWithoutCallback] = struct{}{} + + so.fireOnVertexComplete([]Result{&mockResult{id: "r"}}) + + assert.Equal(t, int32(1), called.Load()) +} + +func TestFireOnVertexComplete_NoJobs(t *testing.T) { + vtx := &mockVertex{dgst: "sha256:jkl", name: "noop-vertex"} + st := &state{ + jobs: make(map[*Job]struct{}), + vtx: vtx, + } + so := &sharedOp{st: st} + + // Should not panic with no jobs + so.fireOnVertexComplete([]Result{&mockResult{id: "r"}}) +} + +func TestSetOnVertexComplete(t *testing.T) { + j := &Job{} + assert.Nil(t, j.onVertexComplete) + + called := false + j.SetOnVertexComplete(func(v Vertex, results []Result) { + called = true + }) + assert.NotNil(t, j.onVertexComplete) + + j.onVertexComplete(nil, nil) + assert.True(t, called) +} From fe763ef9f1d096b970b0adde94a9b241c5e0a9f9 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 14:35:22 -0700 Subject: [PATCH 21/55] Add CI and post-merge GitHub Actions workflows - ci.yml: lint (golangci-lint) + unit tests on PRs to v0.22-base - post-merge.yml: integration tests + Docker image build/push on merge to v0.22-base Made-with: Cursor --- .github/workflows/ci.yml | 45 ++++++++++++++++++ .github/workflows/post-merge.yml | 80 ++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/post-merge.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..1e340447ce30 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + pull_request: + branches: + - v0.22-base + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + GO_VERSION: "1.24" + +jobs: + lint: + name: Lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - uses: golangci/golangci-lint-action@v7 + with: + version: latest + args: --modules-download-mode=vendor + + unit-tests: + name: Unit Tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + - name: Run unit tests + run: go test -mod=vendor -count=1 -timeout 10m ./... + env: + SKIP_INTEGRATION_TESTS: "1" diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml new file mode 100644 index 000000000000..45abb2ce214e --- /dev/null +++ b/.github/workflows/post-merge.yml @@ -0,0 +1,80 @@ +name: Post-Merge + +on: + push: + branches: + - v0.22-base + +permissions: + contents: read + +env: + GO_VERSION: "1.24" + SETUP_BUILDX_VERSION: "edge" + SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" + IMAGE_NAME: "baseten/buildkit" + +jobs: + integration-tests: + name: Integration Tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: crazy-max/ghaction-github-runtime@v3 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + with: + version: ${{ env.SETUP_BUILDX_VERSION }} + driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} + buildkitd-flags: --debug + - name: Build integration test image + uses: docker/bake-action@v6 + with: + targets: integration-tests + set: | + *.cache-from=type=gha,scope=integration-tests + *.cache-to=type=gha,scope=integration-tests,repository=${{ github.repository }},ghtoken=${{ secrets.GITHUB_TOKEN }} + - name: Run integration tests + run: ./hack/test integration + env: + TESTPKGS: ./client ./cmd/buildctl ./solver ./frontend + TESTFLAGS: "-v --parallel=6 --timeout=30m" + GOTESTSUM_FORMAT: standard-verbose + TEST_IMAGE_BUILD: "0" + CACHE_FROM: type=gha,scope=integration-tests + + build-and-push: + name: Build & Push Image + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + with: + version: ${{ env.SETUP_BUILDX_VERSION }} + driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }} + buildkitd-flags: --debug + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Generate image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.IMAGE_NAME }} + tags: | + type=raw,value=v0.22-base + type=sha,prefix=v0.22-base- + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=build-image + cache-to: type=gha,scope=build-image,mode=max From 8d119457f98ebf7a31310313afb80565f1fb9136 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 14:43:00 -0700 Subject: [PATCH 22/55] Fix CI lint and unit test failures - Exclude new gosec rules (G120, G122, G703) that flag upstream code - Skip worker/containerd and worker/runc packages whose TestMain panics without a runtime Made-with: Cursor --- .github/workflows/ci.yml | 4 +++- .golangci.yml | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e340447ce30..3c23d2871f7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,8 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: true - name: Run unit tests - run: go test -mod=vendor -count=1 -timeout 10m ./... + run: | + go test -mod=vendor -count=1 -timeout 10m \ + $(go list -mod=vendor ./... | grep -v -E 'worker/(containerd|runc)$') env: SKIP_INTEGRATION_TESTS: "1" diff --git a/.golangci.yml b/.golangci.yml index 2efd20f86d9f..ba4243f116f1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -63,10 +63,13 @@ linters: gosec: excludes: - G101 + - G115 + - G120 + - G122 - G402 - G504 - G601 - - G115 + - G703 config: G306: "0644" govet: From 157019a10c4177dba41bccd261aa4cac66b88cad Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 14:43:57 -0700 Subject: [PATCH 23/55] Add CODEOWNERS to require runtime-fabric team review Made-with: Cursor --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..2855941f405d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @basetenlabs/runtime-fabric From fa6d0ca77b4454736f19915ebe654f7d8fb1ff7d Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 14:48:39 -0700 Subject: [PATCH 24/55] Fix CI: only-new-issues for lint, exclude privileged test packages Lint: use only-new-issues to avoid flagging upstream code with newer golangci-lint rules. Tests: exclude packages that need privileged mounts (cache, snapshot, source/git, source/http, util/overlay) or runtime daemons (worker/containerd, worker/runc). Made-with: Cursor --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c23d2871f7e..77c630665285 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: with: version: latest args: --modules-download-mode=vendor + only-new-issues: true unit-tests: name: Unit Tests @@ -42,6 +43,6 @@ jobs: - name: Run unit tests run: | go test -mod=vendor -count=1 -timeout 10m \ - $(go list -mod=vendor ./... | grep -v -E 'worker/(containerd|runc)$') + $(go list -mod=vendor ./... | grep -v -E '(worker/(containerd|runc)|cache$|cache/contenthash|snapshot$|source/(git|http)|util/overlay)$') env: SKIP_INTEGRATION_TESTS: "1" From bbf96863288a552f2efaac6c8a36393ba849eb7c Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 15:07:10 -0700 Subject: [PATCH 25/55] Update post-merge image tag format and add explicit target - Tag as baseten/buildkit:v0.22.0-- - Explicitly target the buildkit stage Made-with: Cursor --- .github/workflows/post-merge.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 45abb2ce214e..fa4f13f38d00 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -60,21 +60,19 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Generate image metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.IMAGE_NAME }} - tags: | - type=raw,value=v0.22-base - type=sha,prefix=v0.22-base- + - name: Generate tag + id: tag + run: | + SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) + TIMESTAMP=$(date -u +%Y%m%d%H%M%S) + echo "tag=v0.22.0-${SHORT_SHA}-${TIMESTAMP}" >> $GITHUB_OUTPUT - name: Build and push uses: docker/build-push-action@v6 with: context: . push: true + target: buildkit platforms: linux/amd64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + tags: ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} cache-from: type=gha,scope=build-image cache-to: type=gha,scope=build-image,mode=max From dc3f0d9e2314feb9596623d2aa930abc6629e172 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 15:10:20 -0700 Subject: [PATCH 26/55] Add workflow_dispatch and print pushed image name - Manual trigger from any branch with options to skip integration tests or build independently - Print full image name (e.g. baseten/buildkit:v0.22.0-abc1234-...) at end of build job Made-with: Cursor --- .github/workflows/post-merge.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index fa4f13f38d00..9ff5ad527a08 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -4,6 +4,18 @@ on: push: branches: - v0.22-base + workflow_dispatch: + inputs: + skip_integration_tests: + description: 'Skip integration tests' + required: false + type: boolean + default: false + skip_build: + description: 'Skip image build & push' + required: false + type: boolean + default: false permissions: contents: read @@ -17,6 +29,7 @@ env: jobs: integration-tests: name: Integration Tests + if: ${{ !inputs.skip_integration_tests }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -45,6 +58,7 @@ jobs: build-and-push: name: Build & Push Image + if: ${{ !inputs.skip_build }} runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -76,3 +90,5 @@ jobs: tags: ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} cache-from: type=gha,scope=build-image cache-to: type=gha,scope=build-image,mode=max + - name: Print image + run: echo "Pushed ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}" From 8b8198f4819eae46fc53bc024bb591eb212077b0 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 10 Apr 2026 17:52:57 -0700 Subject: [PATCH 27/55] fix lint --- control/control.go | 2 +- control/control_test.go | 8 ++++---- solver/llbsolver/eager_test.go | 6 +++--- solver/llbsolver/solver.go | 14 ++++++++------ solver/vertex_callback_test.go | 16 ++++++++-------- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/control/control.go b/control/control.go index eeaf49ab3349..589be54ef45d 100644 --- a/control/control.go +++ b/control/control.go @@ -571,7 +571,7 @@ func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.Ex var pushCfg *exporter.EagerPushConfig if mode == llbsolver.EagerExportPush { - push, _ := ex.Attrs[string(exptypes.OptKeyPush)] + push := ex.Attrs[string(exptypes.OptKeyPush)] if push != "true" { return 0, nil, errors.New("eager-export=push requires push=true") } diff --git a/control/control_test.go b/control/control_test.go index cfaeb06392b3..1d001da42739 100644 --- a/control/control_test.go +++ b/control/control_test.go @@ -19,10 +19,10 @@ type mockExporterInstance struct { attrs map[string]string } -func (m *mockExporterInstance) ID() int { return 0 } -func (m *mockExporterInstance) Name() string { return "mock" } +func (m *mockExporterInstance) ID() int { return 0 } +func (m *mockExporterInstance) Name() string { return "mock" } func (m *mockExporterInstance) Config() *exporter.Config { return exporter.NewConfig() } -func (m *mockExporterInstance) Type() string { return m.typ } +func (m *mockExporterInstance) Type() string { return m.typ } func (m *mockExporterInstance) Attrs() map[string]string { return m.attrs } func (m *mockExporterInstance) Export(context.Context, *exporter.Source, exptypes.InlineCache, string) (map[string]string, exporter.DescriptorReference, error) { return nil, nil, nil @@ -286,7 +286,7 @@ func TestResolveEagerExport(t *testing.T) { instances: []exporter.ExporterInstance{ &mockEagerExporterInstance{ mockExporterInstance: mockExporterInstance{typ: client.ExporterImage}, - pushCfg: nil, + pushCfg: nil, }, }, wantErr: "eager-export=push requires a single image name", diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index c80f089649fe..e26d672c9251 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -74,19 +74,19 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} err := ep.wait() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") } func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancelCause(context.Background()) ep := &eagerPipeline{ mode: EagerExportCompress, ctx: ctx, work: make(chan eagerWorkItem, 10), } - cancel() + cancel(nil) ep.wg.Add(1) go ep.worker() diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 63a423bcc31d..b2f860b62393 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -527,20 +527,20 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SessionID = sessionID - createLease := func() error { + createLease := func(ctx context.Context) (context.Context, error) { lm, err := s.leaseManager() if err != nil { - return err + return ctx, err } var done func(context.Context) error ctx, done, err = leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) if err != nil { - return err + return ctx, err } releasers = append(releasers, func() { done(context.WithoutCancel(ctx)) }) - return nil + return ctx, nil } // Set up eager export pipeline before the build starts so the vertex @@ -549,7 +549,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // compressed blobs are GC-protected during the build phase. var eager *eagerPipeline if exp.EagerExport != EagerExportNone && len(exp.Exporters) > 0 { - if err := createLease(); err != nil { + ctx, err = createLease(ctx) + if err != nil { return nil, err } comp := exp.Exporters[0].Config().Compression() @@ -674,7 +675,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // location) — after the build completes but before export. This avoids // adding a disk write before gateway forwarder registration. if eager == nil { - if err := createLease(); err != nil { + ctx, err = createLease(ctx) + if err != nil { return nil, err } } diff --git a/solver/vertex_callback_test.go b/solver/vertex_callback_test.go index 29bb6985f63c..6c4b26a2bf8e 100644 --- a/solver/vertex_callback_test.go +++ b/solver/vertex_callback_test.go @@ -15,20 +15,20 @@ type mockVertex struct { name string } -func (v *mockVertex) Digest() digest.Digest { return v.dgst } -func (v *mockVertex) Sys() any { return nil } -func (v *mockVertex) Options() VertexOptions { return VertexOptions{} } -func (v *mockVertex) Inputs() []Edge { return nil } -func (v *mockVertex) Name() string { return v.name } +func (v *mockVertex) Digest() digest.Digest { return v.dgst } +func (v *mockVertex) Sys() any { return nil } +func (v *mockVertex) Options() VertexOptions { return VertexOptions{} } +func (v *mockVertex) Inputs() []Edge { return nil } +func (v *mockVertex) Name() string { return v.name } type mockResult struct { id string } func (r *mockResult) ID() string { return r.id } -func (r *mockResult) Release(context.Context) error { return nil } -func (r *mockResult) Sys() any { return nil } -func (r *mockResult) Clone() Result { return r } +func (r *mockResult) Release(context.Context) error { return nil } +func (r *mockResult) Sys() any { return nil } +func (r *mockResult) Clone() Result { return r } func TestFireOnVertexComplete_CallsRegisteredCallbacks(t *testing.T) { vtx := &mockVertex{dgst: "sha256:abc", name: "test-vertex"} From 68e0d6248ce55e153c7d8e9b40be71197e576ff9 Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 13 Apr 2026 09:52:52 -0700 Subject: [PATCH 28/55] fix lint --- .golangci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 2efd20f86d9f..179029954547 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -63,6 +63,7 @@ linters: gosec: excludes: - G101 + - G118 - G402 - G504 - G601 @@ -119,6 +120,7 @@ linters: text: if-return paths: - .*\.pb\.go$ + - .*_test\.go$ - examples formatters: enable: From 6b8717692c1d21961268ba56881d259d026dcaee Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 13 Apr 2026 17:12:29 -0700 Subject: [PATCH 29/55] quick fix --- solver/llbsolver/eager.go | 8 +++++++ solver/llbsolver/eager_test.go | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 6f28a5f29943..219819bf0e73 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -7,6 +7,7 @@ import ( "strconv" "sync" + "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/core/remotes" "github.com/moby/buildkit/cache" cacheconfig "github.com/moby/buildkit/cache/config" @@ -184,6 +185,9 @@ func (ep *eagerPipeline) pushBlobs(ctx context.Context, rems []*solver.Remote) e ) for _, desc := range remote.Descriptors { + if !shouldEagerPushDesc(desc) { + continue + } if err := ep.pushBlob(ctx, handler, desc); err != nil { return err } @@ -191,6 +195,10 @@ func (ep *eagerPipeline) pushBlobs(ctx context.Context, rems []*solver.Remote) e return nil } +func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { + return !images.IsNonDistributable(desc.MediaType) +} + // pushBlob pushes a single descriptor, deduplicated by digest across all // concurrent workers via flightcontrol. func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error), desc ocispecs.Descriptor) error { diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index e26d672c9251..d48da18c4946 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -7,8 +7,11 @@ import ( "sync/atomic" "testing" + "github.com/containerd/containerd/v2/core/images" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/util/compression" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -107,6 +110,43 @@ func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { ep.wg.Wait() } +func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { + descs := []ocispecs.Descriptor{ + { + Digest: digest.FromString("push me"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + }, + { + Digest: digest.FromString("skip me"), + MediaType: ocispecs.MediaTypeImageLayerNonDistributableGzip, //nolint:staticcheck // deprecated but still supported + }, + { + Digest: digest.FromString("push me too"), + MediaType: images.MediaTypeDockerSchema2Layer, + }, + } + + var pushed []digest.Digest + ep := &eagerPipeline{} + handler := func(_ context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + pushed = append(pushed, desc.Digest) + return nil, nil + } + + for _, desc := range descs { + if !shouldEagerPushDesc(desc) { + continue + } + err := ep.pushBlob(context.Background(), handler, desc) + require.NoError(t, err) + } + + assert.Equal(t, []digest.Digest{ + digest.FromString("push me"), + digest.FromString("push me too"), + }, pushed) +} + // releaseTracker is a minimal stub that satisfies cache.ImmutableRef // for testing ref lifecycle (Release calls). All other methods panic. type releaseTracker struct { From 468ae2493337223774d418a25c9e11044f18346a Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 13 Apr 2026 17:28:19 -0700 Subject: [PATCH 30/55] add user facing logs --- exporter/containerimage/export.go | 2 +- exporter/containerimage/writer.go | 9 ++++++++- solver/llbsolver/solver.go | 14 +++++++++++++- util/push/push.go | 11 +++++++++-- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 11a57ce2ce05..8c621c928bcd 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -432,7 +432,7 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou addAnnotations(annotations, desc) } } - return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations) + return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, e.eagerExport == exptypes.OptValEagerExportPush, annotations) } func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, s session.Group) (err0 error) { diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index c410892d572a..5153eea76a87 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -368,7 +368,7 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC createIfNeeded := eagerExport == "" eg, ctx := errgroup.WithContext(ctx) - layersDone := progress.OneOff(ctx, "exporting layers") + layersDone := progress.OneOff(ctx, exportLayersProgressID(eagerExport)) out := make([]solver.Remote, len(refs)) @@ -394,6 +394,13 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC return out, err } +func exportLayersProgressID(eagerExport string) string { + if eagerExport != "" { + return "loading eagerly compressed layers" + } + return "exporting layers" +} + // rewriteImageLayerWithEpoch rewrites the file timestamps in the layer blob to match the epoch, and returns a new descriptor that points to // the new blob. // diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index b2f860b62393..174d3a33a4ac 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -635,7 +635,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // Wait for all eager compression/push jobs to finish before running // exporters. The manifest needs final blob digests from every layer. if eager != nil { - if err := eager.wait(); err != nil { + waitDone := progress.OneOff(ctx, eagerWaitProgressID(exp.EagerExport)) + if err := waitDone(eager.wait()); err != nil { return nil, errors.Wrap(err, "eager export pipeline failed") } } @@ -1138,6 +1139,17 @@ func withDescHandlerCacheOpts(ctx context.Context, ref cache.ImmutableRef) conte }) } +func eagerWaitProgressID(mode EagerExportMode) string { + switch mode { + case EagerExportPush: + return "waiting for eager compression and push to finish" + case EagerExportCompress: + return "waiting for eager compression to finish" + default: + return "waiting for eager export to finish" + } +} + func (s *Solver) Status(ctx context.Context, id string, statusChan chan *client.SolveStatus) error { if err := s.history.Status(ctx, id, statusChan); err != nil { if !errors.Is(err, os.ErrNotExist) { diff --git a/util/push/push.go b/util/push/push.go index c859cce8eba1..fa89f9e170ee 100644 --- a/util/push/push.go +++ b/util/push/push.go @@ -72,7 +72,7 @@ func NewPusher(ctx context.Context, sm *session.Manager, sid string, ref string, return Pusher(ctx, r, ref) } -func Push(ctx context.Context, sm *session.Manager, sid string, provider content.Provider, manager content.Manager, dgst digest.Digest, ref string, insecure bool, hosts docker.RegistryHosts, byDigest bool, annotations map[digest.Digest]map[string]string) error { +func Push(ctx context.Context, sm *session.Manager, sid string, provider content.Provider, manager content.Manager, dgst digest.Digest, ref string, insecure bool, hosts docker.RegistryHosts, byDigest bool, eager bool, annotations map[digest.Digest]map[string]string) error { ctx = contentutil.RegisterContentPayloadTypes(ctx) desc := ocispecs.Descriptor{ Digest: dgst, @@ -139,7 +139,7 @@ func Push(ctx context.Context, sm *session.Manager, sid string, provider content return err } - layersDone := progress.OneOff(ctx, "pushing layers") + layersDone := progress.OneOff(ctx, pushLayersProgressID(eager)) err = images.Dispatch(ctx, skipNonDistributableBlobs(images.Handlers(handlers...)), nil, ocispecs.Descriptor{ Digest: dgst, Size: ra.Size(), @@ -158,6 +158,13 @@ func Push(ctx context.Context, sm *session.Manager, sid string, provider content return mfstDone(nil) } +func pushLayersProgressID(eager bool) string { + if eager { + return "pushing any remaining layers" + } + return "pushing layers" +} + // TODO: the containerd function for this is filtering too much, that needs to be fixed. // For now we just carry this. func skipNonDistributableBlobs(f images.HandlerFunc) images.HandlerFunc { From a4b93222303cf7440649a849caf6119be06c47df Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 14 Apr 2026 13:19:52 -0700 Subject: [PATCH 31/55] quick logging fix --- solver/llbsolver/solver.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 174d3a33a4ac..81fe3a000ab2 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -635,8 +635,9 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // Wait for all eager compression/push jobs to finish before running // exporters. The manifest needs final blob digests from every layer. if eager != nil { - waitDone := progress.OneOff(ctx, eagerWaitProgressID(exp.EagerExport)) - if err := waitDone(eager.wait()); err != nil { + if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { + return eager.wait() + }); err != nil { return nil, errors.Wrap(err, "eager export pipeline failed") } } From fd8052fe452dce5621cd9bde2926b823cab0d69a Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 14 Apr 2026 18:36:25 -0700 Subject: [PATCH 32/55] updates --- cache/manager.go | 23 ++++++++++++ cache/refs.go | 42 +++++++++++++++++----- cache/refs_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 9 deletions(-) create mode 100644 cache/refs_test.go diff --git a/cache/manager.go b/cache/manager.go index bd109c9ebafc..12d941d6ae2a 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "maps" + "os" + "path/filepath" "slices" "strings" "sync" @@ -125,6 +127,8 @@ func NewManager(opt ManagerOpt) (Manager, error) { return nil, err } + cm.cleanupParallelExtractDirs() + p, err := newSharableMountPool(opt.MountPoolRoot) if err != nil { return nil, err @@ -340,6 +344,25 @@ func (cm *cacheManager) init(ctx context.Context) error { return nil } +// cleanupParallelExtractDirs removes any leftover temp directories from +// parallel layer extraction that weren't cleaned up (e.g. due to a crash). +func (cm *cacheManager) cleanupParallelExtractDirs() { + entries, err := os.ReadDir(cm.root) + if err != nil { + return + } + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), "buildkit-parallel-extract-") { + p := filepath.Join(cm.root, e.Name()) + if err := os.RemoveAll(p); err != nil { + bklog.G(context.TODO()).Warnf("failed to clean up parallel extract temp dir %s: %v", p, err) + } else { + bklog.G(context.TODO()).Infof("cleaned up leftover parallel extract temp dir %s", p) + } + } + } +} + // IdentityMapping returns the userns remapping used for refs func (cm *cacheManager) IdentityMapping() *user.IdentityMapping { return cm.Snapshotter.IdentityMapping() diff --git a/cache/refs.go b/cache/refs.go index ed7ce2057fcc..23c62a5ee84b 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -72,6 +72,28 @@ func extractFSPath(mounts []mount.Mount) string { return "" } +// isLinearLayerChain reports whether chain is a pure linear Layer/BaseLayer +// stack with no Merge or Diff refs. Parallel extract is only safe for such +// chains because Phase 2's parentID tracking assumes each entry's overlay +// parent is exactly the previous entry in the chain. +func isLinearLayerChain(chain []*immutableRef) bool { + for i, ref := range chain { + switch ref.kind() { + case BaseLayer: + if i != 0 { + return false + } + case Layer: + if i == 0 || ref.layerParent != chain[i-1] { + return false + } + default: + return false + } + } + return true +} + // Ref is a reference to cacheable objects. type Ref interface { Mountable @@ -1056,17 +1078,19 @@ func (sr *immutableRef) Extract(ctx context.Context, s session.Group) (rerr erro if parallelExtractEnabled() && sr.cm.Snapshotter.Name() == "overlayfs" { chain := sr.layerChain() - var needsExtract []*immutableRef - for _, ref := range chain { - if ref.getBlobOnly() { - needsExtract = append(needsExtract, ref) + if isLinearLayerChain(chain) { + var needsExtract []*immutableRef + for _, ref := range chain { + if ref.getBlobOnly() { + needsExtract = append(needsExtract, ref) + } } + if len(needsExtract) > 0 { + bklog.G(ctx).Infof("parallel extract: extracting %d/%d layers in parallel", len(needsExtract), len(chain)) + return sr.parallelExtractLayers(ctx, chain, needsExtract, s) + } + return nil } - if len(needsExtract) > 0 { - bklog.G(ctx).Infof("parallel extract: extracting %d/%d layers in parallel", len(needsExtract), len(chain)) - return sr.parallelExtractLayers(ctx, chain, needsExtract, s) - } - return nil } return sr.unlazy(ctx, sr.descHandlers, sr.progress, s, true, false) diff --git a/cache/refs_test.go b/cache/refs_test.go new file mode 100644 index 000000000000..b1195644e0a6 --- /dev/null +++ b/cache/refs_test.go @@ -0,0 +1,88 @@ +package cache + +import ( + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestImmutableRef(kind refKind, parent *immutableRef) *immutableRef { + ref := &immutableRef{ + cacheRecord: &cacheRecord{ + mu: &sync.Mutex{}, + }, + } + switch kind { + case Layer: + ref.layerParent = parent + case Merge: + ref.mergeParents = []*immutableRef{parent} + case Diff: + ref.diffParents = &diffParents{lower: parent} + } + return ref +} + +func TestIsLinearLayerChain(t *testing.T) { + base := newTestImmutableRef(BaseLayer, nil) + layer1 := newTestImmutableRef(Layer, base) + layer2 := newTestImmutableRef(Layer, layer1) + merge := newTestImmutableRef(Merge, base) + diff := newTestImmutableRef(Diff, base) + + tests := []struct { + name string + chain []*immutableRef + want bool + }{ + {"empty chain", nil, true}, + {"single base layer", []*immutableRef{base}, true}, + {"base + layer", []*immutableRef{base, layer1}, true}, + {"base + layer + layer", []*immutableRef{base, layer1, layer2}, true}, + {"single layer (no base)", []*immutableRef{layer1}, false}, + {"base in wrong position", []*immutableRef{layer1, base}, false}, + {"merge ref", []*immutableRef{base, merge}, false}, + {"diff ref", []*immutableRef{base, diff}, false}, + {"broken parent link", []*immutableRef{base, layer2}, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isLinearLayerChain(tt.chain) + require.Equal(t, tt.want, got) + }) + } +} + +func TestCleanupParallelExtractDirs(t *testing.T) { + root := t.TempDir() + + // Create dirs that should be cleaned up + require.NoError(t, os.Mkdir(filepath.Join(root, "buildkit-parallel-extract-abc123"), 0755)) + require.NoError(t, os.Mkdir(filepath.Join(root, "buildkit-parallel-extract-xyz789"), 0755)) + // Create a file inside one to verify RemoveAll works + require.NoError(t, os.WriteFile(filepath.Join(root, "buildkit-parallel-extract-abc123", "data"), []byte("test"), 0644)) + + // Create dirs that should NOT be cleaned up + require.NoError(t, os.Mkdir(filepath.Join(root, "snapshots"), 0755)) + require.NoError(t, os.Mkdir(filepath.Join(root, "other-dir"), 0755)) + + cm := &cacheManager{root: root} + cm.cleanupParallelExtractDirs() + + entries, err := os.ReadDir(root) + require.NoError(t, err) + + var names []string + for _, e := range entries { + names = append(names, e.Name()) + } + + require.NotContains(t, names, "buildkit-parallel-extract-abc123") + require.NotContains(t, names, "buildkit-parallel-extract-xyz789") + require.Contains(t, names, "snapshots") + require.Contains(t, names, "other-dir") +} From 276c48ef154987fa8e91efbc8ffa638fe57573b0 Mon Sep 17 00:00:00 2001 From: Deepak Nagaraj Date: Wed, 15 Apr 2026 19:00:10 -0700 Subject: [PATCH 33/55] Treat OTel export failures as non-fatal (fixes moby/buildkit#4616) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When OTEL_EXPORTER_OTLP_ENDPOINT is set but the collector is unreachable, buildctl and buildkitd stall for up to 30 seconds (the SDK batch span processor export timeout) during shutdown. Root cause: all three trace-export paths use context.TODO() (unbounded) for operations that block on a dead OTLP endpoint: - tp.Shutdown in buildctl's app.After - closer loop in buildkitd's app.After - Controller.Export synchronous ExportSpans in the gRPC handler Fix: cap all three paths with a 5-second deadline and treat errors as non-fatal (discard or debug-log). Trace telemetry is best-effort — a missing collector should never fail a build or block daemon shutdown. Made-with: Cursor --- cmd/buildctl/common/trace.go | 9 +++- cmd/buildkitd/main.go | 13 +++-- control/control.go | 11 ++-- util/tracing/detect/shutdown_test.go | 76 ++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 util/tracing/detect/shutdown_test.go diff --git a/cmd/buildctl/common/trace.go b/cmd/buildctl/common/trace.go index 47e6ad5e9682..3e40ccb78cbb 100644 --- a/cmd/buildctl/common/trace.go +++ b/cmd/buildctl/common/trace.go @@ -2,7 +2,9 @@ package common import ( "context" + "errors" "os" + "time" "github.com/moby/buildkit/util/appcontext" "github.com/moby/buildkit/util/tracing/delegated" @@ -69,7 +71,12 @@ func AttachAppContext(app *cli.App) error { if span != nil { span.End() } - return tp.Shutdown(context.TODO()) + ctx, cancel := context.WithTimeoutCause(context.Background(), 5*time.Second, errors.New("tracer shutdown timeout")) + defer cancel() + // Trace export is best-effort; don't fail the build if the + // collector is unreachable or slow. + _ = tp.Shutdown(ctx) + return nil } return nil } diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index d3bf198d8fbf..7b94462c0492 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -13,6 +13,7 @@ import ( "slices" "strconv" "strings" + "time" "github.com/containerd/containerd/v2/core/remotes/docker" "github.com/containerd/containerd/v2/defaults" @@ -20,7 +21,6 @@ import ( "github.com/containerd/platforms" sddaemon "github.com/coreos/go-systemd/v22/daemon" "github.com/gofrs/flock" - "github.com/hashicorp/go-multierror" "github.com/moby/buildkit/cache/remotecache" "github.com/moby/buildkit/cache/remotecache/azblob" "github.com/moby/buildkit/cache/remotecache/gha" @@ -399,12 +399,15 @@ func main() { } app.After = func(_ *cli.Context) (err error) { + ctx, cancel := context.WithTimeoutCause(context.Background(), 5*time.Second, errors.New("telemetry shutdown timeout")) + defer cancel() for _, c := range closers { - if e := c(context.TODO()); e != nil { - err = multierror.Append(err, e) - } + // Closers here are telemetry providers (TracerProvider, + // MeterProvider). Failures are non-fatal — an unreachable + // collector should not prevent clean daemon shutdown. + _ = c(ctx) } - return err + return nil } profiler.Attach(app) diff --git a/control/control.go b/control/control.go index 589be54ef45d..12073a7e6558 100644 --- a/control/control.go +++ b/control/control.go @@ -51,9 +51,7 @@ import ( tracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" "golang.org/x/sync/errgroup" "google.golang.org/grpc" - "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -289,11 +287,12 @@ func (c *Controller) Prune(req *controlapi.PruneRequest, stream controlapi.Contr func (c *Controller) Export(ctx context.Context, req *tracev1.ExportTraceServiceRequest) (*tracev1.ExportTraceServiceResponse, error) { if c.opt.TraceCollector == nil { - return nil, status.Errorf(codes.Unavailable, "trace collector not configured") + return &tracev1.ExportTraceServiceResponse{}, nil } - err := c.opt.TraceCollector.ExportSpans(ctx, transform.Spans(req.GetResourceSpans())) - if err != nil { - return nil, err + ctx, cancel := context.WithTimeoutCause(ctx, 5*time.Second, errors.New("trace export timeout")) + defer cancel() + if err := c.opt.TraceCollector.ExportSpans(ctx, transform.Spans(req.GetResourceSpans())); err != nil { + bklog.G(ctx).WithError(err).Debug("trace export to collector failed") } return &tracev1.ExportTraceServiceResponse{}, nil } diff --git a/util/tracing/detect/shutdown_test.go b/util/tracing/detect/shutdown_test.go new file mode 100644 index 000000000000..3f98d54f0f5d --- /dev/null +++ b/util/tracing/detect/shutdown_test.go @@ -0,0 +1,76 @@ +package detect_test + +import ( + "context" + "testing" + "time" + + "github.com/moby/buildkit/util/tracing/detect" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func newTestProvider(t *testing.T) *sdktrace.TracerProvider { + t.Helper() + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://192.0.2.1:4317") // RFC 5737 TEST-NET, guaranteed unreachable + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", "grpc") + + exp, err := detect.NewSpanExporter(context.Background()) + if err != nil { + t.Fatalf("NewSpanExporter: %v", err) + } + if detect.IsNoneSpanExporter(exp) { + t.Fatal("expected OTLP exporter, got none") + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithResource(detect.Resource()), + sdktrace.WithBatcher(exp), + ) + + tracer := tp.Tracer("test") + _, span := tracer.Start(context.Background(), "test-op") + span.End() + + return tp +} + +// TestShutdownStallsWithUnboundedContext reproduces +// https://github.com/moby/buildkit/issues/4616. +// Without a deadline on the context, Shutdown blocks until the BSP export +// timeout expires (default 30 s; reduced here via env for test speed). +func TestShutdownStallsWithUnboundedContext(t *testing.T) { + t.Setenv("OTEL_BSP_EXPORT_TIMEOUT", "2000") // 2 s instead of default 30 s + + tp := newTestProvider(t) + + start := time.Now() + _ = tp.Shutdown(context.TODO()) // the buggy call pattern + elapsed := time.Since(start) + + // Even with the reduced timeout, shutdown still blocks for the full + // export timeout (2 s here, 30 s in production). + if elapsed < 1500*time.Millisecond { + t.Fatalf("expected stall of ~2s, got %v", elapsed) + } + t.Logf("Unbounded shutdown stalled for %v (BSP export timeout)", elapsed) +} + +// TestShutdownRespectsDeadline verifies the fix: passing a bounded context +// makes Shutdown return promptly when the deadline is shorter than the export +// timeout. +func TestShutdownRespectsDeadline(t *testing.T) { + tp := newTestProvider(t) + + deadline := 500 * time.Millisecond + ctx, cancel := context.WithTimeout(context.Background(), deadline) + defer cancel() + + start := time.Now() + err := tp.Shutdown(ctx) + elapsed := time.Since(start) + + if elapsed > deadline+200*time.Millisecond { + t.Fatalf("Shutdown took %v, expected <= %v", elapsed, deadline+200*time.Millisecond) + } + t.Logf("Bounded shutdown completed in %v (err=%v)", elapsed, err) +} From 278e17286ae485b834f9fbdb1243b6931eb576a0 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 16 Apr 2026 11:44:05 -0700 Subject: [PATCH 34/55] add prefer push registry --- control/control.go | 77 ++++++++++++++++++++++++ exporter/containerimage/export.go | 20 ++++-- exporter/containerimage/exptypes/keys.go | 8 +++ exporter/exporter.go | 9 +-- solver/llbsolver/ops/source.go | 37 ++++++++---- solver/llbsolver/solver.go | 30 ++++++++- source/containerimage/pull.go | 59 +++++++++++++++++- 7 files changed, 217 insertions(+), 23 deletions(-) diff --git a/control/control.go b/control/control.go index 589be54ef45d..de51ebcebd3a 100644 --- a/control/control.go +++ b/control/control.go @@ -516,6 +516,11 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* return nil, err } + pushRegCfg, err := resolvePushRegistryConfig(req.Exporters, expis) + if err != nil { + return nil, err + } + resp, err := c.solver.Solve(ctx, req.Ref, req.Session, frontend.SolveRequest{ Frontend: req.Frontend, Definition: req.Definition, @@ -528,6 +533,7 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* EnableSessionExporter: req.EnableSessionExporter, EagerExport: eagerExport, EagerPushConfig: eagerPushCfg, + PushRegistryConfig: pushRegCfg, }, entitlementsFromPB(req.Entitlements), procs, req.Internal, req.SourcePolicy) if err != nil { return nil, err @@ -587,6 +593,77 @@ func resolveEagerExport(rawExporters []*controlapi.Exporter, expis []exporter.Ex return mode, pushCfg, nil } +// resolvePushRegistryConfig scans all exporters for a single image exporter +// with push=true and prefer-push-registry=true, and returns its push registry +// configuration. Other exporter types (e.g. oci tarball, local) are ignored. +// Returns an error if multiple image exporters have the flag set (ambiguous +// destination). +func resolvePushRegistryConfig(rawExporters []*controlapi.Exporter, expis []exporter.ExporterInstance) (*exporter.EagerPushConfig, error) { + var found *exporter.EagerPushConfig + for i, ex := range rawExporters { + cfg, err := exporterPushRegistryConfig(ex, expis[i]) + if err != nil { + return nil, err + } + if cfg == nil { + continue + } + if found != nil { + return nil, errors.New("prefer-push-registry set on multiple image exporters; destination is ambiguous") + } + found = cfg + } + return found, nil +} + +// exporterPushRegistryConfig returns the push registry config for a single +// exporter if it has prefer-push-registry=true, or nil if the flag isn't set +// (or the exporter doesn't support eager push config). Validation errors are +// returned when the flag is set but the exporter is misconfigured. +func exporterPushRegistryConfig(ex *controlapi.Exporter, expi exporter.ExporterInstance) (*exporter.EagerPushConfig, error) { + enabled, err := parseBoolAttr(ex.Attrs, string(exptypes.OptKeyPreferPushRegistry)) + if err != nil || !enabled { + return nil, err + } + if expi.Type() != client.ExporterImage { + return nil, errors.Errorf("prefer-push-registry requires image exporter, got %q", expi.Type()) + } + push, err := parseBoolAttr(ex.Attrs, string(exptypes.OptKeyPush)) + if err != nil { + return nil, err + } + if !push { + return nil, errors.New("prefer-push-registry requires push=true") + } + provider, ok := expi.(exporter.EagerExportProvider) + if !ok { + return nil, nil + } + cfg := provider.EagerPushConfig() + if cfg == nil { + return nil, errors.New("prefer-push-registry requires a single image name") + } + return cfg, nil +} + +// parseBoolAttr returns true if the attr is present and evaluates to a truthy +// value. Follows the same convention as the image exporter: empty value means +// true (for --output type=image,flag shorthand). +func parseBoolAttr(attrs map[string]string, key string) (bool, error) { + v, ok := attrs[key] + if !ok { + return false, nil + } + if v == "" { + return true, nil + } + b, err := strconv.ParseBool(v) + if err != nil { + return false, errors.Wrapf(err, "non-bool value specified for %s", key) + } + return b, nil +} + func (c *Controller) Status(req *controlapi.StatusRequest, stream controlapi.Control_StatusServer) error { if err := sendTimestampHeader(stream); err != nil { return err diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 8c621c928bcd..0abaf90cd904 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -177,6 +177,16 @@ func (e *imageExporter) Resolve(ctx context.Context, id int, opt map[string]stri default: return nil, errors.Errorf("invalid value %q for %s, must be \"compress\" or \"push\"", v, k) } + case exptypes.OptKeyPreferPushRegistry: + if v == "" { + i.preferPushRegistry = true + continue + } + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "non-bool value specified for %s", k) + } + i.preferPushRegistry = b default: if i.meta == nil { i.meta = make(map[string][]byte) @@ -203,6 +213,7 @@ type imageExporterInstance struct { danglingPrefix string danglingEmptyOnly bool eagerExport string // "", "compress", or "push" + preferPushRegistry bool meta map[string][]byte } @@ -228,10 +239,11 @@ func (e *imageExporterInstance) EagerPushConfig() *exporter.EagerPushConfig { return nil } return &exporter.EagerPushConfig{ - TargetName: name, - RegistryHosts: e.opt.RegistryHosts, - Insecure: e.insecure, - ContentStore: e.opt.ImageWriter.ContentStore(), + TargetName: name, + RegistryHosts: e.opt.RegistryHosts, + Insecure: e.insecure, + ContentStore: e.opt.ImageWriter.ContentStore(), + PreferPushRegistry: e.preferPushRegistry, } } diff --git a/exporter/containerimage/exptypes/keys.go b/exporter/containerimage/exptypes/keys.go index 9d5b4d149f8b..043692304d19 100644 --- a/exporter/containerimage/exptypes/keys.go +++ b/exporter/containerimage/exptypes/keys.go @@ -93,6 +93,14 @@ var ( // compress — compress layers as vertices complete; push everything at finalize // push — compress AND push layer blobs as vertices complete; only push manifest at finalize OptKeyEagerExport ImageExporterOptKey = "eager-export" + + // When pulling base image layers, check the push (destination) registry + // first and pull from there if the layer exists. Falls back to the origin + // registry transparently. Useful when the push registry is closer than + // the origin (e.g., ECR in the same region vs Docker Hub). + // Requires push=true so the destination registry is known. + // Value: bool + OptKeyPreferPushRegistry ImageExporterOptKey = "prefer-push-registry" ) const ( diff --git a/exporter/exporter.go b/exporter/exporter.go index 1b884d22107b..a78d4ec18a26 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -60,10 +60,11 @@ func (c *Config) Compression() compression.Config { // EagerPushConfig holds the registry details needed to push individual layer // blobs during the build, before the final manifest is assembled. type EagerPushConfig struct { - TargetName string - RegistryHosts docker.RegistryHosts - Insecure bool - ContentStore content.Store + TargetName string + RegistryHosts docker.RegistryHosts + Insecure bool + ContentStore content.Store + PreferPushRegistry bool } // EagerExportProvider is an optional interface that ExporterInstances can diff --git a/solver/llbsolver/ops/source.go b/solver/llbsolver/ops/source.go index d6d1712a442b..388094550ab9 100644 --- a/solver/llbsolver/ops/source.go +++ b/solver/llbsolver/ops/source.go @@ -5,6 +5,7 @@ import ( "strings" "sync" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/llbsolver/ops/opsutils" @@ -18,17 +19,18 @@ import ( const sourceCacheType = "buildkit.source.v0" type SourceOp struct { - mu sync.Mutex - op *pb.Op_Source - platform *pb.Platform - sm *source.Manager - src source.SourceInstance - sessM *session.Manager - w worker.Worker - vtx solver.Vertex - parallelism *semaphore.Weighted - pin string - id source.Identifier + mu sync.Mutex + op *pb.Op_Source + platform *pb.Platform + sm *source.Manager + src source.SourceInstance + sessM *session.Manager + w worker.Worker + vtx solver.Vertex + parallelism *semaphore.Weighted + pin string + id source.Identifier + eagerPushCfg *exporter.EagerPushConfig } var _ solver.Op = &SourceOp{} @@ -48,6 +50,10 @@ func NewSourceOp(vtx solver.Vertex, op *pb.Op_Source, platform *pb.Platform, sm }, nil } +func (s *SourceOp) SetEagerPushConfig(cfg *exporter.EagerPushConfig) { + s.eagerPushCfg = cfg +} + func (s *SourceOp) IsProvenanceProvider() {} func (s *SourceOp) Pin() (source.Identifier, string) { @@ -68,6 +74,11 @@ func (s *SourceOp) instance(ctx context.Context) (source.SourceInstance, error) if err != nil { return nil, err } + if s.eagerPushCfg != nil { + if setter, ok := src.(pushRegistryConfigSetter); ok { + setter.SetEagerPushConfig(s.eagerPushCfg) + } + } s.src = src s.id = id return s.src, nil @@ -112,6 +123,10 @@ func (s *SourceOp) Exec(ctx context.Context, g session.Group, _ []solver.Result) return []solver.Result{worker.NewWorkerRefResult(ref, s.w)}, nil } +type pushRegistryConfigSetter interface { + SetEagerPushConfig(*exporter.EagerPushConfig) +} + func (s *SourceOp) Acquire(ctx context.Context) (solver.ReleaseFunc, error) { if s.parallelism == nil { return func() {}, nil diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 81fe3a000ab2..0ffde75d2d4e 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -51,8 +51,9 @@ import ( ) const ( - keyEntitlements = "llb.entitlements" - keySourcePolicy = "llb.sourcepolicy" + keyEntitlements = "llb.entitlements" + keySourcePolicy = "llb.sourcepolicy" + keyEagerPushConfig = "llb.eagerpushconfig" ) // EagerExportMode controls whether layer compression and/or pushing @@ -71,6 +72,7 @@ type ExporterRequest struct { EnableSessionExporter bool EagerExport EagerExportMode EagerPushConfig *exporter.EagerPushConfig // non-nil when EagerExport == EagerExportPush + PushRegistryConfig *exporter.EagerPushConfig // non-nil when prefer-push-registry=true } type RemoteCacheExporter struct { @@ -153,10 +155,28 @@ func (s *Solver) resolver() solver.ResolveOpFunc { if err != nil { return nil, err } - return w.ResolveOp(v, s.Bridge(b), s.sm) + op, err := w.ResolveOp(v, s.Bridge(b), s.sm) + if err != nil { + return nil, err + } + if setter, ok := op.(eagerPushConfigSetter); ok { + var pushCfg *exporter.EagerPushConfig + b.EachValue(context.TODO(), keyEagerPushConfig, func(v any) error { + pushCfg, _ = v.(*exporter.EagerPushConfig) + return nil + }) + if pushCfg != nil { + setter.SetEagerPushConfig(pushCfg) + } + } + return op, nil } } +type eagerPushConfigSetter interface { + SetEagerPushConfig(*exporter.EagerPushConfig) +} + func (s *Solver) bridge(b solver.Builder) *provenanceBridge { return &provenanceBridge{llbBridge: &llbBridge{ builder: b, @@ -561,6 +581,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro j.SetOnVertexComplete(eager.onVertexComplete) } + if exp.PushRegistryConfig != nil { + j.SetValue(keyEagerPushConfig, exp.PushRegistryConfig) + } + br := s.bridge(j) var fwd gateway.LLBBridgeForwarder if s.gatewayForwarder != nil && req.Definition == nil && req.Frontend == "" { diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index dde923c3b860..39c0016b6b3c 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -18,9 +18,12 @@ import ( "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb/sourceresolver" + "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/estargz" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/imageutil" @@ -47,7 +50,8 @@ type puller struct { layerLimit *int vtx solver.Vertex ResolverType - store sourceresolver.ResolveImageConfigOptStore + store sourceresolver.ResolveImageConfigOptStore + eagerPushCfg *exporter.EagerPushConfig g flightcontrol.Group[struct{}] cacheKeyErr error @@ -60,6 +64,10 @@ type puller struct { *pull.Puller } +func (p *puller) SetEagerPushConfig(cfg *exporter.EagerPushConfig) { + p.eagerPushCfg = cfg +} + func mainManifestKey(desc ocispecs.Descriptor, platform ocispecs.Platform, layerLimit *int) (digest.Digest, error) { dt, err := json.Marshal(struct { Digest digest.Digest @@ -136,6 +144,10 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach p.manifest.Descriptors = p.manifest.Descriptors[:*ll] } + if p.eagerPushCfg != nil && p.eagerPushCfg.PreferPushRegistry && p.ResolverType == ResolverTypeRegistry && len(p.manifest.Descriptors) > 0 { + p.wrapProviderWithPushFallback(g) + } + if len(p.manifest.Descriptors) > 0 { progressController := &controller.Controller{ WriterFactory: progressFactory, @@ -283,6 +295,51 @@ func (p *puller) Snapshot(ctx context.Context, g session.Group) (ir cache.Immuta return current, nil } +// wrapProviderWithPushFallback wraps the manifest's Provider so that when +// a layer blob is lazily fetched, it first tries the push (destination) +// registry. If the layer already exists there, it is pulled from the closer +// registry instead of the origin. On any failure, the original provider is +// used transparently. +func (p *puller) wrapProviderWithPushFallback(g session.Group) { + cfg := p.eagerPushCfg + pushResolver := resolver.DefaultPool.GetResolver(cfg.RegistryHosts, cfg.TargetName, "pull", p.SessionManager, g) + origProvider := p.manifest.Provider + pushRef := cfg.TargetName + + bklog.L.Infof("prefer-push-registry: wrapping provider for %s with fallback to push registry %s", p.Ref, pushRef) + + p.manifest.Provider = func(g session.Group) content.Provider { + return &pushFallbackProvider{ + pushResolver: pushResolver.WithSession(g), + pushRef: pushRef, + origin: origProvider(g), + } + } +} + +// pushFallbackProvider tries fetching a blob from the push registry first, +// falling back to the origin provider on any error (404, auth, network, etc.). +type pushFallbackProvider struct { + pushResolver remotes.Resolver + pushRef string + origin content.Provider +} + +func (p *pushFallbackProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { + fetcher, err := p.pushResolver.Fetcher(ctx, p.pushRef) + if err == nil { + ra, fetchErr := contentutil.FromFetcher(fetcher).ReaderAt(ctx, desc) + if fetchErr == nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s pulled from push registry %s", desc.Digest, p.pushRef) + return ra, nil + } + bklog.G(ctx).Infof("prefer-push-registry: layer %s not in push registry %s (%v), falling back to origin", desc.Digest, p.pushRef, fetchErr) + } else { + bklog.G(ctx).Infof("prefer-push-registry: could not get fetcher for %s (%v), falling back to origin for layer %s", p.pushRef, err, desc.Digest) + } + return p.origin.ReaderAt(ctx, desc) +} + // cacheKeyFromConfig returns a stable digest from image config. If image config // is a known oci image we will use chainID of layers. func cacheKeyFromConfig(dt []byte, layerLimit *int) (digest.Digest, error) { From fece4ec71641c4c466833f9fd3d654ff192c24d9 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 16 Apr 2026 15:42:43 -0700 Subject: [PATCH 35/55] probe push registry blob before committing to it Fix two issues: - gofmt formatting in exporter/exporter.go and solver/llbsolver/solver.go - pushFallbackProvider declared "hit" too early: containerd's httpReadSeeker defers the actual HTTP request until first Read, so ReaderAt() returning nil err did not actually confirm the blob existed in the push registry. This caused build failures mid-extract when the blob was missing. Now do a 1-byte probe read to force the lazy open before returning the reader. Made-with: Cursor --- exporter/exporter.go | 10 +++++----- solver/llbsolver/solver.go | 6 +++--- source/containerimage/pull.go | 32 +++++++++++++++++++++++--------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/exporter/exporter.go b/exporter/exporter.go index a78d4ec18a26..a4c611742186 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -60,11 +60,11 @@ func (c *Config) Compression() compression.Config { // EagerPushConfig holds the registry details needed to push individual layer // blobs during the build, before the final manifest is assembled. type EagerPushConfig struct { - TargetName string - RegistryHosts docker.RegistryHosts - Insecure bool - ContentStore content.Store - PreferPushRegistry bool + TargetName string + RegistryHosts docker.RegistryHosts + Insecure bool + ContentStore content.Store + PreferPushRegistry bool } // EagerExportProvider is an optional interface that ExporterInstances can diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 0ffde75d2d4e..3c66961687b5 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -51,9 +51,9 @@ import ( ) const ( - keyEntitlements = "llb.entitlements" - keySourcePolicy = "llb.sourcepolicy" - keyEagerPushConfig = "llb.eagerpushconfig" + keyEntitlements = "llb.entitlements" + keySourcePolicy = "llb.sourcepolicy" + keyEagerPushConfig = "llb.eagerpushconfig" ) // EagerExportMode controls whether layer compression and/or pushing diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 39c0016b6b3c..9ad213a093c4 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -3,6 +3,7 @@ package containerimage import ( "context" "encoding/json" + "io" "maps" "runtime" "time" @@ -327,17 +328,30 @@ type pushFallbackProvider struct { func (p *pushFallbackProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { fetcher, err := p.pushResolver.Fetcher(ctx, p.pushRef) - if err == nil { - ra, fetchErr := contentutil.FromFetcher(fetcher).ReaderAt(ctx, desc) - if fetchErr == nil { - bklog.G(ctx).Infof("prefer-push-registry: layer %s pulled from push registry %s", desc.Digest, p.pushRef) - return ra, nil - } - bklog.G(ctx).Infof("prefer-push-registry: layer %s not in push registry %s (%v), falling back to origin", desc.Digest, p.pushRef, fetchErr) - } else { + if err != nil { bklog.G(ctx).Infof("prefer-push-registry: could not get fetcher for %s (%v), falling back to origin for layer %s", p.pushRef, err, desc.Digest) + return p.origin.ReaderAt(ctx, desc) } - return p.origin.ReaderAt(ctx, desc) + + ra, fetchErr := contentutil.FromFetcher(fetcher).ReaderAt(ctx, desc) + if fetchErr != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s not in push registry %s (%v), falling back to origin", desc.Digest, p.pushRef, fetchErr) + return p.origin.ReaderAt(ctx, desc) + } + + // Force the lazy httpReadSeeker to actually open the HTTP connection. Without + // this probe, fetcher construction and the ReaderAt wrapper succeed even if + // the blob doesn't exist in the push registry — the 404 only surfaces later + // when the extractor reads, by which point we can't transparently fall back. + probe := make([]byte, 1) + if _, probeErr := ra.ReadAt(probe, 0); probeErr != nil && !errors.Is(probeErr, io.EOF) { + ra.Close() + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe failed against %s (%v), falling back to origin", desc.Digest, p.pushRef, probeErr) + return p.origin.ReaderAt(ctx, desc) + } + + bklog.G(ctx).Infof("prefer-push-registry: layer %s pulled from push registry %s", desc.Digest, p.pushRef) + return ra, nil } // cacheKeyFromConfig returns a stable digest from image config. If image config From 8685c83df45919e9d4da845c7b8d3df29d522104 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 16 Apr 2026 16:02:02 -0700 Subject: [PATCH 36/55] add timeout for the probe --- source/containerimage/pull.go | 72 +++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 9ad213a093c4..430fc95f422b 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -5,6 +5,7 @@ import ( "encoding/json" "io" "maps" + "os" "runtime" "time" @@ -326,27 +327,42 @@ type pushFallbackProvider struct { origin content.Provider } +const ( + pushRegistryProbeTimeoutEnv = "BUILDKIT_PREFER_PUSH_REGISTRY_PROBE_TIMEOUT" + defaultPushRegistryProbeTimeout = 500 * time.Millisecond +) + +// pushRegistryProbeTimeout bounds how long we'll wait for the push registry +// to respond to the existence probe before falling back to origin. Configurable +// via BUILDKIT_PREFER_PUSH_REGISTRY_PROBE_TIMEOUT (Go duration format, e.g. "750ms"). +var pushRegistryProbeTimeout = func() time.Duration { + v := os.Getenv(pushRegistryProbeTimeoutEnv) + if v == "" { + return defaultPushRegistryProbeTimeout + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + bklog.L.Warnf("prefer-push-registry: invalid %s=%q, using default %s", pushRegistryProbeTimeoutEnv, v, defaultPushRegistryProbeTimeout) + return defaultPushRegistryProbeTimeout + } + bklog.L.Infof("prefer-push-registry: probe timeout overridden to %s via %s", d, pushRegistryProbeTimeoutEnv) + return d +}() + func (p *pushFallbackProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { - fetcher, err := p.pushResolver.Fetcher(ctx, p.pushRef) - if err != nil { - bklog.G(ctx).Infof("prefer-push-registry: could not get fetcher for %s (%v), falling back to origin for layer %s", p.pushRef, err, desc.Digest) + if err := p.probe(ctx, desc); err != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe against %s failed (%v), falling back to origin", desc.Digest, p.pushRef, err) return p.origin.ReaderAt(ctx, desc) } - ra, fetchErr := contentutil.FromFetcher(fetcher).ReaderAt(ctx, desc) - if fetchErr != nil { - bklog.G(ctx).Infof("prefer-push-registry: layer %s not in push registry %s (%v), falling back to origin", desc.Digest, p.pushRef, fetchErr) + fetcher, err := p.pushResolver.Fetcher(ctx, p.pushRef) + if err != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe succeeded but fetcher failed for %s (%v), falling back to origin", desc.Digest, p.pushRef, err) return p.origin.ReaderAt(ctx, desc) } - - // Force the lazy httpReadSeeker to actually open the HTTP connection. Without - // this probe, fetcher construction and the ReaderAt wrapper succeed even if - // the blob doesn't exist in the push registry — the 404 only surfaces later - // when the extractor reads, by which point we can't transparently fall back. - probe := make([]byte, 1) - if _, probeErr := ra.ReadAt(probe, 0); probeErr != nil && !errors.Is(probeErr, io.EOF) { - ra.Close() - bklog.G(ctx).Infof("prefer-push-registry: layer %s probe failed against %s (%v), falling back to origin", desc.Digest, p.pushRef, probeErr) + ra, err := contentutil.FromFetcher(fetcher).ReaderAt(ctx, desc) + if err != nil { + bklog.G(ctx).Infof("prefer-push-registry: layer %s probe succeeded but ReaderAt failed for %s (%v), falling back to origin", desc.Digest, p.pushRef, err) return p.origin.ReaderAt(ctx, desc) } @@ -354,6 +370,32 @@ func (p *pushFallbackProvider) ReaderAt(ctx context.Context, desc ocispecs.Descr return ra, nil } +// probe forces an actual HTTP request against the push registry to confirm +// the blob exists. Bounded by pushRegistryProbeTimeout. We can't reuse the +// resulting ReaderAt because its context (with the short timeout) is captured +// inside containerd's httpReadSeeker; the caller must construct a fresh one +// with the original context for the real read. +func (p *pushFallbackProvider) probe(ctx context.Context, desc ocispecs.Descriptor) error { + probeCtx, cancel := context.WithTimeout(ctx, pushRegistryProbeTimeout) + defer cancel() + + fetcher, err := p.pushResolver.Fetcher(probeCtx, p.pushRef) + if err != nil { + return err + } + ra, err := contentutil.FromFetcher(fetcher).ReaderAt(probeCtx, desc) + if err != nil { + return err + } + defer ra.Close() + + probe := make([]byte, 1) + if _, err := ra.ReadAt(probe, 0); err != nil && !errors.Is(err, io.EOF) { + return err + } + return nil +} + // cacheKeyFromConfig returns a stable digest from image config. If image config // is a known oci image we will use chainID of layers. func cacheKeyFromConfig(dt []byte, layerLimit *int) (digest.Digest, error) { From ba31bb35e0d54f7e6e7a6558bf96d08c0fc991e2 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 16 Apr 2026 16:13:11 -0700 Subject: [PATCH 37/55] add tests for pushFallbackProvider and fix lint - Use context.WithTimeoutCause to satisfy the forbidigo lint that bans context.WithTimeout in this repo. - New pull_test.go covers: hit, 404 miss, fetcher error, empty-blob hit (EOF on probe is treated as success), probe timeout fallback, and isolation of the returned ReaderAt's context from the probe deadline. Made-with: Cursor --- source/containerimage/pull.go | 2 +- source/containerimage/pull_test.go | 271 +++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+), 1 deletion(-) create mode 100644 source/containerimage/pull_test.go diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 430fc95f422b..ac7cb6766b36 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -376,7 +376,7 @@ func (p *pushFallbackProvider) ReaderAt(ctx context.Context, desc ocispecs.Descr // inside containerd's httpReadSeeker; the caller must construct a fresh one // with the original context for the real read. func (p *pushFallbackProvider) probe(ctx context.Context, desc ocispecs.Descriptor) error { - probeCtx, cancel := context.WithTimeout(ctx, pushRegistryProbeTimeout) + probeCtx, cancel := context.WithTimeoutCause(ctx, pushRegistryProbeTimeout, errors.WithStack(context.DeadlineExceeded)) defer cancel() fetcher, err := p.pushResolver.Fetcher(probeCtx, p.pushRef) diff --git a/source/containerimage/pull_test.go b/source/containerimage/pull_test.go new file mode 100644 index 000000000000..20aa6902ac98 --- /dev/null +++ b/source/containerimage/pull_test.go @@ -0,0 +1,271 @@ +package containerimage + +import ( + "bytes" + "context" + "errors" + "io" + "sync/atomic" + "testing" + "time" + + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/remotes" + digest "github.com/opencontainers/go-digest" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/require" +) + +const testPushRef = "push.example.com/test:latest" + +func testDesc(payload []byte) ocispecs.Descriptor { + return ocispecs.Descriptor{ + Digest: digest.FromBytes(payload), + Size: int64(len(payload)), + } +} + +// fakeResolver implements just enough of remotes.Resolver to back +// pushFallbackProvider in tests. Only Fetcher is exercised. +type fakeResolver struct { + fetcherFn func(ctx context.Context, ref string) (remotes.Fetcher, error) +} + +func (r *fakeResolver) Resolve(ctx context.Context, ref string) (string, ocispecs.Descriptor, error) { + return ref, ocispecs.Descriptor{}, errors.New("not implemented") +} +func (r *fakeResolver) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + return r.fetcherFn(ctx, ref) +} +func (r *fakeResolver) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + return nil, errors.New("not implemented") +} + +// fetcherFunc adapts a function to remotes.Fetcher for terse test setup. +type fetcherFunc func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) + +func (f fetcherFunc) Fetch(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return f(ctx, desc) +} + +// originProvider serves a fixed payload and counts ReaderAt invocations so +// tests can assert whether the fallback was exercised. +type originProvider struct { + payload []byte + calls int32 +} + +func (o *originProvider) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) { + atomic.AddInt32(&o.calls, 1) + return &bytesReaderAt{data: o.payload}, nil +} + +type bytesReaderAt struct { + data []byte +} + +func (b *bytesReaderAt) ReadAt(p []byte, off int64) (int, error) { + if off >= int64(len(b.data)) { + return 0, io.EOF + } + n := copy(p, b.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil +} +func (b *bytesReaderAt) Close() error { return nil } +func (b *bytesReaderAt) Size() int64 { return int64(len(b.data)) } + +// seekableBuffer wraps bytes.Reader as an io.ReadCloser+Seeker so it works +// with the FromFetcher readerAt wrapper which seeks for offset reads. +type seekableBuffer struct { + *bytes.Reader +} + +func (s *seekableBuffer) Close() error { return nil } + +func newSeekableBuffer(data []byte) io.ReadCloser { + return &seekableBuffer{Reader: bytes.NewReader(data)} +} + +// blockingReader blocks Read until ctx is done, simulating a slow registry. +type blockingReader struct { + ctx context.Context +} + +func (b *blockingReader) Read(p []byte) (int, error) { + <-b.ctx.Done() + return 0, b.ctx.Err() +} +func (b *blockingReader) Close() error { return nil } + +func TestPushFallbackProvider_Hit(t *testing.T) { + t.Parallel() + + pushPayload := []byte("from-push-registry") + originPayload := []byte("from-origin") + + var fetcherCalls int32 + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + atomic.AddInt32(&fetcherCalls, 1) + return newSeekableBuffer(pushPayload), nil + }), nil + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(pushPayload)) + require.NoError(t, err) + defer ra.Close() + + require.Equal(t, int32(0), atomic.LoadInt32(&origin.calls), "origin should not be called on hit") + // Probe + real read both call Fetcher exactly once. + require.Equal(t, int32(2), atomic.LoadInt32(&fetcherCalls)) + + got := make([]byte, len(pushPayload)) + n, err := ra.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, pushPayload, got[:n]) +} + +func TestPushFallbackProvider_NotFound(t *testing.T) { + t.Parallel() + + originPayload := []byte("from-origin") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return nil, errors.New("blob not found") + }), nil + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(originPayload)) + require.NoError(t, err) + defer ra.Close() + + require.Equal(t, int32(1), atomic.LoadInt32(&origin.calls), "origin should be called on miss") + + got := make([]byte, len(originPayload)) + n, err := ra.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, originPayload, got[:n]) +} + +func TestPushFallbackProvider_FetcherError(t *testing.T) { + t.Parallel() + + originPayload := []byte("from-origin") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return nil, errors.New("auth failed") + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(originPayload)) + require.NoError(t, err) + defer ra.Close() + + require.Equal(t, int32(1), atomic.LoadInt32(&origin.calls), "origin should be called when fetcher errors") +} + +func TestPushFallbackProvider_EmptyBlobIsHit(t *testing.T) { + t.Parallel() + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return newSeekableBuffer(nil), nil + }), nil + }, + } + origin := &originProvider{payload: []byte("origin-fallback")} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(nil)) + require.NoError(t, err) + defer ra.Close() + + // EOF on the 1-byte probe of an empty blob is not a "missing" signal — + // the blob exists, it's just empty. We should commit to the push reader. + require.Equal(t, int32(0), atomic.LoadInt32(&origin.calls)) +} + +func TestPushFallbackProvider_ProbeTimesOut(t *testing.T) { + t.Parallel() + + prevTimeout := pushRegistryProbeTimeout + pushRegistryProbeTimeout = 50 * time.Millisecond + defer func() { pushRegistryProbeTimeout = prevTimeout }() + + originPayload := []byte("from-origin") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return &blockingReader{ctx: ctx}, nil + }), nil + }, + } + origin := &originProvider{payload: originPayload} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + start := time.Now() + ra, err := p.ReaderAt(context.Background(), testDesc(originPayload)) + elapsed := time.Since(start) + + require.NoError(t, err) + defer ra.Close() + require.Equal(t, int32(1), atomic.LoadInt32(&origin.calls), "origin should be called when probe times out") + require.Less(t, elapsed, 1*time.Second, "should fall back well before the parent ctx deadline") + require.GreaterOrEqual(t, elapsed, 50*time.Millisecond, "should wait for the probe timeout") +} + +func TestPushFallbackProvider_ReturnedReaderIsolatedFromProbeTimeout(t *testing.T) { + t.Parallel() + + prevTimeout := pushRegistryProbeTimeout + pushRegistryProbeTimeout = 50 * time.Millisecond + defer func() { pushRegistryProbeTimeout = prevTimeout }() + + pushPayload := []byte("from-push-registry") + + push := &fakeResolver{ + fetcherFn: func(ctx context.Context, ref string) (remotes.Fetcher, error) { + return fetcherFunc(func(ctx context.Context, desc ocispecs.Descriptor) (io.ReadCloser, error) { + return newSeekableBuffer(pushPayload), nil + }), nil + }, + } + origin := &originProvider{payload: []byte("origin-fallback")} + + p := &pushFallbackProvider{pushResolver: push, pushRef: testPushRef, origin: origin} + + ra, err := p.ReaderAt(context.Background(), testDesc(pushPayload)) + require.NoError(t, err) + defer ra.Close() + + // Sleep past the probe timeout. If the returned ReaderAt's underlying + // context were the (now-canceled) probeCtx, the read below would fail. + time.Sleep(2 * pushRegistryProbeTimeout) + + got := make([]byte, len(pushPayload)) + n, err := ra.ReadAt(got, 0) + require.NoError(t, err) + require.Equal(t, pushPayload, got[:n]) +} From cad552fa2521661198dbcbec144a36ac4db995f3 Mon Sep 17 00:00:00 2001 From: Amy Date: Fri, 17 Apr 2026 18:10:51 -0700 Subject: [PATCH 38/55] fix bug --- solver/llbsolver/eager.go | 105 ++++++++++++++++++++----- solver/llbsolver/eager_test.go | 137 ++++++++++++++++++++++++++++++++- 2 files changed, 220 insertions(+), 22 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 219819bf0e73..2b8efa684b4c 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -42,6 +42,21 @@ type eagerPipeline struct { work chan eagerWorkItem wg sync.WaitGroup + // done is closed by wait() to signal shutdown to producers and workers. + // It is never reopened, so any onVertexComplete invocation that races + // in after wait() observes a closed channel and bails out instead of + // sending into work (which would have panicked when work was closed + // in the prior implementation). + done chan struct{} + waitOnce sync.Once + + // closeMu serializes producers (RLock during send) against shutdown + // (Lock when wait() flips closed). This eliminates the race window + // where an onVertexComplete invocation could land an item in work + // after wait() finished draining and returned. + closeMu sync.RWMutex + closed bool + // ctx carries the lease so compressed blobs are GC-protected. ctx context.Context @@ -88,6 +103,7 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio pusher: pusher, ctx: ctx, work: make(chan eagerWorkItem, 256), + done: make(chan struct{}), } numWorkers := eagerWorkerCount() @@ -105,25 +121,45 @@ func (ep *eagerPipeline) worker() { select { case <-ep.ctx.Done(): return - case item, ok := <-ep.work: - if !ok { - return - } - if err := ep.processRef(item.ref); err != nil { - ep.mu.Lock() - if ep.firstErr == nil { - ep.firstErr = err + case <-ep.done: + // wait() was called. Drain any items already queued so + // vertices that fired before shutdown still get compressed + // and pushed, then exit. + for { + select { + case item := <-ep.work: + ep.handleItem(item) + default: + return } - ep.mu.Unlock() } - item.ref.Release(context.TODO()) + case item := <-ep.work: + ep.handleItem(item) } } } +func (ep *eagerPipeline) handleItem(item eagerWorkItem) { + if err := ep.processRef(item.ref); err != nil { + ep.mu.Lock() + if ep.firstErr == nil { + ep.firstErr = err + } + ep.mu.Unlock() + } + item.ref.Release(context.TODO()) +} + // onVertexComplete is the callback registered on the solver Job. It extracts // ImmutableRefs from vertex results, clones them for safe async use, and // sends them to the worker pool for compression. +// +// Late fires (after wait()) are expected: orphaned loadCache goroutines +// spawned by the scheduler can complete after the owning Solve has already +// returned. When that happens, ep.done is closed and we release the cloned +// ref instead of sending it. Dropping these is safe because a vertex's full +// chain is processed via GetRemotes from any descendant fire that happened +// before wait(), and orphans by definition aren't on the final image's path. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -135,12 +171,22 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } cloned := workerRef.ImmutableRef.Clone() + + ep.closeMu.RLock() + if ep.closed { + ep.closeMu.RUnlock() + cloned.Release(context.TODO()) + continue + } + select { case ep.work <- eagerWorkItem{ref: cloned}: case <-ep.ctx.Done(): cloned.Release(context.TODO()) + ep.closeMu.RUnlock() return } + ep.closeMu.RUnlock() } } @@ -210,17 +256,38 @@ func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Cont return err } -// wait closes the work channel and blocks until all workers finish. +// wait signals shutdown via ep.done and blocks until all workers finish. // Returns the first error encountered by any worker. +// +// ep.work is intentionally never closed because onVertexComplete can be +// invoked from orphan scheduler goroutines that outlive the originating +// Solve (e.g. a speculative loadCache that completes after the owning +// build has moved past eg.Wait). Closing ep.work would panic those late +// senders. Instead we close ep.done, which late senders observe and use +// to bail out cleanly. func (ep *eagerPipeline) wait() error { - close(ep.work) + ep.waitOnce.Do(func() { + // closeMu.Lock waits for all in-flight producers to finish their + // send (under RLock) and blocks new producers from entering until + // closed is set. This guarantees no producer can land an item in + // ep.work after the drain below completes. + ep.closeMu.Lock() + ep.closed = true + close(ep.done) + ep.closeMu.Unlock() + }) ep.wg.Wait() - // Release any refs left in the channel (e.g. if workers exited early - // due to context cancellation). - for item := range ep.work { - item.ref.Release(context.TODO()) + // Drain anything still in ep.work that workers didn't pick up before + // exiting. With closeMu protecting the send path, no further items + // can be added once we get here. + for { + select { + case item := <-ep.work: + item.ref.Release(context.TODO()) + default: + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.firstErr + } } - ep.mu.Lock() - defer ep.mu.Unlock() - return ep.firstErr } diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index d48da18c4946..ff6d2695b21d 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -4,12 +4,15 @@ import ( "context" "os" "runtime" + "sync" "sync/atomic" "testing" "github.com/containerd/containerd/v2/core/images" "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/compression" + "github.com/moby/buildkit/worker" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" @@ -51,6 +54,7 @@ func TestNewEagerPipeline_PushRequiresConfig(t *testing.T) { func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { ep := &eagerPipeline{ work: make(chan eagerWorkItem), + done: make(chan struct{}), } ep.firstErr = assert.AnError @@ -61,6 +65,7 @@ func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { ep := &eagerPipeline{ work: make(chan eagerWorkItem), + done: make(chan struct{}), } err := ep.wait() @@ -71,6 +76,7 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { var released atomic.Int32 ep := &eagerPipeline{ work: make(chan eagerWorkItem, 10), + done: make(chan struct{}), } ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} @@ -81,12 +87,23 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") } +func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { + ep := &eagerPipeline{ + work: make(chan eagerWorkItem), + done: make(chan struct{}), + } + + require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(), "second wait must not panic from double-close of done") +} + func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { ctx, cancel := context.WithCancelCause(context.Background()) ep := &eagerPipeline{ mode: EagerExportCompress, ctx: ctx, work: make(chan eagerWorkItem, 10), + done: make(chan struct{}), } cancel(nil) @@ -96,20 +113,110 @@ func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { ep.wg.Wait() } -func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { +func TestEagerPipeline_WorkerExitsOnDoneClose(t *testing.T) { ep := &eagerPipeline{ mode: EagerExportCompress, ctx: context.Background(), work: make(chan eagerWorkItem), + done: make(chan struct{}), } ep.wg.Add(1) go ep.worker() - close(ep.work) + close(ep.done) ep.wg.Wait() } +// TestEagerPipeline_OnVertexCompleteAfterWait is the regression test for the +// "send on closed channel" panic that crashed buildkitd-v2-1 on 2026-04-17. +// A late fire from an orphan scheduler goroutine (e.g. a speculative +// loadCache that finished after the owning Solve returned) used to panic +// because wait() closed ep.work. With ep.done as the shutdown signal it +// must instead release the cloned ref and return cleanly. +func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { + ep := &eagerPipeline{ + ctx: context.Background(), + work: make(chan eagerWorkItem, 10), + done: make(chan struct{}), + } + require.NoError(t, ep.wait()) + + var released atomic.Int32 + res := newWorkerRefResult(&releaseTracker{released: &released}) + + require.NotPanics(t, func() { + ep.onVertexComplete(nil, []solver.Result{res}) + }, "late onVertexComplete after wait() must not panic") + + assert.Equal(t, int32(1), released.Load(), + "cloned ref from a late fire must be released, not leaked") +} + +// TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent verifies the fix +// holds under the realistic load pattern: many orphan goroutines firing +// concurrently after wait() has been called. +func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { + ep := &eagerPipeline{ + ctx: context.Background(), + work: make(chan eagerWorkItem, 10), + done: make(chan struct{}), + } + require.NoError(t, ep.wait()) + + var released atomic.Int32 + const fires = 100 + + var wg sync.WaitGroup + wg.Add(fires) + for range fires { + go func() { + defer wg.Done() + res := newWorkerRefResult(&releaseTracker{released: &released}) + ep.onVertexComplete(nil, []solver.Result{res}) + }() + } + wg.Wait() + + assert.Equal(t, int32(fires), released.Load(), + "every cloned ref from late fires must be released") +} + +// TestEagerPipeline_OnVertexCompleteRacingWait verifies the panic doesn't +// reappear when fires happen concurrently with wait() being called. +func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { + ep := &eagerPipeline{ + ctx: context.Background(), + work: make(chan eagerWorkItem, 256), + done: make(chan struct{}), + } + + var released atomic.Int32 + const fires = 200 + + var wg sync.WaitGroup + wg.Add(fires) + for range fires { + go func() { + defer wg.Done() + res := newWorkerRefResult(&releaseTracker{released: &released}) + ep.onVertexComplete(nil, []solver.Result{res}) + }() + } + + require.NotPanics(t, func() { + require.NoError(t, ep.wait()) + }, "wait() racing with concurrent fires must not panic") + + wg.Wait() + + // Items not handed to a worker (because there are no workers in this + // test) are drained by wait() and released. Items that took the + // done branch are released by onVertexComplete. Either way every + // cloned ref should be released exactly once. + assert.Equal(t, int32(fires), released.Load()) +} + func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { descs := []ocispecs.Descriptor{ { @@ -148,7 +255,8 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { } // releaseTracker is a minimal stub that satisfies cache.ImmutableRef -// for testing ref lifecycle (Release calls). All other methods panic. +// for testing ref lifecycle (Release / Clone calls). All other methods +// panic via the embedded interface. type releaseTracker struct { cache.ImmutableRef released *atomic.Int32 @@ -158,3 +266,26 @@ func (r *releaseTracker) Release(context.Context) error { r.released.Add(1) return nil } + +// Clone returns a sibling that shares the released counter so the test can +// verify both the original and the cloned ref end up released. +// onVertexComplete only ever calls Release on the clone, so the original +// is left to the caller (the test) to release. +func (r *releaseTracker) Clone() cache.ImmutableRef { + return &releaseTracker{released: r.released} +} + +func (r *releaseTracker) ID() string { return "release-tracker" } + +// fakeWorker is the bare minimum worker.Worker that worker.WorkerRef.ID() +// can call without panicking. +type fakeWorker struct{ worker.Worker } + +func (fakeWorker) ID() string { return "fake-worker" } + +// newWorkerRefResult builds a solver.Result whose Sys() returns a +// *worker.WorkerRef wrapping the given ImmutableRef, matching the shape +// onVertexComplete expects. +func newWorkerRefResult(ref cache.ImmutableRef) solver.Result { + return worker.NewWorkerRefResult(ref, fakeWorker{}) +} From a790b184037034c723fe81274a20341a9693ca1c Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 20 Apr 2026 09:21:45 -0700 Subject: [PATCH 39/55] more cleanup --- solver/llbsolver/eager.go | 102 ++++++++++++++------------------- solver/llbsolver/eager_test.go | 34 +++++------ 2 files changed, 56 insertions(+), 80 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 2b8efa684b4c..72f52119d2c6 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -42,20 +42,15 @@ type eagerPipeline struct { work chan eagerWorkItem wg sync.WaitGroup - // done is closed by wait() to signal shutdown to producers and workers. - // It is never reopened, so any onVertexComplete invocation that races - // in after wait() observes a closed channel and bails out instead of - // sending into work (which would have panicked when work was closed - // in the prior implementation). - done chan struct{} - waitOnce sync.Once - // closeMu serializes producers (RLock during send) against shutdown - // (Lock when wait() flips closed). This eliminates the race window - // where an onVertexComplete invocation could land an item in work - // after wait() finished draining and returned. - closeMu sync.RWMutex - closed bool + // (Lock when wait() closes the work channel). Producers check closed + // under RLock before sending, so no producer can ever send on a + // closed channel — eliminating the "send on closed channel" panic + // that occurred when orphan scheduler goroutines (e.g. speculative + // loadCache) fired onVertexComplete after wait() had returned. + closeMu sync.RWMutex + closed bool + waitOnce sync.Once // ctx carries the lease so compressed blobs are GC-protected. ctx context.Context @@ -103,7 +98,6 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio pusher: pusher, ctx: ctx, work: make(chan eagerWorkItem, 256), - done: make(chan struct{}), } numWorkers := eagerWorkerCount() @@ -121,45 +115,33 @@ func (ep *eagerPipeline) worker() { select { case <-ep.ctx.Done(): return - case <-ep.done: - // wait() was called. Drain any items already queued so - // vertices that fired before shutdown still get compressed - // and pushed, then exit. - for { - select { - case item := <-ep.work: - ep.handleItem(item) - default: - return + case item, ok := <-ep.work: + if !ok { + return + } + if err := ep.processRef(item.ref); err != nil { + ep.mu.Lock() + if ep.firstErr == nil { + ep.firstErr = err } + ep.mu.Unlock() } - case item := <-ep.work: - ep.handleItem(item) - } - } -} - -func (ep *eagerPipeline) handleItem(item eagerWorkItem) { - if err := ep.processRef(item.ref); err != nil { - ep.mu.Lock() - if ep.firstErr == nil { - ep.firstErr = err + item.ref.Release(context.TODO()) } - ep.mu.Unlock() } - item.ref.Release(context.TODO()) } // onVertexComplete is the callback registered on the solver Job. It extracts // ImmutableRefs from vertex results, clones them for safe async use, and // sends them to the worker pool for compression. // -// Late fires (after wait()) are expected: orphaned loadCache goroutines -// spawned by the scheduler can complete after the owning Solve has already -// returned. When that happens, ep.done is closed and we release the cloned -// ref instead of sending it. Dropping these is safe because a vertex's full -// chain is processed via GetRemotes from any descendant fire that happened -// before wait(), and orphans by definition aren't on the final image's path. +// Late fires (after wait()) are expected: orphan scheduler goroutines (e.g. +// speculative loadCache calls) can fire after the owning Solve has already +// finished. closeMu.RLock + the closed flag make the "am I still open?" +// check atomic with the send: if closed is true the cloned ref is released +// and dropped. Dropping these is safe because a vertex's full chain is +// processed via GetRemotes from any descendant fire that happened before +// wait(), and orphans by definition aren't on the final image's path. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -256,33 +238,33 @@ func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Cont return err } -// wait signals shutdown via ep.done and blocks until all workers finish. -// Returns the first error encountered by any worker. +// wait closes ep.work and blocks until all workers finish. Returns the +// first error encountered by any worker. // -// ep.work is intentionally never closed because onVertexComplete can be -// invoked from orphan scheduler goroutines that outlive the originating -// Solve (e.g. a speculative loadCache that completes after the owning -// build has moved past eg.Wait). Closing ep.work would panic those late -// senders. Instead we close ep.done, which late senders observe and use -// to bail out cleanly. +// closeMu.Lock waits for all in-flight producers to release their RLock +// before setting closed and closing the channel. This guarantees no +// producer is mid-send when the channel is closed, so the "send on closed +// channel" panic cannot occur. Producers that arrive after closeMu.Lock +// returns observe closed=true under RLock and release their cloned ref +// without ever touching the channel. func (ep *eagerPipeline) wait() error { ep.waitOnce.Do(func() { - // closeMu.Lock waits for all in-flight producers to finish their - // send (under RLock) and blocks new producers from entering until - // closed is set. This guarantees no producer can land an item in - // ep.work after the drain below completes. ep.closeMu.Lock() ep.closed = true - close(ep.done) + close(ep.work) ep.closeMu.Unlock() }) ep.wg.Wait() - // Drain anything still in ep.work that workers didn't pick up before - // exiting. With closeMu protecting the send path, no further items - // can be added once we get here. + // If ctx was cancelled, workers may have exited without fully draining + // the channel. Release any leftover refs so their leases aren't leaked. for { select { - case item := <-ep.work: + case item, ok := <-ep.work: + if !ok { + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.firstErr + } item.ref.Release(context.TODO()) default: ep.mu.Lock() diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index ff6d2695b21d..989a73274304 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -54,7 +54,6 @@ func TestNewEagerPipeline_PushRequiresConfig(t *testing.T) { func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { ep := &eagerPipeline{ work: make(chan eagerWorkItem), - done: make(chan struct{}), } ep.firstErr = assert.AnError @@ -65,7 +64,6 @@ func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { ep := &eagerPipeline{ work: make(chan eagerWorkItem), - done: make(chan struct{}), } err := ep.wait() @@ -76,7 +74,6 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { var released atomic.Int32 ep := &eagerPipeline{ work: make(chan eagerWorkItem, 10), - done: make(chan struct{}), } ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} @@ -90,11 +87,10 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { ep := &eagerPipeline{ work: make(chan eagerWorkItem), - done: make(chan struct{}), } require.NoError(t, ep.wait()) - require.NoError(t, ep.wait(), "second wait must not panic from double-close of done") + require.NoError(t, ep.wait(), "second wait must not panic from double-close of work") } func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { @@ -103,7 +99,6 @@ func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { mode: EagerExportCompress, ctx: ctx, work: make(chan eagerWorkItem, 10), - done: make(chan struct{}), } cancel(nil) @@ -113,18 +108,17 @@ func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { ep.wg.Wait() } -func TestEagerPipeline_WorkerExitsOnDoneClose(t *testing.T) { +func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { ep := &eagerPipeline{ mode: EagerExportCompress, ctx: context.Background(), work: make(chan eagerWorkItem), - done: make(chan struct{}), } ep.wg.Add(1) go ep.worker() - close(ep.done) + close(ep.work) ep.wg.Wait() } @@ -132,13 +126,14 @@ func TestEagerPipeline_WorkerExitsOnDoneClose(t *testing.T) { // "send on closed channel" panic that crashed buildkitd-v2-1 on 2026-04-17. // A late fire from an orphan scheduler goroutine (e.g. a speculative // loadCache that finished after the owning Solve returned) used to panic -// because wait() closed ep.work. With ep.done as the shutdown signal it -// must instead release the cloned ref and return cleanly. +// because wait() closed ep.work while the producer was mid-send. With +// closeMu gating the send path, the closed flag is observed before the +// channel is ever touched, so the late fire releases the cloned ref and +// returns cleanly. func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { ep := &eagerPipeline{ ctx: context.Background(), work: make(chan eagerWorkItem, 10), - done: make(chan struct{}), } require.NoError(t, ep.wait()) @@ -160,7 +155,6 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { ep := &eagerPipeline{ ctx: context.Background(), work: make(chan eagerWorkItem, 10), - done: make(chan struct{}), } require.NoError(t, ep.wait()) @@ -182,13 +176,14 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { "every cloned ref from late fires must be released") } -// TestEagerPipeline_OnVertexCompleteRacingWait verifies the panic doesn't -// reappear when fires happen concurrently with wait() being called. +// TestEagerPipeline_OnVertexCompleteRacingWait verifies no panic and no +// leaks when fires happen concurrently with wait() being called. This is +// the scenario that exposed the leak window in an earlier attempt at the +// fix (an unprotected two-channel design). func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { ep := &eagerPipeline{ ctx: context.Background(), work: make(chan eagerWorkItem, 256), - done: make(chan struct{}), } var released atomic.Int32 @@ -210,10 +205,9 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { wg.Wait() - // Items not handed to a worker (because there are no workers in this - // test) are drained by wait() and released. Items that took the - // done branch are released by onVertexComplete. Either way every - // cloned ref should be released exactly once. + // Every cloned ref should be released exactly once: either the sender + // got its item into the channel before close (drained by wait()), or + // it saw closed=true under RLock and released the ref itself. assert.Equal(t, int32(fires), released.Load()) } From 7eb5d370ea1dc0d5c9c57a46e160e4a0323cb18c Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 20 Apr 2026 09:35:36 -0700 Subject: [PATCH 40/55] update comments --- solver/llbsolver/eager.go | 36 ++++++++---------------- solver/llbsolver/eager_test.go | 51 ++++++++-------------------------- 2 files changed, 23 insertions(+), 64 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 72f52119d2c6..ffb9a55cd54d 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -42,12 +42,9 @@ type eagerPipeline struct { work chan eagerWorkItem wg sync.WaitGroup - // closeMu serializes producers (RLock during send) against shutdown - // (Lock when wait() closes the work channel). Producers check closed - // under RLock before sending, so no producer can ever send on a - // closed channel — eliminating the "send on closed channel" panic - // that occurred when orphan scheduler goroutines (e.g. speculative - // loadCache) fired onVertexComplete after wait() had returned. + // closeMu gates producers (RLock) against shutdown (Lock). Producers + // check closed under RLock before sending, ensuring no send can race + // with the close of work. closeMu sync.RWMutex closed bool waitOnce sync.Once @@ -133,15 +130,9 @@ func (ep *eagerPipeline) worker() { // onVertexComplete is the callback registered on the solver Job. It extracts // ImmutableRefs from vertex results, clones them for safe async use, and -// sends them to the worker pool for compression. -// -// Late fires (after wait()) are expected: orphan scheduler goroutines (e.g. -// speculative loadCache calls) can fire after the owning Solve has already -// finished. closeMu.RLock + the closed flag make the "am I still open?" -// check atomic with the send: if closed is true the cloned ref is released -// and dropped. Dropping these is safe because a vertex's full chain is -// processed via GetRemotes from any descendant fire that happened before -// wait(), and orphans by definition aren't on the final image's path. +// sends them to the worker pool for compression. Fires that arrive after +// wait() observe closed=true under RLock and release the clone instead of +// sending. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -239,14 +230,9 @@ func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Cont } // wait closes ep.work and blocks until all workers finish. Returns the -// first error encountered by any worker. -// -// closeMu.Lock waits for all in-flight producers to release their RLock -// before setting closed and closing the channel. This guarantees no -// producer is mid-send when the channel is closed, so the "send on closed -// channel" panic cannot occur. Producers that arrive after closeMu.Lock -// returns observe closed=true under RLock and release their cloned ref -// without ever touching the channel. +// first error encountered by any worker. closeMu.Lock waits for in-flight +// producers to release RLock before flipping closed and closing the +// channel, so no producer can be mid-send during the close. func (ep *eagerPipeline) wait() error { ep.waitOnce.Do(func() { ep.closeMu.Lock() @@ -255,8 +241,8 @@ func (ep *eagerPipeline) wait() error { ep.closeMu.Unlock() }) ep.wg.Wait() - // If ctx was cancelled, workers may have exited without fully draining - // the channel. Release any leftover refs so their leases aren't leaked. + // Release any leftover refs in case workers exited via ctx cancellation + // before draining the channel. for { select { case item, ok := <-ep.work: diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index 989a73274304..f2c851b50acb 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -90,7 +90,7 @@ func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { } require.NoError(t, ep.wait()) - require.NoError(t, ep.wait(), "second wait must not panic from double-close of work") + require.NoError(t, ep.wait(), "second wait must not panic") } func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { @@ -122,14 +122,8 @@ func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { ep.wg.Wait() } -// TestEagerPipeline_OnVertexCompleteAfterWait is the regression test for the -// "send on closed channel" panic that crashed buildkitd-v2-1 on 2026-04-17. -// A late fire from an orphan scheduler goroutine (e.g. a speculative -// loadCache that finished after the owning Solve returned) used to panic -// because wait() closed ep.work while the producer was mid-send. With -// closeMu gating the send path, the closed flag is observed before the -// channel is ever touched, so the late fire releases the cloned ref and -// returns cleanly. +// Late fires of onVertexComplete must release the clone instead of sending +// into the (now closed) work channel. func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { ep := &eagerPipeline{ ctx: context.Background(), @@ -142,15 +136,11 @@ func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { require.NotPanics(t, func() { ep.onVertexComplete(nil, []solver.Result{res}) - }, "late onVertexComplete after wait() must not panic") - - assert.Equal(t, int32(1), released.Load(), - "cloned ref from a late fire must be released, not leaked") + }) + assert.Equal(t, int32(1), released.Load()) } -// TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent verifies the fix -// holds under the realistic load pattern: many orphan goroutines firing -// concurrently after wait() has been called. +// Many concurrent late fires after wait() must all release cleanly. func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { ep := &eagerPipeline{ ctx: context.Background(), @@ -172,14 +162,11 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { } wg.Wait() - assert.Equal(t, int32(fires), released.Load(), - "every cloned ref from late fires must be released") + assert.Equal(t, int32(fires), released.Load()) } -// TestEagerPipeline_OnVertexCompleteRacingWait verifies no panic and no -// leaks when fires happen concurrently with wait() being called. This is -// the scenario that exposed the leak window in an earlier attempt at the -// fix (an unprotected two-channel design). +// Fires racing concurrently with wait() must not panic and must not leak: +// every clone is either drained by wait() or released by the sender. func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { ep := &eagerPipeline{ ctx: context.Background(), @@ -201,13 +188,9 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { require.NotPanics(t, func() { require.NoError(t, ep.wait()) - }, "wait() racing with concurrent fires must not panic") - + }) wg.Wait() - // Every cloned ref should be released exactly once: either the sender - // got its item into the channel before close (drained by wait()), or - // it saw closed=true under RLock and released the ref itself. assert.Equal(t, int32(fires), released.Load()) } @@ -248,9 +231,8 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { }, pushed) } -// releaseTracker is a minimal stub that satisfies cache.ImmutableRef -// for testing ref lifecycle (Release / Clone calls). All other methods -// panic via the embedded interface. +// releaseTracker is a minimal cache.ImmutableRef stub that counts +// Release calls. Clones share the counter. type releaseTracker struct { cache.ImmutableRef released *atomic.Int32 @@ -261,25 +243,16 @@ func (r *releaseTracker) Release(context.Context) error { return nil } -// Clone returns a sibling that shares the released counter so the test can -// verify both the original and the cloned ref end up released. -// onVertexComplete only ever calls Release on the clone, so the original -// is left to the caller (the test) to release. func (r *releaseTracker) Clone() cache.ImmutableRef { return &releaseTracker{released: r.released} } func (r *releaseTracker) ID() string { return "release-tracker" } -// fakeWorker is the bare minimum worker.Worker that worker.WorkerRef.ID() -// can call without panicking. type fakeWorker struct{ worker.Worker } func (fakeWorker) ID() string { return "fake-worker" } -// newWorkerRefResult builds a solver.Result whose Sys() returns a -// *worker.WorkerRef wrapping the given ImmutableRef, matching the shape -// onVertexComplete expects. func newWorkerRefResult(ref cache.ImmutableRef) solver.Result { return worker.NewWorkerRefResult(ref, fakeWorker{}) } From c6374ef58e95b0ebb6382f306b1278eeac937c94 Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 20 Apr 2026 16:03:20 -0700 Subject: [PATCH 41/55] remove the lock on the channel send --- solver/llbsolver/eager.go | 53 +++++++++++++++------------ solver/llbsolver/eager_test.go | 65 +++++++++++++++++++++++++++++----- 2 files changed, 88 insertions(+), 30 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index ffb9a55cd54d..ff9a9b35eabc 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -40,14 +40,16 @@ type eagerPipeline struct { pushCfg *exporter.EagerPushConfig work chan eagerWorkItem + done chan struct{} wg sync.WaitGroup - // closeMu gates producers (RLock) against shutdown (Lock). Producers - // check closed under RLock before sending, ensuring no send can race - // with the close of work. - closeMu sync.RWMutex - closed bool - waitOnce sync.Once + // closeMu gates new senders from entering shutdown. Senders increment + // senderWg while holding the mutex so wait() can stop admission before + // waiting for all in-flight send attempts to finish. + closeMu sync.Mutex + closing bool + senderWg sync.WaitGroup + waitOnce sync.Once // ctx carries the lease so compressed blobs are GC-protected. ctx context.Context @@ -95,6 +97,7 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio pusher: pusher, ctx: ctx, work: make(chan eagerWorkItem, 256), + done: make(chan struct{}), } numWorkers := eagerWorkerCount() @@ -131,8 +134,7 @@ func (ep *eagerPipeline) worker() { // onVertexComplete is the callback registered on the solver Job. It extracts // ImmutableRefs from vertex results, clones them for safe async use, and // sends them to the worker pool for compression. Fires that arrive after -// wait() observe closed=true under RLock and release the clone instead of -// sending. +// shutdown starts are rejected before enqueue and release their clones. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -143,23 +145,26 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re continue } - cloned := workerRef.ImmutableRef.Clone() - - ep.closeMu.RLock() - if ep.closed { - ep.closeMu.RUnlock() - cloned.Release(context.TODO()) + ep.closeMu.Lock() + if ep.closing { + ep.closeMu.Unlock() continue } + ep.senderWg.Add(1) + ep.closeMu.Unlock() + cloned := workerRef.ImmutableRef.Clone() select { case ep.work <- eagerWorkItem{ref: cloned}: + ep.senderWg.Done() + case <-ep.done: + cloned.Release(context.TODO()) + ep.senderWg.Done() case <-ep.ctx.Done(): cloned.Release(context.TODO()) - ep.closeMu.RUnlock() + ep.senderWg.Done() return } - ep.closeMu.RUnlock() } } @@ -229,16 +234,20 @@ func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Cont return err } -// wait closes ep.work and blocks until all workers finish. Returns the -// first error encountered by any worker. closeMu.Lock waits for in-flight -// producers to release RLock before flipping closed and closing the -// channel, so no producer can be mid-send during the close. +// wait shuts down new senders, waits for in-flight send attempts to finish, +// then closes ep.work and waits for workers to exit. func (ep *eagerPipeline) wait() error { ep.waitOnce.Do(func() { ep.closeMu.Lock() - ep.closed = true - close(ep.work) + if ep.done == nil { + ep.done = make(chan struct{}) + } + ep.closing = true ep.closeMu.Unlock() + + close(ep.done) + ep.senderWg.Wait() + close(ep.work) }) ep.wg.Wait() // Release any leftover refs in case workers exited via ctx cancellation diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index f2c851b50acb..aff6b0d651d0 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -125,26 +125,31 @@ func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { // Late fires of onVertexComplete must release the clone instead of sending // into the (now closed) work channel. func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { + var cloned atomic.Int32 ep := &eagerPipeline{ ctx: context.Background(), work: make(chan eagerWorkItem, 10), + done: make(chan struct{}), } require.NoError(t, ep.wait()) var released atomic.Int32 - res := newWorkerRefResult(&releaseTracker{released: &released}) + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) require.NotPanics(t, func() { ep.onVertexComplete(nil, []solver.Result{res}) }) - assert.Equal(t, int32(1), released.Load()) + assert.Zero(t, cloned.Load()) + assert.Zero(t, released.Load()) } -// Many concurrent late fires after wait() must all release cleanly. +// Many concurrent late fires after wait() must be rejected before cloning. func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { + var cloned atomic.Int32 ep := &eagerPipeline{ ctx: context.Background(), work: make(chan eagerWorkItem, 10), + done: make(chan struct{}), } require.NoError(t, ep.wait()) @@ -156,21 +161,61 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { for range fires { go func() { defer wg.Done() - res := newWorkerRefResult(&releaseTracker{released: &released}) + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) ep.onVertexComplete(nil, []solver.Result{res}) }() } wg.Wait() - assert.Equal(t, int32(fires), released.Load()) + assert.Zero(t, cloned.Load()) + assert.Zero(t, released.Load()) +} + +// A sender admitted before wait() but blocked on a full queue must take the +// done path, release its clone, and exit without panic. +func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) { + var cloned atomic.Int32 + var released atomic.Int32 + ep := &eagerPipeline{ + ctx: context.Background(), + work: make(chan eagerWorkItem, 1), + done: make(chan struct{}), + } + ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} + + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) + + var senderWg sync.WaitGroup + senderWg.Add(1) + go func() { + defer senderWg.Done() + ep.onVertexComplete(nil, []solver.Result{res}) + }() + + for range 1000 { + if cloned.Load() == 1 { + break + } + runtime.Gosched() + } + require.Equal(t, int32(1), cloned.Load()) + + require.NotPanics(t, func() { + require.NoError(t, ep.wait()) + }) + senderWg.Wait() + + assert.Equal(t, int32(2), released.Load()) } // Fires racing concurrently with wait() must not panic and must not leak: -// every clone is either drained by wait() or released by the sender. +// every admitted clone is either drained by wait() or released by the sender. func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { + var cloned atomic.Int32 ep := &eagerPipeline{ ctx: context.Background(), work: make(chan eagerWorkItem, 256), + done: make(chan struct{}), } var released atomic.Int32 @@ -181,7 +226,7 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { for range fires { go func() { defer wg.Done() - res := newWorkerRefResult(&releaseTracker{released: &released}) + res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) ep.onVertexComplete(nil, []solver.Result{res}) }() } @@ -191,7 +236,7 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { }) wg.Wait() - assert.Equal(t, int32(fires), released.Load()) + assert.Equal(t, cloned.Load(), released.Load()) } func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { @@ -236,6 +281,7 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { type releaseTracker struct { cache.ImmutableRef released *atomic.Int32 + cloned *atomic.Int32 } func (r *releaseTracker) Release(context.Context) error { @@ -244,6 +290,9 @@ func (r *releaseTracker) Release(context.Context) error { } func (r *releaseTracker) Clone() cache.ImmutableRef { + if r.cloned != nil { + r.cloned.Add(1) + } return &releaseTracker{released: r.released} } From 75f46218198c041d09be220a546a6a687afc3b00 Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 20 Apr 2026 16:13:09 -0700 Subject: [PATCH 42/55] fix eager pipeline gofmt lint failure Format the updated sender-gated shutdown fields so the PR passes golangci-lint after switching away from the RWMutex send path. Made-with: Cursor --- solver/llbsolver/eager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index ff9a9b35eabc..5ff47bbce389 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -46,10 +46,10 @@ type eagerPipeline struct { // closeMu gates new senders from entering shutdown. Senders increment // senderWg while holding the mutex so wait() can stop admission before // waiting for all in-flight send attempts to finish. - closeMu sync.Mutex - closing bool - senderWg sync.WaitGroup - waitOnce sync.Once + closeMu sync.Mutex + closing bool + senderWg sync.WaitGroup + waitOnce sync.Once // ctx carries the lease so compressed blobs are GC-protected. ctx context.Context From 43f01d62b4163f27017abb1688d3433a1ad1c1b3 Mon Sep 17 00:00:00 2001 From: Amy Date: Wed, 22 Apr 2026 12:53:21 -0700 Subject: [PATCH 43/55] add logging --- cache/blobs.go | 6 +++++- cache/refs.go | 15 +++++++++++++++ exporter/containerimage/export.go | 11 +++++++++-- exporter/containerimage/writer.go | 5 ++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cache/blobs.go b/cache/blobs.go index 09836772d08a..9bafb8f1c2e2 100644 --- a/cache/blobs.go +++ b/cache/blobs.go @@ -100,7 +100,11 @@ func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool return nil, nil } if !createIfNeeded { - return nil, errors.WithStack(ErrNoBlobs) + bklog.G(ctx).Warnf( + "computeBlobChain: ErrNoBlobs (createIfNeeded=false) ref=%s kind=%s blobOnly=%t snapshotID=%s compression=%s", + sr.ID(), sr.kind(), sr.getBlobOnly(), sr.getSnapshotID(), comp.Type, + ) + return nil, errors.Wrapf(ErrNoBlobs, "ref %s (kind=%s, snapshotID=%s, blobOnly=%t)", sr.ID(), sr.kind(), sr.getSnapshotID(), sr.getBlobOnly()) } l, ctx, err := leaseutil.NewLease(ctx, sr.cm.LeaseManager, leaseutil.MakeTemporary) diff --git a/cache/refs.go b/cache/refs.go index 23c62a5ee84b..89beaaac64e2 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -267,6 +267,21 @@ const ( Diff ) +func (k refKind) String() string { + switch k { + case BaseLayer: + return "BaseLayer" + case Layer: + return "Layer" + case Merge: + return "Merge" + case Diff: + return "Diff" + default: + return fmt.Sprintf("refKind(%d)", int(k)) + } +} + func (cr *cacheRecord) kind() refKind { if len(cr.mergeParents) > 0 { return Merge diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 0abaf90cd904..e1678e17d145 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -27,6 +27,7 @@ import ( "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/leaseutil" @@ -371,7 +372,10 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source eg.Go(func() error { remotes, err := ref.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) if err != nil { - return err + if errors.Is(err, cache.ErrNoBlobs) { + bklog.G(ctx).Warnf("imageExporter unlazy: ErrNoBlobs for top-level ref=%s", ref.ID()) + } + return errors.Wrapf(err, "imageExporter unlazy: top-level ref %s", ref.ID()) } remote := remotes[0] if unlazier, ok := remote.Provider.(cache.Unlazier); ok { @@ -436,7 +440,10 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou for _, ref := range refs { remotes, err := ref.GetRemotes(ctx, false, e.opts.RefCfg, false, session.NewGroup(sessionID)) if err != nil { - return err + if errors.Is(err, cache.ErrNoBlobs) { + bklog.G(ctx).Warnf("pushImage: ErrNoBlobs for top-level ref=%s targetName=%s", ref.ID(), targetName) + } + return errors.Wrapf(err, "pushImage: top-level ref %s", ref.ID()) } remote := remotes[0] for _, desc := range remote.Descriptors { diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 5153eea76a87..c40b8917aac8 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -380,7 +380,10 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, refCfg cacheconfig.RefC eg.Go(func() error { remotes, err := ref.GetRemotes(ctx, createIfNeeded, refCfg, false, s) if err != nil { - return err + if errors.Is(err, cache.ErrNoBlobs) { + bklog.G(ctx).Warnf("exportLayers: ErrNoBlobs for top-level ref=%s eagerExport=%q createIfNeeded=%t", ref.ID(), eagerExport, createIfNeeded) + } + return errors.Wrapf(err, "exportLayers: top-level ref %s", ref.ID()) } remote := remotes[0] out[i] = *remote From 64022205aaffe5c7abcdd164a0a17ff788f64d8c Mon Sep 17 00:00:00 2001 From: Amy Date: Wed, 22 Apr 2026 13:26:56 -0700 Subject: [PATCH 44/55] more logging --- cache/manager.go | 5 +++++ cache/refs.go | 9 +++++++++ cache/remote.go | 12 ++++++++++++ worker/base/worker.go | 13 +++++++++++++ 4 files changed, 39 insertions(+) diff --git a/cache/manager.go b/cache/manager.go index 12d941d6ae2a..d8bd58e88503 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -211,9 +211,14 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, if p != nil { releaseParent = true } + existingImageRefs := ref.getImageRefs() if err := setImageRefMetadata(ref.cacheMetadata, opts...); err != nil { return nil, errors.Wrapf(err, "failed to append image ref metadata to ref %s", ref.ID()) } + bklog.G(ctx).Infof( + "DEDUP-HIT blobchain=%s reusingRef=%s blobDigest=%s existingImageRefs=%v newImageRefs=%v", + blobChainID, ref.ID(), ref.getBlob(), existingImageRefs, ref.getImageRefs(), + ) return ref, nil } diff --git a/cache/refs.go b/cache/refs.go index 89beaaac64e2..7a6fff9c998c 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -1422,6 +1422,15 @@ func (sr *immutableRef) unlazyLayer(ctx context.Context, dhs DescHandlers, pg pr } dh := dhs[desc.Digest] + dhRef := "" + if dh != nil { + dhRef = dh.Ref + } + bklog.G(ctx).Infof( + "UNLAZY-LAYER ref=%s digest=%s ensureContentStore=%t dhFound=%t dh.Ref=%q dhsLen=%d imageRefs=%v", + sr.ID(), desc.Digest, ensureContentStore, dh != nil, dhRef, len(dhs), sr.getImageRefs(), + ) + eg.Go(func() error { // unlazies if needed, otherwise a no-op return lazyRefProvider{ diff --git a/cache/remote.go b/cache/remote.go index c0e3cc6b48c2..ff58869de0ed 100644 --- a/cache/remote.go +++ b/cache/remote.go @@ -338,6 +338,10 @@ func (p lazyRefProvider) Unlazy(ctx context.Context) error { if p.dh == nil { // shouldn't happen, if you have a lazy immutable ref it already should be validated // that descriptor handlers exist for it + bklog.G(ctx).Warnf( + "UNLAZY-NIL-DH ref=%s digest=%s imageRefs=%v", + p.ref.ID(), p.desc.Digest, p.ref.getImageRefs(), + ) return struct{}{}, errors.New("unexpected nil descriptor handler") } @@ -350,11 +354,19 @@ func (p lazyRefProvider) Unlazy(ctx context.Context) error { // For now, just pull down the whole content and then return a ReaderAt from the local content // store. If efficient partial reads are desired in the future, something more like a "tee" // that caches remote partial reads to a local store may need to replace this. + bklog.G(ctx).Infof( + "UNLAZY-FETCH ref=%s digest=%s mediaType=%s size=%d dh.Ref=%q imageRefs=%v", + p.ref.ID(), p.desc.Digest, p.desc.MediaType, p.desc.Size, p.dh.Ref, p.ref.getImageRefs(), + ) err := contentutil.Copy(ctx, p.ref.cm.ContentStore, &pullprogress.ProviderWithProgress{ Provider: p.dh.Provider(p.session), Manager: p.ref.cm.ContentStore, }, p.desc, p.dh.Ref, logs.LoggerFromContext(ctx)) if err != nil { + bklog.G(ctx).Warnf( + "UNLAZY-FETCH-ERR ref=%s digest=%s dh.Ref=%q imageRefs=%v err=%v", + p.ref.ID(), p.desc.Digest, p.dh.Ref, p.ref.getImageRefs(), err, + ) return struct{}{}, err } diff --git a/worker/base/worker.go b/worker/base/worker.go index 1922b9dc67c6..f0ffdabe2eca 100644 --- a/worker/base/worker.go +++ b/worker/base/worker.go @@ -317,8 +317,21 @@ func (w *Worker) LoadRef(ctx context.Context, id string, hidden bool) (cache.Imm } } } + recoveredRefs := make(map[string]string, len(descHandlers)) + for d, h := range descHandlers { + recoveredRefs[string(d)] = h.Ref + } + bklog.G(ctx).Infof( + "LOADREF-RECOVER id=%s missingDigests=%v recoveredFromSolver=%v", + id, []digest.Digest(needsRemoteProviders), recoveredRefs, + ) opts = append(opts, descHandlers) ref, err = w.CacheMgr.Get(ctx, id, pg, opts...) + } else { + bklog.G(ctx).Warnf( + "LOADREF-RECOVER-NOGETTER id=%s missingDigests=%v (no CacheOptGetter in ctx)", + id, []digest.Digest(needsRemoteProviders), + ) } } if err != nil { From cd0caac72740ab844a2ce8eebd1f24e11007df28 Mon Sep 17 00:00:00 2001 From: Amy Date: Wed, 22 Apr 2026 18:15:08 -0700 Subject: [PATCH 45/55] add a span for eager compress + push --- solver/llbsolver/solver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 3c66961687b5..f0bc1aa7837a 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -660,7 +660,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // exporters. The manifest needs final blob digests from every layer. if eager != nil { if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { - return eager.wait() + span, ctx := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) + err := eager.wait() + tracing.FinishWithError(span, err) + return err }); err != nil { return nil, errors.Wrap(err, "eager export pipeline failed") } From 750c0c93018c21015c1680665e4e250924fa682d Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 23 Apr 2026 10:53:41 -0700 Subject: [PATCH 46/55] make this an info level log --- control/control.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control/control.go b/control/control.go index d8ec83ec6f3a..39abe468f3cb 100644 --- a/control/control.go +++ b/control/control.go @@ -779,7 +779,7 @@ func (c *Controller) gc() { } <-done if size > 0 { - bklog.G(ctx).Debugf("gc cleaned up %d bytes", size) + bklog.G(ctx).Infof("gc cleaned up %d bytes", size) go c.throttledReleaseUnreferenced() } } From b55e8c5d52f1877503083f7fcd59b7d5fc5f0187 Mon Sep 17 00:00:00 2001 From: Amy Date: Thu, 23 Apr 2026 10:58:20 -0700 Subject: [PATCH 47/55] fix lint: ineffectual assignment to ctx Made-with: Cursor --- solver/llbsolver/solver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index f0bc1aa7837a..bf710ec8ac8a 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -660,7 +660,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // exporters. The manifest needs final blob digests from every layer. if eager != nil { if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { - span, ctx := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) + span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) err := eager.wait() tracing.FinishWithError(span, err) return err From cc85e1aeffc82a2bd3d5cc6d94480a3c0980428a Mon Sep 17 00:00:00 2001 From: Amy Date: Mon, 27 Apr 2026 17:05:53 -0700 Subject: [PATCH 48/55] fix eager export --- solver/llbsolver/eager.go | 597 +++++++++++++++++++++++++++++---- solver/llbsolver/eager_test.go | 288 +++++++++++++--- solver/llbsolver/solver.go | 11 +- 3 files changed, 778 insertions(+), 118 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 5ff47bbce389..d84bd6819a32 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -6,12 +6,15 @@ import ( "runtime" "strconv" "sync" + "sync/atomic" + "time" "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/core/remotes" "github.com/moby/buildkit/cache" cacheconfig "github.com/moby/buildkit/cache/config" "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/frontend" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/bklog" @@ -25,23 +28,56 @@ import ( "github.com/pkg/errors" ) -const defaultEagerWorkers = 4 +// defaultEagerWorkers and defaultEagerPushWorkers control the size of the +// compress and push pools respectively. They are deliberately set to 100 so +// that a single ref's full descriptor chain (typically 5-10 layers) can fan +// out into the push pool without serialization, matching what vanilla +// buildkit gets for free via images.Dispatch. +const ( + defaultEagerWorkers = 100 + defaultEagerPushWorkers = 100 +) type eagerWorkItem struct { ref cache.ImmutableRef } -// eagerPipeline manages background compression (and optionally push) of layer -// blobs as build vertices complete, rather than deferring all work to finalize. +// eagerPushItem is a single push descriptor handed to the push pool. The +// per-ref WaitGroup lets the dispatching compress worker block on completion +// of all its descriptor pushes; the per-ref errCh surfaces the first error +// from any of them. +type eagerPushItem struct { + refID string + desc ocispecs.Descriptor + handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error) + wg *sync.WaitGroup + errCh chan<- error +} + +// eagerPipeline manages background compression and pushing of layer blobs as +// build vertices complete, rather than deferring all work to finalize. +// +// It is split into two worker pools: +// - Compress pool: receives refs from the solver vertex callback, runs +// GetRemotes (which performs blob compression for the full parent chain), +// and dispatches each pushable descriptor onto the push pool. +// - Push pool: receives individual descriptors and uploads them to the +// registry. Decoupling lets a single ref fan out parallel pushes across +// its descriptor chain (instead of serializing them in one goroutine), +// which matches the parallelism that vanilla buildkit gets from +// images.Dispatch. type eagerPipeline struct { mode EagerExportMode refCfg cacheconfig.RefConfig sessionID string pushCfg *exporter.EagerPushConfig - work chan eagerWorkItem - done chan struct{} - wg sync.WaitGroup + compressWork chan eagerWorkItem + pushWork chan eagerPushItem + done chan struct{} + + compressWG sync.WaitGroup + pushWG sync.WaitGroup // closeMu gates new senders from entering shutdown. Senders increment // senderWg while holding the mutex so wait() can stop admission before @@ -57,20 +93,62 @@ type eagerPipeline struct { // pusher is created at pipeline init when mode is EagerExportPush. pusher remotes.Pusher - // pushDedup prevents concurrent pushes of the same digest. - pushDedup flightcontrol.Group[struct{}] + // pushDedup coalesces concurrent pushes of the same digest. After a push + // completes, pushedDigests records the digest so subsequent calls + // short-circuit immediately rather than re-running the flightcontrol + // closure (which would emit redundant logs and consume registry + // semaphore slots for no-op HEAD checks). + pushDedup flightcontrol.Group[struct{}] + pushedDigests sync.Map + + // keepRefIDs is the set of ImmutableRef.ID()s whose blobs are part of + // the final exported image manifest. It is populated once, by wait(), + // after the frontend has fully resolved its result. While nil, every + // ref's blobs are eligible for compression+push (the previous + // behaviour). Once non-nil, processRef and pushDescriptor skip refs + // whose ID is not in the set, and waitWithKeepSet cancels in-flight + // pushes whose only requesters are non-kept refs. + keepRefIDs atomic.Pointer[map[string]struct{}] + // inflight tracks per-digest cancel funcs and the set of refIDs that + // requested each digest. Used by waitWithKeepSet to cancel uploads + // whose every requester turns out to be non-kept. + inflight sync.Map // map[string]*pushTracker mu sync.Mutex firstErr error } +// pushTracker holds bookkeeping for a single digest's eager push: +// - refIDs: every refID that asked for this digest (kept or not). Used +// by cancelNonKeptInflight to decide whether any kept ref needs the +// blob — if so, the closure stays alive. +// - cancels: the cancel func of every in-flight pushDescriptor caller +// (one per refID), keyed by refID. Cancellation must hit *every* +// waiter because flightcontrol's closure-ctx is a sharedContext that +// only fires done when *all* waiter ctxs are done (see +// util/flightcontrol.sharedContext.checkDone). Cancelling just one +// leaves the closure running. +type pushTracker struct { + mu sync.Mutex + refIDs map[string]struct{} + cancels map[string]context.CancelFunc +} + func eagerWorkerCount() int { - if s := os.Getenv("BUILDKIT_EAGER_EXPORT_WORKERS"); s != "" { + return envWorkerCount("BUILDKIT_EAGER_EXPORT_WORKERS", defaultEagerWorkers) +} + +func eagerPushWorkerCount() int { + return envWorkerCount("BUILDKIT_EAGER_PUSH_WORKERS", defaultEagerPushWorkers) +} + +func envWorkerCount(env string, fallback int) int { + if s := os.Getenv(env); s != "" { if n, err := strconv.Atoi(s); err == nil && n > 0 { return n } } - return max(defaultEagerWorkers, runtime.NumCPU()) + return max(fallback, runtime.NumCPU()) } func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compression.Config, sessionID string, sm *session.Manager, pushCfg *exporter.EagerPushConfig) (*eagerPipeline, error) { @@ -92,49 +170,86 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio refCfg: cacheconfig.RefConfig{ Compression: comp, }, - sessionID: sessionID, - pushCfg: pushCfg, - pusher: pusher, - ctx: ctx, - work: make(chan eagerWorkItem, 256), - done: make(chan struct{}), + sessionID: sessionID, + pushCfg: pushCfg, + pusher: pusher, + ctx: ctx, + compressWork: make(chan eagerWorkItem, 256), + pushWork: make(chan eagerPushItem, 1024), + done: make(chan struct{}), + } + + numCompress := eagerWorkerCount() + ep.compressWG.Add(numCompress) + for range numCompress { + go ep.compressWorker() } - numWorkers := eagerWorkerCount() - ep.wg.Add(numWorkers) - for range numWorkers { - go ep.worker() + if mode == EagerExportPush { + numPush := eagerPushWorkerCount() + ep.pushWG.Add(numPush) + for range numPush { + go ep.pushWorker() + } + bklog.G(ctx).Infof("eager pipeline started compress_workers=%d push_workers=%d", numCompress, numPush) + } else { + bklog.G(ctx).Infof("eager pipeline started compress_workers=%d (no push)", numCompress) } return ep, nil } -func (ep *eagerPipeline) worker() { - defer ep.wg.Done() +func (ep *eagerPipeline) compressWorker() { + defer ep.compressWG.Done() for { select { case <-ep.ctx.Done(): return - case item, ok := <-ep.work: + case item, ok := <-ep.compressWork: if !ok { return } if err := ep.processRef(item.ref); err != nil { - ep.mu.Lock() - if ep.firstErr == nil { - ep.firstErr = err - } - ep.mu.Unlock() + ep.recordErr(err) } item.ref.Release(context.TODO()) } } } +func (ep *eagerPipeline) pushWorker() { + defer ep.pushWG.Done() + for { + // Don't bail out on ctx.Done here: we need to drain pushWork so + // the per-ref WaitGroup counters reach zero, otherwise compress + // workers can deadlock in dispatchPushes during shutdown. + item, ok := <-ep.pushWork + if !ok { + return + } + if err := ep.pushDescriptor(ep.ctx, item); err != nil { + ep.recordErr(err) + select { + case item.errCh <- err: + default: + } + } + item.wg.Done() + } +} + +func (ep *eagerPipeline) recordErr(err error) { + ep.mu.Lock() + if ep.firstErr == nil { + ep.firstErr = err + } + ep.mu.Unlock() +} + // onVertexComplete is the callback registered on the solver Job. It extracts // ImmutableRefs from vertex results, clones them for safe async use, and -// sends them to the worker pool for compression. Fires that arrive after -// shutdown starts are rejected before enqueue and release their clones. +// sends them to the compress pool. Fires that arrive after shutdown starts +// are rejected before enqueue and release their clones. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -155,7 +270,7 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re cloned := workerRef.ImmutableRef.Clone() select { - case ep.work <- eagerWorkItem{ref: cloned}: + case ep.compressWork <- eagerWorkItem{ref: cloned}: ep.senderWg.Done() case <-ep.done: cloned.Release(context.TODO()) @@ -168,36 +283,163 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } } -// processRef compresses a single ref's blob (and its parent chain) and -// optionally pushes it. Parent compression is deduplicated by flightcontrol -// inside computeBlobChain, so overlapping parent chains across workers are -// only compressed once. +// computeEagerKeepSet walks the resolved frontend Result and returns the +// set of ImmutableRef.ID()s that belong to the final exported image — +// including every layer in each output ref's parent chain. +// +// Why the chain matters: each layer is its own ImmutableRef with its own +// ID, *equal to* the ID of the intermediate vertex that produced it. The +// solver's onVertexComplete fires once per vertex during the build, so +// the eager pipeline pushes layer L_n via vertex V_n's processRef long +// before V_final completes. If we only kept V_final.ID() we'd filter +// every intermediate vertex's processRef, and all the real layer pushes +// would get deferred to wait() — defeating the entire point of eager +// export. +// +// res must already be fully resolved: every ResultProxy.Result(ctx) call +// must have completed without error. The caller in solver.go ensures +// this via eg.Wait() right before invoking us. +// +// If res is nil or contains zero refs, returns an empty map (== filter +// everything). Errors resolving individual refs are logged but not +// returned: a partial keep-set is safer than failing the build, since a +// missing entry just costs some wasted bandwidth, not correctness. +func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]struct{} { + keep := make(map[string]struct{}) + if res == nil { + return keep + } + res.EachRef(func(rp solver.ResultProxy) error { + if rp == nil { + return nil + } + cached, err := rp.Result(ctx) + if err != nil { + bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to resolve ref") + return nil + } + workerRef, ok := cached.Sys().(*worker.WorkerRef) + if !ok || workerRef.ImmutableRef == nil { + return nil + } + // LayerChain walks BaseLayer → ... → tip, including the tip + // itself. Each entry is a *clone* of the underlying ref — we + // only need its ID, then must release. + chain := workerRef.ImmutableRef.LayerChain() + for _, layer := range chain { + if layer == nil { + continue + } + keep[layer.ID()] = struct{}{} + } + if err := chain.Release(context.WithoutCancel(ctx)); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to release chain clones") + } + return nil + }) + return keep +} + +// keepSet returns the current keep-set, or nil if none has been set yet. +// While nil, no filtering is applied (every ref is eligible). +func (ep *eagerPipeline) keepSet() map[string]struct{} { + if p := ep.keepRefIDs.Load(); p != nil { + return *p + } + return nil +} + +// isKept reports whether refID should be retained. If no keep-set has been +// installed yet, every ref is considered kept. +func (ep *eagerPipeline) isKept(refID string) bool { + keep := ep.keepSet() + if keep == nil { + return true + } + _, ok := keep[refID] + return ok +} + +// trackerFor returns (creating if necessary) the pushTracker for digest. +func (ep *eagerPipeline) trackerFor(digest string) *pushTracker { + if v, ok := ep.inflight.Load(digest); ok { + return v.(*pushTracker) + } + nt := &pushTracker{refIDs: make(map[string]struct{})} + actual, _ := ep.inflight.LoadOrStore(digest, nt) + return actual.(*pushTracker) +} + +// processRef compresses a single ref's blob (and its parent chain) and, in +// push mode, dispatches each pushable descriptor onto the push pool. Parent +// compression is deduplicated by flightcontrol inside computeBlobChain, so +// overlapping parent chains across workers are only compressed once. func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { ctx := ep.ctx s := session.NewGroup(ep.sessionID) + refID := ref.ID() + + // If the keep-set is already installed (e.g. queued items processed + // during shutdown drain), skip non-kept refs without paying for + // compression. + if !ep.isKept(refID) { + bklog.G(ctx).Debugf("eager compress skipped (filtered) ref=%s", refID) + return nil + } - bklog.G(ctx).Infof("eager compress starting for ref %s", ref.ID()) - remotes, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) + bklog.G(ctx).Infof("eager compress starting ref=%s", refID) + compressStart := time.Now() + rems, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) if err != nil { - bklog.G(ctx).WithError(err).Warnf("eager compress failed for ref %s", ref.ID()) + bklog.G(ctx).WithError(err).Warnf("eager compress failed ref=%s", refID) return err } - bklog.G(ctx).Infof("eager compress done for ref %s", ref.ID()) + compressDur := time.Since(compressStart).Round(time.Millisecond) - if ep.mode == EagerExportPush { - if err := ep.pushBlobs(ctx, remotes); err != nil { - bklog.G(ctx).WithError(err).Warnf("eager push failed for ref %s", ref.ID()) - return err - } - bklog.G(ctx).Infof("eager push done for ref %s", ref.ID()) + descCount, totalBytes := summarizeDescriptors(rems) + bklog.G(ctx).Infof("eager compress done ref=%s descriptors=%d bytes=%d duration=%s", + refID, descCount, totalBytes, compressDur) + + if ep.mode != EagerExportPush { + return nil + } + + // Re-check after the (potentially long) compress: the keep-set may + // have been installed while we were inside GetRemotes. + if !ep.isKept(refID) { + bklog.G(ctx).Debugf("eager push skipped after compress (filtered) ref=%s", refID) + return nil + } + + pushStart := time.Now() + if err := ep.dispatchPushes(ctx, refID, rems); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager push failed ref=%s", refID) + return err } + bklog.G(ctx).Infof("eager push complete ref=%s descriptors=%d bytes=%d duration=%s", + refID, descCount, totalBytes, time.Since(pushStart).Round(time.Millisecond)) return nil } -// pushBlobs pushes each layer descriptor from the GetRemotes result to the -// registry. Pushes are deduplicated by digest via flightcontrol — if two -// workers try to push the same blob concurrently, only one upload happens. -func (ep *eagerPipeline) pushBlobs(ctx context.Context, rems []*solver.Remote) error { +func summarizeDescriptors(rems []*solver.Remote) (count int, totalBytes int64) { + if len(rems) == 0 { + return 0, 0 + } + for _, desc := range rems[0].Descriptors { + if !shouldEagerPushDesc(desc) { + continue + } + count++ + totalBytes += desc.Size + } + return +} + +// dispatchPushes sends every pushable descriptor in a ref's chain to the +// push pool and waits for all of them to complete. Within a single ref this +// fans out to up to len(Descriptors) parallel uploads (gated by the push +// pool size and the registry's concurrency cap). +func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems []*solver.Remote) error { if len(rems) == 0 { return nil } @@ -208,36 +450,169 @@ func (ep *eagerPipeline) pushBlobs(ctx context.Context, rems []*solver.Remote) e nil, ) + var wg sync.WaitGroup + errCh := make(chan error, len(remote.Descriptors)) + + enqueued := 0 for _, desc := range remote.Descriptors { if !shouldEagerPushDesc(desc) { continue } - if err := ep.pushBlob(ctx, handler, desc); err != nil { - return err + wg.Add(1) + item := eagerPushItem{ + refID: refID, + desc: desc, + handler: handler, + wg: &wg, + errCh: errCh, + } + select { + case ep.pushWork <- item: + enqueued++ + case <-ctx.Done(): + wg.Done() + waitWithCtx(ctx, &wg) + return context.Cause(ctx) + } + } + + if err := waitWithCtx(ctx, &wg); err != nil { + return err + } + + close(errCh) + for e := range errCh { + if e != nil { + return e } } return nil } +// waitWithCtx blocks until wg reaches zero or ctx is cancelled. On +// cancellation it returns the ctx error without waiting (the WaitGroup +// counter may still be non-zero; it is safe to leak as the process is +// shutting down). On normal completion it returns nil. +func waitWithCtx(ctx context.Context, wg *sync.WaitGroup) error { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return context.Cause(ctx) + } +} + func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { return !images.IsNonDistributable(desc.MediaType) } -// pushBlob pushes a single descriptor, deduplicated by digest across all -// concurrent workers via flightcontrol. -func (ep *eagerPipeline) pushBlob(ctx context.Context, handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error), desc ocispecs.Descriptor) error { - _, err := ep.pushDedup.Do(ctx, desc.Digest.String(), func(ctx context.Context) (struct{}, error) { - bklog.G(ctx).Infof("eager pushing blob %s (%d bytes)", desc.Digest, desc.Size) - _, err := handler(ctx, desc) - return struct{}{}, err +// pushDescriptor uploads a single descriptor, deduplicated by digest. After +// the first successful push of a digest in this build, subsequent calls +// short-circuit at the pushedDigests check before entering the +// flightcontrol closure — so they emit no log, acquire no registry +// semaphore slot, and do no HEAD round trip. +// +// pushDescriptor also participates in the keep-set filter: it tracks every +// refID requesting each digest, lets cancelNonKeptInflight abort uploads +// whose requesters are all non-kept, and skips outright when the keep-set +// is already installed and item.refID is not in it. +// +// Kept refs that arrive *after* a cancellation pass simply re-enter +// flightcontrol with a fresh ctx: either the prior closure has already +// torn down (g.m[digest] is gone) and we start a new push, or the prior +// closure is still live and our kept ctx keeps it alive past +// cancelNonKeptInflight's reach. +func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) error { + digest := item.desc.Digest.String() + if _, done := ep.pushedDigests.Load(digest); done { + bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } + + // Always record the requester first so cancelNonKeptInflight can + // decide whether any kept ref needs this digest, even if we filter + // below. + tracker := ep.trackerFor(digest) + tracker.mu.Lock() + tracker.refIDs[item.refID] = struct{}{} + tracker.mu.Unlock() + + if !ep.isKept(item.refID) { + bklog.G(ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } + + pushCtx, cancel := context.WithCancel(ctx) + defer cancel() + + tracker.mu.Lock() + if tracker.cancels == nil { + tracker.cancels = make(map[string]context.CancelFunc) + } + tracker.cancels[item.refID] = cancel + tracker.mu.Unlock() + defer func() { + tracker.mu.Lock() + delete(tracker.cancels, item.refID) + tracker.mu.Unlock() + }() + + start := time.Now() + bklog.G(ctx).Infof("eager push starting ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + + _, err := ep.pushDedup.Do(pushCtx, digest, func(ctx context.Context) (struct{}, error) { + _, herr := item.handler(ctx, item.desc) + return struct{}{}, herr }) - return err + if err != nil { + // Distinguish a deliberate keep-set cancellation from a real + // failure: only the former has pushCtx done while the parent + // ctx is still alive. + if pushCtx.Err() != nil && ctx.Err() == nil { + bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil + } + return err + } + + ep.pushedDigests.Store(digest, struct{}{}) + bklog.G(ctx).Infof("eager push done ref=%s digest=%s size=%d duration=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil } -// wait shuts down new senders, waits for in-flight send attempts to finish, -// then closes ep.work and waits for workers to exit. -func (ep *eagerPipeline) wait() error { +// wait shuts down both pools in order: stop new compress senders, drain +// compressWork, then drain pushWork. Compress workers are guaranteed to +// have all dispatched their pushes by the time compressWG.Wait returns +// (because dispatchPushes blocks on the per-ref WaitGroup), so closing +// pushWork at that point is safe. +// +// The optional keep-set is the canonical set of ImmutableRef.ID()s that +// belong to the final image manifest, computed by the caller from the +// resolved frontend Result. When non-nil: +// - any in-flight push whose every requester is non-kept is cancelled +// immediately, freeing its registry-channel slot; +// - any items still queued in pushWork or compressWork for non-kept refs +// are skipped on dequeue; +// - kept refs continue to compress and push as normal. +// +// Pass nil to keep every ref (back-compat for non-gateway frontends or +// callers that haven't computed a keep-set). +func (ep *eagerPipeline) wait(keep map[string]struct{}) error { ep.waitOnce.Do(func() { + if keep != nil { + cp := keep + ep.keepRefIDs.Store(&cp) + n := ep.cancelNonKeptInflight(keep) + bklog.G(ep.ctx).Infof("eager wait keep_set_size=%d cancelled_inflight=%d", len(keep), n) + } + ep.closeMu.Lock() if ep.done == nil { ep.done = make(chan struct{}) @@ -247,24 +622,100 @@ func (ep *eagerPipeline) wait() error { close(ep.done) ep.senderWg.Wait() - close(ep.work) + close(ep.compressWork) + ep.compressWG.Wait() + ep.drainCompress() + + if ep.pushWork != nil { + close(ep.pushWork) + ep.pushWG.Wait() + ep.drainPushWork() + } + }) + ep.mu.Lock() + defer ep.mu.Unlock() + return ep.firstErr +} + +// cancelNonKeptInflight walks every digest currently being pushed and +// cancels the upload if no requester is in the keep-set. Returns the +// number of digests for which at least one cancel was issued. +// +// CRITICAL: flightcontrol's closure ctx is a sharedContext whose Done() +// only fires once *every* registered waiter ctx is done (see +// util/flightcontrol.sharedContext.checkDone). Cancelling a single +// waiter is not enough — the closure keeps running, the upload finishes, +// and we get a ~6 GB freebie we never wanted. So we cancel every waiter +// for the digest in one pass. +// +// A digest with at least one kept requester is left entirely alone: that +// kept waiter's pushDescriptor still needs the closure's result, and +// flightcontrol will deliver it. +// +// Cancelled uploads return from flightcontrol.Do with a ctx error; +// pushDescriptor recognises that as an intentional skip (pushCtx.Err +// non-nil while the parent ctx is alive) and converts it to a nil error +// so the build doesn't fail. +func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { + var digestsCancelled int + ep.inflight.Range(func(key, val any) bool { + digest := key.(string) + tracker := val.(*pushTracker) + + tracker.mu.Lock() + defer tracker.mu.Unlock() + + for refID := range tracker.refIDs { + if _, ok := keep[refID]; ok { + return true + } + } + if len(tracker.cancels) == 0 { + // Nothing in flight — queued items will be filtered by + // isKept() at dequeue. + return true + } + n := len(tracker.cancels) + for refID, cancelFn := range tracker.cancels { + cancelFn() + delete(tracker.cancels, refID) + } + digestsCancelled++ + bklog.G(ep.ctx).Infof("eager push cancel digest=%s requesters=%d cancelled_waiters=%d", + digest, len(tracker.refIDs), n) + return true }) - ep.wg.Wait() - // Release any leftover refs in case workers exited via ctx cancellation - // before draining the channel. + return digestsCancelled +} + +// drainCompress releases any refs left in compressWork after workers have +// exited (e.g. via ctx cancellation before the channel was drained). +func (ep *eagerPipeline) drainCompress() { for { select { - case item, ok := <-ep.work: + case item, ok := <-ep.compressWork: if !ok { - ep.mu.Lock() - defer ep.mu.Unlock() - return ep.firstErr + return } item.ref.Release(context.TODO()) default: - ep.mu.Lock() - defer ep.mu.Unlock() - return ep.firstErr + return + } + } +} + +// drainPushWork marks any leftover push items' WaitGroups as Done so that +// dispatching compress workers (if any are still blocked) can unblock. +func (ep *eagerPipeline) drainPushWork() { + for { + select { + case item, ok := <-ep.pushWork: + if !ok { + return + } + item.wg.Done() + default: + return } } } diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index aff6b0d651d0..72d2d5ba81c2 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -45,6 +45,16 @@ func TestEagerWorkerCount_EnvNegative(t *testing.T) { assert.Equal(t, max(defaultEagerWorkers, runtime.NumCPU()), eagerWorkerCount()) } +func TestEagerPushWorkerCount_Default(t *testing.T) { + os.Unsetenv("BUILDKIT_EAGER_PUSH_WORKERS") + assert.Equal(t, max(defaultEagerPushWorkers, runtime.NumCPU()), eagerPushWorkerCount()) +} + +func TestEagerPushWorkerCount_EnvOverride(t *testing.T) { + t.Setenv("BUILDKIT_EAGER_PUSH_WORKERS", "5") + assert.Equal(t, 5, eagerPushWorkerCount()) +} + func TestNewEagerPipeline_PushRequiresConfig(t *testing.T) { _, err := newEagerPipeline(context.Background(), EagerExportPush, compression.Config{}, "", nil, nil) require.Error(t, err) @@ -53,73 +63,86 @@ func TestNewEagerPipeline_PushRequiresConfig(t *testing.T) { func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { ep := &eagerPipeline{ - work: make(chan eagerWorkItem), + compressWork: make(chan eagerWorkItem), } ep.firstErr = assert.AnError - err := ep.wait() + err := ep.wait(nil) assert.Equal(t, assert.AnError, err) } func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { ep := &eagerPipeline{ - work: make(chan eagerWorkItem), + compressWork: make(chan eagerWorkItem), } - err := ep.wait() + err := ep.wait(nil) assert.NoError(t, err) } func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { var released atomic.Int32 ep := &eagerPipeline{ - work: make(chan eagerWorkItem, 10), + compressWork: make(chan eagerWorkItem, 10), } - ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} - ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} + ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} + ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} - err := ep.wait() + err := ep.wait(nil) require.NoError(t, err) assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") } func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { ep := &eagerPipeline{ - work: make(chan eagerWorkItem), + compressWork: make(chan eagerWorkItem), } - require.NoError(t, ep.wait()) - require.NoError(t, ep.wait(), "second wait must not panic") + require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait(nil), "second wait must not panic") } -func TestEagerPipeline_WorkerExitsOnContextCancel(t *testing.T) { +func TestEagerPipeline_CompressWorkerExitsOnContextCancel(t *testing.T) { ctx, cancel := context.WithCancelCause(context.Background()) ep := &eagerPipeline{ - mode: EagerExportCompress, - ctx: ctx, - work: make(chan eagerWorkItem, 10), + mode: EagerExportCompress, + ctx: ctx, + compressWork: make(chan eagerWorkItem, 10), } cancel(nil) - ep.wg.Add(1) - go ep.worker() - ep.wg.Wait() + ep.compressWG.Add(1) + go ep.compressWorker() + ep.compressWG.Wait() } -func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { +func TestEagerPipeline_CompressWorkerExitsOnChannelClose(t *testing.T) { ep := &eagerPipeline{ - mode: EagerExportCompress, - ctx: context.Background(), - work: make(chan eagerWorkItem), + mode: EagerExportCompress, + ctx: context.Background(), + compressWork: make(chan eagerWorkItem), } - ep.wg.Add(1) - go ep.worker() + ep.compressWG.Add(1) + go ep.compressWorker() - close(ep.work) - ep.wg.Wait() + close(ep.compressWork) + ep.compressWG.Wait() +} + +func TestEagerPipeline_PushWorkerExitsOnChannelClose(t *testing.T) { + ep := &eagerPipeline{ + ctx: context.Background(), + pushWork: make(chan eagerPushItem), + } + + ep.pushWG.Add(1) + go ep.pushWorker() + + close(ep.pushWork) + ep.pushWG.Wait() } // Late fires of onVertexComplete must release the clone instead of sending @@ -127,11 +150,11 @@ func TestEagerPipeline_WorkerExitsOnChannelClose(t *testing.T) { func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ - ctx: context.Background(), - work: make(chan eagerWorkItem, 10), - done: make(chan struct{}), + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 10), + done: make(chan struct{}), } - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) var released atomic.Int32 res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) @@ -147,11 +170,11 @@ func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ - ctx: context.Background(), - work: make(chan eagerWorkItem, 10), - done: make(chan struct{}), + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 10), + done: make(chan struct{}), } - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) var released atomic.Int32 const fires = 100 @@ -177,11 +200,11 @@ func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) var cloned atomic.Int32 var released atomic.Int32 ep := &eagerPipeline{ - ctx: context.Background(), - work: make(chan eagerWorkItem, 1), - done: make(chan struct{}), + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 1), + done: make(chan struct{}), } - ep.work <- eagerWorkItem{ref: &releaseTracker{released: &released}} + ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) @@ -201,7 +224,7 @@ func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) require.Equal(t, int32(1), cloned.Load()) require.NotPanics(t, func() { - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) }) senderWg.Wait() @@ -213,9 +236,9 @@ func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ - ctx: context.Background(), - work: make(chan eagerWorkItem, 256), - done: make(chan struct{}), + ctx: context.Background(), + compressWork: make(chan eagerWorkItem, 256), + done: make(chan struct{}), } var released atomic.Int32 @@ -232,7 +255,7 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { } require.NotPanics(t, func() { - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) }) wg.Wait() @@ -266,7 +289,11 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { if !shouldEagerPushDesc(desc) { continue } - err := ep.pushBlob(context.Background(), handler, desc) + err := ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "test-ref", + desc: desc, + handler: handler, + }) require.NoError(t, err) } @@ -276,6 +303,179 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { }, pushed) } +// pushDescriptor must short-circuit on the second call for the same digest, +// without invoking the handler again. This is the fix for the noisy +// "eager pushing blob" log entries that appeared once per shared parent +// blob per ref. +func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("shared-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 1234, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + for range 5 { + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "ref-test", + desc: desc, + handler: handler, + })) + } + assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once across repeated calls for the same digest") +} + +// pushDescriptor must skip — without invoking the handler — when the +// keep-set is installed and the requesting refID is not in it. +func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("intermediate-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 4096, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) + + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "intermediate-ref", + desc: desc, + handler: handler, + })) + assert.Zero(t, calls.Load(), "non-kept ref must not invoke the push handler") + + // And the digest must still appear in the inflight tracker so + // cancelNonKeptInflight can reason about it. + v, ok := ep.inflight.Load(desc.Digest.String()) + require.True(t, ok) + tracker := v.(*pushTracker) + tracker.mu.Lock() + _, hasRef := tracker.refIDs["intermediate-ref"] + tracker.mu.Unlock() + assert.True(t, hasRef, "tracker must record requester even when filtered") +} + +// pushDescriptor must still push when at least one kept ref requests the +// digest, even if a non-kept ref also asked for it. +func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("shared-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 4096, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) + + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "intermediate-ref", + desc: desc, + handler: handler, + })) + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "final-ref", + desc: desc, + handler: handler, + })) + assert.Equal(t, int32(1), calls.Load(), "kept-ref call must drive exactly one push") +} + +// cancelNonKeptInflight must cancel every waiter for a digest whose +// requesters are all non-kept, and must leave digests with at least one +// kept requester completely alone (because the kept waiter's +// flightcontrol.Do still needs the closure result). +func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { + ep := &eagerPipeline{ctx: context.Background()} + + keep := map[string]struct{}{"final-ref": {}} + + // Digest A: two non-kept requesters in flight. Both cancels must fire, + // matching flightcontrol's all-waiters-must-cancel semantics. + var cancelA1, cancelA2 atomic.Bool + ep.inflight.Store("digest-a", &pushTracker{ + refIDs: map[string]struct{}{"int-ref-1": {}, "int-ref-2": {}}, + cancels: map[string]context.CancelFunc{ + "int-ref-1": func() { cancelA1.Store(true) }, + "int-ref-2": func() { cancelA2.Store(true) }, + }, + }) + // Digest B: requested by both kept and non-kept — must be spared + // even though the non-kept waiter has a cancel registered. + var cancelB atomic.Bool + ep.inflight.Store("digest-b", &pushTracker{ + refIDs: map[string]struct{}{"int-ref-3": {}, "final-ref": {}}, + cancels: map[string]context.CancelFunc{ + "int-ref-3": func() { cancelB.Store(true) }, + }, + }) + // Digest C: non-kept requester but no cancel yet (still queued). + // Must not be counted and must not panic. + ep.inflight.Store("digest-c", &pushTracker{ + refIDs: map[string]struct{}{"int-ref-4": {}}, + }) + + n := ep.cancelNonKeptInflight(keep) + assert.Equal(t, 1, n, "exactly one digest's waiters should be cancelled") + assert.True(t, cancelA1.Load(), "digest A waiter 1 must be cancelled") + assert.True(t, cancelA2.Load(), "digest A waiter 2 must be cancelled") + assert.False(t, cancelB.Load(), "digest B must be spared (kept requester)") + + trackerA, _ := ep.inflight.Load("digest-a") + assert.Empty(t, trackerA.(*pushTracker).cancels, "cancels map should be drained after cancellation") +} + +// After cancelNonKeptInflight has fired, a kept ref that arrives later +// must not be skipped — it still needs to push the blob (the previous +// closure was cancelled before completion). +func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("retry-after-cancel"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 4096, + } + var calls atomic.Int32 + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + calls.Add(1) + return nil, nil + } + + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) + + // Simulate a prior cancel pass: tracker exists with the requester + // recorded and an empty cancels map (the in-flight waiter already + // got cancelled and unregistered itself). + ep.inflight.Store(desc.Digest.String(), &pushTracker{ + refIDs: map[string]struct{}{"int-ref": {}}, + cancels: map[string]context.CancelFunc{}, + }) + + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + refID: "final-ref", + desc: desc, + handler: handler, + })) + assert.Equal(t, int32(1), calls.Load(), "kept ref must drive a fresh push after a prior cancellation") +} + // releaseTracker is a minimal cache.ImmutableRef stub that counts // Release calls. Clones share the counter. type releaseTracker struct { diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index bf710ec8ac8a..9d67c8920fbd 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -659,9 +659,18 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // Wait for all eager compression/push jobs to finish before running // exporters. The manifest needs final blob digests from every layer. if eager != nil { + // Compute the keep-set: the set of ImmutableRef.ID()s that + // belong to the final exported result. Any ref that the + // solver's vertex callback fired for but that didn't end up in + // res (e.g. intermediate stages from a multi-stage Dockerfile, + // or the cache_warmer pattern's source layers) is filtered out + // — its compressed blobs stay in the local content store, but + // no registry bandwidth is spent on it. With a gateway + // frontend, res is final by this point so the set is complete. + keep := computeEagerKeepSet(ctx, res) if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) - err := eager.wait() + err := eager.wait(keep) tracing.FinishWithError(span, err) return err }); err != nil { From 9811c249c43bad03f3550280c071fd05022d962d Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 28 Apr 2026 09:58:48 -0700 Subject: [PATCH 49/55] adjust comments --- solver/llbsolver/eager.go | 207 +++++++-------------------------- solver/llbsolver/eager_test.go | 49 +++----- solver/llbsolver/solver.go | 25 +--- 3 files changed, 60 insertions(+), 221 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index d84bd6819a32..6f4e89e635c0 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -28,11 +28,8 @@ import ( "github.com/pkg/errors" ) -// defaultEagerWorkers and defaultEagerPushWorkers control the size of the -// compress and push pools respectively. They are deliberately set to 100 so -// that a single ref's full descriptor chain (typically 5-10 layers) can fan -// out into the push pool without serialization, matching what vanilla -// buildkit gets for free via images.Dispatch. +// Keep the pools large enough for a ref's descriptor chain to fan out without +// serializing pushes. const ( defaultEagerWorkers = 100 defaultEagerPushWorkers = 100 @@ -42,10 +39,7 @@ type eagerWorkItem struct { ref cache.ImmutableRef } -// eagerPushItem is a single push descriptor handed to the push pool. The -// per-ref WaitGroup lets the dispatching compress worker block on completion -// of all its descriptor pushes; the per-ref errCh surfaces the first error -// from any of them. +// eagerPushItem is one descriptor handed to the push pool. type eagerPushItem struct { refID string desc ocispecs.Descriptor @@ -54,18 +48,8 @@ type eagerPushItem struct { errCh chan<- error } -// eagerPipeline manages background compression and pushing of layer blobs as -// build vertices complete, rather than deferring all work to finalize. -// -// It is split into two worker pools: -// - Compress pool: receives refs from the solver vertex callback, runs -// GetRemotes (which performs blob compression for the full parent chain), -// and dispatches each pushable descriptor onto the push pool. -// - Push pool: receives individual descriptors and uploads them to the -// registry. Decoupling lets a single ref fan out parallel pushes across -// its descriptor chain (instead of serializing them in one goroutine), -// which matches the parallelism that vanilla buildkit gets from -// images.Dispatch. +// eagerPipeline compresses refs as vertices complete and optionally pushes +// their descriptors through a separate pool for per-chain parallelism. type eagerPipeline struct { mode EagerExportMode refCfg cacheconfig.RefConfig @@ -79,55 +63,34 @@ type eagerPipeline struct { compressWG sync.WaitGroup pushWG sync.WaitGroup - // closeMu gates new senders from entering shutdown. Senders increment - // senderWg while holding the mutex so wait() can stop admission before - // waiting for all in-flight send attempts to finish. + // closeMu stops new senders from entering shutdown races. closeMu sync.Mutex closing bool senderWg sync.WaitGroup waitOnce sync.Once - // ctx carries the lease so compressed blobs are GC-protected. + // ctx carries the lease that keeps compressed blobs GC-protected. ctx context.Context - // pusher is created at pipeline init when mode is EagerExportPush. pusher remotes.Pusher - // pushDedup coalesces concurrent pushes of the same digest. After a push - // completes, pushedDigests records the digest so subsequent calls - // short-circuit immediately rather than re-running the flightcontrol - // closure (which would emit redundant logs and consume registry - // semaphore slots for no-op HEAD checks). + // pushDedup coalesces concurrent pushes; pushedDigests skips digests that + // already succeeded in this build. pushDedup flightcontrol.Group[struct{}] pushedDigests sync.Map - // keepRefIDs is the set of ImmutableRef.ID()s whose blobs are part of - // the final exported image manifest. It is populated once, by wait(), - // after the frontend has fully resolved its result. While nil, every - // ref's blobs are eligible for compression+push (the previous - // behaviour). Once non-nil, processRef and pushDescriptor skip refs - // whose ID is not in the set, and waitWithKeepSet cancels in-flight - // pushes whose only requesters are non-kept refs. + // keepRefIDs is nil until wait() installs the final image ref set. After + // that, non-kept refs are skipped or cancelled. keepRefIDs atomic.Pointer[map[string]struct{}] - // inflight tracks per-digest cancel funcs and the set of refIDs that - // requested each digest. Used by waitWithKeepSet to cancel uploads - // whose every requester turns out to be non-kept. + // inflight tracks requesters and cancel funcs per digest. inflight sync.Map // map[string]*pushTracker mu sync.Mutex firstErr error } -// pushTracker holds bookkeeping for a single digest's eager push: -// - refIDs: every refID that asked for this digest (kept or not). Used -// by cancelNonKeptInflight to decide whether any kept ref needs the -// blob — if so, the closure stays alive. -// - cancels: the cancel func of every in-flight pushDescriptor caller -// (one per refID), keyed by refID. Cancellation must hit *every* -// waiter because flightcontrol's closure-ctx is a sharedContext that -// only fires done when *all* waiter ctxs are done (see -// util/flightcontrol.sharedContext.checkDone). Cancelling just one -// leaves the closure running. +// pushTracker records every ref that requested a digest and every active +// waiter. flightcontrol only cancels the shared push when all waiters cancel. type pushTracker struct { mu sync.Mutex refIDs map[string]struct{} @@ -220,9 +183,7 @@ func (ep *eagerPipeline) compressWorker() { func (ep *eagerPipeline) pushWorker() { defer ep.pushWG.Done() for { - // Don't bail out on ctx.Done here: we need to drain pushWork so - // the per-ref WaitGroup counters reach zero, otherwise compress - // workers can deadlock in dispatchPushes during shutdown. + // Drain pushWork even after ctx cancellation so per-ref waiters finish. item, ok := <-ep.pushWork if !ok { return @@ -246,10 +207,7 @@ func (ep *eagerPipeline) recordErr(err error) { ep.mu.Unlock() } -// onVertexComplete is the callback registered on the solver Job. It extracts -// ImmutableRefs from vertex results, clones them for safe async use, and -// sends them to the compress pool. Fires that arrive after shutdown starts -// are rejected before enqueue and release their clones. +// onVertexComplete clones finished refs and enqueues them for eager work. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -283,27 +241,9 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } } -// computeEagerKeepSet walks the resolved frontend Result and returns the -// set of ImmutableRef.ID()s that belong to the final exported image — -// including every layer in each output ref's parent chain. -// -// Why the chain matters: each layer is its own ImmutableRef with its own -// ID, *equal to* the ID of the intermediate vertex that produced it. The -// solver's onVertexComplete fires once per vertex during the build, so -// the eager pipeline pushes layer L_n via vertex V_n's processRef long -// before V_final completes. If we only kept V_final.ID() we'd filter -// every intermediate vertex's processRef, and all the real layer pushes -// would get deferred to wait() — defeating the entire point of eager -// export. -// -// res must already be fully resolved: every ResultProxy.Result(ctx) call -// must have completed without error. The caller in solver.go ensures -// this via eg.Wait() right before invoking us. -// -// If res is nil or contains zero refs, returns an empty map (== filter -// everything). Errors resolving individual refs are logged but not -// returned: a partial keep-set is safer than failing the build, since a -// missing entry just costs some wasted bandwidth, not correctness. +// computeEagerKeepSet returns every ref ID that contributes to the final image, +// including parent-chain layers. Each layer has its own vertex/ref ID, so +// keeping only the final ref would filter out the eager layer pushes. func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]struct{} { keep := make(map[string]struct{}) if res == nil { @@ -322,9 +262,7 @@ func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]s if !ok || workerRef.ImmutableRef == nil { return nil } - // LayerChain walks BaseLayer → ... → tip, including the tip - // itself. Each entry is a *clone* of the underlying ref — we - // only need its ID, then must release. + // LayerChain returns clones, so release them after reading IDs. chain := workerRef.ImmutableRef.LayerChain() for _, layer := range chain { if layer == nil { @@ -340,8 +278,7 @@ func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]s return keep } -// keepSet returns the current keep-set, or nil if none has been set yet. -// While nil, no filtering is applied (every ref is eligible). +// keepSet returns nil until filtering is enabled. func (ep *eagerPipeline) keepSet() map[string]struct{} { if p := ep.keepRefIDs.Load(); p != nil { return *p @@ -349,8 +286,7 @@ func (ep *eagerPipeline) keepSet() map[string]struct{} { return nil } -// isKept reports whether refID should be retained. If no keep-set has been -// installed yet, every ref is considered kept. +// isKept treats every ref as kept until a keep-set is installed. func (ep *eagerPipeline) isKept(refID string) bool { keep := ep.keepSet() if keep == nil { @@ -370,18 +306,14 @@ func (ep *eagerPipeline) trackerFor(digest string) *pushTracker { return actual.(*pushTracker) } -// processRef compresses a single ref's blob (and its parent chain) and, in -// push mode, dispatches each pushable descriptor onto the push pool. Parent -// compression is deduplicated by flightcontrol inside computeBlobChain, so -// overlapping parent chains across workers are only compressed once. +// processRef compresses a ref's chain and, in push mode, dispatches pushable +// descriptors. Parent-chain compression is deduplicated lower in the cache. func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { ctx := ep.ctx s := session.NewGroup(ep.sessionID) refID := ref.ID() - // If the keep-set is already installed (e.g. queued items processed - // during shutdown drain), skip non-kept refs without paying for - // compression. + // Skip queued work that became irrelevant before compression started. if !ep.isKept(refID) { bklog.G(ctx).Debugf("eager compress skipped (filtered) ref=%s", refID) return nil @@ -404,8 +336,7 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { return nil } - // Re-check after the (potentially long) compress: the keep-set may - // have been installed while we were inside GetRemotes. + // The keep-set may have been installed during GetRemotes. if !ep.isKept(refID) { bklog.G(ctx).Debugf("eager push skipped after compress (filtered) ref=%s", refID) return nil @@ -435,10 +366,7 @@ func summarizeDescriptors(rems []*solver.Remote) (count int, totalBytes int64) { return } -// dispatchPushes sends every pushable descriptor in a ref's chain to the -// push pool and waits for all of them to complete. Within a single ref this -// fans out to up to len(Descriptors) parallel uploads (gated by the push -// pool size and the registry's concurrency cap). +// dispatchPushes fans a ref's pushable descriptors out to the push pool. func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems []*solver.Remote) error { if len(rems) == 0 { return nil @@ -489,10 +417,7 @@ func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems return nil } -// waitWithCtx blocks until wg reaches zero or ctx is cancelled. On -// cancellation it returns the ctx error without waiting (the WaitGroup -// counter may still be non-zero; it is safe to leak as the process is -// shutting down). On normal completion it returns nil. +// waitWithCtx returns early on context cancellation instead of blocking shutdown. func waitWithCtx(ctx context.Context, wg *sync.WaitGroup) error { done := make(chan struct{}) go func() { @@ -511,22 +436,8 @@ func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { return !images.IsNonDistributable(desc.MediaType) } -// pushDescriptor uploads a single descriptor, deduplicated by digest. After -// the first successful push of a digest in this build, subsequent calls -// short-circuit at the pushedDigests check before entering the -// flightcontrol closure — so they emit no log, acquire no registry -// semaphore slot, and do no HEAD round trip. -// -// pushDescriptor also participates in the keep-set filter: it tracks every -// refID requesting each digest, lets cancelNonKeptInflight abort uploads -// whose requesters are all non-kept, and skips outright when the keep-set -// is already installed and item.refID is not in it. -// -// Kept refs that arrive *after* a cancellation pass simply re-enter -// flightcontrol with a fresh ctx: either the prior closure has already -// torn down (g.m[digest] is gone) and we start a new push, or the prior -// closure is still live and our kept ctx keeps it alive past -// cancelNonKeptInflight's reach. +// pushDescriptor deduplicates by digest and records requesters so wait() can +// cancel pushes that only serve non-final refs. func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) error { digest := item.desc.Digest.String() if _, done := ep.pushedDigests.Load(digest); done { @@ -534,9 +445,7 @@ func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) return nil } - // Always record the requester first so cancelNonKeptInflight can - // decide whether any kept ref needs this digest, even if we filter - // below. + // Record before filtering so cancellation can see all requesters. tracker := ep.trackerFor(digest) tracker.mu.Lock() tracker.refIDs[item.refID] = struct{}{} @@ -570,9 +479,7 @@ func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) return struct{}{}, herr }) if err != nil { - // Distinguish a deliberate keep-set cancellation from a real - // failure: only the former has pushCtx done while the parent - // ctx is still alive. + // A cancelled pushCtx with a live parent means keep-set filtering won. if pushCtx.Err() != nil && ctx.Err() == nil { bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) @@ -587,23 +494,8 @@ func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) return nil } -// wait shuts down both pools in order: stop new compress senders, drain -// compressWork, then drain pushWork. Compress workers are guaranteed to -// have all dispatched their pushes by the time compressWG.Wait returns -// (because dispatchPushes blocks on the per-ref WaitGroup), so closing -// pushWork at that point is safe. -// -// The optional keep-set is the canonical set of ImmutableRef.ID()s that -// belong to the final image manifest, computed by the caller from the -// resolved frontend Result. When non-nil: -// - any in-flight push whose every requester is non-kept is cancelled -// immediately, freeing its registry-channel slot; -// - any items still queued in pushWork or compressWork for non-kept refs -// are skipped on dequeue; -// - kept refs continue to compress and push as normal. -// -// Pass nil to keep every ref (back-compat for non-gateway frontends or -// callers that haven't computed a keep-set). +// wait installs the optional keep-set, cancels in-flight pushes that only serve +// non-kept refs, then drains the compress and push pools in order. func (ep *eagerPipeline) wait(keep map[string]struct{}) error { ep.waitOnce.Do(func() { if keep != nil { @@ -637,25 +529,9 @@ func (ep *eagerPipeline) wait(keep map[string]struct{}) error { return ep.firstErr } -// cancelNonKeptInflight walks every digest currently being pushed and -// cancels the upload if no requester is in the keep-set. Returns the -// number of digests for which at least one cancel was issued. -// -// CRITICAL: flightcontrol's closure ctx is a sharedContext whose Done() -// only fires once *every* registered waiter ctx is done (see -// util/flightcontrol.sharedContext.checkDone). Cancelling a single -// waiter is not enough — the closure keeps running, the upload finishes, -// and we get a ~6 GB freebie we never wanted. So we cancel every waiter -// for the digest in one pass. -// -// A digest with at least one kept requester is left entirely alone: that -// kept waiter's pushDescriptor still needs the closure's result, and -// flightcontrol will deliver it. -// -// Cancelled uploads return from flightcontrol.Do with a ctx error; -// pushDescriptor recognises that as an intentional skip (pushCtx.Err -// non-nil while the parent ctx is alive) and converts it to a nil error -// so the build doesn't fail. +// cancelNonKeptInflight cancels digests whose requesters are all non-kept. +// All waiters for a digest must be cancelled, or flightcontrol keeps the +// shared push running. func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { var digestsCancelled int ep.inflight.Range(func(key, val any) bool { @@ -671,8 +547,7 @@ func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { } } if len(tracker.cancels) == 0 { - // Nothing in flight — queued items will be filtered by - // isKept() at dequeue. + // Queued items will be filtered at dequeue. return true } n := len(tracker.cancels) @@ -688,8 +563,7 @@ func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { return digestsCancelled } -// drainCompress releases any refs left in compressWork after workers have -// exited (e.g. via ctx cancellation before the channel was drained). +// drainCompress releases refs left behind after workers exit. func (ep *eagerPipeline) drainCompress() { for { select { @@ -704,8 +578,7 @@ func (ep *eagerPipeline) drainCompress() { } } -// drainPushWork marks any leftover push items' WaitGroups as Done so that -// dispatching compress workers (if any are still blocked) can unblock. +// drainPushWork unblocks any dispatchers still waiting on queued push items. func (ep *eagerPipeline) drainPushWork() { for { select { diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index 72d2d5ba81c2..efac37439a71 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -145,8 +145,7 @@ func TestEagerPipeline_PushWorkerExitsOnChannelClose(t *testing.T) { ep.pushWG.Wait() } -// Late fires of onVertexComplete must release the clone instead of sending -// into the (now closed) work channel. +// Late callbacks must be ignored after wait() starts shutdown. func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ @@ -166,7 +165,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { assert.Zero(t, released.Load()) } -// Many concurrent late fires after wait() must be rejected before cloning. +// Concurrent late callbacks must not clone refs. func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ @@ -194,8 +193,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { assert.Zero(t, released.Load()) } -// A sender admitted before wait() but blocked on a full queue must take the -// done path, release its clone, and exit without panic. +// Blocked senders must release clones when wait() closes the done channel. func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) { var cloned atomic.Int32 var released atomic.Int32 @@ -231,8 +229,7 @@ func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) assert.Equal(t, int32(2), released.Load()) } -// Fires racing concurrently with wait() must not panic and must not leak: -// every admitted clone is either drained by wait() or released by the sender. +// Callbacks racing with wait() must not panic or leak clones. func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ @@ -303,10 +300,7 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { }, pushed) } -// pushDescriptor must short-circuit on the second call for the same digest, -// without invoking the handler again. This is the fix for the noisy -// "eager pushing blob" log entries that appeared once per shared parent -// blob per ref. +// Repeated pushes of the same digest should only call the handler once. func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("shared-blob"), @@ -330,8 +324,7 @@ func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once across repeated calls for the same digest") } -// pushDescriptor must skip — without invoking the handler — when the -// keep-set is installed and the requesting refID is not in it. +// Non-kept refs should be tracked but not pushed. func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("intermediate-blob"), @@ -355,8 +348,6 @@ func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { })) assert.Zero(t, calls.Load(), "non-kept ref must not invoke the push handler") - // And the digest must still appear in the inflight tracker so - // cancelNonKeptInflight can reason about it. v, ok := ep.inflight.Load(desc.Digest.String()) require.True(t, ok) tracker := v.(*pushTracker) @@ -366,8 +357,7 @@ func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { assert.True(t, hasRef, "tracker must record requester even when filtered") } -// pushDescriptor must still push when at least one kept ref requests the -// digest, even if a non-kept ref also asked for it. +// A kept requester should push even if the digest was first requested by a non-kept ref. func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("shared-blob"), @@ -397,17 +387,13 @@ func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { assert.Equal(t, int32(1), calls.Load(), "kept-ref call must drive exactly one push") } -// cancelNonKeptInflight must cancel every waiter for a digest whose -// requesters are all non-kept, and must leave digests with at least one -// kept requester completely alone (because the kept waiter's -// flightcontrol.Do still needs the closure result). +// cancelNonKeptInflight cancels only digests with no kept requesters. func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { ep := &eagerPipeline{ctx: context.Background()} keep := map[string]struct{}{"final-ref": {}} - // Digest A: two non-kept requesters in flight. Both cancels must fire, - // matching flightcontrol's all-waiters-must-cancel semantics. + // Digest A: all requesters are non-kept, so every waiter must cancel. var cancelA1, cancelA2 atomic.Bool ep.inflight.Store("digest-a", &pushTracker{ refIDs: map[string]struct{}{"int-ref-1": {}, "int-ref-2": {}}, @@ -416,8 +402,7 @@ func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { "int-ref-2": func() { cancelA2.Store(true) }, }, }) - // Digest B: requested by both kept and non-kept — must be spared - // even though the non-kept waiter has a cancel registered. + // Digest B: one kept requester keeps the shared push alive. var cancelB atomic.Bool ep.inflight.Store("digest-b", &pushTracker{ refIDs: map[string]struct{}{"int-ref-3": {}, "final-ref": {}}, @@ -425,8 +410,7 @@ func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { "int-ref-3": func() { cancelB.Store(true) }, }, }) - // Digest C: non-kept requester but no cancel yet (still queued). - // Must not be counted and must not panic. + // Digest C: queued work has no waiter to cancel. ep.inflight.Store("digest-c", &pushTracker{ refIDs: map[string]struct{}{"int-ref-4": {}}, }) @@ -441,9 +425,7 @@ func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { assert.Empty(t, trackerA.(*pushTracker).cancels, "cancels map should be drained after cancellation") } -// After cancelNonKeptInflight has fired, a kept ref that arrives later -// must not be skipped — it still needs to push the blob (the previous -// closure was cancelled before completion). +// A kept ref arriving after cancellation should start a fresh push. func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("retry-after-cancel"), @@ -460,9 +442,7 @@ func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { keep := map[string]struct{}{"final-ref": {}} ep.keepRefIDs.Store(&keep) - // Simulate a prior cancel pass: tracker exists with the requester - // recorded and an empty cancels map (the in-flight waiter already - // got cancelled and unregistered itself). + // Simulate a prior cancel pass that already removed its waiter. ep.inflight.Store(desc.Digest.String(), &pushTracker{ refIDs: map[string]struct{}{"int-ref": {}}, cancels: map[string]context.CancelFunc{}, @@ -476,8 +456,7 @@ func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { assert.Equal(t, int32(1), calls.Load(), "kept ref must drive a fresh push after a prior cancellation") } -// releaseTracker is a minimal cache.ImmutableRef stub that counts -// Release calls. Clones share the counter. +// releaseTracker counts Release calls across clones. type releaseTracker struct { cache.ImmutableRef released *atomic.Int32 diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 9d67c8920fbd..87bb2c8f8db1 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -56,8 +56,7 @@ const ( keyEagerPushConfig = "llb.eagerpushconfig" ) -// EagerExportMode controls whether layer compression and/or pushing -// happens concurrently with the build, rather than after all vertices complete. +// EagerExportMode controls whether layers are handled during the build. type EagerExportMode int const ( @@ -563,10 +562,8 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return ctx, nil } - // Set up eager export pipeline before the build starts so the vertex - // completion callback can kick off compression/push during the build. - // When eager export is active, the lease must be created early so - // compressed blobs are GC-protected during the build phase. + // Start eager export before the build so completed vertices can enqueue work. + // The lease is needed early to protect compressed blobs during the build. var eager *eagerPipeline if exp.EagerExport != EagerExportNone && len(exp.Exporters) > 0 { ctx, err = createLease(ctx) @@ -656,17 +653,9 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - // Wait for all eager compression/push jobs to finish before running - // exporters. The manifest needs final blob digests from every layer. + // Exporters need the final layer digests, so wait for eager work first. if eager != nil { - // Compute the keep-set: the set of ImmutableRef.ID()s that - // belong to the final exported result. Any ref that the - // solver's vertex callback fired for but that didn't end up in - // res (e.g. intermediate stages from a multi-stage Dockerfile, - // or the cache_warmer pattern's source layers) is filtered out - // — its compressed blobs stay in the local content store, but - // no registry bandwidth is spent on it. With a gateway - // frontend, res is final by this point so the set is complete. + // Filter out completed vertices that are not part of the final result. keep := computeEagerKeepSet(ctx, res) if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) @@ -709,9 +698,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } - // When eager export is not active, the lease is created here (the original - // location) — after the build completes but before export. This avoids - // adding a disk write before gateway forwarder registration. + // Without eager export, keep the original lease timing. if eager == nil { ctx, err = createLease(ctx) if err != nil { From 238ab9d352fe23b9a22a58c08de9d47597c91e89 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 28 Apr 2026 13:28:22 -0700 Subject: [PATCH 50/55] cleanup --- solver/llbsolver/eager.go | 155 +++++++++++++-------------------- solver/llbsolver/eager_test.go | 84 +++++++++--------- solver/llbsolver/solver.go | 6 +- 3 files changed, 108 insertions(+), 137 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 6f4e89e635c0..64301052f343 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -44,8 +44,7 @@ type eagerPushItem struct { refID string desc ocispecs.Descriptor handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error) - wg *sync.WaitGroup - errCh chan<- error + result chan<- error } // eagerPipeline compresses refs as vertices complete and optionally pushes @@ -79,9 +78,9 @@ type eagerPipeline struct { pushDedup flightcontrol.Group[struct{}] pushedDigests sync.Map - // keepRefIDs is nil until wait() installs the final image ref set. After - // that, non-kept refs are skipped or cancelled. - keepRefIDs atomic.Pointer[map[string]struct{}] + // exportRefIDs is nil until applyExportRefs installs the final export refs. + // After that, non-export refs are skipped or cancelled. + exportRefIDs atomic.Pointer[map[string]struct{}] // inflight tracks requesters and cancel funcs per digest. inflight sync.Map // map[string]*pushTracker @@ -188,14 +187,12 @@ func (ep *eagerPipeline) pushWorker() { if !ok { return } - if err := ep.pushDescriptor(ep.ctx, item); err != nil { + if err := ep.pushDescriptor(item); err != nil { ep.recordErr(err) - select { - case item.errCh <- err: - default: - } + item.result <- err + } else { + item.result <- nil } - item.wg.Done() } } @@ -241,13 +238,13 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } } -// computeEagerKeepSet returns every ref ID that contributes to the final image, +// computeEagerExportRefs returns every ref ID that contributes to the final image, // including parent-chain layers. Each layer has its own vertex/ref ID, so -// keeping only the final ref would filter out the eager layer pushes. -func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]struct{} { - keep := make(map[string]struct{}) +// exporting only the final ref would filter out the eager layer pushes. +func computeEagerExportRefs(ctx context.Context, res *frontend.Result) map[string]struct{} { + exportRefs := make(map[string]struct{}) if res == nil { - return keep + return exportRefs } res.EachRef(func(rp solver.ResultProxy) error { if rp == nil { @@ -255,7 +252,7 @@ func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]s } cached, err := rp.Result(ctx) if err != nil { - bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to resolve ref") + bklog.G(ctx).WithError(err).Warnf("eager export ref set: failed to resolve ref") return nil } workerRef, ok := cached.Sys().(*worker.WorkerRef) @@ -268,31 +265,23 @@ func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]s if layer == nil { continue } - keep[layer.ID()] = struct{}{} + exportRefs[layer.ID()] = struct{}{} } if err := chain.Release(context.WithoutCancel(ctx)); err != nil { - bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to release chain clones") + bklog.G(ctx).WithError(err).Warnf("eager export ref set: failed to release chain clones") } return nil }) - return keep -} - -// keepSet returns nil until filtering is enabled. -func (ep *eagerPipeline) keepSet() map[string]struct{} { - if p := ep.keepRefIDs.Load(); p != nil { - return *p - } - return nil + return exportRefs } -// isKept treats every ref as kept until a keep-set is installed. -func (ep *eagerPipeline) isKept(refID string) bool { - keep := ep.keepSet() - if keep == nil { +// isExportRef treats every ref as exportable until an export ref set is installed. +func (ep *eagerPipeline) isExportRef(refID string) bool { + exportRefs := ep.exportRefIDs.Load() + if exportRefs == nil { return true } - _, ok := keep[refID] + _, ok := (*exportRefs)[refID] return ok } @@ -314,7 +303,7 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { refID := ref.ID() // Skip queued work that became irrelevant before compression started. - if !ep.isKept(refID) { + if !ep.isExportRef(refID) { bklog.G(ctx).Debugf("eager compress skipped (filtered) ref=%s", refID) return nil } @@ -336,8 +325,8 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { return nil } - // The keep-set may have been installed during GetRemotes. - if !ep.isKept(refID) { + // The export ref set may have been installed during GetRemotes. + if !ep.isExportRef(refID) { bklog.G(ctx).Debugf("eager push skipped after compress (filtered) ref=%s", refID) return nil } @@ -373,98 +362,75 @@ func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems } remote := rems[0] + resultCh := make(chan error, len(remote.Descriptors)) handler := retryhandler.New( limited.PushHandler(ep.pusher, remote.Provider, ep.pushCfg.TargetName), nil, ) - var wg sync.WaitGroup - errCh := make(chan error, len(remote.Descriptors)) - enqueued := 0 for _, desc := range remote.Descriptors { if !shouldEagerPushDesc(desc) { continue } - wg.Add(1) item := eagerPushItem{ refID: refID, desc: desc, handler: handler, - wg: &wg, - errCh: errCh, + result: resultCh, } select { case ep.pushWork <- item: enqueued++ case <-ctx.Done(): - wg.Done() - waitWithCtx(ctx, &wg) return context.Cause(ctx) } } - if err := waitWithCtx(ctx, &wg); err != nil { - return err - } - - close(errCh) - for e := range errCh { - if e != nil { - return e + for range enqueued { + select { + case err := <-resultCh: + if err != nil { + return err + } + case <-ctx.Done(): + return context.Cause(ctx) } } return nil } -// waitWithCtx returns early on context cancellation instead of blocking shutdown. -func waitWithCtx(ctx context.Context, wg *sync.WaitGroup) error { - done := make(chan struct{}) - go func() { - wg.Wait() - close(done) - }() - select { - case <-done: - return nil - case <-ctx.Done(): - return context.Cause(ctx) - } -} - func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { return !images.IsNonDistributable(desc.MediaType) } // pushDescriptor deduplicates by digest and records requesters so wait() can // cancel pushes that only serve non-final refs. -func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) error { +func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { + ctx := ep.ctx digest := item.desc.Digest.String() if _, done := ep.pushedDigests.Load(digest); done { bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) return nil } - // Record before filtering so cancellation can see all requesters. tracker := ep.trackerFor(digest) + pushCtx, cancel := context.WithCancel(ctx) + defer cancel() + tracker.mu.Lock() tracker.refIDs[item.refID] = struct{}{} - tracker.mu.Unlock() - - if !ep.isKept(item.refID) { + if !ep.isExportRef(item.refID) { + tracker.mu.Unlock() bklog.G(ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) return nil } - - pushCtx, cancel := context.WithCancel(ctx) - defer cancel() - - tracker.mu.Lock() if tracker.cancels == nil { tracker.cancels = make(map[string]context.CancelFunc) } tracker.cancels[item.refID] = cancel tracker.mu.Unlock() + defer func() { tracker.mu.Lock() delete(tracker.cancels, item.refID) @@ -479,7 +445,7 @@ func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) return struct{}{}, herr }) if err != nil { - // A cancelled pushCtx with a live parent means keep-set filtering won. + // A cancelled pushCtx with a live parent means export ref set filtering won. if pushCtx.Err() != nil && ctx.Err() == nil { bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) @@ -494,17 +460,20 @@ func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) return nil } -// wait installs the optional keep-set, cancels in-flight pushes that only serve -// non-kept refs, then drains the compress and push pools in order. -func (ep *eagerPipeline) wait(keep map[string]struct{}) error { - ep.waitOnce.Do(func() { - if keep != nil { - cp := keep - ep.keepRefIDs.Store(&cp) - n := ep.cancelNonKeptInflight(keep) - bklog.G(ep.ctx).Infof("eager wait keep_set_size=%d cancelled_inflight=%d", len(keep), n) - } +// applyExportRefs installs the final export refs and cancels in-flight pushes that +// only serve non-export refs. +func (ep *eagerPipeline) applyExportRefs(exportRefs map[string]struct{}) int { + if exportRefs == nil { + return 0 + } + cp := exportRefs + ep.exportRefIDs.Store(&cp) + return ep.cancelNonExportInflight(exportRefs) +} +// wait drains the compress and push pools in order. +func (ep *eagerPipeline) wait() error { + ep.waitOnce.Do(func() { ep.closeMu.Lock() if ep.done == nil { ep.done = make(chan struct{}) @@ -529,10 +498,10 @@ func (ep *eagerPipeline) wait(keep map[string]struct{}) error { return ep.firstErr } -// cancelNonKeptInflight cancels digests whose requesters are all non-kept. -// All waiters for a digest must be cancelled, or flightcontrol keeps the +// cancelNonExportInflight cancels digests whose requesters are all non-export. +// All waiters for a digest must be cancelled, or flightcontrol leaves the // shared push running. -func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { +func (ep *eagerPipeline) cancelNonExportInflight(exportRefs map[string]struct{}) int { var digestsCancelled int ep.inflight.Range(func(key, val any) bool { digest := key.(string) @@ -542,7 +511,7 @@ func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { defer tracker.mu.Unlock() for refID := range tracker.refIDs { - if _, ok := keep[refID]; ok { + if _, ok := exportRefs[refID]; ok { return true } } @@ -586,7 +555,7 @@ func (ep *eagerPipeline) drainPushWork() { if !ok { return } - item.wg.Done() + item.result <- nil default: return } diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index efac37439a71..c9485123b4b2 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -67,7 +67,7 @@ func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { } ep.firstErr = assert.AnError - err := ep.wait(nil) + err := ep.wait() assert.Equal(t, assert.AnError, err) } @@ -76,7 +76,7 @@ func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { compressWork: make(chan eagerWorkItem), } - err := ep.wait(nil) + err := ep.wait() assert.NoError(t, err) } @@ -89,7 +89,7 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} - err := ep.wait(nil) + err := ep.wait() require.NoError(t, err) assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") } @@ -99,8 +99,8 @@ func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { compressWork: make(chan eagerWorkItem), } - require.NoError(t, ep.wait(nil)) - require.NoError(t, ep.wait(nil), "second wait must not panic") + require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(), "second wait must not panic") } func TestEagerPipeline_CompressWorkerExitsOnContextCancel(t *testing.T) { @@ -153,7 +153,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { compressWork: make(chan eagerWorkItem, 10), done: make(chan struct{}), } - require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait()) var released atomic.Int32 res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) @@ -173,7 +173,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { compressWork: make(chan eagerWorkItem, 10), done: make(chan struct{}), } - require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait()) var released atomic.Int32 const fires = 100 @@ -222,7 +222,7 @@ func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) require.Equal(t, int32(1), cloned.Load()) require.NotPanics(t, func() { - require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait()) }) senderWg.Wait() @@ -252,7 +252,7 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { } require.NotPanics(t, func() { - require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait()) }) wg.Wait() @@ -276,7 +276,7 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { } var pushed []digest.Digest - ep := &eagerPipeline{} + ep := &eagerPipeline{ctx: context.Background()} handler := func(_ context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { pushed = append(pushed, desc.Digest) return nil, nil @@ -286,7 +286,7 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { if !shouldEagerPushDesc(desc) { continue } - err := ep.pushDescriptor(context.Background(), eagerPushItem{ + err := ep.pushDescriptor(eagerPushItem{ refID: "test-ref", desc: desc, handler: handler, @@ -313,9 +313,9 @@ func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { return nil, nil } - ep := &eagerPipeline{} + ep := &eagerPipeline{ctx: context.Background()} for range 5 { - require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + require.NoError(t, ep.pushDescriptor(eagerPushItem{ refID: "ref-test", desc: desc, handler: handler, @@ -324,8 +324,8 @@ func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once across repeated calls for the same digest") } -// Non-kept refs should be tracked but not pushed. -func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { +// Non-export refs should be tracked but not pushed. +func TestEagerPushDescriptor_SkipsNonExportRef(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("intermediate-blob"), MediaType: ocispecs.MediaTypeImageLayerGzip, @@ -337,16 +337,16 @@ func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { return nil, nil } - ep := &eagerPipeline{} - keep := map[string]struct{}{"final-ref": {}} - ep.keepRefIDs.Store(&keep) + ep := &eagerPipeline{ctx: context.Background()} + exportRefs := map[string]struct{}{"final-ref": {}} + ep.exportRefIDs.Store(&exportRefs) - require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + require.NoError(t, ep.pushDescriptor(eagerPushItem{ refID: "intermediate-ref", desc: desc, handler: handler, })) - assert.Zero(t, calls.Load(), "non-kept ref must not invoke the push handler") + assert.Zero(t, calls.Load(), "non-export ref must not invoke the push handler") v, ok := ep.inflight.Load(desc.Digest.String()) require.True(t, ok) @@ -357,8 +357,8 @@ func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { assert.True(t, hasRef, "tracker must record requester even when filtered") } -// A kept requester should push even if the digest was first requested by a non-kept ref. -func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { +// An export requester should push even if the digest was first requested by a non-export ref. +func TestEagerPushDescriptor_PushesWhenAnyRequesterExported(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("shared-blob"), MediaType: ocispecs.MediaTypeImageLayerGzip, @@ -370,30 +370,30 @@ func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { return nil, nil } - ep := &eagerPipeline{} - keep := map[string]struct{}{"final-ref": {}} - ep.keepRefIDs.Store(&keep) + ep := &eagerPipeline{ctx: context.Background()} + exportRefs := map[string]struct{}{"final-ref": {}} + ep.exportRefIDs.Store(&exportRefs) - require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + require.NoError(t, ep.pushDescriptor(eagerPushItem{ refID: "intermediate-ref", desc: desc, handler: handler, })) - require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + require.NoError(t, ep.pushDescriptor(eagerPushItem{ refID: "final-ref", desc: desc, handler: handler, })) - assert.Equal(t, int32(1), calls.Load(), "kept-ref call must drive exactly one push") + assert.Equal(t, int32(1), calls.Load(), "export-ref call must drive exactly one push") } -// cancelNonKeptInflight cancels only digests with no kept requesters. -func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { +// cancelNonExportInflight cancels only digests with no export requesters. +func TestEagerPipeline_CancelNonExportInflight(t *testing.T) { ep := &eagerPipeline{ctx: context.Background()} - keep := map[string]struct{}{"final-ref": {}} + exportRefs := map[string]struct{}{"final-ref": {}} - // Digest A: all requesters are non-kept, so every waiter must cancel. + // Digest A: all requesters are non-export, so every waiter must cancel. var cancelA1, cancelA2 atomic.Bool ep.inflight.Store("digest-a", &pushTracker{ refIDs: map[string]struct{}{"int-ref-1": {}, "int-ref-2": {}}, @@ -402,7 +402,7 @@ func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { "int-ref-2": func() { cancelA2.Store(true) }, }, }) - // Digest B: one kept requester keeps the shared push alive. + // Digest B: one export requester leaves the shared push alive. var cancelB atomic.Bool ep.inflight.Store("digest-b", &pushTracker{ refIDs: map[string]struct{}{"int-ref-3": {}, "final-ref": {}}, @@ -415,18 +415,18 @@ func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { refIDs: map[string]struct{}{"int-ref-4": {}}, }) - n := ep.cancelNonKeptInflight(keep) + n := ep.cancelNonExportInflight(exportRefs) assert.Equal(t, 1, n, "exactly one digest's waiters should be cancelled") assert.True(t, cancelA1.Load(), "digest A waiter 1 must be cancelled") assert.True(t, cancelA2.Load(), "digest A waiter 2 must be cancelled") - assert.False(t, cancelB.Load(), "digest B must be spared (kept requester)") + assert.False(t, cancelB.Load(), "digest B must be spared (export requester)") trackerA, _ := ep.inflight.Load("digest-a") assert.Empty(t, trackerA.(*pushTracker).cancels, "cancels map should be drained after cancellation") } -// A kept ref arriving after cancellation should start a fresh push. -func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { +// An export ref arriving after cancellation should start a fresh push. +func TestEagerPushDescriptor_ExportRefAfterCancelStillPushes(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("retry-after-cancel"), MediaType: ocispecs.MediaTypeImageLayerGzip, @@ -438,9 +438,9 @@ func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { return nil, nil } - ep := &eagerPipeline{} - keep := map[string]struct{}{"final-ref": {}} - ep.keepRefIDs.Store(&keep) + ep := &eagerPipeline{ctx: context.Background()} + exportRefs := map[string]struct{}{"final-ref": {}} + ep.exportRefIDs.Store(&exportRefs) // Simulate a prior cancel pass that already removed its waiter. ep.inflight.Store(desc.Digest.String(), &pushTracker{ @@ -448,12 +448,12 @@ func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { cancels: map[string]context.CancelFunc{}, }) - require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ + require.NoError(t, ep.pushDescriptor(eagerPushItem{ refID: "final-ref", desc: desc, handler: handler, })) - assert.Equal(t, int32(1), calls.Load(), "kept ref must drive a fresh push after a prior cancellation") + assert.Equal(t, int32(1), calls.Load(), "export ref must drive a fresh push after a prior cancellation") } // releaseTracker counts Release calls across clones. diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 87bb2c8f8db1..9588081dbece 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -656,10 +656,12 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // Exporters need the final layer digests, so wait for eager work first. if eager != nil { // Filter out completed vertices that are not part of the final result. - keep := computeEagerKeepSet(ctx, res) + exportRefs := computeEagerExportRefs(ctx, res) + cancelled := eager.applyExportRefs(exportRefs) + bklog.G(ctx).Infof("eager export_refs=%d cancelled_inflight=%d", len(exportRefs), cancelled) if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) - err := eager.wait(keep) + err := eager.wait() tracing.FinishWithError(span, err) return err }); err != nil { From 0244ef5c2944a56efe922cadd10c266bdf91d841 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 28 Apr 2026 14:14:35 -0700 Subject: [PATCH 51/55] address comments --- solver/llbsolver/eager.go | 39 +++++++++++++++----- solver/llbsolver/eager_test.go | 67 ++++++++++++++++++++++++++++++++++ solver/llbsolver/solver.go | 11 ++++++ 3 files changed, 107 insertions(+), 10 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 64301052f343..6ead23c20bb2 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -69,13 +69,14 @@ type eagerPipeline struct { waitOnce sync.Once // ctx carries the lease that keeps compressed blobs GC-protected. - ctx context.Context + ctx context.Context + cancel context.CancelCauseFunc pusher remotes.Pusher // pushDedup coalesces concurrent pushes; pushedDigests skips digests that // already succeeded in this build. - pushDedup flightcontrol.Group[struct{}] + pushDedup flightcontrol.Group[bool] pushedDigests sync.Map // exportRefIDs is nil until applyExportRefs installs the final export refs. @@ -127,6 +128,7 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio } } + pipelineCtx, cancel := context.WithCancelCause(ctx) ep := &eagerPipeline{ mode: mode, refCfg: cacheconfig.RefConfig{ @@ -135,7 +137,8 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio sessionID: sessionID, pushCfg: pushCfg, pusher: pusher, - ctx: ctx, + ctx: pipelineCtx, + cancel: cancel, compressWork: make(chan eagerWorkItem, 256), pushWork: make(chan eagerPushItem, 1024), done: make(chan struct{}), @@ -362,6 +365,8 @@ func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems } remote := rems[0] + // Buffered to the descriptor count so workers can report completion even if + // dispatchPushes returns early on context cancellation. resultCh := make(chan error, len(remote.Descriptors)) handler := retryhandler.New( limited.PushHandler(ep.pusher, remote.Provider, ep.pushCfg.TargetName), @@ -415,8 +420,8 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { } tracker := ep.trackerFor(digest) - pushCtx, cancel := context.WithCancel(ctx) - defer cancel() + pushCtx, cancel := context.WithCancelCause(ctx) + defer cancel(errors.WithStack(context.Canceled)) tracker.mu.Lock() tracker.refIDs[item.refID] = struct{}{} @@ -428,7 +433,7 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { if tracker.cancels == nil { tracker.cancels = make(map[string]context.CancelFunc) } - tracker.cancels[item.refID] = cancel + tracker.cancels[item.refID] = func() { cancel(errors.WithStack(context.Canceled)) } tracker.mu.Unlock() defer func() { @@ -440,21 +445,31 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { start := time.Now() bklog.G(ctx).Infof("eager push starting ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) - _, err := ep.pushDedup.Do(pushCtx, digest, func(ctx context.Context) (struct{}, error) { + pushed, err := ep.pushDedup.Do(pushCtx, digest, func(ctx context.Context) (bool, error) { + if _, done := ep.pushedDigests.Load(digest); done { + return false, nil + } _, herr := item.handler(ctx, item.desc) - return struct{}{}, herr + if herr != nil { + return false, herr + } + ep.pushedDigests.Store(digest, struct{}{}) + return true, nil }) if err != nil { // A cancelled pushCtx with a live parent means export ref set filtering won. - if pushCtx.Err() != nil && ctx.Err() == nil { + if context.Cause(pushCtx) != nil && context.Cause(ctx) == nil { bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) return nil } return err } + if !pushed { + bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } - ep.pushedDigests.Store(digest, struct{}{}) bklog.G(ctx).Infof("eager push done ref=%s digest=%s size=%d duration=%s", item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) return nil @@ -474,6 +489,10 @@ func (ep *eagerPipeline) applyExportRefs(exportRefs map[string]struct{}) int { // wait drains the compress and push pools in order. func (ep *eagerPipeline) wait() error { ep.waitOnce.Do(func() { + if ep.cancel != nil { + defer ep.cancel(errors.WithStack(context.Canceled)) + } + ep.closeMu.Lock() if ep.done == nil { ep.done = make(chan struct{}) diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index c9485123b4b2..a173f60843c6 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -145,6 +145,32 @@ func TestEagerPipeline_PushWorkerExitsOnChannelClose(t *testing.T) { ep.pushWG.Wait() } +func TestEagerPipeline_PushWorkerErrorReturnedByWait(t *testing.T) { + ep := &eagerPipeline{ + ctx: context.Background(), + compressWork: make(chan eagerWorkItem), + pushWork: make(chan eagerPushItem, 1), + } + ep.pushWG.Add(1) + go ep.pushWorker() + + resultCh := make(chan error, 1) + ep.pushWork <- eagerPushItem{ + refID: "test-ref", + desc: ocispecs.Descriptor{ + Digest: digest.FromString("push-error"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + }, + handler: func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + return nil, assert.AnError + }, + result: resultCh, + } + + require.Equal(t, assert.AnError, <-resultCh) + require.Equal(t, assert.AnError, ep.wait()) +} + // Late callbacks must be ignored after wait() starts shutdown. func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { var cloned atomic.Int32 @@ -324,6 +350,47 @@ func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once across repeated calls for the same digest") } +func TestEagerPushDescriptor_DedupsConcurrentPushes(t *testing.T) { + desc := ocispecs.Descriptor{ + Digest: digest.FromString("shared-blob"), + MediaType: ocispecs.MediaTypeImageLayerGzip, + Size: 1234, + } + var calls atomic.Int32 + started := make(chan struct{}) + release := make(chan struct{}) + handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { + if calls.Add(1) == 1 { + close(started) + } + <-release + return nil, nil + } + + ep := &eagerPipeline{ctx: context.Background()} + errCh := make(chan error, 2) + go func() { + errCh <- ep.pushDescriptor(eagerPushItem{ + refID: "ref-a", + desc: desc, + handler: handler, + }) + }() + <-started + go func() { + errCh <- ep.pushDescriptor(eagerPushItem{ + refID: "ref-b", + desc: desc, + handler: handler, + }) + }() + close(release) + + require.NoError(t, <-errCh) + require.NoError(t, <-errCh) + assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once for concurrent pushes of the same digest") +} + // Non-export refs should be tracked but not pushed. func TestEagerPushDescriptor_SkipsNonExportRef(t *testing.T) { desc := ocispecs.Descriptor{ diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 9588081dbece..3f1355f00b89 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -565,6 +565,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // Start eager export before the build so completed vertices can enqueue work. // The lease is needed early to protect compressed blobs during the build. var eager *eagerPipeline + eagerWaited := false if exp.EagerExport != EagerExportNone && len(exp.Exporters) > 0 { ctx, err = createLease(ctx) if err != nil { @@ -576,6 +577,15 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro return nil, err } j.SetOnVertexComplete(eager.onVertexComplete) + defer func() { + if eagerWaited { + return + } + eager.cancel(errors.WithStack(context.Canceled)) + if err := eager.wait(); err != nil { + bklog.G(ctx).WithError(err).Warnf("eager export cleanup failed") + } + }() } if exp.PushRegistryConfig != nil { @@ -662,6 +672,7 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) err := eager.wait() + eagerWaited = true tracing.FinishWithError(span, err) return err }); err != nil { From 2d9d471519f1c2d9dc5023dd2823219473aaa990 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 28 Apr 2026 14:23:00 -0700 Subject: [PATCH 52/55] don't be too spammy --- solver/llbsolver/eager.go | 48 +++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 6ead23c20bb2..335c3f17f755 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -23,9 +23,12 @@ import ( pushutil "github.com/moby/buildkit/util/push" "github.com/moby/buildkit/util/resolver/limited" "github.com/moby/buildkit/util/resolver/retryhandler" + "github.com/moby/buildkit/util/tracing" "github.com/moby/buildkit/worker" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) // Keep the pools large enough for a ref's descriptor chain to fan out without @@ -311,18 +314,20 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { return nil } - bklog.G(ctx).Infof("eager compress starting ref=%s", refID) - compressStart := time.Now() - rems, err := ref.GetRemotes(ctx, true, ep.refCfg, false, s) + compressSpan, compressCtx := tracing.StartSpan(ctx, "eager compress ref", trace.WithAttributes( + attribute.String("ref.id", refID), + )) + rems, err := ref.GetRemotes(compressCtx, true, ep.refCfg, false, s) + descCount, totalBytes := summarizeDescriptors(rems) + compressSpan.SetAttributes( + attribute.Int("descriptors", descCount), + attribute.Int64("bytes", totalBytes), + ) + tracing.FinishWithError(compressSpan, err) if err != nil { bklog.G(ctx).WithError(err).Warnf("eager compress failed ref=%s", refID) return err } - compressDur := time.Since(compressStart).Round(time.Millisecond) - - descCount, totalBytes := summarizeDescriptors(rems) - bklog.G(ctx).Infof("eager compress done ref=%s descriptors=%d bytes=%d duration=%s", - refID, descCount, totalBytes, compressDur) if ep.mode != EagerExportPush { return nil @@ -414,20 +419,27 @@ func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { ctx := ep.ctx digest := item.desc.Digest.String() + span, pushCtx := tracing.StartSpan(ctx, "eager push descriptor", trace.WithAttributes( + attribute.String("ref.id", item.refID), + attribute.String("digest", digest), + attribute.Int64("size", item.desc.Size), + )) if _, done := ep.pushedDigests.Load(digest); done { - bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + span.SetAttributes(attribute.Bool("deduped", true)) + span.End() return nil } tracker := ep.trackerFor(digest) - pushCtx, cancel := context.WithCancelCause(ctx) + pushCtx, cancel := context.WithCancelCause(pushCtx) defer cancel(errors.WithStack(context.Canceled)) tracker.mu.Lock() tracker.refIDs[item.refID] = struct{}{} if !ep.isExportRef(item.refID) { tracker.mu.Unlock() - bklog.G(ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + span.SetAttributes(attribute.Bool("filtered", true)) + span.End() return nil } if tracker.cancels == nil { @@ -442,9 +454,6 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { tracker.mu.Unlock() }() - start := time.Now() - bklog.G(ctx).Infof("eager push starting ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) - pushed, err := ep.pushDedup.Do(pushCtx, digest, func(ctx context.Context) (bool, error) { if _, done := ep.pushedDigests.Load(digest); done { return false, nil @@ -459,19 +468,20 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { if err != nil { // A cancelled pushCtx with a live parent means export ref set filtering won. if context.Cause(pushCtx) != nil && context.Cause(ctx) == nil { - bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", - item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + span.SetAttributes(attribute.Bool("cancelled", true)) + span.End() return nil } + tracing.FinishWithError(span, err) return err } if !pushed { - bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + span.SetAttributes(attribute.Bool("deduped", true)) + span.End() return nil } - bklog.G(ctx).Infof("eager push done ref=%s digest=%s size=%d duration=%s", - item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + span.End() return nil } From 7afc8952a996c190a5a1081a386cad854234dc02 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 28 Apr 2026 15:06:10 -0700 Subject: [PATCH 53/55] add comments --- solver/llbsolver/eager.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 335c3f17f755..b442e3c44160 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -50,8 +50,10 @@ type eagerPushItem struct { result chan<- error } -// eagerPipeline compresses refs as vertices complete and optionally pushes -// their descriptors through a separate pool for per-chain parallelism. +// eagerPipeline receives completed vertex refs, compresses their layer chains, +// and optionally fans out pushable descriptors to a separate push pool. Once +// the final export refs are known, it filters/cancels non-export work and +// drains both pools. type eagerPipeline struct { mode EagerExportMode refCfg cacheconfig.RefConfig @@ -92,8 +94,12 @@ type eagerPipeline struct { firstErr error } -// pushTracker records every ref that requested a digest and every active -// waiter. flightcontrol only cancels the shared push when all waiters cancel. +// pushTracker is per digest. refIDs records every ref that requested the blob, +// including refs later filtered out of the final export; cancels records only +// active push waiters. When the final export refs are known, a digest is +// cancelled only if none of its requesters are part of the export. If it is +// cancelled, every active waiter must be cancelled because flightcontrol keeps +// the shared push running until all waiter contexts are done. type pushTracker struct { mu sync.Mutex refIDs map[string]struct{} @@ -434,6 +440,8 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { pushCtx, cancel := context.WithCancelCause(pushCtx) defer cancel(errors.WithStack(context.Canceled)) + // Register requester and cancel func atomically with export-ref filtering + // so cancelNonExportInflight cannot miss an about-to-start push. tracker.mu.Lock() tracker.refIDs[item.refID] = struct{}{} if !ep.isExportRef(item.refID) { From f60202c4c80fe81b3842d6d1eb32e8836188c186 Mon Sep 17 00:00:00 2001 From: Amy Date: Tue, 28 Apr 2026 18:22:24 -0700 Subject: [PATCH 54/55] address comments --- solver/llbsolver/eager.go | 27 ++++++++++++--------------- solver/llbsolver/solver.go | 10 +++++++--- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index b442e3c44160..09c8bcd41108 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -34,8 +34,8 @@ import ( // Keep the pools large enough for a ref's descriptor chain to fan out without // serializing pushes. const ( - defaultEagerWorkers = 100 - defaultEagerPushWorkers = 100 + defaultEagerWorkers = 128 + defaultEagerPushWorkers = 128 ) type eagerWorkItem struct { @@ -253,19 +253,18 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re // computeEagerExportRefs returns every ref ID that contributes to the final image, // including parent-chain layers. Each layer has its own vertex/ref ID, so // exporting only the final ref would filter out the eager layer pushes. -func computeEagerExportRefs(ctx context.Context, res *frontend.Result) map[string]struct{} { +func computeEagerExportRefs(ctx context.Context, res *frontend.Result) (map[string]struct{}, error) { exportRefs := make(map[string]struct{}) if res == nil { - return exportRefs + return exportRefs, nil } - res.EachRef(func(rp solver.ResultProxy) error { + if err := res.EachRef(func(rp solver.ResultProxy) error { if rp == nil { return nil } cached, err := rp.Result(ctx) if err != nil { - bklog.G(ctx).WithError(err).Warnf("eager export ref set: failed to resolve ref") - return nil + return errors.Wrap(err, "eager export ref set: failed to resolve ref") } workerRef, ok := cached.Sys().(*worker.WorkerRef) if !ok || workerRef.ImmutableRef == nil { @@ -283,8 +282,10 @@ func computeEagerExportRefs(ctx context.Context, res *frontend.Result) map[strin bklog.G(ctx).WithError(err).Warnf("eager export ref set: failed to release chain clones") } return nil - }) - return exportRefs + }); err != nil { + return nil, err + } + return exportRefs, nil } // isExportRef treats every ref as exportable until an export ref set is installed. @@ -430,9 +431,9 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { attribute.String("digest", digest), attribute.Int64("size", item.desc.Size), )) + defer span.End() if _, done := ep.pushedDigests.Load(digest); done { span.SetAttributes(attribute.Bool("deduped", true)) - span.End() return nil } @@ -447,7 +448,6 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { if !ep.isExportRef(item.refID) { tracker.mu.Unlock() span.SetAttributes(attribute.Bool("filtered", true)) - span.End() return nil } if tracker.cancels == nil { @@ -477,19 +477,16 @@ func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { // A cancelled pushCtx with a live parent means export ref set filtering won. if context.Cause(pushCtx) != nil && context.Cause(ctx) == nil { span.SetAttributes(attribute.Bool("cancelled", true)) - span.End() return nil } - tracing.FinishWithError(span, err) + span.RecordError(err) return err } if !pushed { span.SetAttributes(attribute.Bool("deduped", true)) - span.End() return nil } - span.End() return nil } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index 3f1355f00b89..49d34e42a063 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -666,9 +666,13 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro // Exporters need the final layer digests, so wait for eager work first. if eager != nil { // Filter out completed vertices that are not part of the final result. - exportRefs := computeEagerExportRefs(ctx, res) - cancelled := eager.applyExportRefs(exportRefs) - bklog.G(ctx).Infof("eager export_refs=%d cancelled_inflight=%d", len(exportRefs), cancelled) + exportRefs, err := computeEagerExportRefs(ctx, res) + if err != nil { + bklog.G(ctx).WithError(err).Warnf("failed to compute eager export refs; skipping eager export filtering") + } else { + cancelled := eager.applyExportRefs(exportRefs) + bklog.G(ctx).Infof("eager export_refs=%d cancelled_inflight=%d", len(exportRefs), cancelled) + } if err := inBuilderContext(ctx, j, eagerWaitProgressID(exp.EagerExport), "", func(ctx context.Context, _ session.Group) error { span, _ := tracing.StartSpan(ctx, eagerWaitProgressID(exp.EagerExport)) err := eager.wait() From 384fe01aee5af42a3edd0e645e84e5ad5d61eca7 Mon Sep 17 00:00:00 2001 From: Amy Date: Wed, 29 Apr 2026 14:17:16 -0700 Subject: [PATCH 55/55] more improvements --- solver/llbsolver/eager.go | 483 +++++++++++++++++++++++---------- solver/llbsolver/eager_test.go | 212 ++++++--------- 2 files changed, 415 insertions(+), 280 deletions(-) diff --git a/solver/llbsolver/eager.go b/solver/llbsolver/eager.go index 09c8bcd41108..fe2b1f93e10d 100644 --- a/solver/llbsolver/eager.go +++ b/solver/llbsolver/eager.go @@ -19,7 +19,6 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/compression" - "github.com/moby/buildkit/util/flightcontrol" pushutil "github.com/moby/buildkit/util/push" "github.com/moby/buildkit/util/resolver/limited" "github.com/moby/buildkit/util/resolver/retryhandler" @@ -42,18 +41,28 @@ type eagerWorkItem struct { ref cache.ImmutableRef } -// eagerPushItem is one descriptor handed to the push pool. +// eagerPushItem is a single unique push descriptor handed to the push pool. +// Duplicate descriptor requests are tracked per digest and do not enqueue +// additional work. type eagerPushItem struct { refID string desc ocispecs.Descriptor handler func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error) - result chan<- error + attempt uint64 } -// eagerPipeline receives completed vertex refs, compresses their layer chains, -// and optionally fans out pushable descriptors to a separate push pool. Once -// the final export refs are known, it filters/cancels non-export work and -// drains both pools. +// eagerPipeline manages background compression and pushing of layer blobs as +// build vertices complete, rather than deferring all work to finalize. +// +// It is split into two worker pools: +// - Compress pool: receives refs from the solver vertex callback, runs +// GetRemotes (which performs blob compression for the full parent chain), +// and dispatches each pushable descriptor onto the push pool. +// - Push pool: receives individual descriptors and uploads them to the +// registry. Decoupling lets a single ref fan out parallel pushes across +// its descriptor chain (instead of serializing them in one goroutine), +// which matches the parallelism that vanilla buildkit gets from +// images.Dispatch. type eagerPipeline struct { mode EagerExportMode refCfg cacheconfig.RefConfig @@ -67,43 +76,50 @@ type eagerPipeline struct { compressWG sync.WaitGroup pushWG sync.WaitGroup - // closeMu stops new senders from entering shutdown races. + // closeMu gates new senders from entering shutdown. Senders increment + // senderWg while holding the mutex so wait() can stop admission before + // waiting for all in-flight send attempts to finish. closeMu sync.Mutex closing bool senderWg sync.WaitGroup waitOnce sync.Once - // ctx carries the lease that keeps compressed blobs GC-protected. - ctx context.Context - cancel context.CancelCauseFunc + // ctx carries the lease so compressed blobs are GC-protected. + ctx context.Context + cancelCause context.CancelCauseFunc + // pusher is created at pipeline init when mode is EagerExportPush. pusher remotes.Pusher - // pushDedup coalesces concurrent pushes; pushedDigests skips digests that - // already succeeded in this build. - pushDedup flightcontrol.Group[bool] - pushedDigests sync.Map - - // exportRefIDs is nil until applyExportRefs installs the final export refs. - // After that, non-export refs are skipped or cancelled. - exportRefIDs atomic.Pointer[map[string]struct{}] - // inflight tracks requesters and cancel funcs per digest. + // keepRefIDs is the set of ImmutableRef.ID()s whose blobs are part of + // the final exported image manifest. It is populated once, by wait(), + // after the frontend has fully resolved its result. While nil, every + // ref's blobs are eligible for compression+push. Once non-nil, processRef + // and requestPushDescriptor skip refs + // whose ID is not in the set, and wait() cancels in-flight + // pushes whose only requesters are non-kept refs. + keepRefIDs atomic.Pointer[map[string]struct{}] + // inflight tracks per-digest cancel funcs and the set of refIDs that + // requested each digest. Used by wait() to cancel uploads + // whose every requester turns out to be non-kept. inflight sync.Map // map[string]*pushTracker mu sync.Mutex firstErr error } -// pushTracker is per digest. refIDs records every ref that requested the blob, -// including refs later filtered out of the final export; cancels records only -// active push waiters. When the final export refs are known, a digest is -// cancelled only if none of its requesters are part of the export. If it is -// cancelled, every active waiter must be cancelled because flightcontrol keeps -// the shared push running until all waiter contexts are done. +// pushTracker holds bookkeeping for a single digest's eager push. A digest is +// enqueued at most once at a time; duplicate requesters are recorded here but do +// not occupy push worker slots. type pushTracker struct { - mu sync.Mutex - refIDs map[string]struct{} - cancels map[string]context.CancelFunc + mu sync.Mutex + refIDs map[string]struct{} + enqueued bool + pushing bool + pushed bool + cancel context.CancelFunc + cancelled bool + attempt uint64 } func eagerWorkerCount() int { @@ -147,7 +163,7 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio pushCfg: pushCfg, pusher: pusher, ctx: pipelineCtx, - cancel: cancel, + cancelCause: cancel, compressWork: make(chan eagerWorkItem, 256), pushWork: make(chan eagerPushItem, 1024), done: make(chan struct{}), @@ -173,6 +189,12 @@ func newEagerPipeline(ctx context.Context, mode EagerExportMode, comp compressio return ep, nil } +func (ep *eagerPipeline) cancel(err error) { + if ep.cancelCause != nil { + ep.cancelCause(err) + } +} + func (ep *eagerPipeline) compressWorker() { defer ep.compressWG.Done() for { @@ -194,16 +216,15 @@ func (ep *eagerPipeline) compressWorker() { func (ep *eagerPipeline) pushWorker() { defer ep.pushWG.Done() for { - // Drain pushWork even after ctx cancellation so per-ref waiters finish. + // Don't bail out on ctx.Done here: wait() closes pushWork only after + // compression has finished enqueueing, and the worker should drain any + // remaining items so cancellation bookkeeping can settle. item, ok := <-ep.pushWork if !ok { return } - if err := ep.pushDescriptor(item); err != nil { + if err := ep.pushDescriptor(ep.ctx, item); err != nil { ep.recordErr(err) - item.result <- err - } else { - item.result <- nil } } } @@ -216,7 +237,10 @@ func (ep *eagerPipeline) recordErr(err error) { ep.mu.Unlock() } -// onVertexComplete clones finished refs and enqueues them for eager work. +// onVertexComplete is the callback registered on the solver Job. It extracts +// ImmutableRefs from vertex results, clones them for safe async use, and +// sends them to the compress pool. Fires that arrive after shutdown starts +// are rejected before enqueue and release their clones. func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Result) { for _, res := range results { if res == nil { @@ -250,54 +274,100 @@ func (ep *eagerPipeline) onVertexComplete(vtx solver.Vertex, results []solver.Re } } -// computeEagerExportRefs returns every ref ID that contributes to the final image, -// including parent-chain layers. Each layer has its own vertex/ref ID, so -// exporting only the final ref would filter out the eager layer pushes. -func computeEagerExportRefs(ctx context.Context, res *frontend.Result) (map[string]struct{}, error) { - exportRefs := make(map[string]struct{}) +// computeEagerKeepSet walks the resolved frontend Result and returns the +// set of ImmutableRef.ID()s that belong to the final exported image — +// including every layer in each output ref's parent chain. +// +// Why the chain matters: each layer is its own ImmutableRef with its own +// ID, *equal to* the ID of the intermediate vertex that produced it. The +// solver's onVertexComplete fires once per vertex during the build, so +// the eager pipeline pushes layer L_n via vertex V_n's processRef long +// before V_final completes. If we only kept V_final.ID() we'd filter +// every intermediate vertex's processRef, and all the real layer pushes +// would get deferred to wait() — defeating the entire point of eager +// export. +// +// res must already be fully resolved: every ResultProxy.Result(ctx) call +// must have completed without error. The caller in solver.go ensures +// this via eg.Wait() right before invoking us. +// +// If res is nil or contains zero refs, returns an empty map (== filter +// everything). Errors resolving individual refs are logged but not +// returned: a partial keep-set is safer than failing the build, since a +// missing entry just costs some wasted bandwidth, not correctness. +func computeEagerKeepSet(ctx context.Context, res *frontend.Result) map[string]struct{} { + keep := make(map[string]struct{}) if res == nil { - return exportRefs, nil + return keep } - if err := res.EachRef(func(rp solver.ResultProxy) error { + res.EachRef(func(rp solver.ResultProxy) error { if rp == nil { return nil } cached, err := rp.Result(ctx) if err != nil { - return errors.Wrap(err, "eager export ref set: failed to resolve ref") + bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to resolve ref") + return nil } workerRef, ok := cached.Sys().(*worker.WorkerRef) if !ok || workerRef.ImmutableRef == nil { return nil } - // LayerChain returns clones, so release them after reading IDs. + // LayerChain walks BaseLayer → ... → tip, including the tip + // itself. Each entry is a *clone* of the underlying ref — we + // only need its ID, then must release. chain := workerRef.ImmutableRef.LayerChain() for _, layer := range chain { if layer == nil { continue } - exportRefs[layer.ID()] = struct{}{} + keep[layer.ID()] = struct{}{} } if err := chain.Release(context.WithoutCancel(ctx)); err != nil { - bklog.G(ctx).WithError(err).Warnf("eager export ref set: failed to release chain clones") + bklog.G(ctx).WithError(err).Warnf("eager keep-set: failed to release chain clones") } return nil - }); err != nil { - return nil, err + }) + return keep +} + +func computeEagerExportRefs(ctx context.Context, res *frontend.Result) (map[string]struct{}, error) { + return computeEagerKeepSet(ctx, res), nil +} + +// keepSet returns the current keep-set, or nil if none has been set yet. +// While nil, no filtering is applied (every ref is eligible). +func (ep *eagerPipeline) keepSet() map[string]struct{} { + if p := ep.keepRefIDs.Load(); p != nil { + return *p } - return exportRefs, nil + return nil } -// isExportRef treats every ref as exportable until an export ref set is installed. -func (ep *eagerPipeline) isExportRef(refID string) bool { - exportRefs := ep.exportRefIDs.Load() - if exportRefs == nil { +// isKept reports whether refID should be retained. If no keep-set has been +// installed yet, every ref is considered kept. +func (ep *eagerPipeline) isKept(refID string) bool { + keep := ep.keepSet() + if keep == nil { return true } - _, ok := (*exportRefs)[refID] + _, ok := keep[refID] return ok } +func (ep *eagerPipeline) hasKeptRequester(refIDs map[string]struct{}) bool { + keep := ep.keepSet() + if keep == nil { + return len(refIDs) > 0 + } + for refID := range refIDs { + if _, ok := keep[refID]; ok { + return true + } + } + return false +} + // trackerFor returns (creating if necessary) the pushTracker for digest. func (ep *eagerPipeline) trackerFor(digest string) *pushTracker { if v, ok := ep.inflight.Load(digest); ok { @@ -308,19 +378,25 @@ func (ep *eagerPipeline) trackerFor(digest string) *pushTracker { return actual.(*pushTracker) } -// processRef compresses a ref's chain and, in push mode, dispatches pushable -// descriptors. Parent-chain compression is deduplicated lower in the cache. +// processRef compresses a single ref's blob (and its parent chain) and, in +// push mode, dispatches each pushable descriptor onto the push pool. Parent +// compression is deduplicated by flightcontrol inside computeBlobChain, so +// overlapping parent chains across workers are only compressed once. func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { ctx := ep.ctx s := session.NewGroup(ep.sessionID) refID := ref.ID() - // Skip queued work that became irrelevant before compression started. - if !ep.isExportRef(refID) { + // If the keep-set is already installed (e.g. queued items processed + // during shutdown drain), skip non-kept refs without paying for + // compression. + if !ep.isKept(refID) { bklog.G(ctx).Debugf("eager compress skipped (filtered) ref=%s", refID) return nil } + bklog.G(ctx).Infof("eager compress starting ref=%s", refID) + compressStart := time.Now() compressSpan, compressCtx := tracing.StartSpan(ctx, "eager compress ref", trace.WithAttributes( attribute.String("ref.id", refID), )) @@ -335,13 +411,16 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { bklog.G(ctx).WithError(err).Warnf("eager compress failed ref=%s", refID) return err } + bklog.G(ctx).Infof("eager compress done ref=%s descriptors=%d bytes=%d duration=%s", + refID, descCount, totalBytes, time.Since(compressStart).Round(time.Millisecond)) if ep.mode != EagerExportPush { return nil } - // The export ref set may have been installed during GetRemotes. - if !ep.isExportRef(refID) { + // Re-check after the (potentially long) compress: the keep-set may + // have been installed while we were inside GetRemotes. + if !ep.isKept(refID) { bklog.G(ctx).Debugf("eager push skipped after compress (filtered) ref=%s", refID) return nil } @@ -351,7 +430,7 @@ func (ep *eagerPipeline) processRef(ref cache.ImmutableRef) error { bklog.G(ctx).WithError(err).Warnf("eager push failed ref=%s", refID) return err } - bklog.G(ctx).Infof("eager push complete ref=%s descriptors=%d bytes=%d duration=%s", + bklog.G(ctx).Infof("eager push dispatched ref=%s descriptors=%d bytes=%d duration=%s", refID, descCount, totalBytes, time.Since(pushStart).Round(time.Millisecond)) return nil } @@ -370,16 +449,16 @@ func summarizeDescriptors(rems []*solver.Remote) (count int, totalBytes int64) { return } -// dispatchPushes fans a ref's pushable descriptors out to the push pool. +// dispatchPushes sends every not-yet-enqueued pushable descriptor in a ref's +// chain to the push pool. It deliberately does not wait for completion; final +// export waits for the push pool after all compression workers have finished +// enqueueing. func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems []*solver.Remote) error { if len(rems) == 0 { return nil } remote := rems[0] - // Buffered to the descriptor count so workers can report completion even if - // dispatchPushes returns early on context cancellation. - resultCh := make(chan error, len(remote.Descriptors)) handler := retryhandler.New( limited.PushHandler(ep.pusher, remote.Provider, ep.pushCfg.TargetName), nil, @@ -390,30 +469,26 @@ func (ep *eagerPipeline) dispatchPushes(ctx context.Context, refID string, rems if !shouldEagerPushDesc(desc) { continue } + attempt, shouldEnqueue := ep.requestPushDescriptor(refID, desc) + if !shouldEnqueue { + continue + } item := eagerPushItem{ refID: refID, desc: desc, handler: handler, - result: resultCh, + attempt: attempt, } select { case ep.pushWork <- item: enqueued++ case <-ctx.Done(): + ep.unmarkQueuedDescriptor(refID, desc, attempt) return context.Cause(ctx) } } - for range enqueued { - select { - case err := <-resultCh: - if err != nil { - return err - } - case <-ctx.Done(): - return context.Cause(ctx) - } - } + bklog.G(ctx).Debugf("eager push dispatched ref=%s enqueued=%d", refID, enqueued) return nil } @@ -421,91 +496,192 @@ func shouldEagerPushDesc(desc ocispecs.Descriptor) bool { return !images.IsNonDistributable(desc.MediaType) } -// pushDescriptor deduplicates by digest and records requesters so wait() can -// cancel pushes that only serve non-final refs. -func (ep *eagerPipeline) pushDescriptor(item eagerPushItem) error { - ctx := ep.ctx +// requestPushDescriptor records that refID needs desc and returns true only +// for the first requester that should enqueue actual push work. Duplicate +// requesters are bookkeeping only: they do not occupy push worker slots. +func (ep *eagerPipeline) requestPushDescriptor(refID string, desc ocispecs.Descriptor) (uint64, bool) { + digest := desc.Digest.String() + tracker := ep.trackerFor(digest) + + tracker.mu.Lock() + defer tracker.mu.Unlock() + + tracker.refIDs[refID] = struct{}{} + + if tracker.pushed { + bklog.G(ep.ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", refID, digest, desc.Size) + return 0, false + } + if !ep.hasKeptRequester(tracker.refIDs) { + bklog.G(ep.ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", refID, digest, desc.Size) + return 0, false + } + if tracker.enqueued || tracker.pushing { + bklog.G(ep.ctx).Debugf("eager push already in progress ref=%s digest=%s size=%d", refID, digest, desc.Size) + return 0, false + } + + tracker.attempt++ + tracker.enqueued = true + tracker.cancelled = false + return tracker.attempt, true +} + +func (ep *eagerPipeline) unmarkQueuedDescriptor(refID string, desc ocispecs.Descriptor, attempt uint64) { + digest := desc.Digest.String() + if v, ok := ep.inflight.Load(digest); ok { + tracker := v.(*pushTracker) + tracker.mu.Lock() + if tracker.attempt == attempt && !tracker.pushing && !tracker.pushed { + tracker.enqueued = false + } + delete(tracker.refIDs, refID) + tracker.mu.Unlock() + } +} + +// pushDescriptor uploads a single descriptor. Deduplication has already +// happened at enqueue time, so push workers only handle real push attempts. If +// a kept requester arrives after a non-kept upload was cancelled, it can enqueue +// a fresh item once the cancelled attempt clears the tracker state. +func (ep *eagerPipeline) pushDescriptor(ctx context.Context, item eagerPushItem) error { digest := item.desc.Digest.String() - span, pushCtx := tracing.StartSpan(ctx, "eager push descriptor", trace.WithAttributes( + parentCtx := ctx + span, spanCtx := tracing.StartSpan(ctx, "eager push descriptor", trace.WithAttributes( attribute.String("ref.id", item.refID), attribute.String("digest", digest), attribute.Int64("size", item.desc.Size), )) - defer span.End() - if _, done := ep.pushedDigests.Load(digest); done { - span.SetAttributes(attribute.Bool("deduped", true)) - return nil - } - + ctx = spanCtx tracker := ep.trackerFor(digest) - pushCtx, cancel := context.WithCancelCause(pushCtx) - defer cancel(errors.WithStack(context.Canceled)) - // Register requester and cancel func atomically with export-ref filtering - // so cancelNonExportInflight cannot miss an about-to-start push. tracker.mu.Lock() + if item.attempt != 0 && tracker.attempt != item.attempt { + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("deduped", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Debugf("eager push skipped stale attempt ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil + } tracker.refIDs[item.refID] = struct{}{} - if !ep.isExportRef(item.refID) { + if tracker.pushed { + tracker.enqueued = false tracker.mu.Unlock() - span.SetAttributes(attribute.Bool("filtered", true)) + span.SetAttributes(attribute.Bool("deduped", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Debugf("eager push deduped ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) return nil } - if tracker.cancels == nil { - tracker.cancels = make(map[string]context.CancelFunc) + if !ep.hasKeptRequester(tracker.refIDs) { + tracker.enqueued = false + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("filtered", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Debugf("eager push skipped (filtered) ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + return nil } - tracker.cancels[item.refID] = func() { cancel(errors.WithStack(context.Canceled)) } - tracker.mu.Unlock() + pushCtx, cancel := context.WithCancel(ctx) + defer cancel() + + tracker.enqueued = false + tracker.pushing = true + tracker.cancel = cancel + tracker.cancelled = false + tracker.mu.Unlock() defer func() { tracker.mu.Lock() - delete(tracker.cancels, item.refID) + if tracker.attempt == item.attempt { + tracker.cancel = nil + tracker.pushing = false + } tracker.mu.Unlock() }() - pushed, err := ep.pushDedup.Do(pushCtx, digest, func(ctx context.Context) (bool, error) { - if _, done := ep.pushedDigests.Load(digest); done { - return false, nil - } - _, herr := item.handler(ctx, item.desc) - if herr != nil { - return false, herr + start := time.Now() + bklog.G(ctx).Infof("eager push starting ref=%s digest=%s size=%d", item.refID, digest, item.desc.Size) + + _, err := item.handler(pushCtx, item.desc) + if err == nil && pushCtx.Err() != nil && parentCtx.Err() == nil { + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.pushing = false + tracker.enqueued = false + tracker.cancel = nil + tracker.cancelled = true } - ep.pushedDigests.Store(digest, struct{}{}) - return true, nil - }) + tracker.mu.Unlock() + span.SetAttributes(attribute.Bool("cancelled", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil + } if err != nil { - // A cancelled pushCtx with a live parent means export ref set filtering won. - if context.Cause(pushCtx) != nil && context.Cause(ctx) == nil { + // Distinguish a deliberate keep-set cancellation from a real + // failure: only the former has pushCtx done while the parent + // ctx is still alive. + if pushCtx.Err() != nil && parentCtx.Err() == nil { + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.pushing = false + tracker.enqueued = false + tracker.cancel = nil + tracker.cancelled = true + } + tracker.mu.Unlock() span.SetAttributes(attribute.Bool("cancelled", true)) + tracing.FinishWithError(span, nil) + bklog.G(ctx).Infof("eager push cancelled (filtered) ref=%s digest=%s size=%d after=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) return nil } - span.RecordError(err) + tracing.FinishWithError(span, err) return err } - if !pushed { - span.SetAttributes(attribute.Bool("deduped", true)) - return nil - } - - return nil -} -// applyExportRefs installs the final export refs and cancels in-flight pushes that -// only serve non-export refs. -func (ep *eagerPipeline) applyExportRefs(exportRefs map[string]struct{}) int { - if exportRefs == nil { - return 0 + tracker.mu.Lock() + if tracker.attempt == item.attempt { + tracker.pushed = true + tracker.pushing = false + tracker.enqueued = false + tracker.cancel = nil + tracker.cancelled = false } - cp := exportRefs - ep.exportRefIDs.Store(&cp) - return ep.cancelNonExportInflight(exportRefs) + tracker.mu.Unlock() + tracing.FinishWithError(span, nil) + bklog.G(ctx).Infof("eager push done ref=%s digest=%s size=%d duration=%s", + item.refID, digest, item.desc.Size, time.Since(start).Round(time.Millisecond)) + return nil } -// wait drains the compress and push pools in order. -func (ep *eagerPipeline) wait() error { +// wait shuts down both pools in order: stop new compress senders, drain +// compressWork, then drain pushWork. Compress workers enqueue push work without +// waiting for it; once compressWG.Wait returns, every reachable descriptor has +// either been enqueued or deduped, so closing pushWork is safe. +// +// The optional keep-set is the canonical set of ImmutableRef.ID()s that +// belong to the final image manifest, computed by the caller from the +// resolved frontend Result. When non-nil: +// - any in-flight push whose every requester is non-kept is cancelled +// immediately, freeing its registry-channel slot; +// - any items still queued in pushWork or compressWork for non-kept refs +// are skipped on dequeue; +// - kept refs continue to compress and push as normal. +// +// Pass nil to keep every ref (back-compat for non-gateway frontends or +// callers that haven't computed a keep-set). +func (ep *eagerPipeline) wait(keepOpt ...map[string]struct{}) error { ep.waitOnce.Do(func() { - if ep.cancel != nil { - defer ep.cancel(errors.WithStack(context.Canceled)) + var keep map[string]struct{} + if len(keepOpt) > 0 { + keep = keepOpt[0] + } + if keep != nil { + cp := keep + ep.keepRefIDs.Store(&cp) + n := ep.cancelNonKeptInflight(keep) + bklog.G(ep.ctx).Infof("eager wait keep_set_size=%d cancelled_inflight=%d", len(keep), n) } ep.closeMu.Lock() @@ -532,10 +708,14 @@ func (ep *eagerPipeline) wait() error { return ep.firstErr } -// cancelNonExportInflight cancels digests whose requesters are all non-export. -// All waiters for a digest must be cancelled, or flightcontrol leaves the -// shared push running. -func (ep *eagerPipeline) cancelNonExportInflight(exportRefs map[string]struct{}) int { +// cancelNonKeptInflight walks every digest currently being pushed and +// cancels the upload if no requester is in the keep-set. Returns the +// number of digests for which at least one cancel was issued. +// +// A digest with at least one kept requester is left entirely alone. A digest +// whose requesters are all non-kept can have its single active push cancelled; +// queued items will observe the keep-set when dequeued and skip without pushing. +func (ep *eagerPipeline) cancelNonKeptInflight(keep map[string]struct{}) int { var digestsCancelled int ep.inflight.Range(func(key, val any) bool { digest := key.(string) @@ -545,28 +725,37 @@ func (ep *eagerPipeline) cancelNonExportInflight(exportRefs map[string]struct{}) defer tracker.mu.Unlock() for refID := range tracker.refIDs { - if _, ok := exportRefs[refID]; ok { + if _, ok := keep[refID]; ok { return true } } - if len(tracker.cancels) == 0 { - // Queued items will be filtered at dequeue. + if tracker.cancel == nil { + // Nothing in flight — queued items will be filtered by + // isKept() at dequeue. return true } - n := len(tracker.cancels) - for refID, cancelFn := range tracker.cancels { - cancelFn() - delete(tracker.cancels, refID) - } + tracker.cancel() + tracker.cancel = nil + tracker.pushing = false + tracker.enqueued = false + tracker.cancelled = true + tracker.attempt++ digestsCancelled++ bklog.G(ep.ctx).Infof("eager push cancel digest=%s requesters=%d cancelled_waiters=%d", - digest, len(tracker.refIDs), n) + digest, len(tracker.refIDs), 1) return true }) return digestsCancelled } -// drainCompress releases refs left behind after workers exit. +func (ep *eagerPipeline) applyExportRefs(exportRefs map[string]struct{}) int { + cp := exportRefs + ep.keepRefIDs.Store(&cp) + return ep.cancelNonKeptInflight(exportRefs) +} + +// drainCompress releases any refs left in compressWork after workers have +// exited (e.g. via ctx cancellation before the channel was drained). func (ep *eagerPipeline) drainCompress() { for { select { @@ -581,7 +770,7 @@ func (ep *eagerPipeline) drainCompress() { } } -// drainPushWork unblocks any dispatchers still waiting on queued push items. +// drainPushWork clears any leftover queued push items after push workers exit. func (ep *eagerPipeline) drainPushWork() { for { select { @@ -589,7 +778,7 @@ func (ep *eagerPipeline) drainPushWork() { if !ok { return } - item.result <- nil + ep.unmarkQueuedDescriptor(item.refID, item.desc, item.attempt) default: return } diff --git a/solver/llbsolver/eager_test.go b/solver/llbsolver/eager_test.go index a173f60843c6..38fd2a1a6765 100644 --- a/solver/llbsolver/eager_test.go +++ b/solver/llbsolver/eager_test.go @@ -67,7 +67,7 @@ func TestEagerPipeline_WaitReturnsFirstError(t *testing.T) { } ep.firstErr = assert.AnError - err := ep.wait() + err := ep.wait(nil) assert.Equal(t, assert.AnError, err) } @@ -76,7 +76,7 @@ func TestEagerPipeline_WaitReturnsNilWhenNoError(t *testing.T) { compressWork: make(chan eagerWorkItem), } - err := ep.wait() + err := ep.wait(nil) assert.NoError(t, err) } @@ -89,7 +89,7 @@ func TestEagerPipeline_WaitDrainsLeftoverRefs(t *testing.T) { ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} ep.compressWork <- eagerWorkItem{ref: &releaseTracker{released: &released}} - err := ep.wait() + err := ep.wait(nil) require.NoError(t, err) assert.Equal(t, int32(2), released.Load(), "leftover refs should be released by wait()") } @@ -99,8 +99,8 @@ func TestEagerPipeline_WaitIsIdempotent(t *testing.T) { compressWork: make(chan eagerWorkItem), } - require.NoError(t, ep.wait()) - require.NoError(t, ep.wait(), "second wait must not panic") + require.NoError(t, ep.wait(nil)) + require.NoError(t, ep.wait(nil), "second wait must not panic") } func TestEagerPipeline_CompressWorkerExitsOnContextCancel(t *testing.T) { @@ -145,33 +145,8 @@ func TestEagerPipeline_PushWorkerExitsOnChannelClose(t *testing.T) { ep.pushWG.Wait() } -func TestEagerPipeline_PushWorkerErrorReturnedByWait(t *testing.T) { - ep := &eagerPipeline{ - ctx: context.Background(), - compressWork: make(chan eagerWorkItem), - pushWork: make(chan eagerPushItem, 1), - } - ep.pushWG.Add(1) - go ep.pushWorker() - - resultCh := make(chan error, 1) - ep.pushWork <- eagerPushItem{ - refID: "test-ref", - desc: ocispecs.Descriptor{ - Digest: digest.FromString("push-error"), - MediaType: ocispecs.MediaTypeImageLayerGzip, - }, - handler: func(context.Context, ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { - return nil, assert.AnError - }, - result: resultCh, - } - - require.Equal(t, assert.AnError, <-resultCh) - require.Equal(t, assert.AnError, ep.wait()) -} - -// Late callbacks must be ignored after wait() starts shutdown. +// Late fires of onVertexComplete must release the clone instead of sending +// into the (now closed) work channel. func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ @@ -179,7 +154,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { compressWork: make(chan eagerWorkItem, 10), done: make(chan struct{}), } - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) var released atomic.Int32 res := newWorkerRefResult(&releaseTracker{released: &released, cloned: &cloned}) @@ -191,7 +166,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait(t *testing.T) { assert.Zero(t, released.Load()) } -// Concurrent late callbacks must not clone refs. +// Many concurrent late fires after wait() must be rejected before cloning. func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ @@ -199,7 +174,7 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { compressWork: make(chan eagerWorkItem, 10), done: make(chan struct{}), } - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) var released atomic.Int32 const fires = 100 @@ -219,7 +194,8 @@ func TestEagerPipeline_OnVertexCompleteAfterWait_Concurrent(t *testing.T) { assert.Zero(t, released.Load()) } -// Blocked senders must release clones when wait() closes the done channel. +// A sender admitted before wait() but blocked on a full queue must take the +// done path, release its clone, and exit without panic. func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) { var cloned atomic.Int32 var released atomic.Int32 @@ -248,14 +224,15 @@ func TestEagerPipeline_OnVertexCompleteBlockedSenderReleasedOnWait(t *testing.T) require.Equal(t, int32(1), cloned.Load()) require.NotPanics(t, func() { - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) }) senderWg.Wait() assert.Equal(t, int32(2), released.Load()) } -// Callbacks racing with wait() must not panic or leak clones. +// Fires racing concurrently with wait() must not panic and must not leak: +// every admitted clone is either drained by wait() or released by the sender. func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { var cloned atomic.Int32 ep := &eagerPipeline{ @@ -278,7 +255,7 @@ func TestEagerPipeline_OnVertexCompleteRacingWait(t *testing.T) { } require.NotPanics(t, func() { - require.NoError(t, ep.wait()) + require.NoError(t, ep.wait(nil)) }) wg.Wait() @@ -302,7 +279,7 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { } var pushed []digest.Digest - ep := &eagerPipeline{ctx: context.Background()} + ep := &eagerPipeline{} handler := func(_ context.Context, desc ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { pushed = append(pushed, desc.Digest) return nil, nil @@ -312,7 +289,7 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { if !shouldEagerPushDesc(desc) { continue } - err := ep.pushDescriptor(eagerPushItem{ + err := ep.pushDescriptor(context.Background(), eagerPushItem{ refID: "test-ref", desc: desc, handler: handler, @@ -326,7 +303,10 @@ func TestEagerPushSkipsNonDistributableDescriptors(t *testing.T) { }, pushed) } -// Repeated pushes of the same digest should only call the handler once. +// pushDescriptor must short-circuit on the second call for the same digest, +// without invoking the handler again. This is the fix for the noisy +// "eager pushing blob" log entries that appeared once per shared parent +// blob per ref. func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("shared-blob"), @@ -339,9 +319,9 @@ func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { return nil, nil } - ep := &eagerPipeline{ctx: context.Background()} + ep := &eagerPipeline{} for range 5 { - require.NoError(t, ep.pushDescriptor(eagerPushItem{ + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ refID: "ref-test", desc: desc, handler: handler, @@ -350,49 +330,9 @@ func TestEagerPushDescriptor_DedupsAfterSuccess(t *testing.T) { assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once across repeated calls for the same digest") } -func TestEagerPushDescriptor_DedupsConcurrentPushes(t *testing.T) { - desc := ocispecs.Descriptor{ - Digest: digest.FromString("shared-blob"), - MediaType: ocispecs.MediaTypeImageLayerGzip, - Size: 1234, - } - var calls atomic.Int32 - started := make(chan struct{}) - release := make(chan struct{}) - handler := func(_ context.Context, _ ocispecs.Descriptor) ([]ocispecs.Descriptor, error) { - if calls.Add(1) == 1 { - close(started) - } - <-release - return nil, nil - } - - ep := &eagerPipeline{ctx: context.Background()} - errCh := make(chan error, 2) - go func() { - errCh <- ep.pushDescriptor(eagerPushItem{ - refID: "ref-a", - desc: desc, - handler: handler, - }) - }() - <-started - go func() { - errCh <- ep.pushDescriptor(eagerPushItem{ - refID: "ref-b", - desc: desc, - handler: handler, - }) - }() - close(release) - - require.NoError(t, <-errCh) - require.NoError(t, <-errCh) - assert.Equal(t, int32(1), calls.Load(), "handler must run exactly once for concurrent pushes of the same digest") -} - -// Non-export refs should be tracked but not pushed. -func TestEagerPushDescriptor_SkipsNonExportRef(t *testing.T) { +// pushDescriptor must skip — without invoking the handler — when the +// keep-set is installed and the requesting refID is not in it. +func TestEagerPushDescriptor_SkipsNonKeptRef(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("intermediate-blob"), MediaType: ocispecs.MediaTypeImageLayerGzip, @@ -404,17 +344,19 @@ func TestEagerPushDescriptor_SkipsNonExportRef(t *testing.T) { return nil, nil } - ep := &eagerPipeline{ctx: context.Background()} - exportRefs := map[string]struct{}{"final-ref": {}} - ep.exportRefIDs.Store(&exportRefs) + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) - require.NoError(t, ep.pushDescriptor(eagerPushItem{ + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ refID: "intermediate-ref", desc: desc, handler: handler, })) - assert.Zero(t, calls.Load(), "non-export ref must not invoke the push handler") + assert.Zero(t, calls.Load(), "non-kept ref must not invoke the push handler") + // And the digest must still appear in the inflight tracker so + // cancelNonKeptInflight can reason about it. v, ok := ep.inflight.Load(desc.Digest.String()) require.True(t, ok) tracker := v.(*pushTracker) @@ -424,8 +366,9 @@ func TestEagerPushDescriptor_SkipsNonExportRef(t *testing.T) { assert.True(t, hasRef, "tracker must record requester even when filtered") } -// An export requester should push even if the digest was first requested by a non-export ref. -func TestEagerPushDescriptor_PushesWhenAnyRequesterExported(t *testing.T) { +// pushDescriptor must still push when at least one kept ref requests the +// digest, even if a non-kept ref also asked for it. +func TestEagerPushDescriptor_PushesWhenAnyRequesterKept(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("shared-blob"), MediaType: ocispecs.MediaTypeImageLayerGzip, @@ -437,63 +380,65 @@ func TestEagerPushDescriptor_PushesWhenAnyRequesterExported(t *testing.T) { return nil, nil } - ep := &eagerPipeline{ctx: context.Background()} - exportRefs := map[string]struct{}{"final-ref": {}} - ep.exportRefIDs.Store(&exportRefs) + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) - require.NoError(t, ep.pushDescriptor(eagerPushItem{ + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ refID: "intermediate-ref", desc: desc, handler: handler, })) - require.NoError(t, ep.pushDescriptor(eagerPushItem{ + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ refID: "final-ref", desc: desc, handler: handler, })) - assert.Equal(t, int32(1), calls.Load(), "export-ref call must drive exactly one push") + assert.Equal(t, int32(1), calls.Load(), "kept-ref call must drive exactly one push") } -// cancelNonExportInflight cancels only digests with no export requesters. -func TestEagerPipeline_CancelNonExportInflight(t *testing.T) { +// cancelNonKeptInflight must cancel the active push for a digest whose +// requesters are all non-kept, and must leave digests with at least one +// kept requester completely alone. +func TestEagerPipeline_CancelNonKeptInflight(t *testing.T) { ep := &eagerPipeline{ctx: context.Background()} - exportRefs := map[string]struct{}{"final-ref": {}} + keep := map[string]struct{}{"final-ref": {}} - // Digest A: all requesters are non-export, so every waiter must cancel. - var cancelA1, cancelA2 atomic.Bool + // Digest A: two non-kept requesters share one active push. + var cancelA atomic.Bool ep.inflight.Store("digest-a", &pushTracker{ - refIDs: map[string]struct{}{"int-ref-1": {}, "int-ref-2": {}}, - cancels: map[string]context.CancelFunc{ - "int-ref-1": func() { cancelA1.Store(true) }, - "int-ref-2": func() { cancelA2.Store(true) }, - }, + refIDs: map[string]struct{}{"int-ref-1": {}, "int-ref-2": {}}, + cancel: func() { cancelA.Store(true) }, + pushing: true, }) - // Digest B: one export requester leaves the shared push alive. + // Digest B: requested by both kept and non-kept — must be spared + // even though it has an active push. var cancelB atomic.Bool ep.inflight.Store("digest-b", &pushTracker{ - refIDs: map[string]struct{}{"int-ref-3": {}, "final-ref": {}}, - cancels: map[string]context.CancelFunc{ - "int-ref-3": func() { cancelB.Store(true) }, - }, + refIDs: map[string]struct{}{"int-ref-3": {}, "final-ref": {}}, + cancel: func() { cancelB.Store(true) }, + pushing: true, }) - // Digest C: queued work has no waiter to cancel. + // Digest C: non-kept requester but no cancel yet (still queued). + // Must not be counted and must not panic. ep.inflight.Store("digest-c", &pushTracker{ refIDs: map[string]struct{}{"int-ref-4": {}}, }) - n := ep.cancelNonExportInflight(exportRefs) - assert.Equal(t, 1, n, "exactly one digest's waiters should be cancelled") - assert.True(t, cancelA1.Load(), "digest A waiter 1 must be cancelled") - assert.True(t, cancelA2.Load(), "digest A waiter 2 must be cancelled") - assert.False(t, cancelB.Load(), "digest B must be spared (export requester)") + n := ep.cancelNonKeptInflight(keep) + assert.Equal(t, 1, n, "exactly one digest's active push should be cancelled") + assert.True(t, cancelA.Load(), "digest A active push must be cancelled") + assert.False(t, cancelB.Load(), "digest B must be spared (kept requester)") trackerA, _ := ep.inflight.Load("digest-a") - assert.Empty(t, trackerA.(*pushTracker).cancels, "cancels map should be drained after cancellation") + assert.Nil(t, trackerA.(*pushTracker).cancel, "cancel should be cleared after cancellation") } -// An export ref arriving after cancellation should start a fresh push. -func TestEagerPushDescriptor_ExportRefAfterCancelStillPushes(t *testing.T) { +// After cancelNonKeptInflight has fired, a kept ref that arrives later +// must not be skipped — it still needs to push the blob (the previous +// closure was cancelled before completion). +func TestEagerPushDescriptor_KeptRefAfterCancelStillPushes(t *testing.T) { desc := ocispecs.Descriptor{ Digest: digest.FromString("retry-after-cancel"), MediaType: ocispecs.MediaTypeImageLayerGzip, @@ -505,25 +450,26 @@ func TestEagerPushDescriptor_ExportRefAfterCancelStillPushes(t *testing.T) { return nil, nil } - ep := &eagerPipeline{ctx: context.Background()} - exportRefs := map[string]struct{}{"final-ref": {}} - ep.exportRefIDs.Store(&exportRefs) + ep := &eagerPipeline{} + keep := map[string]struct{}{"final-ref": {}} + ep.keepRefIDs.Store(&keep) - // Simulate a prior cancel pass that already removed its waiter. + // Simulate a prior cancel pass: tracker exists with the requester + // recorded and no active push. ep.inflight.Store(desc.Digest.String(), &pushTracker{ - refIDs: map[string]struct{}{"int-ref": {}}, - cancels: map[string]context.CancelFunc{}, + refIDs: map[string]struct{}{"int-ref": {}}, }) - require.NoError(t, ep.pushDescriptor(eagerPushItem{ + require.NoError(t, ep.pushDescriptor(context.Background(), eagerPushItem{ refID: "final-ref", desc: desc, handler: handler, })) - assert.Equal(t, int32(1), calls.Load(), "export ref must drive a fresh push after a prior cancellation") + assert.Equal(t, int32(1), calls.Load(), "kept ref must drive a fresh push after a prior cancellation") } -// releaseTracker counts Release calls across clones. +// releaseTracker is a minimal cache.ImmutableRef stub that counts +// Release calls. Clones share the counter. type releaseTracker struct { cache.ImmutableRef released *atomic.Int32