From 1a73bafe56c756d30f12aee86d4610b5e1748bdf Mon Sep 17 00:00:00 2001 From: Zachary Spar Date: Sat, 13 Jun 2026 00:27:13 -0400 Subject: [PATCH] feat: support custom BuildKit frontends --- .gitignore | 3 +- pkg/build/build.go | 16 ++- pkg/build/buildopts.go | 211 +++++++++++++++++++++------------- pkg/build/buildopts_test.go | 100 ++++++++++++++++ pkg/build/frontend.go | 48 ++++++++ pkg/build/syntax.go | 94 +++++++++++++++ pkg/build/syntax_test.go | 198 +++++++++++++++++++++++++++++++ pkg/fileutils/tarxfer.go | 4 +- pkg/fileutils/tarxfer_test.go | 58 +++++++++- pkg/fssync/fssync.go | 8 +- pkg/fssync/walk.go | 2 +- 11 files changed, 648 insertions(+), 94 deletions(-) create mode 100644 pkg/build/buildopts_test.go create mode 100644 pkg/build/syntax.go create mode 100644 pkg/build/syntax_test.go diff --git a/.gitignore b/.gitignore index 297d6f6..1b8ba75 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ container-builder-shim bin .build .swiftpm -.local \ No newline at end of file +.local +*.test \ No newline at end of file diff --git a/pkg/build/build.go b/pkg/build/build.go index be5be4c..5206da2 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -113,8 +113,13 @@ func Build(ctx context.Context, opts *BOpts) error { if _, ok := export.Attrs["name"]; !ok { export.Attrs["name"] = opts.Tag } - if _, ok := export.Attrs["annotation-index-descriptor.com.apple.containerization.image.name"]; !ok { - export.Attrs["annotation-index-descriptor.com.apple.containerization.image.name"] = opts.Tag + // The apple containerization annotation is only injected for the + // native Dockerfile path, which produces a manifest list via frontend.go. + // External frontends control their own manifest metadata. + if opts.frontend.native() { + if _, ok := export.Attrs["annotation-index-descriptor.com.apple.containerization.image.name"]; !ok { + export.Attrs["annotation-index-descriptor.com.apple.containerization.image.name"] = opts.Tag + } } exportsWithOutput = append(exportsWithOutput, export) } @@ -143,7 +148,10 @@ func Build(ctx context.Context, opts *BOpts) error { KeyContentStoreName: opts.ContentStore, } - if len(opts.Dockerignore) > 0 { + solveOpt.FrontendAttrs["local.metadatatransfer"] = "false" + solveOpt.FrontendAttrs["local.differ"] = "none" + + if len(opts.Dockerignore) > 0 || !opts.frontend.native() { solveOpt.FrontendAttrs["filename"] = filepath.Join(DockerfileStaging, "Dockerfile") } @@ -171,7 +179,6 @@ func Build(ctx context.Context, opts *BOpts) error { for k, v := range opts.Labels { solveOpt.FrontendAttrs["label:"+k] = v } - solveOpt.Frontend = "dockerfile.v1" if len(opts.SSH) > 0 { sshProvider, err := sshprovider.NewSSHAgentProvider(opts.SSH) @@ -181,6 +188,7 @@ func Build(ctx context.Context, opts *BOpts) error { solveOpt.Session = append(solveOpt.Session, sshProvider) } + solveOpt.Frontend = "dockerfile.v1" _, err = buildkit.Build(opts.Context(ctx), solveOpt, "", frontend, opts.ProgressWriter.Status()) <-opts.ProgressWriter.Done() return err diff --git a/pkg/build/buildopts.go b/pkg/build/buildopts.go index fb77643..a6a5f4e 100644 --- a/pkg/build/buildopts.go +++ b/pkg/build/buildopts.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "encoding/base64" + "fmt" "path/filepath" "strings" @@ -73,6 +74,15 @@ const ( KeyOutput = "outputs" // Unique build identifier. KeyBuildID = "build-id" + // BuildKit frontend name (e.g. dockerfile.v0, gateway.v0). + KeyFrontend = "frontend" + // Frontend option key=value pairs passed to BuildKit. + KeyFrontendOpt = "frontend-opt" +) + +const ( + frontendDockerfileV0 = "dockerfile.v0" + frontendGatewayV0 = "gateway.v0" ) const ( @@ -83,6 +93,32 @@ const ( DockerfileStaging = fssync.DockerfileStaging ) +func mapExtract(contextMap map[string][]string, key string) map[string]string { + values, ok := contextMap[key] + if !ok { + return map[string]string{} + } + args := map[string]string{} + for _, label := range values { + parts := strings.SplitN(label, "=", 2) + switch len(parts) { + case 1: + args[parts[0]] = "" + case 2: + args[parts[0]] = parts[1] + } + } + return args +} + +func first(contextMap map[string][]string, key string) (string, bool) { + values, ok := contextMap[key] + if !ok || len(values) == 0 { + return "", false + } + return values[0], true +} + var keyBOpts = struct{}{} type BOpts struct { @@ -109,24 +145,18 @@ type BOpts struct { FSSync *fssync.FSSyncProxy Stdio *stdio.StdioProxy + frontend frontendSettings + basePath string } func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][]string) (*BOpts, error) { - first := func(key string) (string, bool) { - values, ok := contextMap[key] - if !ok { - return "", false - } - return values[0], true - } - - buildID, ok := first(KeyBuildID) + buildID, ok := first(contextMap, KeyBuildID) if !ok { return nil, ErrMissingBuildID } - dockerfileBase64Bytes, ok := first(KeyDockerfile) + dockerfileBase64Bytes, ok := first(contextMap, KeyDockerfile) if !ok { return nil, ErrMissingContextDockerfile } @@ -136,10 +166,10 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] return nil, err } - dockerignoreBase64Bytes, ok := first(KeyDockerignore) + dockerignoreBase64Bytes, dockerignoreProvided := first(contextMap, KeyDockerignore) dockerignoreBytes := []byte{} - if ok { + if dockerignoreProvided { dockerignoreBytes, err = base64.StdEncoding.DecodeString(dockerignoreBase64Bytes) if err != nil { return nil, err @@ -148,7 +178,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] dockerignoreBytes = append(dockerignoreBytes, []byte("\n"+DockerfileStaging)...) } - progress, ok := first(KeyProgress) + progress, ok := first(contextMap, KeyProgress) if !ok { progress = "auto" } @@ -159,17 +189,17 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] } noCache := false - if _, ok := first(KeyNoCache); ok { + if _, ok := first(contextMap, KeyNoCache); ok { noCache = true } - tag, ok := first(KeyTag) + tag, ok := first(contextMap, KeyTag) if !ok { return nil, ErrMissingContextRef } ctxDir := "." - if c, ok := first(KeyContext); ok { + if c, ok := first(contextMap, KeyContext); ok { ctxDir = c } @@ -198,27 +228,10 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] } target := "" - if tStr, ok := first(KeyTarget); ok { + if tStr, ok := first(contextMap, KeyTarget); ok { target = tStr } - mapExtract := func(key string) map[string]string { - values, ok := contextMap[key] - if !ok { - return map[string]string{} - } - args := map[string]string{} - for _, label := range values { - parts := strings.SplitN(label, "=", 2) - switch len(parts) { - case 1: - args[parts[0]] = "" - case 2: - args[parts[0]] = parts[1] - } - } - return args - } mapExtractB64 := func(key string) (map[string][]byte, error) { values, ok := contextMap[key] if !ok { @@ -270,8 +283,16 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] return agentConfigs } - labels := mapExtract(KeyLabels) - buildArgs := mapExtract(KeyBuildArgs) + labels := mapExtract(contextMap, KeyLabels) + buildArgs := mapExtract(contextMap, KeyBuildArgs) + frontend := resolveFrontend(dockerfileBytes, buildArgs, contextMap) + if err := validateFrontend(frontend); err != nil { + return nil, err + } + if !frontend.native() { + dockerignoreBytes = ensureDockerfileStagingExcluded(dockerignoreBytes) + } + stageBuildDefinition := dockerignoreProvided || !frontend.native() secrets, err := mapExtractB64(KeySecrets) if err != nil { return nil, err @@ -286,16 +307,86 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] return nil, err } - dockerfile, err := parser.Parse(bytes.NewReader(dockerfileBytes)) + var addedGlobs []string + if frontend.native() { + addedGlobs, buildArgs, err = parseDockerfileMetadata(dockerfileBytes, buildArgs) + if err != nil { + return nil, err + } + } + + pw, err := progresswriter.NewPrinter(ctx, stdioProxy, progress) if err != nil { return nil, err } - _, metaArgs, err := instructions.Parse(dockerfile.AST, nil) + fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes, stageBuildDefinition) + if err != nil { + return nil, err + } + + contentProxy, err := content.NewContentStoreProxy() if err != nil { return nil, err } + bopts := &BOpts{ + BuildID: buildID, + Dockerfile: dockerfileBytes, + Dockerignore: dockerignoreBytes, + Tag: tag, + BuildPlatforms: bps, + Platforms: pls, + ContextDir: ctxDir, + ContentStore: contentProxy, + FSSync: fssyncProxy, + NoCache: noCache, + Resolver: resolver.NewResolverProxy(), + ProgressWriter: pw, + Stdio: stdioProxy, + Target: target, + Labels: labels, + BuildArgs: buildArgs, + Secrets: secrets, + SSH: ssh, + CacheIn: cacheIn, + CacheOut: cacheOut, + Outputs: outputs, + frontend: frontend, + basePath: filepath.Join(basePath, buildID), + } + + return bopts, nil +} + +func ensureDockerfileStagingExcluded(dockerignore []byte) []byte { + if bytes.Contains(dockerignore, []byte(DockerfileStaging)) { + return dockerignore + } + if len(dockerignore) == 0 { + return []byte(DockerfileStaging + "\n") + } + return append(dockerignore, []byte("\n"+DockerfileStaging)...) +} + +func validateFrontend(frontend frontendSettings) error { + if frontend.Name == frontendDockerfileV0 { + return fmt.Errorf("frontend %q is not supported; use # syntax= or BUILDKIT_SYNTAX instead", frontendDockerfileV0) + } + return nil +} + +func parseDockerfileMetadata(dockerfileBytes []byte, buildArgs map[string]string) ([]string, map[string]string, error) { + dockerfile, err := parser.Parse(bytes.NewReader(dockerfileBytes)) + if err != nil { + return nil, buildArgs, err + } + + _, metaArgs, err := instructions.Parse(dockerfile.AST, nil) + if err != nil { + return nil, buildArgs, err + } + for _, metaArg := range metaArgs { for _, arg := range metaArg.Args { // Only use the dockerfile meta arg if the user did not overwrite it @@ -305,18 +396,13 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] // Expand with prior args and strip shell quotes resolved, err := shell.NewLex('\\').ProcessWordWithMatches(arg.ValueString(), utils.NewMapGetter(buildArgs)) if err != nil { - return nil, err + return nil, buildArgs, err } // Save the resolved value for later use buildArgs[arg.Key] = resolved.Result } } - pw, err := progresswriter.NewPrinter(ctx, stdioProxy, progress) - if err != nil { - return nil, err - } - // addedGlobs is the fallback value for followpaths when BuildKit does not // supply it. Pre-compute it by scanning the Dockerfile AST for COPY, ADD, // and RUN --mount=type=bind source paths so the host packs only the files @@ -350,42 +436,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] } } - fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes) - if err != nil { - return nil, err - } - - contentProxy, err := content.NewContentStoreProxy() - if err != nil { - return nil, err - } - - bopts := &BOpts{ - BuildID: buildID, - Dockerfile: dockerfileBytes, - Dockerignore: dockerignoreBytes, - Tag: tag, - BuildPlatforms: bps, - Platforms: pls, - ContextDir: ctxDir, - ContentStore: contentProxy, - FSSync: fssyncProxy, - NoCache: noCache, - Resolver: resolver.NewResolverProxy(), - ProgressWriter: pw, - Stdio: stdioProxy, - Target: target, - Labels: labels, - BuildArgs: buildArgs, - Secrets: secrets, - SSH: ssh, - CacheIn: cacheIn, - CacheOut: cacheOut, - Outputs: outputs, - basePath: filepath.Join(basePath, buildID), - } - - return bopts, nil + return addedGlobs, buildArgs, nil } func (b *BOpts) Context(parent context.Context) context.Context { diff --git a/pkg/build/buildopts_test.go b/pkg/build/buildopts_test.go new file mode 100644 index 0000000..5437db7 --- /dev/null +++ b/pkg/build/buildopts_test.go @@ -0,0 +1,100 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025-2026 Apple Inc. and the container-builder-shim project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +package build + +import ( + "testing" +) + +const yamlfileSample = `# syntax=ghcr.io/builderhub/yamlfile:latest +apiVersion: v1alpha1 +targets: + myapp: + from: alpine:latest + steps: + - run: echo hello +` + +func TestExternalFrontendSkipsDockerfileParse(t *testing.T) { + frontend := resolveFrontend([]byte(yamlfileSample), map[string]string{}, map[string][]string{}) + if frontend.native() { + t.Fatal("expected external frontend for yamlfile syntax header") + } + + _, _, err := parseDockerfileMetadata([]byte(yamlfileSample), map[string]string{}) + if err == nil { + t.Fatal("parseDockerfileMetadata should fail on yamlfile content") + } +} + +func TestNativeFrontendParsesDockerfileMetadata(t *testing.T) { + dockerfile := `FROM alpine:latest +RUN echo hi +` + frontend := resolveFrontend([]byte(dockerfile), map[string]string{}, map[string][]string{}) + if !frontend.native() { + t.Fatalf("expected native frontend, got %+v", frontend) + } + + globs, _, err := parseDockerfileMetadata([]byte(dockerfile), map[string]string{}) + if err != nil { + t.Fatalf("parseDockerfileMetadata() error = %v", err) + } + if len(globs) != 0 { + t.Fatalf("parseDockerfileMetadata() globs = %v, want empty", globs) + } +} + +func TestEnsureDockerfileStagingExcluded(t *testing.T) { + tests := []struct { + name string + input []byte + want string + }{ + { + name: "empty input adds staging exclude", + input: nil, + want: DockerfileStaging + "\n", + }, + { + name: "already present is unchanged", + input: []byte("node_modules\n" + DockerfileStaging + "\n"), + want: "node_modules\n" + DockerfileStaging + "\n", + }, + { + name: "appends staging exclude", + input: []byte("node_modules\n"), + want: "node_modules\n\n" + DockerfileStaging, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := string(ensureDockerfileStagingExcluded(tt.input)) + if got != tt.want { + t.Fatalf("ensureDockerfileStagingExcluded() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestValidateFrontendRejectsDockerfileV0(t *testing.T) { + err := validateFrontend(frontendSettings{Name: frontendDockerfileV0}) + if err == nil { + t.Fatal("expected error for dockerfile.v0 frontend") + } +} diff --git a/pkg/build/frontend.go b/pkg/build/frontend.go index 94ebb07..a336045 100644 --- a/pkg/build/frontend.go +++ b/pkg/build/frontend.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "maps" "strconv" "strings" "sync" @@ -38,6 +39,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/moby/buildkit/frontend/dockerui" gateway "github.com/moby/buildkit/frontend/gateway/client" + gwpb "github.com/moby/buildkit/frontend/gateway/pb" "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/progress/progresswriter" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" @@ -52,6 +54,10 @@ func frontend(ctx context.Context, c gateway.Client) (*gateway.Result, error) { return nil, err } + if !bopts.frontend.native() { + return forwardGateway(ctx, c, bopts.frontend) + } + res := gateway.NewResult() expPlatforms := &exptypes.Platforms{ Platforms: make([]exptypes.Platform, len(bopts.Platforms)), @@ -76,6 +82,7 @@ func frontend(ctx context.Context, c gateway.Client) (*gateway.Result, error) { states, err := resolveStates(ctx, bopts, pl, clog) if err != nil { plErrCh <- err + return } ref, cfgJSON, err := solvePlatform(ctx, bopts, pl, c, states) @@ -481,3 +488,44 @@ func globalArgs(buildPlatform, targetPlatform ocispecs.Platform, buildArgs map[s } return utils.NewMapGetter(args) } + +func forwardGateway(ctx context.Context, c gateway.Client, settings frontendSettings) (*gateway.Result, error) { + base := c.BuildOpts().Opts + opts := maps.Clone(base) + if opts == nil { + opts = map[string]string{} + } + if settings.Cmdline != "" { + opts["cmdline"] = settings.Cmdline + } + if settings.Source != "" { + opts["source"] = settings.Source + } + for k, v := range settings.Opts { + opts[k] = v + } + + gwcaps := c.BuildOpts().Caps + var frontendInputs map[string]*pb.Definition + if (&gwcaps).Supports(gwpb.CapFrontendInputs) == nil { + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, err + } + + frontendInputs = make(map[string]*pb.Definition) + for name, state := range inputs { + def, err := state.Marshal(ctx) + if err != nil { + return nil, err + } + frontendInputs[name] = def.ToPB() + } + } + + return c.Solve(ctx, gateway.SolveRequest{ + Frontend: frontendGatewayV0, + FrontendOpt: opts, + FrontendInputs: frontendInputs, + }) +} diff --git a/pkg/build/syntax.go b/pkg/build/syntax.go new file mode 100644 index 0000000..81f44c2 --- /dev/null +++ b/pkg/build/syntax.go @@ -0,0 +1,94 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025-2026 Apple Inc. and the container-builder-shim project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +package build + +import ( + "strings" + + "github.com/moby/buildkit/frontend/dockerfile/parser" +) + +const ( + keyBuildkitSyntax = "BUILDKIT_SYNTAX" +) + +type frontendSettings struct { + Name string // "dockerfile.v0", "gateway.v0", or "" for the native dockerfile path + Source string // gateway image ref when applicable + Cmdline string + Opts map[string]string // extra FrontendAttrs +} + +func (f frontendSettings) native() bool { + return f.Name == "" +} + +func resolveFrontend(dockerfile []byte, buildArgs map[string]string, contextMap map[string][]string) frontendSettings { + opts := mapExtract(contextMap, KeyFrontendOpt) + if len(opts) == 0 { + opts = nil + } + + if name, ok := first(contextMap, KeyFrontend); ok { + settings := frontendSettings{ + Name: name, + Opts: opts, + } + if source, ok := opts["source"]; ok { + settings.Source = source + } + if cmdline, ok := opts["cmdline"]; ok { + settings.Cmdline = cmdline + } + return settings + } + + if cmdline, ok := buildArgs[keyBuildkitSyntax]; ok { + cmdline = strings.TrimSpace(cmdline) + if cmdline != "" { + ref, _ := splitSyntaxCmdline(cmdline) + if ref != "" { + return frontendSettings{ + Name: frontendGatewayV0, + Source: ref, + Cmdline: cmdline, + Opts: opts, + } + } + } + } + + if ref, cmdline, _, ok := parser.DetectSyntax(dockerfile); ok { + ref = strings.TrimSpace(ref) + if ref != "" { + return frontendSettings{ + Name: frontendGatewayV0, + Source: ref, + Cmdline: cmdline, + Opts: opts, + } + } + } + + return frontendSettings{} +} + +func splitSyntaxCmdline(cmdline string) (ref, full string) { + cmdline = strings.TrimSpace(cmdline) + parts := strings.SplitN(cmdline, " ", 2) + return parts[0], cmdline +} diff --git a/pkg/build/syntax_test.go b/pkg/build/syntax_test.go new file mode 100644 index 0000000..539c19a --- /dev/null +++ b/pkg/build/syntax_test.go @@ -0,0 +1,198 @@ +//===----------------------------------------------------------------------===// +// Copyright © 2025-2026 Apple Inc. and the container-builder-shim project authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//===----------------------------------------------------------------------===// + +package build + +import ( + "reflect" + "testing" +) + +func TestResolveFrontend(t *testing.T) { + tests := []struct { + name string + dockerfile string + buildArgs map[string]string + contextMap map[string][]string + want frontendSettings + wantNative bool + }{ + { + name: "plain dockerfile uses native path", + dockerfile: `FROM alpine:latest +RUN echo hi`, + buildArgs: map[string]string{}, + contextMap: map[string][]string{}, + want: frontendSettings{}, + wantNative: true, + }, + { + name: "yamlfile syntax header", + dockerfile: `# syntax=ghcr.io/builderhub/yamlfile:latest +apiVersion: v1alpha1 +targets: + myapp: + from: alpine:latest`, + buildArgs: map[string]string{}, + contextMap: map[string][]string{}, + want: frontendSettings{ + Name: frontendGatewayV0, + Source: "ghcr.io/builderhub/yamlfile:latest", + Cmdline: "ghcr.io/builderhub/yamlfile:latest", + }, + wantNative: false, + }, + { + name: "BUILDKIT_SYNTAX build arg", + dockerfile: `FROM alpine:latest`, + buildArgs: map[string]string{ + keyBuildkitSyntax: "ghcr.io/builderhub/yamlfile:latest", + }, + contextMap: map[string][]string{}, + want: frontendSettings{ + Name: frontendGatewayV0, + Source: "ghcr.io/builderhub/yamlfile:latest", + Cmdline: "ghcr.io/builderhub/yamlfile:latest", + }, + wantNative: false, + }, + { + name: "labs syntax directive", + dockerfile: `# syntax=docker/dockerfile:1.7-labs +FROM alpine:latest +RUN --mount=type=cache echo hi`, + buildArgs: map[string]string{}, + contextMap: map[string][]string{}, + want: frontendSettings{ + Name: frontendGatewayV0, + Source: "docker/dockerfile:1.7-labs", + Cmdline: "docker/dockerfile:1.7-labs", + }, + wantNative: false, + }, + { + name: "explicit gateway frontend", + dockerfile: `FROM alpine:latest`, + buildArgs: map[string]string{}, + contextMap: map[string][]string{ + KeyFrontend: {"gateway.v0"}, + KeyFrontendOpt: {"source=ghcr.io/builderhub/yamlfile:latest"}, + }, + want: frontendSettings{ + Name: frontendGatewayV0, + Source: "ghcr.io/builderhub/yamlfile:latest", + Opts: map[string]string{ + "source": "ghcr.io/builderhub/yamlfile:latest", + }, + }, + wantNative: false, + }, + { + name: "explicit dockerfile.v0 frontend", + dockerfile: `FROM alpine:latest`, + buildArgs: map[string]string{}, + contextMap: map[string][]string{ + KeyFrontend: {"dockerfile.v0"}, + }, + want: frontendSettings{ + Name: frontendDockerfileV0, + }, + wantNative: false, + }, + { + name: "BUILDKIT_SYNTAX takes priority over file syntax", + dockerfile: `# syntax=docker/dockerfile:1.7-labs +FROM alpine:latest`, + buildArgs: map[string]string{ + keyBuildkitSyntax: "ghcr.io/custom/frontend:v1", + }, + contextMap: map[string][]string{}, + want: frontendSettings{ + Name: frontendGatewayV0, + Source: "ghcr.io/custom/frontend:v1", + Cmdline: "ghcr.io/custom/frontend:v1", + }, + wantNative: false, + }, + { + name: "explicit frontend metadata takes priority over BUILDKIT_SYNTAX", + dockerfile: `FROM alpine:latest`, + buildArgs: map[string]string{ + keyBuildkitSyntax: "ghcr.io/custom/frontend:v1", + }, + contextMap: map[string][]string{ + KeyFrontend: {"dockerfile.v0"}, + }, + want: frontendSettings{ + Name: frontendDockerfileV0, + }, + wantNative: false, + }, + { + name: "explicit frontend takes priority over file syntax header", + dockerfile: `# syntax=ghcr.io/builderhub/yamlfile:latest +FROM alpine:latest`, + buildArgs: map[string]string{}, + contextMap: map[string][]string{ + KeyFrontend: {"gateway.v0"}, + KeyFrontendOpt: {"source=ghcr.io/custom/explicit:latest"}, + }, + want: frontendSettings{ + Name: frontendGatewayV0, + Source: "ghcr.io/custom/explicit:latest", + Opts: map[string]string{ + "source": "ghcr.io/custom/explicit:latest", + }, + }, + wantNative: false, + }, + { + name: "empty BUILDKIT_SYNTAX uses native path", + dockerfile: `FROM alpine:latest +RUN echo hi`, + buildArgs: map[string]string{ + keyBuildkitSyntax: "", + }, + contextMap: map[string][]string{}, + want: frontendSettings{}, + wantNative: true, + }, + { + name: "whitespace BUILDKIT_SYNTAX uses native path", + dockerfile: `FROM alpine:latest +RUN echo hi`, + buildArgs: map[string]string{ + keyBuildkitSyntax: " ", + }, + contextMap: map[string][]string{}, + want: frontendSettings{}, + wantNative: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := resolveFrontend([]byte(tt.dockerfile), tt.buildArgs, tt.contextMap) + + if got.native() != tt.wantNative { + t.Errorf("resolveFrontend().native() = %v, want %v", got.native(), tt.wantNative) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("resolveFrontend() = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/pkg/fileutils/tarxfer.go b/pkg/fileutils/tarxfer.go index c87faad..c188ae2 100644 --- a/pkg/fileutils/tarxfer.go +++ b/pkg/fileutils/tarxfer.go @@ -50,7 +50,7 @@ func NewTarReceiver(cacheBase string, demux *stream.Demultiplexer) *Receiver { return &Receiver{demux: demux, cacheBase: cacheBase} } -func (r *Receiver) Receive(ctx context.Context, dockerfile []byte, dockerignore []byte, fn fs.WalkDirFunc) (string, error) { +func (r *Receiver) Receive(ctx context.Context, dockerfile []byte, dockerignore []byte, stageBuildDefinition bool, fn fs.WalkDirFunc) (string, error) { errCh := make(chan error, 1) hashCh := make(chan string, 1) dataCh := make(chan []byte) @@ -95,7 +95,7 @@ func (r *Receiver) Receive(ctx context.Context, dockerfile []byte, dockerignore _ = os.Remove(tarFile) } - if len(dockerignore) > 0 { + if stageBuildDefinition && len(dockerfile) > 0 { if err := stageDockerfiles(ctx, cacheDir, dockerfile, dockerignore); err != nil { return "", err } diff --git a/pkg/fileutils/tarxfer_test.go b/pkg/fileutils/tarxfer_test.go index 42042c6..0d45683 100644 --- a/pkg/fileutils/tarxfer_test.go +++ b/pkg/fileutils/tarxfer_test.go @@ -95,7 +95,7 @@ func TestReceiver_Receive_Success(t *testing.T) { return nil } - checksum, err := r.Receive(ctx, []byte{}, []byte{}, walkFn) + checksum, err := r.Receive(ctx, []byte{}, []byte{}, false, walkFn) if err != nil { t.Fatalf("Receive returned error: %v", err) } @@ -181,7 +181,7 @@ func TestReceiver_Receive_OverflowsDemuxChannel(t *testing.T) { } start := time.Now() - checksum, err := r.Receive(ctx, []byte{}, []byte{}, walkFn) + checksum, err := r.Receive(ctx, []byte{}, []byte{}, false, walkFn) elapsed := time.Since(start) if err != nil { t.Fatalf("Receive failed after %v: %v", elapsed, err) @@ -211,7 +211,7 @@ func TestReceiver_Receive_ServerError(t *testing.T) { tmpDir := t.TempDir() r := NewTarReceiver(tmpDir, demux) - _, err := r.Receive(ctx, []byte{}, []byte{}, func(string, fs.DirEntry, error) error { return nil }) + _, err := r.Receive(ctx, []byte{}, []byte{}, false, func(string, fs.DirEntry, error) error { return nil }) if err == nil { t.Fatalf("expected server error, got nil") } @@ -219,3 +219,55 @@ func TestReceiver_Receive_ServerError(t *testing.T) { t.Fatalf("unexpected error: %v", err) } } + +func TestReceiver_Receive_StagesDockerfile(t *testing.T) { + archive, err := makeTar() + if err != nil { + t.Fatalf("makeTar: %v", err) + } + + hashBytes := sha256.Sum256(archive) + hash := hex.EncodeToString(hashBytes[:]) + header := archive[:512] + body := archive[512:] + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + demux := newDemux(ctx) + + _ = demux.Accept(btPacket([]byte{}, false, map[string]string{"hash": hash})) + _ = demux.Accept(btPacket(header, false, nil)) + _ = demux.Accept(btPacket(body, true, nil)) + + tmpDir := t.TempDir() + r := NewTarReceiver(tmpDir, demux) + + dockerfile := []byte("# syntax=ghcr.io/builderhub/yamlfile:latest\nFROM alpine\n") + var visited []string + walkFn := func(p string, _ fs.DirEntry, _ error) error { + visited = append(visited, p) + return nil + } + + checksum, err := r.Receive(ctx, dockerfile, []byte{}, true, walkFn) + if err != nil { + t.Fatalf("Receive returned error: %v", err) + } + + stagedPath := filepath.Join(DockerfileStaging, "Dockerfile") + found := false + for _, p := range visited { + if p == stagedPath { + found = true + break + } + } + if !found { + t.Fatalf("staged dockerfile not in walk output: %v", visited) + } + + cacheDir := filepath.Join(tmpDir, checksum) + if fi, err := os.Stat(filepath.Join(cacheDir, stagedPath)); err != nil || !fi.Mode().IsRegular() { + t.Fatalf("staged dockerfile missing on disk: %v", err) + } +} diff --git a/pkg/fssync/fssync.go b/pkg/fssync/fssync.go index 820f5ce..754492c 100644 --- a/pkg/fssync/fssync.go +++ b/pkg/fssync/fssync.go @@ -56,12 +56,13 @@ type FSSyncProxy struct { addedGlobs []string - dockerfile []byte - dockerignore []byte + dockerfile []byte + dockerignore []byte + stageBuildDefinition bool } func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string, - dockerfile []byte, dockerignore []byte) (*FSSyncProxy, error) { + dockerfile []byte, dockerignore []byte, stageBuildDefinition bool) (*FSSyncProxy, error) { f := new(FSSyncProxy) f.contextDir = contextDir @@ -70,6 +71,7 @@ func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string, f.dockerfile = dockerfile f.dockerignore = dockerignore + f.stageBuildDefinition = stageBuildDefinition return f, nil } diff --git a/pkg/fssync/walk.go b/pkg/fssync/walk.go index 52290e6..42f660a 100644 --- a/pkg/fssync/walk.go +++ b/pkg/fssync/walk.go @@ -134,7 +134,7 @@ func (f *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error { switch walkMeta.Mode { case ModeTAR: receiver := fileutils.NewTarReceiver(f.fsPath, demux) - checksum, err := receiver.Receive(ctx, f.proxy.dockerfile, f.proxy.dockerignore, + checksum, err := receiver.Receive(ctx, f.proxy.dockerfile, f.proxy.dockerignore, f.proxy.stageBuildDefinition, func(path string, d fs.DirEntry, err error) error { excluded, err := excludeMatcher.MatchesOrParentMatches(path) if excluded {