diff --git a/pkg/build/build.go b/pkg/build/build.go index 0178a4e..7ea5e2b 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -137,8 +137,8 @@ func Build(ctx context.Context, opts *BOpts) error { KeyContentStoreName: opts.ContentStore, } - if opts.HiddenDockerDir != "" { - solveOpt.FrontendAttrs["filename"] = filepath.Join(opts.HiddenDockerDir, "Dockerfile") + if len(opts.Dockerignore) > 0 { + solveOpt.FrontendAttrs["filename"] = filepath.Join(DockerfileStaging, "Dockerfile") } if opts.NoCache { diff --git a/pkg/build/buildopts.go b/pkg/build/buildopts.go index 20fa8db..8658717 100644 --- a/pkg/build/buildopts.go +++ b/pkg/build/buildopts.go @@ -42,9 +42,8 @@ const ( KeyContentStoreName = "container" // Base64-encoded Dockerfile contents. KeyDockerfile = "dockerfile" - // Hidden directory for the dockerfile and dockerignore to be placed. - // This is provided when docker specific ignore file is found, which might live outside the build context. - KeyHiddenDockerDir = "hidden-docker-dir" + // Base64-encoded Dockerignore contents. + KeyDockerignore = "dockerignore" // Image reference (name:tag) to assign to the built image. KeyTag = "tag" // Target platforms to build the image for. @@ -74,28 +73,32 @@ const ( ) const ( + // Used to share built artifacts outside VM GlobalExportPath = "/var/lib/container-builder-shim/exports" + // If KeyDockerignore argument is provided, Dockerfile and ignore file are + // staged at DockerfileStaging directory, and buildkit uses them. + DockerfileStaging = fssync.DockerfileStaging ) var keyBOpts = struct{}{} type BOpts struct { - BuildID string - Dockerfile []byte - Tag string - ContextDir string - HiddenDockerDir string - BuildPlatforms []ocispecs.Platform - Platforms []ocispecs.Platform - NoCache bool - Target string - BuildArgs map[string]string - Secrets map[string][]byte - CacheIn []string - CacheOut []string - Outputs []string - Labels map[string]string - ProgressWriter progresswriter.Writer + BuildID string + Dockerfile []byte + Dockerignore []byte + Tag string + ContextDir string + BuildPlatforms []ocispecs.Platform + Platforms []ocispecs.Platform + NoCache bool + Target string + BuildArgs map[string]string + Secrets map[string][]byte + CacheIn []string + CacheOut []string + Outputs []string + Labels map[string]string + ProgressWriter progresswriter.Writer ContentStore *content.ContentStoreProxy Resolver *resolver.ResolverProxy @@ -129,7 +132,17 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] return nil, err } - hiddenDockerDir, _ := first(KeyHiddenDockerDir) + dockerignoreBase64Bytes, ok := first(KeyDockerignore) + + dockerignoreBytes := []byte{} + if ok { + dockerignoreBytes, err = base64.StdEncoding.DecodeString(dockerignoreBase64Bytes) + if err != nil { + return nil, err + } + + dockerignoreBytes = append(dockerignoreBytes, []byte("\n"+DockerfileStaging)...) + } progress, ok := first(KeyProgress) if !ok { @@ -301,7 +314,7 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] } } - fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs) + fssyncProxy, err := fssync.NewFSSyncProxy(".", basePath, addedGlobs, dockerfileBytes, dockerignoreBytes) if err != nil { return nil, err } @@ -312,27 +325,27 @@ func NewBuildOpts(ctx context.Context, basePath string, contextMap map[string][] } bopts := &BOpts{ - BuildID: buildID, - Dockerfile: dockerfileBytes, - Tag: tag, - BuildPlatforms: bps, - Platforms: pls, - ContextDir: ctxDir, - HiddenDockerDir: hiddenDockerDir, - ContentStore: contentProxy, - FSSync: fssyncProxy, - NoCache: noCache, - Resolver: resolver.NewResolverProxy(), - ProgressWriter: pw, - Stdio: stdioProxy, - Target: target, - Labels: labels, - BuildArgs: buildArgs, - Secrets: secrets, - CacheIn: cacheIn, - CacheOut: cacheOut, - Outputs: outputs, - basePath: filepath.Join(basePath, buildID), + BuildID: buildID, + Dockerfile: dockerfileBytes, + Dockerignore: dockerignoreBytes, + Tag: tag, + BuildPlatforms: bps, + Platforms: pls, + ContextDir: ctxDir, + ContentStore: contentProxy, + FSSync: fssyncProxy, + NoCache: noCache, + Resolver: resolver.NewResolverProxy(), + ProgressWriter: pw, + Stdio: stdioProxy, + Target: target, + Labels: labels, + BuildArgs: buildArgs, + Secrets: secrets, + CacheIn: cacheIn, + CacheOut: cacheOut, + Outputs: outputs, + basePath: filepath.Join(basePath, buildID), } return bopts, nil diff --git a/pkg/fileutils/tarxfer.go b/pkg/fileutils/tarxfer.go index 964e88c..b3145b0 100644 --- a/pkg/fileutils/tarxfer.go +++ b/pkg/fileutils/tarxfer.go @@ -29,6 +29,8 @@ import ( "github.com/apple/container-builder-shim/pkg/stream" ) +const DockerfileStaging = ".com.apple.container" + // Receiver streams a remote tar archive, caches it under cacheBase and calls fn // for every entry that is a regular file, directory or symlink. type Receiver struct { @@ -40,7 +42,7 @@ func NewTarReceiver(cacheBase string, demux *stream.Demultiplexer) *Receiver { return &Receiver{demux: demux, cacheBase: cacheBase} } -func (r *Receiver) Receive(ctx context.Context, fn fs.WalkDirFunc) (string, error) { +func (r *Receiver) Receive(ctx context.Context, dockerfile []byte, dockerignore []byte, fn fs.WalkDirFunc) (string, error) { errCh := make(chan error, 1) hashCh := make(chan string, 1) dataCh := make(chan []byte) @@ -85,6 +87,12 @@ func (r *Receiver) Receive(ctx context.Context, fn fs.WalkDirFunc) (string, erro _ = os.Remove(tarFile) } + if len(dockerignore) > 0 { + if err := stageDockerfiles(ctx, cacheDir, dockerfile, dockerignore); err != nil { + return "", err + } + } + return checksum, filepath.Walk(cacheDir, func(p string, info os.FileInfo, _ error) error { rel, err := filepath.Rel(cacheDir, p) if err != nil || rel == "." { @@ -270,6 +278,9 @@ func unpackTar(ctx context.Context, tarFile, dest string) error { if err != nil { return err } + if hdr.Name == DockerfileStaging { + return fmt.Errorf("cannot use reserved path: %s", hdr.Name) + } target := filepath.Join(dest, hdr.Name) if !strings.HasPrefix(target, filepath.Clean(dest)+string(os.PathSeparator)) { return fmt.Errorf("invalid tar path: %s", hdr.Name) @@ -310,3 +321,28 @@ func unpackTar(ctx context.Context, tarFile, dest string) error { } return nil } + +func stageDockerfiles(ctx context.Context, cacheDir string, dockerfile []byte, dockerignore []byte) error { + staging := filepath.Join(cacheDir, DockerfileStaging) + if err := os.MkdirAll(staging, 0o755); err != nil { + return err + } + + dockerfilePath := filepath.Join(staging, "Dockerfile") + f, err := os.OpenFile(dockerfilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + f.Write(dockerfile) + f.Close() + + dockerignorePath := filepath.Join(staging, "Dockerfile.dockerignore") + f, err = os.OpenFile(dockerignorePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + f.Write(dockerignore) + f.Close() + + return nil +} diff --git a/pkg/fileutils/tarxfer_test.go b/pkg/fileutils/tarxfer_test.go index 294c11c..e36de8f 100644 --- a/pkg/fileutils/tarxfer_test.go +++ b/pkg/fileutils/tarxfer_test.go @@ -95,7 +95,7 @@ func TestReceiver_Receive_Success(t *testing.T) { return nil } - checksum, err := r.Receive(ctx, walkFn) + checksum, err := r.Receive(ctx, []byte{}, []byte{}, walkFn) if err != nil { t.Fatalf("Receive returned error: %v", err) } @@ -125,7 +125,7 @@ func TestReceiver_Receive_ServerError(t *testing.T) { tmpDir := t.TempDir() r := NewTarReceiver(tmpDir, demux) - _, err := r.Receive(ctx, func(string, fs.DirEntry, error) error { return nil }) + _, err := r.Receive(ctx, []byte{}, []byte{}, func(string, fs.DirEntry, error) error { return nil }) if err == nil { t.Fatalf("expected server error, got nil") } diff --git a/pkg/fssync/diffcopy.go b/pkg/fssync/diffcopy.go index 5f6d6d6..2546da2 100644 --- a/pkg/fssync/diffcopy.go +++ b/pkg/fssync/diffcopy.go @@ -17,10 +17,12 @@ package fssync import ( + "bytes" "context" "io" "io/fs" "os" + "path/filepath" "sync" "syscall" @@ -141,12 +143,27 @@ func (s *sender) queue(id uint32) error { } func (s *sender) sendFile(h *sendHandle) error { - f, err := s.fs.Open(h.path) - if err == nil { - defer f.Close() - buf := bufPool.Get().(*[]byte) - defer bufPool.Put(buf) - if _, err := io.CopyBuffer(&fileSender{sender: s, id: h.id}, struct{ io.Reader }{f}, *buf); err != nil { + var r io.Reader + buf := bufPool.Get().(*[]byte) + defer bufPool.Put(buf) + + switch h.path { + case filepath.Join(DockerfileStaging, "Dockerfile"): + r = bytes.NewReader(s.fs.proxy.dockerfile) + case filepath.Join(DockerfileStaging, "Dockerfile.dockerignore"): + r = bytes.NewReader(s.fs.proxy.dockerignore) + } + + if r == nil { + f, err := s.fs.Open(h.path) + if err == nil { + defer f.Close() + if _, err := io.CopyBuffer(&fileSender{sender: s, id: h.id}, struct{ io.Reader }{f}, *buf); err != nil { + return err + } + } + } else { + if _, err := io.CopyBuffer(&fileSender{sender: s, id: h.id}, r, *buf); err != nil { return err } } @@ -183,6 +200,7 @@ func (s *sender) walk(ctx context.Context) error { if err != nil { return err } + return errors.Wrapf(s.conn.SendMsg(&types.Packet{Type: types.PACKET_STAT}), "failed to send last stat") } diff --git a/pkg/fssync/fssync.go b/pkg/fssync/fssync.go index 0d094f1..595d676 100644 --- a/pkg/fssync/fssync.go +++ b/pkg/fssync/fssync.go @@ -25,11 +25,14 @@ import ( "github.com/moby/buildkit/session/filesync" "github.com/apple/container-builder-shim/pkg/api" + "github.com/apple/container-builder-shim/pkg/fileutils" "github.com/apple/container-builder-shim/pkg/stream" "google.golang.org/grpc" ) +const DockerfileStaging = fileutils.DockerfileStaging + var ( _ stream.Stage = &FSSyncProxy{} _ session.Attachable = &FSSyncProxy{} @@ -46,14 +49,21 @@ type FSSyncProxy struct { basePath string addedGlobs []string + + dockerfile []byte + dockerignore []byte } -func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string) (*FSSyncProxy, error) { +func NewFSSyncProxy(contextDir string, basePath string, addedGlobs []string, + dockerfile []byte, dockerignore []byte) (*FSSyncProxy, error) { f := new(FSSyncProxy) f.contextDir = contextDir f.basePath = filepath.Join(basePath, f.String()) f.addedGlobs = addedGlobs + + f.dockerfile = dockerfile + f.dockerignore = dockerignore return f, nil } diff --git a/pkg/fssync/walk.go b/pkg/fssync/walk.go index 53df3f5..7715f6a 100644 --- a/pkg/fssync/walk.go +++ b/pkg/fssync/walk.go @@ -124,14 +124,15 @@ func (f *FS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error { switch walkMeta.Mode { case ModeTAR: receiver := fileutils.NewTarReceiver(f.fsPath, demux) - checksum, err := receiver.Receive(ctx, func(path string, d fs.DirEntry, err error) error { - excluded, err := excludeMatcher.MatchesOrParentMatches(path) - if excluded { - return nil - } - - return fn(path, d, err) - }) + checksum, err := receiver.Receive(ctx, f.proxy.dockerfile, f.proxy.dockerignore, + func(path string, d fs.DirEntry, err error) error { + excluded, err := excludeMatcher.MatchesOrParentMatches(path) + if excluded { + return nil + } + + return fn(path, d, err) + }) if err != nil { return err }