From 048792d1c2760a5fcbd4d780279e63f6074e1cb0 Mon Sep 17 00:00:00 2001 From: dberkov Date: Sun, 19 Jul 2026 21:33:40 -0700 Subject: [PATCH 1/8] Add node-local OCI image layer cache (imagecache), replacing memorypullcache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Content-addressed pool of unpacked image layers, shared by every actor on the node; actor rootfs becomes an overlayfs mount (cached layers as read- only lowers, bundle-local upper) instead of a full re-untar per run. atelet (no capabilities) pulls and unpacks; the privileged ateoms finalize whiteouts and mount. Tag refs resolve via one HEAD and become cacheable; pull memory is O(stream buffers); the cache survives restarts. Phase 1 of #463. Fixes #437, #166, #228. Validated: kind + GKE counter demos (gvisor and microvm), suspend/resume (oci_unpack ~3ms vs ~15-20s), 411 SWE-bench-scale images pulled and unpacked with 0 failures, root-gated unit tests for the privileged paths. Known gaps: no GC yet (Phase 2, see internal/imagecache/README.md); upgrade ordering — deploy new ateoms before/with the new atelet. --- .github/workflows/pr-workflow.yaml | 5 + .../memorypullcache/memorypullcache.go | 235 ------- .../memorypullcache/memorypullcache_test.go | 84 --- cmd/atelet/main.go | 33 +- cmd/atelet/oci.go | 276 ++------- cmd/atelet/oci_test.go | 499 --------------- cmd/ateom-gvisor/main.go | 33 +- cmd/ateom-microvm/checkpoint.go | 8 + cmd/ateom-microvm/run.go | 10 + internal/ateompath/ateompath.go | 6 + internal/imagecache/README.md | 179 ++++++ internal/imagecache/bundle_linux.go | 215 +++++++ internal/imagecache/bundle_linux_test.go | 210 +++++++ internal/imagecache/imagecache.go | 546 +++++++++++++++++ internal/imagecache/imagecache_test.go | 298 +++++++++ internal/imagecache/mountinfo.go | 61 ++ internal/imagecache/mountinfo_test.go | 73 +++ internal/imagecache/options_test.go | 69 +++ internal/imagecache/removeall.go | 40 ++ internal/imagecache/spec.go | 142 +++++ internal/imagecache/spec_test.go | 145 +++++ internal/imagecache/unpack.go | 259 ++++++++ internal/imagecache/unpack_test.go | 572 ++++++++++++++++++ tools/validate-image-cache/README.md | 102 ++++ tools/validate-image-cache/main.go | 263 ++++++++ .../internal/httptest/httptest.go | 104 ++++ .../pkg/registry/README.md | 14 + .../pkg/registry/blobs.go | 540 +++++++++++++++++ .../pkg/registry/blobs_disk.go | 84 +++ .../pkg/registry/error.go | 79 +++ .../pkg/registry/manifest.go | 444 ++++++++++++++ .../pkg/registry/registry.go | 144 +++++ .../go-containerregistry/pkg/registry/tls.go | 29 + vendor/modules.txt | 2 + 34 files changed, 4729 insertions(+), 1074 deletions(-) delete mode 100644 cmd/atelet/internal/memorypullcache/memorypullcache.go delete mode 100644 cmd/atelet/internal/memorypullcache/memorypullcache_test.go create mode 100644 internal/imagecache/README.md create mode 100644 internal/imagecache/bundle_linux.go create mode 100644 internal/imagecache/bundle_linux_test.go create mode 100644 internal/imagecache/imagecache.go create mode 100644 internal/imagecache/imagecache_test.go create mode 100644 internal/imagecache/mountinfo.go create mode 100644 internal/imagecache/mountinfo_test.go create mode 100644 internal/imagecache/options_test.go create mode 100644 internal/imagecache/removeall.go create mode 100644 internal/imagecache/spec.go create mode 100644 internal/imagecache/spec_test.go create mode 100644 internal/imagecache/unpack.go create mode 100644 internal/imagecache/unpack_test.go create mode 100644 tools/validate-image-cache/README.md create mode 100644 tools/validate-image-cache/main.go create mode 100644 vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/README.md create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/blobs_disk.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/error.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/registry.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/tls.go diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 5a6c99eec..48023baf7 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -33,6 +33,11 @@ jobs: with: go-version-file: 'go.mod' - run: go test -v ./... + # The imagecache consumer half (overlay mounts, whiteout mknod, trusted.* + # xattrs) is guarded by root-only tests that skip for the unprivileged + # runner user above; run that one package again as root so they execute. + - name: imagecache privileged tests + run: sudo -E env "PATH=$PATH" go test -v -count=1 ./internal/imagecache/ - name: verify run: hack/verify-all.sh # One kind cluster exercises BOTH runtimes. Free x86-64 ubuntu-latest runners diff --git a/cmd/atelet/internal/memorypullcache/memorypullcache.go b/cmd/atelet/internal/memorypullcache/memorypullcache.go deleted file mode 100644 index 32e84bf26..000000000 --- a/cmd/atelet/internal/memorypullcache/memorypullcache.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memorypullcache - -import ( - "bytes" - "context" - "fmt" - "io" - "log/slog" - "net" - "runtime" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/remote" - "k8s.io/utils/lru" -) - -type MemoryPullCache struct { - gcpAuthenticator authn.Authenticator - - localhostRegistryReplacement string - - // Map from hexadecimal sha256 hash of image to the cached image (tarball + config env). - cache *lru.Cache -} - -type cachedImage struct { - tar []byte - cfg v1.Config -} - -func NewMemoryPullCache(ctx context.Context, gcpAuthenticator authn.Authenticator, localhostRegistryReplacement string) (*MemoryPullCache, error) { - c := &MemoryPullCache{ - // TODO: Need a smarter cache with bounds on total consumed size, not - // just number of entries. Potentially also try to share the cache - // across ateoms on the same machine. - // - // It would have to be a directory with files named after the sha256 - // hash. The benefit would be that a read might be found in the - // filesystem cache, or perhaps the folder could be on SSD. - // - // From the perspective of stable operation, without hidden kernel - // caches that could fill up or have weird behavior, it might be better - // to just have two levels. Store some images in ateom memory, and the - // rest are kept in a shared GCS cache. - cache: lru.New(256), - localhostRegistryReplacement: localhostRegistryReplacement, - } - - c.gcpAuthenticator = gcpAuthenticator - - return c, nil -} - -// Fetch returns the image's extracted filesystem tarball and its OCI image config. -func (c *MemoryPullCache) Fetch(ctx context.Context, ref string) (io.ReadCloser, *v1.Config, error) { - // when running in kind we need to rewrite the registry endpoint similar to the - // containerd mirror config used in https://kind.sigs.k8s.io/docs/user/local-registry/ - // for now we have simple opt-in support to rewrite local registries - rewritten := false - if c.localhostRegistryReplacement != "" { - newRef := c.rewriteLocalRegistry(ref) - if newRef != ref { - ref = newRef - rewritten = true - } - } - var nameOpts []name.Option - // match docker behavior, permit http image pulls for local registries - // this avoids needing to distribute TLS certs all around for local development - if rewritten || isLocalRegistry(ref) { - nameOpts = append(nameOpts, name.Insecure) - } - - parsedRef, err := name.ParseReference(ref, nameOpts...) - if err != nil { - return nil, nil, fmt.Errorf("while parsing reference: %w", err) - } - - // If the image ref included a digest, check for a hit in the pull cache. - requestedDigest, digestWasIncluded := parsedRef.(name.Digest) - if digestWasIncluded { - slog.InfoContext( - ctx, - "Ref includes digest, checking for cache hit", - slog.String("ref", ref), - slog.String("digest", requestedDigest.DigestStr()), - ) - - if vAny, ok := c.cache.Get(requestedDigest.DigestStr()); ok { - slog.InfoContext( - ctx, - "Cache hit", - slog.String("ref", ref), - slog.String("digest", requestedDigest.DigestStr()), - ) - img := vAny.(*cachedImage) - return io.NopCloser(bytes.NewReader(img.tar)), &img.cfg, nil - } - } - - slog.InfoContext( - ctx, - "Cache miss", - slog.String("ref", ref), - ) - - // If we didn't have a cache hit, we are on the slow path of pulling the - // image from the registry. This is a chatty process, with multiple round - // trips to the registry. - - var remoteOptions []remote.Option - remoteOptions = append(remoteOptions, - // Propagate caller ctx into go-containerregistry so cancellation tears - // down in-flight layer-blob HTTP requests instead of letting them run - // to completion in background goroutines (which retain partial-blob - // buffers and amplify atelet RSS during ResumeActor death loops). - remote.WithContext(ctx), - remote.WithPlatform(v1.Platform{ - Architecture: runtime.GOARCH, - OS: "linux", - }), - ) - - registry := parsedRef.Context().Registry.RegistryStr() - if registry == "gcr.io" || strings.HasSuffix(registry, ".gcr.io") || registry == "pkg.dev" || strings.HasSuffix(registry, ".pkg.dev") { - if c.gcpAuthenticator != nil { - remoteOptions = append(remoteOptions, remote.WithAuth(c.gcpAuthenticator)) - } - } - - img, err := remote.Image(parsedRef, remoteOptions...) - if err != nil { - return nil, nil, fmt.Errorf("in remote.Image: %w", err) - } - - var imageCfg v1.Config - cfg, cfgErr := img.ConfigFile() - if cfgErr != nil { - return nil, nil, fmt.Errorf("while reading image config: %w", cfgErr) - } - imageCfg = cfg.Config - - size, err := img.Size() - if err != nil { - return nil, nil, fmt.Errorf("in img.Size(): %w", err) - } - - if size > 100*1024*1024 { - slog.InfoContext(ctx, - "Image is too large to cache", - slog.String("ref", ref), - slog.Int64("size", size), - ) - return mutate.Extract(img), &imageCfg, err - } - - tarData := mutate.Extract(img) - defer tarData.Close() - - memData, err := io.ReadAll(tarData) - if err != nil { - return nil, nil, fmt.Errorf("while reading image: %w", err) - } - - if digestWasIncluded { - // If the user requested multi-arch image, the digest they request will - // not be the same as the digest of the image we actually downloaded - // from the registry. We need to place the cache entry under the digest - // they requested. - c.cache.Add(requestedDigest.DigestStr(), &cachedImage{tar: memData, cfg: imageCfg}) - slog.InfoContext( - ctx, - "Populated image cache", - slog.String("ref", ref), - slog.String("digest", requestedDigest.DigestStr()), - ) - } - - return io.NopCloser(bytes.NewReader(memData)), &imageCfg, nil -} - -func registryHost(ref string) string { - parts := strings.SplitN(ref, "/", 2) - reg, err := name.NewRegistry(parts[0], name.Insecure) - if err != nil { - return "" - } - hostPart := reg.Name() - if h, _, err := net.SplitHostPort(hostPart); err == nil { - return h - } - return hostPart -} - -func isLocalhostOrLoopback(host string) bool { - if host == "localhost" { - return true - } - if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { - return true - } - return false -} - -func isLocalRegistry(ref string) bool { - // by default docker permits localhost and 127.0.0.0/8 - // we also permit IPv6 loopback here - return isLocalhostOrLoopback(registryHost(ref)) -} - -func (c *MemoryPullCache) rewriteLocalRegistry(ref string) string { - if isLocalRegistry(ref) { - parts := strings.SplitN(ref, "/", 2) - return c.localhostRegistryReplacement + "/" + parts[1] - } - return ref -} diff --git a/cmd/atelet/internal/memorypullcache/memorypullcache_test.go b/cmd/atelet/internal/memorypullcache/memorypullcache_test.go deleted file mode 100644 index 5cae666c5..000000000 --- a/cmd/atelet/internal/memorypullcache/memorypullcache_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2026 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memorypullcache - -import ( - "testing" -) - -func TestIsLocalRegistry(t *testing.T) { - tests := []struct { - ref string - want bool - }{ - {ref: "localhost/foo", want: true}, - {ref: "localhost:5001/foo", want: true}, - {ref: "127.0.0.1/foo", want: true}, - {ref: "127.0.0.1:5001/foo", want: true}, - {ref: "127.0.0.2/foo", want: true}, - {ref: "127.0.0.2:8080/foo", want: true}, - {ref: "kind-registry/foo", want: false}, - {ref: "kind-registry:5000/foo", want: false}, - {ref: "my-registry.local/foo", want: false}, - {ref: "my-registry.local:8080/foo", want: false}, - {ref: "gcr.io/foo", want: false}, - {ref: "example.com/foo", want: false}, - {ref: "foo", want: false}, - {ref: "", want: false}, - } - - for _, tt := range tests { - t.Run(tt.ref, func(t *testing.T) { - got := isLocalRegistry(tt.ref) - if got != tt.want { - t.Errorf("isLocalRegistry(%q) = %v; want %v", tt.ref, got, tt.want) - } - }) - } -} - -func TestRewriteLocalRegistry(t *testing.T) { - c := &MemoryPullCache{ - localhostRegistryReplacement: "kind-registry:5000", - } - - tests := []struct { - ref string - want string - }{ - {ref: "localhost/foo", want: "kind-registry:5000/foo"}, - {ref: "localhost:5001/foo", want: "kind-registry:5000/foo"}, - {ref: "localhost:8080/foo", want: "kind-registry:5000/foo"}, - {ref: "127.0.0.1/foo", want: "kind-registry:5000/foo"}, - {ref: "127.0.0.1:3000/foo", want: "kind-registry:5000/foo"}, - {ref: "127.0.0.2/foo", want: "kind-registry:5000/foo"}, - {ref: "127.0.0.2:8080/foo", want: "kind-registry:5000/foo"}, - {ref: "kind-registry/foo", want: "kind-registry/foo"}, - {ref: "kind-registry:5000/foo", want: "kind-registry:5000/foo"}, - {ref: "my-registry.local/foo", want: "my-registry.local/foo"}, - {ref: "gcr.io/foo", want: "gcr.io/foo"}, - {ref: "foo", want: "foo"}, - {ref: "", want: ""}, - } - - for _, tt := range tests { - t.Run(tt.ref, func(t *testing.T) { - got := c.rewriteLocalRegistry(tt.ref) - if got != tt.want { - t.Errorf("rewriteLocalRegistry(%q) = %q; want %q", tt.ref, got, tt.want) - } - }) - } -} diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 33a0d5331..a826361d5 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -30,10 +30,10 @@ import ( "cloud.google.com/go/storage" "github.com/agent-substrate/substrate/cmd/atelet/internal/ategcs" - "github.com/agent-substrate/substrate/cmd/atelet/internal/memorypullcache" "github.com/agent-substrate/substrate/internal/ateerrors" "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateletpb" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" @@ -67,6 +67,7 @@ var ( gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") + imageCacheDir = pflag.String("image-cache-dir", ateompath.ImageCacheDir, "Directory for the node-local OCI image layer cache. Must be on the volume shared with the ateom pods (the cached layers are their overlay lowerdirs), and on a disk sized for both capacity and IOPS: unpack throughput is gated by the volume's IOPS.") showVersion = pflag.Bool("version", false, "Print version and exit.") ) @@ -113,9 +114,12 @@ func main() { } } - pullCache, err := memorypullcache.NewMemoryPullCache(ctx, gcpRegistryAuthn, *localhostRegistryReplacement) + imageCache, err := imagecache.New(*imageCacheDir, + imagecache.WithAuthenticator(gcpRegistryAuthn), + imagecache.WithLocalhostRegistryReplacement(*localhostRegistryReplacement), + ) if err != nil { - serverboot.Fatal(ctx, "Failed to create pull cache", err) + serverboot.Fatal(ctx, "Failed to open image cache", err) } anonGCSClient, err := storage.NewClient(ctx, option.WithoutAuthentication()) @@ -165,7 +169,7 @@ func main() { ateomDialer, wrappedAnonGCS, wrappedGCS, - pullCache, + imageCache, ) lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) @@ -188,7 +192,7 @@ type AteomHerder struct { ateletpb.UnimplementedAteomHerderServer ateomDialer *AteomDialer - pullCache *memorypullcache.MemoryPullCache + imageCache *imagecache.Store anonGCSClient ategcs.ObjectStorage gcsClient ategcs.ObjectStorage } @@ -201,11 +205,11 @@ func NewService( ateomDialer *AteomDialer, anonGCSClient ategcs.ObjectStorage, gcsClient ategcs.ObjectStorage, - pullCache *memorypullcache.MemoryPullCache, + imageCache *imagecache.Store, ) *AteomHerder { wms := &AteomHerder{ ateomDialer: ateomDialer, - pullCache: pullCache, + imageCache: imageCache, anonGCSClient: anonGCSClient, gcsClient: gcsClient, } @@ -694,7 +698,7 @@ func (s *AteomHerder) prepareOCIBundles( if err := prepareOCIDirectory( gCtx, - s.pullCache, + s.imageCache, actorUID, "pause", spec.GetPauseImage(), @@ -727,7 +731,7 @@ func (s *AteomHerder) prepareOCIBundles( g.Go(func() error { if err := prepareOCIDirectory( gCtx, - s.pullCache, + s.imageCache, actorUID, ctr.GetName(), ctr.GetImage(), @@ -1009,11 +1013,14 @@ func writeFileAtomic(path string, data []byte, perm os.FileMode) error { func resetActorDirs(actorUID string) error { // Explicitly leave runsc logs dir untouched. - // removeAllWritable, not os.RemoveAll: the bundle holds unpacked actor-image - // rootfs whose directories keep the image's (possibly read-only) modes, which - // atelet can't remove as plain root without first making them writable. + // RemoveAllWritable, not os.RemoveAll: the bundle's upper dir can hold + // copied-up actor-image directories keeping the image's (possibly + // read-only) modes, which atelet can't remove as plain root without first + // making them writable. (The rootfs itself is just an empty mountpoint + // here: the overlay is mounted in the ateom pod's mount namespace, not + // atelet's, and is detached by ateom at teardown.) bundleDir := ateompath.OCIBundleDir(actorUID) - if err := removeAllWritable(bundleDir); err != nil { + if err := imagecache.RemoveAllWritable(bundleDir); err != nil { return wrapFileSystemErr("while deleting bundle dir: %w", err) } if err := os.MkdirAll(bundleDir, 0o700); err != nil { diff --git a/cmd/atelet/oci.go b/cmd/atelet/oci.go index aceb4e54d..9caca4928 100644 --- a/cmd/atelet/oci.go +++ b/cmd/atelet/oci.go @@ -15,23 +15,16 @@ package main import ( - "archive/tar" "context" "encoding/json" - "errors" "fmt" - "io" - "io/fs" - "log/slog" "os" "path" - "path/filepath" - "sort" "strings" - "github.com/agent-substrate/substrate/cmd/atelet/internal/memorypullcache" "github.com/agent-substrate/substrate/internal/ateerrors" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/opencontainers/runtime-spec/specs-go" "go.opentelemetry.io/otel" @@ -57,7 +50,7 @@ const ( ActorIDFileName = "actor-id" ) -func prepareOCIDirectory(ctx context.Context, pullCache *memorypullcache.MemoryPullCache, actorUID, containerName, ref string, command, args []string, env []string, annotations map[string]string, netns string, identityDir string, durableDirVolumeMounts []*ateletpb.VolumeMount) error { +func prepareOCIDirectory(ctx context.Context, imageCache *imagecache.Store, actorUID, containerName, ref string, command, args []string, env []string, annotations map[string]string, netns string, identityDir string, durableDirVolumeMounts []*ateletpb.VolumeMount) error { tracer := otel.Tracer("prepareOCIDirectory") ctx, span := tracer.Start(ctx, "prepareOCIDirectory") @@ -65,41 +58,51 @@ func prepareOCIDirectory(ctx context.Context, pullCache *memorypullcache.MemoryP defer span.End() bundlePath := ateompath.OCIBundlePath(actorUID, containerName) - rootPath := path.Join(bundlePath, "rootfs") - if err := removeAllWritable(rootPath); err != nil { - return fmt.Errorf("while clearing rootfs %q: %w", rootPath, err) + // Clear any previous bundle contents (belt and suspenders: resetActorDirs + // already wiped the bundle dir on the Run/Restore path). + if err := imagecache.RemoveAllWritable(bundlePath); err != nil { + return fmt.Errorf("while clearing bundle %q: %w", bundlePath, err) } - if err := os.MkdirAll(rootPath, 0o700); err != nil { - return fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) + // The bundle's rootfs is composed by ateom as an overlay mount just before + // the workload runs: the cached image layers are the read-only lowerdirs, + // and the bundle-local upper/work hold this actor's private writes (wiped + // between runs, preserving the pristine-rootfs-per-run contract the old + // full re-untar provided). atelet only prepares the (empty) directories — + // it deliberately runs with no capabilities, so it cannot mount. + for _, d := range []string{"rootfs", "upper", "work"} { + if err := os.MkdirAll(path.Join(bundlePath, d), 0o700); err != nil { + return fmt.Errorf("in os.MkdirAll for container bundle dir: %w", err) + } } - tarData, imageCfg, err := pullCache.Fetch(ctx, ref) + img, err := imageCache.EnsureImage(ctx, ref) if err != nil { - return fmt.Errorf("in pullCache.Fetch: %w", err) + return fmt.Errorf("in imageCache.EnsureImage: %w", err) } - defer tarData.Close() - // Argv and env need only the image config, so resolve them before the - // untar to fail fast on an invalid container config. - resolvedArgs, err := resolveProcessArgs(imageCfg, command, args) + // Argv and env need only the image config; resolve them before writing + // any spec so an invalid container config fails fast. + resolvedArgs, err := resolveProcessArgs(&img.Config, command, args) if err != nil { return fmt.Errorf("while resolving process args for container %q: %w", containerName, err) } - resolvedEnv := resolveActorEnv(imageCfg, env) - - if err := untar(ctx, tarData, rootPath); err != nil { - return fmt.Errorf("in untar: %w", err) - } + resolvedEnv := resolveActorEnv(&img.Config, env) - // Bind-mount the per-actor identity directory so the workload can read its - // own name at IdentityMountPath/ActorIDFileName. The bind target must exist - // in the rootfs for the mount to attach. + // The identity bind target must exist in the rootfs for the mount to + // attach; ateom creates it through the mounted overlay (it lands in the + // actor's upper) so the workload can read its own name at + // IdentityMountPath/ActorIDFileName. + var extraDirs []string if identityDir != "" { - if err := createMountPoint(rootPath, IdentityMountPath); err != nil { - return fmt.Errorf("while creating identity mount point: %w", err) - } + extraDirs = append(extraDirs, IdentityMountPath) + } + if err := imagecache.WriteSpec(bundlePath, &imagecache.OverlaySpec{ + Layers: img.LayerDirs, + ExtraDirs: extraDirs, + }); err != nil { + return fmt.Errorf("while writing overlay spec: %w", err) } ociSpec := buildActorOCISpec(actorUID, resolvedArgs, resolvedEnv, annotations, netns, identityDir, durableDirVolumeMounts) @@ -298,214 +301,3 @@ func buildActorOCISpec(actorUID string, args []string, env []string, annotations return spec } - -// createMountPoint creates the directory mountPath (an absolute in-rootfs -// path) to serve as a bind-mount target. It uses os.Root so the operation is -// confined to rootPath: a symlink planted by the image cannot redirect the -// write outside the extracted rootfs (same protection untar relies on). -func createMountPoint(rootPath, mountPath string) error { - root, err := os.OpenRoot(rootPath) - if err != nil { - return fmt.Errorf("opening rootfs %q: %w", rootPath, err) - } - defer root.Close() - - rel := strings.TrimPrefix(mountPath, "/") - if err := root.MkdirAll(rel, 0o755); err != nil { - return fmt.Errorf("creating mount dir %q: %w", rel, err) - } - return nil -} - -func validateTarName(name string) (cleaned string, skip bool, err error) { - if name == "" { - return "", true, nil - } - cleaned = filepath.Clean(name) - if cleaned == "." { - return "", true, nil - } - cleaned = strings.TrimPrefix(cleaned, "/") - if cleaned == "" || cleaned == "." { - return "", true, nil - } - if !filepath.IsLocal(cleaned) { - return "", false, fmt.Errorf("not a local path: %q", name) - } - return cleaned, false, nil -} - -func untar(ctx context.Context, tarData io.Reader, rootPath string) error { - tracer := otel.Tracer("ateom-gvisor") - ctx, span := tracer.Start(ctx, "untar") - defer span.End() - - // os.Root confines file operations to rootPath: ".." components and - // out-of-tree symlinks are refused by the kernel. - root, err := os.OpenRoot(rootPath) - if err != nil { - return fmt.Errorf("while opening rootfs %q as os.Root: %w", rootPath, err) - } - defer root.Close() - - // Directories are created owner-writable during extraction (so their children - // can be written even when the image marks them read-only, e.g. ko ships - // /ko-app as 0555) and their real modes are restored afterwards. This lets - // atelet, running as plain root, unpack arbitrary actor images without - // CAP_DAC_OVERRIDE. Keyed by name so a repeated dir entry's last mode wins. - dirModes := map[string]os.FileMode{} - - tarReader := tar.NewReader(tarData) - for { - hdr, err := tarReader.Next() - if errors.Is(err, io.EOF) { - break - } else if err != nil { - return fmt.Errorf("in tarReader.Next: %w", err) - } - - name, skip, err := validateTarName(hdr.Name) - if err != nil { - return fmt.Errorf("invalid tar entry: %w", err) - } - if skip { - continue - } - - mode := hdr.FileInfo().Mode().Perm() - - switch hdr.Typeflag { - case tar.TypeReg: // Regular file - // Same "later entry wins" handling: if any entry exists at the target path, - // remove it first. This ensures that: - // 1. If it's a symlink, we don't write through it (security vulnerability / incorrectness). - // 2. If it's a hardlink, we unlink it instead of truncating the shared inode. - // 3. If it's a directory, we recursively remove it so we can write the file. - if _, err := root.Lstat(name); err == nil { - if err := root.RemoveAll(name); err != nil { - return fmt.Errorf("while replacing existing path at %q before regular file: %w", name, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("while checking existing path at %q before regular file: %w", name, err) - } - - // Stream directly from tarReader to target file to avoid buffering in memory. - outFile, err := root.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, mode) - if err != nil { - return fmt.Errorf("while creating file %q: %w", name, err) - } - - _, err = io.Copy(outFile, tarReader) - closeErr := outFile.Close() - - if err != nil { - return fmt.Errorf("while writing contents of %q from tar stream: %w", name, err) - } - if closeErr != nil { - return fmt.Errorf("while closing file %q: %w", name, closeErr) - } - - case tar.TypeDir: - // Create owner-writable so children can be written even when the image - // marks the dir read-only; the real mode is restored after extraction - // (see dirModes / the restore pass below). - err := root.Mkdir(name, mode|0o700) - if errors.Is(err, os.ErrExist) { - // OCI layers can repeat a directory entry (real ko images do); the - // existing dir is already owner-writable, so let the later entry's - // mode win at restore time. - } else if err != nil { - return fmt.Errorf("while creating directory=%q, mode=%v: %w", name, mode, err) - } - dirModes[name] = mode - - case tar.TypeSymlink: - // OCI image layers may re-define the same path across layers (e.g. - // an earlier layer creates /var/run as a directory and a later - // layer re-declares it as a symlink to /run). Standard tar-extract - // semantics are "later entry wins": replace any existing entry. - if existing, err := root.Lstat(name); err == nil { - // If it's already the same symlink, skip the unlink+symlink pair. - if existing.Mode()&os.ModeSymlink != 0 { - if cur, rerr := root.Readlink(name); rerr == nil && cur == hdr.Linkname { - continue - } - } - // Root.RemoveAll removes the symlink entry itself; it does NOT - // traverse and remove the directory the symlink points to. - // That's the desired semantic here — replace this path's - // entry without touching whatever the prior symlink targeted. - if err := root.RemoveAll(name); err != nil { - return fmt.Errorf("while replacing existing path at %q before symlink: %w", name, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("while checking existing path at %q before symlink: %w", name, err) - } - if err := root.Symlink(hdr.Linkname, name); err != nil { - return fmt.Errorf("while creating symlink src=%q target=%q: %w", name, hdr.Linkname, err) - } - - case tar.TypeLink: - linkname, linkSkip, err := validateTarName(hdr.Linkname) - if err != nil { - return fmt.Errorf("invalid hardlink target for %q: %w", name, err) - } - if linkSkip { - return fmt.Errorf("invalid hardlink target for %q: empty", name) - } - // Same "later entry wins" handling as TypeSymlink: replace existing entry. - if _, err := root.Lstat(name); err == nil { - if err := root.RemoveAll(name); err != nil { - return fmt.Errorf("while replacing existing path at %q before hardlink: %w", name, err) - } - } else if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("while checking existing path at %q before hardlink: %w", name, err) - } - if err := root.Link(linkname, name); err != nil { - return fmt.Errorf("while creating hardlink src=%q target=%q: %w", name, linkname, err) - } - - default: - tfStr := string([]byte{hdr.Typeflag}) - slog.ErrorContext(ctx, "Unhandled tar entry typeflag", slog.String("typeflag", tfStr), slog.Any("hdr", hdr)) - return fmt.Errorf("unhandled tar entry typeflag %q", tfStr) - } - - } - - // Restore the image's intended directory modes now that every child exists. - // Deepest paths first: a child's path is always longer than its parent's, so - // length-descending order guarantees a directory is restored before any of its - // ancestors — restoring a parent to a non-traversable mode then can't block - // restoring its children. - dirs := make([]string, 0, len(dirModes)) - for name := range dirModes { - dirs = append(dirs, name) - } - sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) - for _, name := range dirs { - if err := root.Chmod(name, dirModes[name]); err != nil { - return fmt.Errorf("while restoring mode %v on directory %q: %w", dirModes[name], name, err) - } - } - - return nil -} - -// removeAllWritable removes path and everything under it, first making every -// directory owner-writable so its children can be unlinked. atelet runs as plain -// root (no CAP_DAC_OVERRIDE), so it cannot remove entries inside an image-defined -// read-only directory (e.g. ko's restored 0555 /ko-app) — os.RemoveAll alone -// fails there with EACCES. atelet owns these files, so chmod needs no capability. -func removeAllWritable(path string) error { - // Make dirs traversable/writable top-down (WalkDir visits a directory before - // reading it, so chmod here lets the walk descend into otherwise-unreadable - // dirs). Best-effort: ignore errors and let os.RemoveAll surface real ones. - _ = filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error { - if err == nil && d.IsDir() { - _ = os.Chmod(p, 0o700) - } - return nil - }) - return os.RemoveAll(path) -} diff --git a/cmd/atelet/oci_test.go b/cmd/atelet/oci_test.go index cdc12dcbd..e07c6c159 100644 --- a/cmd/atelet/oci_test.go +++ b/cmd/atelet/oci_test.go @@ -15,14 +15,8 @@ package main import ( - "archive/tar" - "bytes" - "context" "errors" - "os" - "path/filepath" "slices" - "strings" "testing" "github.com/agent-substrate/substrate/internal/ateerrors" @@ -31,62 +25,6 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" ) -type tarEntry struct { - name string - typeflag byte - mode int64 - body string - linkname string -} - -func defaultMode(typeflag byte) int64 { - switch typeflag { - case tar.TypeDir: - return 0o755 - case tar.TypeSymlink: - return 0o777 - default: - return 0o644 - } -} - -func buildTar(t *testing.T, entries []tarEntry) []byte { - t.Helper() - var buf bytes.Buffer - tw := tar.NewWriter(&buf) - for _, e := range entries { - mode := e.mode - if mode == 0 { - mode = defaultMode(e.typeflag) - } - hdr := &tar.Header{ - Name: e.name, - Typeflag: e.typeflag, - Mode: mode, - Size: int64(len(e.body)), - Linkname: e.linkname, - } - if err := tw.WriteHeader(hdr); err != nil { - t.Fatalf("tar.WriteHeader(%+v): %v", hdr, err) - } - if e.body != "" { - if _, err := tw.Write([]byte(e.body)); err != nil { - t.Fatalf("tar.Write(%q): %v", e.name, err) - } - } - } - if err := tw.Close(); err != nil { - t.Fatalf("tar.Close: %v", err) - } - return buf.Bytes() -} - -func runUntar(t *testing.T, entries []tarEntry) (string, error) { - t.Helper() - dir := t.TempDir() - return dir, untar(context.Background(), bytes.NewReader(buildTar(t, entries)), dir) -} - // With an identity dir, a read-only bind mount appears at IdentityMountPath. func TestBuildActorOCISpec_IdentityMount(t *testing.T) { spec := buildActorOCISpec( @@ -299,440 +237,3 @@ func TestBuildActorOCISpec_DurableDirVolumeMounts(t *testing.T) { } } } - -func TestCreateMountPoint(t *testing.T) { - t.Run("creates target inside rootfs", func(t *testing.T) { - root := t.TempDir() - if err := createMountPoint(root, IdentityMountPath); err != nil { - t.Fatalf("createMountPoint: %v", err) - } - info, err := os.Stat(filepath.Join(root, "run", "ate")) - if err != nil { - t.Fatalf("mount point not created in rootfs: %v", err) - } - if !info.IsDir() { - t.Errorf("mount point must be a directory to host the identity bind mount") - } - }) - - t.Run("refuses symlink escaping the rootfs", func(t *testing.T) { - root := t.TempDir() - outside := t.TempDir() - // A malicious image could ship /run as a symlink pointing out of the - // rootfs; os.Root must refuse to follow it. - if err := os.Symlink(outside, filepath.Join(root, "run")); err != nil { - t.Fatalf("planting symlink: %v", err) - } - if err := createMountPoint(root, IdentityMountPath); err == nil { - t.Errorf("expected error when /run escapes the rootfs, got nil") - } - // Nothing may be created through the escaping symlink. - if entries, err := os.ReadDir(outside); err != nil { - t.Errorf("reading outside dir: %v", err) - } else if len(entries) != 0 { - t.Errorf("write escaped the rootfs: %s is not empty (%d entries)", outside, len(entries)) - } - }) -} - -func TestValidateTarName(t *testing.T) { - tests := []struct { - name string - input string - wantClean string - wantSkip bool - wantErr bool - }{ - {name: "regular file", input: "etc/passwd", wantClean: "etc/passwd"}, - {name: "current dir", input: ".", wantSkip: true}, - {name: "empty", input: "", wantSkip: true}, - {name: "trailing slash", input: "etc/", wantClean: "etc"}, - {name: "absolute path", input: "/etc/passwd", wantClean: "etc/passwd"}, - {name: "double slash absolute", input: "//etc/passwd", wantClean: "etc/passwd"}, - {name: "parent escape", input: "../etc/passwd", wantErr: true}, - {name: "parent only", input: "..", wantErr: true}, - {name: "embedded escape", input: "a/../../escape", wantErr: true}, - {name: "ok with dot segments", input: "./a/./b", wantClean: "a/b"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - gotClean, gotSkip, err := validateTarName(tc.input) - if (err != nil) != tc.wantErr { - t.Fatalf("validateTarName(%q) err = %v, wantErr %v", tc.input, err, tc.wantErr) - } - if err != nil { - return - } - if gotSkip != tc.wantSkip { - t.Errorf("skip = %v, want %v", gotSkip, tc.wantSkip) - } - if gotClean != tc.wantClean { - t.Errorf("clean = %q, want %q", gotClean, tc.wantClean) - } - }) - } -} - -func TestUntar_HappyPath(t *testing.T) { - entries := []tarEntry{ - {name: ".", typeflag: tar.TypeDir}, - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/hostname", typeflag: tar.TypeReg, body: "demo\n"}, - {name: "bin/", typeflag: tar.TypeDir}, - {name: "bin/sh", typeflag: tar.TypeReg, mode: 0o755, body: "#!/sh\n"}, - {name: "bin/bash", typeflag: tar.TypeLink, linkname: "bin/sh"}, - {name: "etc/host-link", typeflag: tar.TypeSymlink, linkname: "hostname"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - - if got, err := os.ReadFile(filepath.Join(dir, "etc/hostname")); err != nil { - t.Errorf("read etc/hostname: %v", err) - } else if string(got) != "demo\n" { - t.Errorf("etc/hostname = %q, want %q", got, "demo\n") - } - - if target, err := os.Readlink(filepath.Join(dir, "etc/host-link")); err != nil { - t.Errorf("readlink etc/host-link: %v", err) - } else if target != "hostname" { - t.Errorf("symlink target = %q, want %q", target, "hostname") - } - - srcInfo, err := os.Stat(filepath.Join(dir, "bin/sh")) - if err != nil { - t.Fatalf("stat bin/sh: %v", err) - } - dstInfo, err := os.Stat(filepath.Join(dir, "bin/bash")) - if err != nil { - t.Fatalf("stat bin/bash: %v", err) - } - if !os.SameFile(srcInfo, dstInfo) { - t.Errorf("bin/bash is not a hardlink to bin/sh") - } -} - -func TestUntar_LaterEntryWins(t *testing.T) { - t.Run("dir then symlink", func(t *testing.T) { - entries := []tarEntry{ - {name: "var/", typeflag: tar.TypeDir}, - {name: "var/run/", typeflag: tar.TypeDir}, - {name: "run/", typeflag: tar.TypeDir}, - {name: "run/sock", typeflag: tar.TypeReg, body: "sock"}, - {name: "var/run", typeflag: tar.TypeSymlink, linkname: "../run"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - fi, err := os.Lstat(filepath.Join(dir, "var/run")) - if err != nil { - t.Fatalf("lstat var/run: %v", err) - } - if fi.Mode()&os.ModeSymlink == 0 { - t.Fatalf("var/run not a symlink, mode = %v", fi.Mode()) - } - if got, _ := os.Readlink(filepath.Join(dir, "var/run")); got != "../run" { - t.Errorf("symlink target = %q, want %q", got, "../run") - } - }) - - t.Run("file overwrite", func(t *testing.T) { - entries := []tarEntry{ - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/conf", typeflag: tar.TypeReg, body: "v1"}, - {name: "etc/conf", typeflag: tar.TypeReg, body: "v2"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - if got, _ := os.ReadFile(filepath.Join(dir, "etc/conf")); string(got) != "v2" { - t.Errorf("etc/conf = %q, want %q", got, "v2") - } - }) - - t.Run("symlink retargeted", func(t *testing.T) { - entries := []tarEntry{ - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/x", typeflag: tar.TypeReg, body: "x"}, - {name: "etc/y", typeflag: tar.TypeReg, body: "y"}, - {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, - {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "y"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - if got, _ := os.Readlink(filepath.Join(dir, "etc/link")); got != "y" { - t.Errorf("symlink target = %q, want %q", got, "y") - } - }) - - t.Run("repeated dir entry tolerated", func(t *testing.T) { - entries := []tarEntry{ - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/", typeflag: tar.TypeDir}, - } - if _, err := runUntar(t, entries); err != nil { - t.Errorf("untar: %v", err) - } - }) - - t.Run("identical symlink redeclaration is a no-op", func(t *testing.T) { - entries := []tarEntry{ - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/x", typeflag: tar.TypeReg, body: "x"}, - {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, - {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - if got, _ := os.Readlink(filepath.Join(dir, "etc/link")); got != "x" { - t.Errorf("symlink target = %q, want %q", got, "x") - } - }) - - t.Run("symlink overwritten by file", func(t *testing.T) { - entries := []tarEntry{ - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/x", typeflag: tar.TypeReg, body: "original"}, - {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, - {name: "etc/link", typeflag: tar.TypeReg, body: "replacement"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - fi, err := os.Lstat(filepath.Join(dir, "etc/link")) - if err != nil { - t.Fatalf("lstat etc/link: %v", err) - } - if fi.Mode().IsRegular() { - got, err := os.ReadFile(filepath.Join(dir, "etc/link")) - if err != nil { - t.Fatalf("read etc/link: %v", err) - } - if string(got) != "replacement" { - t.Errorf("etc/link content = %q, want %q", got, "replacement") - } - } else { - t.Errorf("etc/link mode is not regular file: %v", fi.Mode()) - } - // Also verify etc/x was NOT overwritten - gotX, err := os.ReadFile(filepath.Join(dir, "etc/x")) - if err != nil { - t.Fatalf("read etc/x: %v", err) - } - if string(gotX) != "original" { - t.Errorf("etc/x content was overwritten to %q", gotX) - } - }) - - t.Run("file overwritten by symlink", func(t *testing.T) { - entries := []tarEntry{ - {name: "etc/", typeflag: tar.TypeDir}, - {name: "etc/link", typeflag: tar.TypeReg, body: "original-file"}, - {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "target-doesnt-exist"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - fi, err := os.Lstat(filepath.Join(dir, "etc/link")) - if err != nil { - t.Fatalf("lstat etc/link: %v", err) - } - if fi.Mode()&os.ModeSymlink == 0 { - t.Errorf("etc/link mode is not a symlink: %v", fi.Mode()) - } - got, err := os.Readlink(filepath.Join(dir, "etc/link")) - if err != nil { - t.Fatalf("readlink etc/link: %v", err) - } - if got != "target-doesnt-exist" { - t.Errorf("etc/link target = %q, want %q", got, "target-doesnt-exist") - } - }) - - t.Run("hardlink overwritten by file", func(t *testing.T) { - entries := []tarEntry{ - {name: "bin/", typeflag: tar.TypeDir}, - {name: "bin/sh", typeflag: tar.TypeReg, body: "sh-original"}, - {name: "bin/bash", typeflag: tar.TypeLink, linkname: "bin/sh"}, - {name: "bin/bash", typeflag: tar.TypeReg, body: "bash-new"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar: %v", err) - } - gotBash, err := os.ReadFile(filepath.Join(dir, "bin/bash")) - if err != nil { - t.Fatalf("read bin/bash: %v", err) - } - if string(gotBash) != "bash-new" { - t.Errorf("bin/bash content = %q, want %q", gotBash, "bash-new") - } - // Verify bin/sh was NOT modified! - gotSh, err := os.ReadFile(filepath.Join(dir, "bin/sh")) - if err != nil { - t.Fatalf("read bin/sh: %v", err) - } - if string(gotSh) != "sh-original" { - t.Errorf("bin/sh content was overwritten to %q (hardlink was not unlinked)", gotSh) - } - }) -} - -// A read-only directory in the image (e.g. ko ships /ko-app as 0555) must still -// get its child written AND keep the image's mode, so atelet can unpack arbitrary -// actor images as plain root without CAP_DAC_OVERRIDE. removeAllWritable must -// then still be able to delete the restored read-only tree. (Meaningful as a -// non-root test run; as root the dir-permission checks are bypassed.) -func TestUntar_ReadOnlyDir(t *testing.T) { - entries := []tarEntry{ - {name: "ko-app", typeflag: tar.TypeDir, mode: 0o555}, - {name: "ko-app/counter", typeflag: tar.TypeReg, mode: 0o755, body: "bin"}, - } - dir, err := runUntar(t, entries) - if err != nil { - t.Fatalf("untar into read-only dir: %v", err) - } - if got, _ := os.ReadFile(filepath.Join(dir, "ko-app/counter")); string(got) != "bin" { - t.Errorf("ko-app/counter = %q, want %q", got, "bin") - } - info, err := os.Stat(filepath.Join(dir, "ko-app")) - if err != nil { - t.Fatalf("stat ko-app: %v", err) - } - if info.Mode().Perm() != 0o555 { - t.Errorf("ko-app mode = %v, want the image's 0555 preserved", info.Mode().Perm()) - } - // atelet must be able to delete the restored read-only tree (this also lets - // t.TempDir's cleanup succeed, which plain os.RemoveAll could not on 0555). - if err := removeAllWritable(filepath.Join(dir, "ko-app")); err != nil { - t.Errorf("removeAllWritable on restored read-only dir: %v", err) - } -} - -func TestUntar_PathTraversal(t *testing.T) { - tests := []struct { - name string - entry tarEntry - }{ - {name: "parent prefix", entry: tarEntry{name: "../escape", typeflag: tar.TypeReg, body: "x"}}, - {name: "embedded parent", entry: tarEntry{name: "a/b/../../../escape", typeflag: tar.TypeReg, body: "x"}}, - {name: "parent only", entry: tarEntry{name: "..", typeflag: tar.TypeReg, body: "x"}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := runUntar(t, []tarEntry{tc.entry}) - if err == nil { - t.Fatalf("untar(%q) succeeded, want error", tc.entry.name) - } - if !strings.Contains(err.Error(), "invalid tar entry") { - t.Errorf("error = %q, want it to mention 'invalid tar entry'", err.Error()) - } - }) - } -} - -func TestUntar_SymlinkEscape(t *testing.T) { - // CVE-2024-24579 / CVE-2020-27833 pattern: a tar declares a symlink - // pointing outside the rootfs, then a later entry writes through it. - parent := t.TempDir() - rootfsDir := filepath.Join(parent, "rootfs") - if err := os.Mkdir(rootfsDir, 0o755); err != nil { - t.Fatalf("mkdir rootfs: %v", err) - } - hostDir := filepath.Join(parent, "host") - if err := os.Mkdir(hostDir, 0o755); err != nil { - t.Fatalf("mkdir host: %v", err) - } - hostFile := filepath.Join(hostDir, "passwd") - if err := os.WriteFile(hostFile, []byte("original"), 0o644); err != nil { - t.Fatalf("write host file: %v", err) - } - - entries := []tarEntry{ - {name: "etc", typeflag: tar.TypeSymlink, linkname: hostDir}, - {name: "etc/passwd", typeflag: tar.TypeReg, body: "OWNED"}, - } - if err := untar(context.Background(), bytes.NewReader(buildTar(t, entries)), rootfsDir); err == nil { - t.Fatalf("untar succeeded; expected escape via symlink to be refused") - } - - got, err := os.ReadFile(hostFile) - if err != nil { - t.Fatalf("read host file: %v", err) - } - if string(got) != "original" { - t.Errorf("host file modified to %q -- symlink escape was NOT prevented", got) - } -} - -func TestUntar_HardlinkEscape(t *testing.T) { - tests := []struct { - name string - entry tarEntry - }{ - {name: "parent target", entry: tarEntry{name: "etc/passwd", typeflag: tar.TypeLink, linkname: "../host/passwd"}}, - {name: "embedded escape target", entry: tarEntry{name: "etc/passwd", typeflag: tar.TypeLink, linkname: "a/../../host"}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := runUntar(t, []tarEntry{tc.entry}) - if err == nil { - t.Fatalf("untar succeeded, want hardlink escape refused") - } - if !strings.Contains(err.Error(), "invalid hardlink target") { - t.Errorf("error = %q, want it to mention 'invalid hardlink target'", err.Error()) - } - }) - } -} - -func TestUntar_RejectSpecialFiles(t *testing.T) { - tests := []struct { - name string - typeflag byte - }{ - {name: "char device", typeflag: tar.TypeChar}, - {name: "block device", typeflag: tar.TypeBlock}, - {name: "fifo", typeflag: tar.TypeFifo}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := runUntar(t, []tarEntry{{name: "weird", typeflag: tc.typeflag}}) - if err == nil { - t.Fatalf("untar succeeded, want unhandled-typeflag error") - } - if !strings.Contains(err.Error(), "unhandled tar entry typeflag") { - t.Errorf("error = %q, want it to mention 'unhandled tar entry typeflag'", err.Error()) - } - }) - } -} - -func TestUntar_TruncatedArchive(t *testing.T) { - full := buildTar(t, []tarEntry{ - {name: "ok", typeflag: tar.TypeReg, body: "hello"}, - }) - if len(full) < 64 { - t.Fatalf("buildTar produced suspiciously small output: %d bytes", len(full)) - } - truncated := full[:len(full)-64] - - dir := t.TempDir() - err := untar(context.Background(), bytes.NewReader(truncated), dir) - if err == nil { - t.Fatalf("untar on truncated archive succeeded; want error") - } - if !strings.Contains(err.Error(), "in tarReader.Next") && - !strings.Contains(err.Error(), "unexpected EOF") { - t.Errorf("error = %v, want it to surface the underlying tar/copy error", err) - } -} diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index 6d0129452..affedf08e 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -32,6 +32,7 @@ import ( "github.com/agent-substrate/substrate/internal/ateinterceptors" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/contextlogging" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "github.com/agent-substrate/substrate/internal/serverboot" @@ -202,7 +203,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload actorUID: req.GetActorUid(), } - // Create and start pause container + // Create and start pause container. The bundle rootfs is composed here — + // an overlay of the node's cached image layers plus the bundle's private + // upper — because mounting is ateom's job (atelet runs with no + // capabilities); runsc's gofer resolves the mount in this pod's mount + // namespace. + if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), "pause")); err != nil { + return nil, fmt.Errorf("while composing pause rootfs: %w", err) + } if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } @@ -218,6 +226,9 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() + if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ac.GetName())); err != nil { + return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) + } if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { return nil, fmt.Errorf("while creating %q application container: %w", ac.GetName(), err) } @@ -292,6 +303,16 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec "err", err) } + // Detach the overlay rootfs mounts before atelet wipes the bundle dirs + // (deleting a bundle out from under a live mount in this namespace would + // leave the mount orphaned until the pod restarts). Best-effort, same as + // the container cleanup above. + if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(req.GetActorUid())); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after checkpoint", + "actorUID", req.GetActorUid(), + "err", err) + } + s.cleanupActorNetworkOrExit(ctx, "Failed to clean up actor network after checkpoint") // Report exactly the files runsc wrote so atelet ships precisely this set @@ -377,6 +398,13 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore checkpointDir := ateompath.RestoreStateDir(req.GetActorUid()) + // Compose the pause rootfs before create (see RunWorkload). runsc restore + // only needs the rootfs to hold the correct content; whether it came from + // an untar or an overlay of cached layers is transparent to it. + if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), "pause")); err != nil { + return nil, fmt.Errorf("while composing pause rootfs: %w", err) + } + switch req.GetScope() { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: // Create and restore pause container @@ -406,6 +434,9 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore return nil, fmt.Errorf("while starting json log pipe for %q: %w", ac.GetName(), err) } defer pw.Close() + if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ac.GetName())); err != nil { + return nil, fmt.Errorf("while composing %q rootfs: %w", ac.GetName(), err) + } switch req.GetScope() { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: if err := rcmd.cmdCreate(ctx, pw, ac.GetName(), nil); err != nil { diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index 50a19c168..f95e5a7ee 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -27,6 +27,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" ) @@ -207,4 +208,11 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running // Sweep any leftover per-sandbox host-side state + orphaned per-sandbox // processes. This is ateom's own cleanup (process kill + unmount + rm). kata.CleanupSandboxState(ctx, id) + + // Detach the bundle rootfs overlays composed in buildActorContainers, so + // atelet's bundle wipe doesn't strand live mounts in this namespace. + // Best-effort like the rest of teardown. + if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(id)); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays", slog.String("actorUID", id), slog.Any("err", err)) + } } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 6348a30f2..c519debb2 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -30,6 +30,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/third_party/kata/agentpb" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -383,6 +384,15 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom if err != nil { return nil, fmt.Errorf("while preparing kata OCI spec for %q: %w", cn, err) } + // Compose the bundle rootfs from the node's cached image layers (an + // overlay mounted in this pod's namespace; no-op for bundles without an + // overlay spec). Everything downstream — the resolv.conf write below, + // the bind into virtiofsd's shared dir, the read-only remount — then + // sees the composed tree, with host-side writes landing in the bundle's + // private upper. The guest still builds its own tmpfs upper on top. + if err := imagecache.SetupBundleRootfs(bundle); err != nil { + return nil, fmt.Errorf("while composing rootfs for %q: %w", cn, err) + } bundleRootfs := filepath.Join(bundle, "rootfs") // Write cluster DNS into the lower before it's served over virtio-fs: ateom // drops atelet's resolv.conf bind and sends no CreateSandbox.Dns, so without diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index 812c2ba27..42b2105c8 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -28,6 +28,12 @@ const ( var ( // StaticFilesDir holds things like downloaded runsc binaries. StaticFilesDir = filepath.Join(BasePath, "static-files") + + // ImageCacheDir is the node-local OCI image layer cache (see + // internal/imagecache). It lives under BasePath so the cached layer + // directories are visible at the same path in atelet (which writes them) + // and in every ateom pod (which mounts them as overlay lowerdirs). + ImageCacheDir = filepath.Join(BasePath, "image-cache") ) func RunSCBinaryPath(sha256 string) string { diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md new file mode 100644 index 000000000..0c5855990 --- /dev/null +++ b/internal/imagecache/README.md @@ -0,0 +1,179 @@ +# imagecache — node-local OCI image layer cache + +`internal/imagecache` implements substrate's node-local OCI image cache: a +content-addressed pool of **unpacked image layers**, stored once per node and +shared by every actor on it, plus the machinery that composes an actor's +rootfs from those layers as an overlayfs mount instead of extracting the +image on every run. + +It replaces the previous design (an in-memory LRU of flattened image +tarballs in `atelet`, re-untarred into every bundle on every actor +start/resume) and is the Phase 1 implementation of +[#463](https://github.com/agent-substrate/substrate/issues/463), addressing +[#120](https://github.com/agent-substrate/substrate/issues/120), +[#166](https://github.com/agent-substrate/substrate/issues/166), +[#228](https://github.com/agent-substrate/substrate/issues/228) and +[#437](https://github.com/agent-substrate/substrate/issues/437). + +What it buys, concretely: + +- Actor start/resume composes the rootfs with **one overlay mount + (milliseconds)** instead of a full image extraction (tens of seconds for + GB-scale images). `Restore timing breakdown` logs show `oci_unpack` + dropping from ~15–20 s to single-digit milliseconds on warm nodes. +- Layers shared between images are **downloaded and unpacked once per + node**, not once per image; actors sharing layers also share page cache. +- The cache is **on disk and survives atelet restarts and node reboots** + (the old in-memory cache was lost on every restart, and its unbounded heap + retention could OOM atelet — #437). +- **Tag refs are cacheable**: a tag costs one `HEAD` request to resolve to a + manifest digest (the only safe cache key for mutable tags); digest refs + hit the cache with zero network I/O. +- Memory use during pulls is O(stream buffers), independent of image size + (the old `mutate.Extract` path buffered entire flattened images — #120). + +## The privilege split + +The design is shaped by an existing substrate boundary: **atelet runs as +plain root with every Linux capability dropped** ("atelet does no mounts" — +see `manifests/ate-install/atelet.yaml`), while the **ateom worker pods are +privileged** and own all mounts on the node. The module is split accordingly: + +| Half | Runs in | Files | Needs | +|---|---|---|---| +| Store: pull, parse, unpack, record | atelet | `imagecache.go`, `unpack.go`, `spec.go` (portable) | nothing but file I/O | +| Consumer: finalize, mount, unmount | ateom-gvisor / ateom-microvm | `bundle_linux.go` (`//go:build linux`) | `CAP_MKNOD`, `CAP_SYS_ADMIN` | + +The two halves communicate through the filesystem only: the shared cache +directory (on the `/var/lib/ateom-gvisor` hostPath, so the same absolute +paths resolve in every pod) and a small per-bundle spec file. + +Because the consumer mounts the overlay **in its own mount namespace** — +exactly where the workload resolves it (runsc's gofer for gVisor, virtiofsd +for the micro-VM) — no Kubernetes mount-propagation configuration is needed +anywhere. + +## On-disk layout + +``` +/ default: /var/lib/ateom-gvisor/image-cache + version layout version marker ("1") + layers/sha256// + fs/ the unpacked layer tree (an overlay lowerdir) + whiteouts.json whiteout state recorded at unpack time + finalized marker written by FinalizeLayer (consumer side) + manifests/sha256/.json image config + ordered diffID list +``` + +A layer directory that exists is always complete: unpack streams into a +`.tmp-*` sibling and moves it into place with a single atomic rename. +Startup recovery (`New`) therefore only has to sweep orphaned temp dirs and +verify the layout version. An "image" is nothing but a manifest record +listing layer diffIDs in order — layers shared by N images exist once. + +## Pull path (atelet: `Store.EnsureImage`) + +1. **Resolve** the ref. Digest refs are parsed directly; tag refs cost one + `remote.Head`. Localhost/loopback registries are rewritten for kind + (`--localhost-registry-replacement`) and pulled over plain HTTP; gcr.io / + pkg.dev registries get the configured GCP authenticator. +2. **Cache check**: if the manifest record exists and every layer dir is + present, return with no network I/O. Missing layers (only) are re-pulled. +3. **Pull** by resolved digest: layers download in parallel (bounded at 4), + each streamed download → decompress → untar directly into the pool. + Concurrent pulls of the same image or layer are collapsed with + singleflight, so simultaneous actor starts never duplicate work — and + each completed layer lands individually, so an interrupted pull makes + incremental progress across retries. +4. **Unpack** (`unpackLayer`) is the repo's hardened untar: `os.Root` + confinement (path traversal and symlink/hardlink escapes are refused), + "later entry wins" within a layer, read-only-dir handling that works + without `CAP_DAC_OVERRIDE`, and creation of parent directories that the + layer tar omits (they may exist only in lower layers). Whiteout entries + (`.wh.*`) are **not** written into the tree — overlayfs whiteouts are + char devices atelet cannot create — they are recorded in + `whiteouts.json` for the consumer to materialize. +5. **Record**: the image config + diffID list is written under the + requested digest (and the per-platform child digest for multi-arch refs). + +`prepareOCIDirectory` in atelet then writes `rootfs-overlay.json` +(`OverlaySpec`) into the bundle next to `config.json`, listing the layer +directories bottom-first plus any `ExtraDirs` (in-rootfs bind-mount targets, +e.g. the actor identity mount at `/run/ate`), and creates the empty +bundle-local `rootfs/`, `upper/`, and `work/` directories. + +## Compose path (ateom: `SetupBundleRootfs`) + +Called immediately before `runsc create`/`runsc restore` (gVisor) and before +staging the virtio-fs lower (micro-VM): + +1. **`FinalizeLayer`** for each referenced layer — materializes the recorded + whiteouts as 0:0 char devices (`mknod`) and opaque dirs as + `trusted.overlay.opaque=y` xattrs. Once per layer node-wide; idempotent + and safe under concurrent ateom pods (`EEXIST` tolerated, marker written + last). Paths from `whiteouts.json` are re-validated, so a crafted file + cannot escape the layer tree. +2. **Mount** an overlay at `/rootfs`: `lowerdir` is the layer chain + reversed into overlayfs's top-first order (duplicate layers — images can + legitimately list the same diffID twice — are collapsed to the topmost + occurrence, which overlayfs otherwise rejects with `ELOOP`), `upperdir` / + `workdir` are the bundle-local dirs, holding this actor's private writes. +3. **ExtraDirs** are created through the mount (landing in the upper), again + under `os.Root` confinement. + +A bundle without a spec file is left untouched (compatibility with bundles +prepared by a pre-imagecache atelet). A zero-layer spec composes an empty +rootfs with ExtraDirs and no mount. + +Actor semantics are unchanged from the untar era: the upper is wiped by +atelet's `resetActorDirs` between runs, so every run still starts from a +bit-exact, pristine image rootfs — it just costs a mount instead of an +extraction. The micro-VM path is nearly untouched: it bind-mounts the (now +overlay-composed) bundle rootfs into virtiofsd's shared dir and the guest +keeps building its own tmpfs upper, as before. + +**Teardown**: `UnmountAllUnder(bundleDir)` lazily detaches every mount below +an actor's bundle directory (via `/proc/self/mountinfo`) before atelet wipes +it — called from the checkpoint cleanup path in ateom-gvisor and +`teardownActor` in ateom-microvm. + +## Not implemented yet: garbage collection + +**There is no eviction. Layers and manifest records, once cached, are never +deleted from the node VM.** Disk usage grows monotonically with the set of +distinct layers ever pulled on the node, bounded only by the size of the +volume backing the cache root. Operators should size the +`--image-cache-dir` volume accordingly (and note that on GKE, disk size also +gates IOPS, which directly bounds unpack throughput). If a node fills up, +deleting the cache root entirely (or any individual +`layers/sha256/` directory plus the `manifests/` records) while no +actors are starting is safe — the store re-pulls whatever is missing. + +This is Phase 2 of [#463](https://github.com/agent-substrate/substrate/issues/463): +watermark-driven eviction (start evicting at a disk high-watermark, stop at +the low-watermark) with a protection hierarchy — layers referenced by +actively mounted images are never evicted, then preload-pinned images, then +LRU by image last-use — plus cache metrics. Phase 3 adds the control-plane +surface (reporting cached digests for scheduling affinity, and a +`PreloadImage` API with expiring pins). The layer-materializer seam is also +designed so a lazy-pull backend (eStargz/SOCI-style FUSE) can replace the +untar backend later without restructuring. + +## Testing + +- Portable unit tests (run everywhere, including macOS): the unpack security + suite (traversal, symlink/hardlink escape, whiteout capture, later-entry- + wins, read-only dirs, missing parents), spec round-trips, overlay option + assembly (including duplicate-layer dedup), mountinfo parsing, ref + rewriting, options. End-to-end pull tests run against an in-memory + registry (`pkg/registry`). +- Linux-tagged tests (`bundle_linux_test.go`): unprivileged ones cover + escape rejection and specless/zero-layer compose; root-gated ones execute + the real `mknod`/xattr materialization and a full mount → write-isolation + → unmount round trip (the write-isolation assertion — actor writes land in + the bundle upper, never in the shared pool — is the key safety property). + CI runs the package twice: once unprivileged, once under `sudo` so the + root-gated tests execute (see `.github/workflows/pr-workflow.yaml`). +- `tools/validate-image-cache` batch-validates that arbitrary registry + images can be pulled, parsed, and unpacked by the store half — useful for + sweeping large image corpora before relying on them in production. diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go new file mode 100644 index 000000000..31e551ec1 --- /dev/null +++ b/internal/imagecache/bundle_linux.go @@ -0,0 +1,215 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The consumer half of the image cache: everything in this file runs in the +// privileged ateom pods (which own all mounts on the node), never in atelet. + +package imagecache + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + + "golang.org/x/sys/unix" +) + +// SetupBundleRootfs composes the bundle's rootfs from cached layers per the +// bundle's overlay spec: it finalizes each layer (whiteout materialization, +// once per layer node-wide), mounts an overlay at /rootfs with the +// cached layers as read-only lowerdirs and the bundle-local upper/ + work/ +// as the actor's private writable side, and creates the spec's ExtraDirs +// through the mount (so they land in the upper). +// +// A bundle without an overlay spec is left untouched (its rootfs is a plain +// extracted directory). The mount lives in the calling process's mount +// namespace, which is exactly where the workload (runsc's gofer, virtiofsd) +// resolves it. +func SetupBundleRootfs(bundlePath string) error { + spec, err := ReadSpec(bundlePath) + if err != nil { + return err + } + if spec == nil { + return nil + } + + for _, layerDir := range spec.Layers { + if err := FinalizeLayer(layerDir); err != nil { + return fmt.Errorf("while finalizing layer %q: %w", layerDir, err) + } + } + + rootfs := filepath.Join(bundlePath, "rootfs") + upper := filepath.Join(bundlePath, "upper") + work := filepath.Join(bundlePath, "work") + for _, d := range []string{rootfs, upper, work} { + if err := os.MkdirAll(d, 0o700); err != nil { + return fmt.Errorf("while creating %q: %w", d, err) + } + } + + // Detach any stale mount left by a previous incarnation of this bundle + // path (e.g. a run that failed between mount and teardown). EINVAL just + // means nothing was mounted there. + _ = unix.Unmount(rootfs, unix.MNT_DETACH) + + if len(spec.Layers) == 0 { + // Degenerate zero-layer image: the empty rootfs dir plus ExtraDirs is + // all there is. + return createExtraDirs(rootfs, spec.ExtraDirs) + } + + opts, err := overlayMountOptions(spec.Layers, upper, work) + if err != nil { + return err + } + if err := unix.Mount("overlay", rootfs, "overlay", 0, opts); err != nil { + return fmt.Errorf("while mounting overlay rootfs at %q (%s): %w", rootfs, opts, err) + } + + return createExtraDirs(rootfs, spec.ExtraDirs) +} + +// FinalizeLayer materializes the whiteout state recorded at unpack time: +// 0:0 char devices for whiteouts and trusted.overlay.opaque=y on opaque +// dirs. This runs in ateom rather than atelet because mknod needs CAP_MKNOD +// and trusted.* xattrs need CAP_SYS_ADMIN, both of which atelet deliberately +// drops. +// +// Idempotent and safe under concurrent callers (multiple ateom pods share +// the node's pool): EEXIST from mknod is success, setxattr is naturally +// idempotent, and the marker is written last. +func FinalizeLayer(layerDir string) error { + marker := filepath.Join(layerDir, layerFinalizedMarkerName) + if _, err := os.Stat(marker); err == nil { + return nil + } + + wh, err := readWhiteouts(layerDir) + if err != nil { + return err + } + + fsDir := filepath.Join(layerDir, layerFSDirName) + root, err := os.OpenRoot(fsDir) + if err != nil { + return fmt.Errorf("while opening layer fs %q: %w", fsDir, err) + } + defer root.Close() + + for _, p := range wh.Whiteouts { + rel, skip, err := validateTarName(p) + if err != nil { + return fmt.Errorf("invalid whiteout path: %w", err) + } + if skip { + continue + } + if err := mknodWhiteout(root, rel); err != nil { + return fmt.Errorf("while creating whiteout %q: %w", rel, err) + } + } + + for _, p := range wh.Opaques { + rel, skip, err := validateTarName(p) + if err != nil { + return fmt.Errorf("invalid opaque dir path: %w", err) + } + if skip { + continue + } + if err := setOpaque(root, rel); err != nil { + return fmt.Errorf("while marking %q opaque: %w", rel, err) + } + } + + if err := os.WriteFile(marker, nil, 0o600); err != nil { + return fmt.Errorf("while writing finalized marker: %w", err) + } + return nil +} + +// mknodWhiteout creates the overlayfs whiteout (a 0:0 char device) at rel +// inside root, creating parent directories as needed (the whited-out path's +// parent may only exist in a lower layer). +func mknodWhiteout(root *os.Root, rel string) error { + dir, base := filepath.Dir(rel), filepath.Base(rel) + if dir != "." { + if err := root.MkdirAll(dir, 0o755); err != nil { + return err + } + } + df, err := root.Open(dir) + if err != nil { + return err + } + defer df.Close() + if err := unix.Mknodat(int(df.Fd()), base, unix.S_IFCHR, 0); err != nil && !errors.Is(err, os.ErrExist) { + return &os.PathError{Op: "mknodat", Path: rel, Err: err} + } + return nil +} + +// setOpaque marks the directory rel inside root as overlayfs-opaque. +func setOpaque(root *os.Root, rel string) error { + if err := root.MkdirAll(rel, 0o755); err != nil { + return err + } + df, err := root.Open(rel) + if err != nil { + return err + } + defer df.Close() + if err := unix.Fsetxattr(int(df.Fd()), "trusted.overlay.opaque", []byte("y"), 0); err != nil { + return &os.PathError{Op: "fsetxattr", Path: rel, Err: err} + } + return nil +} + +// UnmountAllUnder lazily detaches every mount at or below dir in the calling +// process's mount namespace. It is the teardown counterpart of +// SetupBundleRootfs, keyed by directory rather than by container name so a +// single call cleans up all of an actor's bundle mounts. Missing mounts are +// not an error. +func UnmountAllUnder(dir string) error { + points, err := mountPointsUnder(dir) + if err != nil { + return err + } + // Deepest first, so nested mounts unmount before their parents. + sort.Slice(points, func(i, j int) bool { return len(points[i]) > len(points[j]) }) + var errs []error + for _, p := range points { + if err := unix.Unmount(p, unix.MNT_DETACH); err != nil && !errors.Is(err, unix.EINVAL) && !errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("while unmounting %q: %w", p, err)) + } + } + return errors.Join(errs...) +} + +// mountPointsUnder lists mount points at or below dir per +// /proc/self/mountinfo. +func mountPointsUnder(dir string) ([]string, error) { + f, err := os.Open("/proc/self/mountinfo") + if err != nil { + return nil, fmt.Errorf("while opening mountinfo: %w", err) + } + defer f.Close() + return mountPointsIn(f, dir) +} diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go new file mode 100644 index 000000000..88851b86f --- /dev/null +++ b/internal/imagecache/bundle_linux_test.go @@ -0,0 +1,210 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +// writeLayer builds a layer dir (fs/ tree + whiteouts.json) as the store's +// unpack would. +func writeLayer(t *testing.T, dir string, files map[string]string, wh *whiteoutSet) { + t.Helper() + fs := filepath.Join(dir, layerFSDirName) + if err := os.MkdirAll(fs, 0o755); err != nil { + t.Fatalf("mkdir fs: %v", err) + } + for name, body := range files { + p := filepath.Join(fs, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", name, err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + if wh != nil { + wh.Version = 1 + b, err := json.Marshal(wh) + if err != nil { + t.Fatalf("marshal whiteouts: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, layerWhiteoutsFileName), b, 0o600); err != nil { + t.Fatalf("write whiteouts.json: %v", err) + } + } +} + +// FinalizeLayer materializes whiteout devices (mknod, CAP_MKNOD) and opaque +// xattrs (trusted.*, CAP_SYS_ADMIN); only root has those in a plain test +// environment. Runs in privileged CI / root shells, skips elsewhere. +func TestFinalizeLayer_MaterializesWhiteouts(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (CAP_MKNOD + CAP_SYS_ADMIN for trusted.* xattrs)") + } + dir := t.TempDir() + writeLayer(t, dir, + map[string]string{"kept.txt": "kept"}, + &whiteoutSet{ + Whiteouts: []string{"removed.txt", "sub/dir/removed-deep.txt"}, + Opaques: []string{"opaque-dir"}, + }) + + if err := FinalizeLayer(dir); err != nil { + t.Fatalf("FinalizeLayer: %v", err) + } + + for _, p := range []string{"removed.txt", "sub/dir/removed-deep.txt"} { + fi, err := os.Lstat(filepath.Join(dir, layerFSDirName, p)) + if err != nil { + t.Fatalf("whiteout %q not created: %v", p, err) + } + if fi.Mode()&os.ModeCharDevice == 0 { + t.Errorf("whiteout %q mode = %v, want char device", p, fi.Mode()) + } + var st unix.Stat_t + if err := unix.Stat(filepath.Join(dir, layerFSDirName, p), &st); err == nil && st.Rdev != 0 { + t.Errorf("whiteout %q rdev = %d, want 0:0", p, st.Rdev) + } + } + + var val [8]byte + n, err := unix.Getxattr(filepath.Join(dir, layerFSDirName, "opaque-dir"), "trusted.overlay.opaque", val[:]) + if err != nil || string(val[:n]) != "y" { + t.Errorf("opaque xattr = %q (err=%v), want \"y\"", val[:n], err) + } + + if _, err := os.Stat(filepath.Join(dir, layerFinalizedMarkerName)); err != nil { + t.Errorf("finalized marker missing: %v", err) + } + + // Idempotent: second call is a marker-hit no-op. + if err := FinalizeLayer(dir); err != nil { + t.Errorf("FinalizeLayer (second call): %v", err) + } +} + +// Escape rejection needs no privileges: a crafted whiteouts.json must fail +// validation before any mknod/setxattr is attempted. +func TestFinalizeLayer_RejectsEscapingPaths(t *testing.T) { + t.Run("whiteout escape", func(t *testing.T) { + dir := t.TempDir() + writeLayer(t, dir, nil, &whiteoutSet{Whiteouts: []string{"../escape"}}) + if err := FinalizeLayer(dir); err == nil { + t.Errorf("FinalizeLayer accepted an escaping whiteout path") + } + }) + t.Run("opaque escape", func(t *testing.T) { + dir := t.TempDir() + writeLayer(t, dir, nil, &whiteoutSet{Opaques: []string{"a/../../escape"}}) + if err := FinalizeLayer(dir); err == nil { + t.Errorf("FinalizeLayer accepted an escaping opaque path") + } + }) +} + +// A layer with no whiteouts.json finalizes to just the marker; needs no +// privileges. +func TestFinalizeLayer_NoWhiteouts(t *testing.T) { + dir := t.TempDir() + writeLayer(t, dir, map[string]string{"f": "x"}, nil) + if err := FinalizeLayer(dir); err != nil { + t.Fatalf("FinalizeLayer: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, layerFinalizedMarkerName)); err != nil { + t.Errorf("finalized marker missing: %v", err) + } +} + +// A bundle without an overlay spec must be left untouched. +func TestSetupBundleRootfs_NoSpecIsNoop(t *testing.T) { + bundle := t.TempDir() + if err := SetupBundleRootfs(bundle); err != nil { + t.Fatalf("SetupBundleRootfs: %v", err) + } + if entries, _ := os.ReadDir(bundle); len(entries) != 0 { + t.Errorf("no-spec bundle was modified: %v", entries) + } +} + +// A zero-layer spec composes without any mount: empty rootfs plus ExtraDirs. +// Needs no privileges (the stale-unmount attempt's failure is ignored). +func TestSetupBundleRootfs_ZeroLayers(t *testing.T) { + bundle := t.TempDir() + if err := WriteSpec(bundle, &OverlaySpec{Layers: nil, ExtraDirs: []string{"/run/ate"}}); err != nil { + t.Fatalf("WriteSpec: %v", err) + } + if err := SetupBundleRootfs(bundle); err != nil { + t.Fatalf("SetupBundleRootfs: %v", err) + } + fi, err := os.Stat(filepath.Join(bundle, "rootfs", "run", "ate")) + if err != nil || !fi.IsDir() { + t.Errorf("ExtraDir not created in rootfs: fi=%v err=%v", fi, err) + } + for _, d := range []string{"upper", "work"} { + if fi, err := os.Stat(filepath.Join(bundle, d)); err != nil || !fi.IsDir() { + t.Errorf("bundle dir %q missing: %v", d, err) + } + } +} + +// Full overlay mount + UnmountAllUnder round trip; needs CAP_SYS_ADMIN. +func TestSetupBundleRootfs_MountAndUnmount(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (mount/unmount)") + } + layer := t.TempDir() + writeLayer(t, layer, map[string]string{"from-layer.txt": "hello"}, nil) + + bundle := t.TempDir() + if err := WriteSpec(bundle, &OverlaySpec{Layers: []string{layer}, ExtraDirs: []string{"/run/ate"}}); err != nil { + t.Fatalf("WriteSpec: %v", err) + } + if err := SetupBundleRootfs(bundle); err != nil { + t.Fatalf("SetupBundleRootfs: %v", err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + if got, err := os.ReadFile(filepath.Join(bundle, "rootfs", "from-layer.txt")); err != nil || string(got) != "hello" { + t.Errorf("layer content not visible through overlay: %q (%v)", got, err) + } + if fi, err := os.Stat(filepath.Join(bundle, "rootfs", "run", "ate")); err != nil || !fi.IsDir() { + t.Errorf("ExtraDir missing in overlay: %v", err) + } + // A write through the mount lands in the bundle's upper, not the layer. + if err := os.WriteFile(filepath.Join(bundle, "rootfs", "written.txt"), []byte("w"), 0o644); err != nil { + t.Fatalf("write through overlay: %v", err) + } + if _, err := os.Stat(filepath.Join(bundle, "upper", "written.txt")); err != nil { + t.Errorf("write did not land in upper: %v", err) + } + if _, err := os.Stat(filepath.Join(layer, layerFSDirName, "written.txt")); err == nil { + t.Errorf("write leaked into the shared layer") + } + + if err := UnmountAllUnder(bundle); err != nil { + t.Fatalf("UnmountAllUnder: %v", err) + } + if _, err := os.Stat(filepath.Join(bundle, "rootfs", "from-layer.txt")); err == nil { + t.Errorf("rootfs still shows layer content after unmount") + } +} diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go new file mode 100644 index 000000000..1d589bd1e --- /dev/null +++ b/internal/imagecache/imagecache.go @@ -0,0 +1,546 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package imagecache implements the node-local OCI image cache: a +// content-addressed pool of unpacked image layers shared by every actor on +// the node, plus the per-bundle overlay spec that tells the ateom runtimes +// how to compose an actor rootfs from cached layers. +// +// The work is split along the existing atelet/ateom privilege boundary: +// +// - atelet (plain root, all capabilities dropped) pulls layers and unpacks +// them into the pool (Store.EnsureImage), and writes a rootfs-overlay.json +// next to each bundle's config.json (WriteSpec). Whiteout entries are +// recorded in per-layer metadata rather than materialized, because +// overlayfs whiteouts are char devices (CAP_MKNOD) with trusted.* xattrs +// for opaque dirs (CAP_SYS_ADMIN). +// - ateom (privileged; it already owns every mount on the node) finalizes +// layers — materializing the recorded whiteout state, once per layer — +// and mounts the overlay rootfs (SetupBundleRootfs) just before +// `runsc create` / staging the micro-VM virtio-fs lower. +// +// On-disk layout under the cache root (a directory on the BasePath hostPath, +// so the same absolute paths resolve in atelet and every ateom pod): +// +// version layout version marker +// layers/sha256// +// fs/ the unpacked layer tree (overlay lowerdir) +// whiteouts.json whiteout state recorded at unpack time +// finalized marker written by FinalizeLayer (ateom) +// manifests/sha256/.json +// image config + ordered diffID list +// +// Layers land in the pool via unpack-into-tempdir + atomic rename, so a +// layer directory that exists is always complete; startup recovery only has +// to sweep orphaned temp dirs. +package imagecache + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "os" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/singleflight" +) + +const ( + layoutVersion = "1" + versionFileName = "version" + + layerFSDirName = "fs" + layerWhiteoutsFileName = "whiteouts.json" + layerFinalizedMarkerName = "finalized" + + // layerPullConcurrency bounds concurrent layer download+unpack streams per + // image pull. Memory use is O(stream buffers) per slot, independent of + // layer size. + layerPullConcurrency = 4 +) + +// Store is atelet's handle to the on-disk layer pool. It is safe for +// concurrent use; concurrent pulls of the same image or layer are collapsed. +// The store assumes it is the only writer on the node (one atelet per node). +type Store struct { + root string + + // authenticator, when set, is attached to pulls from registries that use + // GCP credentials (gcr.io / pkg.dev). See remoteOpts. + authenticator authn.Authenticator + + localhostRegistryReplacement string + + // platform overrides the default pull platform (linux/GOARCH). Used by + // validation tooling that runs on a different architecture than the + // nodes it validates for. + platform *v1.Platform + + imageSF singleflight.Group + layerSF singleflight.Group +} + +// Option configures a Store. +type Option func(*Store) + +// WithAuthenticator attaches an authenticator used for gcr.io / pkg.dev +// registries. A nil authenticator is ignored. +func WithAuthenticator(a authn.Authenticator) Option { + return func(s *Store) { s.authenticator = a } +} + +// WithLocalhostRegistryReplacement rewrites localhost/loopback registry refs +// to the given endpoint, mirroring the containerd mirror config used by kind +// local registries (https://kind.sigs.k8s.io/docs/user/local-registry/). +func WithLocalhostRegistryReplacement(replacement string) Option { + return func(s *Store) { s.localhostRegistryReplacement = replacement } +} + +// WithPlatform overrides the pull platform (default: linux/GOARCH). +func WithPlatform(p v1.Platform) Option { + return func(s *Store) { s.platform = &p } +} + +// Image describes one cached, ready-to-compose image. +type Image struct { + // Digest is the manifest digest the caller's ref resolved to (for a + // multi-arch ref, the index digest as requested, not the per-platform + // child). + Digest v1.Hash + // Config is the OCI image config (entrypoint, env, ...). + Config v1.Config + // LayerDirs are the absolute cached layer directories, bottom-most layer + // first. Each contains the unpacked tree under "fs/". + LayerDirs []string +} + +// imageRecord is the persisted form of a cached image, stored under +// manifests//.json. +type imageRecord struct { + Version int `json:"version"` + Config v1.Config `json:"config"` + DiffIDs []string `json:"diffIDs"` +} + +// New opens (creating if needed) the layer pool rooted at root and runs +// startup recovery: verifying the layout version and sweeping temp dirs left +// by unpacks that were in flight when a previous atelet died. +func New(root string, opts ...Option) (*Store, error) { + s := &Store{root: root} + for _, o := range opts { + o(s) + } + + for _, d := range []string{s.layersDir(), s.manifestsDir()} { + if err := os.MkdirAll(d, 0o700); err != nil { + return nil, fmt.Errorf("while creating image cache dir %q: %w", d, err) + } + } + + versionPath := filepath.Join(root, versionFileName) + switch b, err := os.ReadFile(versionPath); { + case err == nil: + if got := strings.TrimSpace(string(b)); got != layoutVersion { + // Fail loudly instead of silently mixing layouts; an operator can + // delete the cache dir to rebuild it (it holds no unique state). + return nil, fmt.Errorf("image cache at %q has layout version %q, this atelet supports %q", root, got, layoutVersion) + } + case errors.Is(err, os.ErrNotExist): + if err := os.WriteFile(versionPath, []byte(layoutVersion+"\n"), 0o600); err != nil { + return nil, fmt.Errorf("while writing image cache version marker: %w", err) + } + default: + return nil, fmt.Errorf("while reading image cache version marker: %w", err) + } + + if err := s.sweepTempDirs(); err != nil { + return nil, err + } + return s, nil +} + +func (s *Store) layersDir() string { return filepath.Join(s.root, "layers", "sha256") } +func (s *Store) manifestsDir() string { return filepath.Join(s.root, "manifests", "sha256") } + +func (s *Store) layerDir(diffID v1.Hash) string { + return filepath.Join(s.root, "layers", diffID.Algorithm, diffID.Hex) +} + +func (s *Store) recordPath(digest v1.Hash) string { + return filepath.Join(s.root, "manifests", digest.Algorithm, digest.Hex+".json") +} + +// sweepTempDirs removes unpack temp dirs orphaned by a crash. A layer dir +// without the temp prefix is always complete (it was moved into place with a +// single rename), so this is the only recovery the pool needs. +func (s *Store) sweepTempDirs() error { + entries, err := os.ReadDir(s.layersDir()) + if err != nil { + return fmt.Errorf("while listing layer pool: %w", err) + } + for _, e := range entries { + if !strings.HasPrefix(e.Name(), ".tmp-") { + continue + } + p := filepath.Join(s.layersDir(), e.Name()) + if err := RemoveAllWritable(p); err != nil { + return fmt.Errorf("while sweeping orphaned layer temp dir %q: %w", p, err) + } + } + return nil +} + +// EnsureImage makes ref's image available in the pool and returns its config +// and ordered layer directories. Digest refs hit the cache with no network +// I/O; tag refs cost one HEAD request to resolve the tag to a manifest +// digest (so tag refs are cacheable, and a moved tag is picked up on the +// next call). +func (s *Store) EnsureImage(ctx context.Context, ref string) (*Image, error) { + parsedRef, err := s.parseRef(ref) + if err != nil { + return nil, fmt.Errorf("while parsing reference: %w", err) + } + + var digest v1.Hash + if d, ok := parsedRef.(name.Digest); ok { + digest, err = v1.NewHash(d.DigestStr()) + if err != nil { + return nil, fmt.Errorf("while parsing digest of %q: %w", ref, err) + } + } else { + // Tag ref: one small HEAD request pins it to an immutable manifest + // digest, which is the only safe cache key for mutable tags. + desc, err := remote.Head(parsedRef, s.remoteOpts(ctx, parsedRef)...) + if err != nil { + return nil, fmt.Errorf("while resolving tag %q to a digest: %w", ref, err) + } + digest = desc.Digest + } + + if img, err := s.cachedImage(digest); err != nil { + return nil, err + } else if img != nil { + slog.InfoContext(ctx, "Image cache hit", slog.String("ref", ref), slog.String("digest", digest.String())) + return img, nil + } + slog.InfoContext(ctx, "Image cache miss", slog.String("ref", ref), slog.String("digest", digest.String())) + + // Collapse concurrent pulls of the same digest (e.g. several containers of + // one actor, or several actors landing at once). The winning call's ctx + // governs the pull; if it is cancelled the waiters fail too and retry at + // the RPC level. + v, err, _ := s.imageSF.Do(digest.String(), func() (any, error) { + return s.pull(ctx, parsedRef, digest) + }) + if err != nil { + return nil, err + } + return v.(*Image), nil +} + +// cachedImage returns the cached image for digest, or nil if the record or +// any of its layer dirs is missing (in which case the caller re-pulls; only +// the missing layers cost anything). +func (s *Store) cachedImage(digest v1.Hash) (*Image, error) { + b, err := os.ReadFile(s.recordPath(digest)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("while reading image record for %s: %w", digest, err) + } + var rec imageRecord + if err := json.Unmarshal(b, &rec); err != nil { + return nil, fmt.Errorf("while decoding image record for %s: %w", digest, err) + } + + layerDirs := make([]string, len(rec.DiffIDs)) + for i, d := range rec.DiffIDs { + diffID, err := v1.NewHash(d) + if err != nil { + return nil, fmt.Errorf("invalid diffID %q in image record for %s: %w", d, digest, err) + } + dir := s.layerDir(diffID) + if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err != nil { + return nil, nil + } + layerDirs[i] = dir + } + return &Image{Digest: digest, Config: rec.Config, LayerDirs: layerDirs}, nil +} + +// pull fetches the image (by its resolved digest, so what is unpacked is +// exactly what was recorded), unpacks every missing layer into the pool, and +// writes the image record. +func (s *Store) pull(ctx context.Context, parsedRef name.Reference, digest v1.Hash) (*Image, error) { + // Re-check under the flight lock: a racing EnsureImage may have completed + // the pull between our cache miss and winning the singleflight slot. + if img, err := s.cachedImage(digest); err != nil { + return nil, err + } else if img != nil { + return img, nil + } + + tStart := time.Now() + digestRef := parsedRef.Context().Digest(digest.String()) + img, err := remote.Image(digestRef, s.remoteOpts(ctx, parsedRef)...) + if err != nil { + return nil, fmt.Errorf("in remote.Image: %w", err) + } + + cfgFile, err := img.ConfigFile() + if err != nil { + return nil, fmt.Errorf("while reading image config: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return nil, fmt.Errorf("while listing image layers: %w", err) + } + + layerDirs := make([]string, len(layers)) + diffIDs := make([]string, len(layers)) + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(layerPullConcurrency) + for i, layer := range layers { + g.Go(func() error { + diffID, err := layer.DiffID() + if err != nil { + return fmt.Errorf("while reading layer diffID: %w", err) + } + dir, err := s.ensureLayer(gctx, diffID, layer) + if err != nil { + return fmt.Errorf("while unpacking layer %s: %w", diffID, err) + } + layerDirs[i], diffIDs[i] = dir, diffID.String() + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + + rec := imageRecord{Version: 1, Config: cfgFile.Config, DiffIDs: diffIDs} + if err := s.writeRecord(digest, rec); err != nil { + return nil, err + } + // For a multi-arch ref the requested digest is the index digest, but the + // layers unpacked belong to the per-platform child manifest. Record the + // image under the child digest too, so refs pinned either way hit. + if actual, err := img.Digest(); err == nil && actual != digest { + if err := s.writeRecord(actual, rec); err != nil { + slog.WarnContext(ctx, "Failed to record image under platform manifest digest", + slog.String("digest", actual.String()), slog.Any("err", err)) + } + } + + slog.InfoContext(ctx, "Image pulled into layer cache", + slog.String("digest", digest.String()), + slog.Int("layers", len(layers)), + slog.Duration("took", time.Since(tStart))) + + return &Image{Digest: digest, Config: cfgFile.Config, LayerDirs: layerDirs}, nil +} + +// ensureLayer makes the unpacked tree for diffID present in the pool, +// collapsing concurrent requests for the same layer across images. +func (s *Store) ensureLayer(ctx context.Context, diffID v1.Hash, layer v1.Layer) (string, error) { + dir := s.layerDir(diffID) + _, err, _ := s.layerSF.Do(diffID.String(), func() (any, error) { + if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { + return nil, nil + } + return nil, s.unpackLayerToPool(ctx, diffID, layer) + }) + if err != nil { + return "", err + } + return dir, nil +} + +// unpackLayerToPool streams the layer (download → decompress → untar) into a +// temp dir and renames it into place, so a layer dir either exists complete +// or not at all. +func (s *Store) unpackLayerToPool(ctx context.Context, diffID v1.Hash, layer v1.Layer) (retErr error) { + tmp, err := os.MkdirTemp(filepath.Dir(s.layerDir(diffID)), ".tmp-"+diffID.Hex[:12]+"-") + if err != nil { + return fmt.Errorf("while creating layer temp dir: %w", err) + } + defer func() { + // No-op once the rename has moved tmp into place. + if _, err := os.Stat(tmp); err == nil { + if rmErr := RemoveAllWritable(tmp); rmErr != nil && retErr == nil { + retErr = fmt.Errorf("while cleaning up layer temp dir: %w", rmErr) + } + } + }() + + fsDir := filepath.Join(tmp, layerFSDirName) + if err := os.Mkdir(fsDir, 0o755); err != nil { + return fmt.Errorf("while creating layer fs dir: %w", err) + } + + rc, err := layer.Uncompressed() + if err != nil { + return fmt.Errorf("while opening layer stream: %w", err) + } + defer rc.Close() + + root, err := os.OpenRoot(fsDir) + if err != nil { + return fmt.Errorf("while opening layer fs dir as os.Root: %w", err) + } + defer root.Close() + + wh, err := unpackLayer(ctx, rc, root) + if err != nil { + return err + } + + whBytes, err := json.Marshal(wh) + if err != nil { + return fmt.Errorf("while encoding layer whiteouts: %w", err) + } + if err := os.WriteFile(filepath.Join(tmp, layerWhiteoutsFileName), whBytes, 0o600); err != nil { + return fmt.Errorf("while writing layer whiteouts: %w", err) + } + + if err := os.Rename(tmp, s.layerDir(diffID)); err != nil { + // A concurrent unpack (another process sharing the pool) may have won; + // its layer is as good as ours. + if _, statErr := os.Stat(filepath.Join(s.layerDir(diffID), layerFSDirName)); statErr == nil { + return nil + } + return fmt.Errorf("while moving layer into pool: %w", err) + } + return nil +} + +func (s *Store) writeRecord(digest v1.Hash, rec imageRecord) error { + b, err := json.MarshalIndent(rec, "", " ") + if err != nil { + return fmt.Errorf("while encoding image record: %w", err) + } + path := s.recordPath(digest) + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".tmp-*") + if err != nil { + return fmt.Errorf("while creating image record temp file: %w", err) + } + defer os.Remove(tmp.Name()) // no-op once the rename succeeds + if _, err := tmp.Write(b); err != nil { + tmp.Close() + return fmt.Errorf("while writing image record: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("while closing image record: %w", err) + } + if err := os.Rename(tmp.Name(), path); err != nil { + return fmt.Errorf("while moving image record into place: %w", err) + } + return nil +} + +// remoteOpts assembles the go-containerregistry options for pulls from +// parsedRef's registry. +func (s *Store) remoteOpts(ctx context.Context, parsedRef name.Reference) []remote.Option { + platform := v1.Platform{ + Architecture: runtime.GOARCH, + OS: "linux", + } + if s.platform != nil { + platform = *s.platform + } + opts := []remote.Option{ + // Propagate caller ctx into go-containerregistry so cancellation tears + // down in-flight layer-blob HTTP requests instead of letting them run + // to completion in background goroutines. + remote.WithContext(ctx), + remote.WithPlatform(platform), + } + registry := parsedRef.Context().Registry.RegistryStr() + if s.authenticator != nil && registryUsesGCPAuth(registry) { + opts = append(opts, remote.WithAuth(s.authenticator)) + } + return opts +} + +func registryUsesGCPAuth(registry string) bool { + return registry == "gcr.io" || strings.HasSuffix(registry, ".gcr.io") || + registry == "pkg.dev" || strings.HasSuffix(registry, ".pkg.dev") +} + +// parseRef applies the localhost-registry rewrite (kind local registries) and +// permits plain-HTTP pulls for localhost/loopback registries, matching docker +// behavior so local development needs no TLS certs. +func (s *Store) parseRef(ref string) (name.Reference, error) { + rewritten := false + if s.localhostRegistryReplacement != "" { + if newRef := s.rewriteLocalRegistry(ref); newRef != ref { + ref = newRef + rewritten = true + } + } + var nameOpts []name.Option + if rewritten || isLocalRegistry(ref) { + nameOpts = append(nameOpts, name.Insecure) + } + return name.ParseReference(ref, nameOpts...) +} + +func registryHost(ref string) string { + parts := strings.SplitN(ref, "/", 2) + reg, err := name.NewRegistry(parts[0], name.Insecure) + if err != nil { + return "" + } + hostPart := reg.Name() + if h, _, err := net.SplitHostPort(hostPart); err == nil { + return h + } + return hostPart +} + +func isLocalhostOrLoopback(host string) bool { + if host == "localhost" { + return true + } + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + return true + } + return false +} + +func isLocalRegistry(ref string) bool { + // By default docker permits localhost and 127.0.0.0/8; we also permit the + // IPv6 loopback here. + return isLocalhostOrLoopback(registryHost(ref)) +} + +func (s *Store) rewriteLocalRegistry(ref string) string { + if isLocalRegistry(ref) { + parts := strings.SplitN(ref, "/", 2) + return s.localhostRegistryReplacement + "/" + parts[1] + } + return ref +} diff --git a/internal/imagecache/imagecache_test.go b/internal/imagecache/imagecache_test.go new file mode 100644 index 000000000..2bb19a98c --- /dev/null +++ b/internal/imagecache/imagecache_test.go @@ -0,0 +1,298 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "archive/tar" + "bytes" + "context" + "io" + "log" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// newTestRegistry starts an in-memory OCI registry and returns its host +// ("127.0.0.1:", which the store treats as a local registry and pulls +// over plain HTTP). +func newTestRegistry(t *testing.T) (*httptest.Server, string) { + t.Helper() + srv := httptest.NewServer(registry.New(registry.Logger(log.New(io.Discard, "", 0)))) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parsing registry URL: %v", err) + } + return srv, u.Host +} + +func layerFromEntries(t *testing.T, entries []tarEntry) v1.Layer { + t.Helper() + b := buildTar(t, entries) + l, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(b)), nil + }) + if err != nil { + t.Fatalf("tarball.LayerFromOpener: %v", err) + } + return l +} + +func pushImage(t *testing.T, ref string, cfg v1.Config, layers ...v1.Layer) v1.Image { + t.Helper() + img, err := mutate.ConfigFile(empty.Image, &v1.ConfigFile{Config: cfg}) + if err != nil { + t.Fatalf("mutate.ConfigFile: %v", err) + } + img, err = mutate.AppendLayers(img, layers...) + if err != nil { + t.Fatalf("mutate.AppendLayers: %v", err) + } + tag, err := name.ParseReference(ref, name.Insecure) + if err != nil { + t.Fatalf("name.ParseReference(%q): %v", ref, err) + } + if err := remote.Write(tag, img); err != nil { + t.Fatalf("remote.Write(%q): %v", ref, err) + } + return img +} + +func newTestStore(t *testing.T, opts ...Option) *Store { + t.Helper() + s, err := New(t.TempDir(), opts...) + if err != nil { + t.Fatalf("New: %v", err) + } + return s +} + +func TestEnsureImage_TagPullAndDigestHit(t *testing.T) { + srv, host := newTestRegistry(t) + ref := host + "/test/app:latest" + + base := layerFromEntries(t, []tarEntry{ + {name: "bin/", typeflag: tar.TypeDir}, + {name: "bin/sh", typeflag: tar.TypeReg, mode: 0o755, body: "#!/sh\n"}, + }) + top := layerFromEntries(t, []tarEntry{ + {name: "app/", typeflag: tar.TypeDir}, + {name: "app/main", typeflag: tar.TypeReg, mode: 0o755, body: "main"}, + {name: "bin/.wh.sh", typeflag: tar.TypeReg}, + {name: "app/.wh..wh..opq", typeflag: tar.TypeReg}, + }) + pushImage(t, ref, v1.Config{Env: []string{"FOO=bar"}}, base, top) + + store := newTestStore(t) + ctx := context.Background() + + // Tag refs must be cacheable: they resolve to a digest via one HEAD. + img, err := store.EnsureImage(ctx, ref) + if err != nil { + t.Fatalf("EnsureImage(tag): %v", err) + } + if len(img.LayerDirs) != 2 { + t.Fatalf("LayerDirs = %v, want 2 entries", img.LayerDirs) + } + if !slices.Equal(img.Config.Env, []string{"FOO=bar"}) { + t.Errorf("Config.Env = %v, want [FOO=bar]", img.Config.Env) + } + + // Bottom-first order: the base layer's file is in LayerDirs[0]. + if _, err := os.Stat(filepath.Join(img.LayerDirs[0], layerFSDirName, "bin/sh")); err != nil { + t.Errorf("base layer content missing: %v", err) + } + if _, err := os.Stat(filepath.Join(img.LayerDirs[1], layerFSDirName, "app/main")); err != nil { + t.Errorf("top layer content missing: %v", err) + } + + // The top layer's whiteout state is recorded beside its tree, not in it. + wh, err := readWhiteouts(img.LayerDirs[1]) + if err != nil { + t.Fatalf("readWhiteouts: %v", err) + } + if want := []string{"bin/sh"}; !slices.Equal(wh.Whiteouts, want) { + t.Errorf("whiteouts = %v, want %v", wh.Whiteouts, want) + } + if want := []string{"app"}; !slices.Equal(wh.Opaques, want) { + t.Errorf("opaques = %v, want %v", wh.Opaques, want) + } + if _, err := os.Lstat(filepath.Join(img.LayerDirs[1], layerFSDirName, "bin/.wh.sh")); err == nil { + t.Errorf("whiteout entry written into the layer tree") + } + + // A digest-pinned ref must now hit the cache with zero registry traffic: + // kill the registry and pull again through a fresh store on the same root + // (which also exercises startup recovery over a populated pool). + srv.Close() + store2, err := New(store.root) + if err != nil { + t.Fatalf("New(existing root): %v", err) + } + img2, err := store2.EnsureImage(ctx, host+"/test/app@"+img.Digest.String()) + if err != nil { + t.Fatalf("EnsureImage(digest, registry down): %v", err) + } + if !slices.Equal(img2.LayerDirs, img.LayerDirs) { + t.Errorf("cache hit LayerDirs = %v, want %v", img2.LayerDirs, img.LayerDirs) + } + if !slices.Equal(img2.Config.Env, img.Config.Env) { + t.Errorf("cache hit Config.Env = %v, want %v", img2.Config.Env, img.Config.Env) + } +} + +// Two images sharing a base layer must share its unpacked tree: the pool +// holds three layer dirs, not four. +func TestEnsureImage_SharedLayersDeduplicated(t *testing.T) { + _, host := newTestRegistry(t) + + shared := layerFromEntries(t, []tarEntry{ + {name: "lib/", typeflag: tar.TypeDir}, + {name: "lib/base.so", typeflag: tar.TypeReg, body: "base"}, + }) + topA := layerFromEntries(t, []tarEntry{{name: "a", typeflag: tar.TypeReg, body: "a"}}) + topB := layerFromEntries(t, []tarEntry{{name: "b", typeflag: tar.TypeReg, body: "b"}}) + pushImage(t, host+"/test/a:latest", v1.Config{}, shared, topA) + pushImage(t, host+"/test/b:latest", v1.Config{}, shared, topB) + + store := newTestStore(t) + ctx := context.Background() + + imgA, err := store.EnsureImage(ctx, host+"/test/a:latest") + if err != nil { + t.Fatalf("EnsureImage(a): %v", err) + } + imgB, err := store.EnsureImage(ctx, host+"/test/b:latest") + if err != nil { + t.Fatalf("EnsureImage(b): %v", err) + } + + if imgA.LayerDirs[0] != imgB.LayerDirs[0] { + t.Errorf("shared base layer not deduplicated: %q vs %q", imgA.LayerDirs[0], imgB.LayerDirs[0]) + } + entries, err := os.ReadDir(store.layersDir()) + if err != nil { + t.Fatalf("ReadDir(layers): %v", err) + } + if len(entries) != 3 { + names := make([]string, 0, len(entries)) + for _, e := range entries { + names = append(names, e.Name()) + } + t.Errorf("layer pool holds %d dirs (%v), want 3 (shared base stored once)", len(entries), names) + } +} + +// Startup recovery must sweep temp dirs orphaned by a crash mid-unpack, and +// reject a cache root with an unknown layout version. +func TestNew_RecoveryAndVersioning(t *testing.T) { + root := t.TempDir() + s, err := New(root) + if err != nil { + t.Fatalf("New: %v", err) + } + orphan := filepath.Join(s.layersDir(), ".tmp-deadbeef-123") + if err := os.MkdirAll(orphan, 0o700); err != nil { + t.Fatalf("planting orphan: %v", err) + } + if _, err := New(root); err != nil { + t.Fatalf("New(recovery): %v", err) + } + if _, err := os.Stat(orphan); !os.IsNotExist(err) { + t.Errorf("orphaned temp dir survived recovery") + } + + if err := os.WriteFile(filepath.Join(root, versionFileName), []byte("99\n"), 0o600); err != nil { + t.Fatalf("writing version marker: %v", err) + } + if _, err := New(root); err == nil { + t.Errorf("New accepted an unknown layout version") + } +} + +func TestIsLocalRegistry(t *testing.T) { + tests := []struct { + ref string + want bool + }{ + {ref: "localhost/foo", want: true}, + {ref: "localhost:5001/foo", want: true}, + {ref: "127.0.0.1/foo", want: true}, + {ref: "127.0.0.1:5001/foo", want: true}, + {ref: "127.0.0.2/foo", want: true}, + {ref: "127.0.0.2:8080/foo", want: true}, + {ref: "kind-registry/foo", want: false}, + {ref: "kind-registry:5000/foo", want: false}, + {ref: "my-registry.local/foo", want: false}, + {ref: "my-registry.local:8080/foo", want: false}, + {ref: "gcr.io/foo", want: false}, + {ref: "example.com/foo", want: false}, + {ref: "foo", want: false}, + {ref: "", want: false}, + } + + for _, tt := range tests { + t.Run(tt.ref, func(t *testing.T) { + got := isLocalRegistry(tt.ref) + if got != tt.want { + t.Errorf("isLocalRegistry(%q) = %v; want %v", tt.ref, got, tt.want) + } + }) + } +} + +func TestRewriteLocalRegistry(t *testing.T) { + s := &Store{localhostRegistryReplacement: "kind-registry:5000"} + + tests := []struct { + ref string + want string + }{ + {ref: "localhost/foo", want: "kind-registry:5000/foo"}, + {ref: "localhost:5001/foo", want: "kind-registry:5000/foo"}, + {ref: "localhost:8080/foo", want: "kind-registry:5000/foo"}, + {ref: "127.0.0.1/foo", want: "kind-registry:5000/foo"}, + {ref: "127.0.0.1:3000/foo", want: "kind-registry:5000/foo"}, + {ref: "127.0.0.2/foo", want: "kind-registry:5000/foo"}, + {ref: "127.0.0.2:8080/foo", want: "kind-registry:5000/foo"}, + {ref: "kind-registry/foo", want: "kind-registry/foo"}, + {ref: "kind-registry:5000/foo", want: "kind-registry:5000/foo"}, + {ref: "my-registry.local/foo", want: "my-registry.local/foo"}, + {ref: "gcr.io/foo", want: "gcr.io/foo"}, + {ref: "foo", want: "foo"}, + {ref: "", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.ref, func(t *testing.T) { + got := s.rewriteLocalRegistry(tt.ref) + if got != tt.want { + t.Errorf("rewriteLocalRegistry(%q) = %q; want %q", tt.ref, got, tt.want) + } + }) + } +} diff --git a/internal/imagecache/mountinfo.go b/internal/imagecache/mountinfo.go new file mode 100644 index 000000000..0f874fa5a --- /dev/null +++ b/internal/imagecache/mountinfo.go @@ -0,0 +1,61 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// mountinfo parsing lives in a portable file (the reader comes in as an +// io.Reader) so the logic is unit-testable off-Linux; only the caller that +// opens /proc/self/mountinfo is linux-tagged. + +package imagecache + +import ( + "bufio" + "fmt" + "io" + "path/filepath" + "strings" +) + +// mountPointsIn returns the mount points at or below dir found in r, which +// must be in /proc/[pid]/mountinfo format: +// +// ID parentID major:minor root MOUNTPOINT options... +func mountPointsIn(r io.Reader, dir string) ([]string, error) { + dir = filepath.Clean(dir) + var points []string + scanner := bufio.NewScanner(r) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) < 5 { + continue + } + mp := unescapeMountPath(fields[4]) + if mp == dir || strings.HasPrefix(mp, dir+"/") { + points = append(points, mp) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("while reading mountinfo: %w", err) + } + return points, nil +} + +// unescapeMountPath decodes the octal escapes mountinfo uses for space, tab, +// newline, and backslash. +func unescapeMountPath(s string) string { + if !strings.Contains(s, `\`) { + return s + } + r := strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`) + return r.Replace(s) +} diff --git a/internal/imagecache/mountinfo_test.go b/internal/imagecache/mountinfo_test.go new file mode 100644 index 000000000..fdaf6db25 --- /dev/null +++ b/internal/imagecache/mountinfo_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "slices" + "strings" + "testing" +) + +func TestMountPointsIn(t *testing.T) { + mountinfo := strings.Join([]string{ + `22 26 0:20 / /sys rw,nosuid,nodev shared:7 - sysfs sysfs rw`, + `40 21 8:1 / /var/lib/ateom-gvisor/actors/a1/bundles/task/rootfs rw shared:1 - overlay overlay rw,lowerdir=/x`, + `41 21 8:1 / /var/lib/ateom-gvisor/actors/a1/bundles/pause/rootfs rw - overlay overlay rw`, + `42 21 8:1 / /var/lib/ateom-gvisor/actors/a2/bundles/task/rootfs rw - overlay overlay rw`, + // Path with an escaped space; mountinfo octal-escapes it. + `43 21 8:1 / /var/lib/ateom-gvisor/actors/a1/bundles/odd\040name/rootfs rw - overlay overlay rw`, + // Prefix-sibling directory that must NOT match a1's subtree. + `44 21 8:1 / /var/lib/ateom-gvisor/actors/a1-sibling/bundles/x/rootfs rw - overlay overlay rw`, + `malformed line`, + ``, + }, "\n") + + got, err := mountPointsIn(strings.NewReader(mountinfo), "/var/lib/ateom-gvisor/actors/a1/bundles") + if err != nil { + t.Fatalf("mountPointsIn: %v", err) + } + want := []string{ + "/var/lib/ateom-gvisor/actors/a1/bundles/task/rootfs", + "/var/lib/ateom-gvisor/actors/a1/bundles/pause/rootfs", + "/var/lib/ateom-gvisor/actors/a1/bundles/odd name/rootfs", + } + if !slices.Equal(got, want) { + t.Errorf("mount points = %v, want %v", got, want) + } +} + +func TestMountPointsIn_ExactDirMatch(t *testing.T) { + got, err := mountPointsIn(strings.NewReader(`40 21 8:1 / /mnt/target rw - ext4 /dev/sda1 rw`), "/mnt/target") + if err != nil { + t.Fatalf("mountPointsIn: %v", err) + } + if !slices.Equal(got, []string{"/mnt/target"}) { + t.Errorf("mount points = %v, want the exact dir itself", got) + } +} + +func TestUnescapeMountPath(t *testing.T) { + tests := []struct{ in, want string }{ + {`/plain/path`, `/plain/path`}, + {`/with\040space`, `/with space`}, + {`/tab\011and\012newline`, "/tab\tand\nnewline"}, + {`/back\134slash`, `/back\slash`}, + } + for _, tc := range tests { + if got := unescapeMountPath(tc.in); got != tc.want { + t.Errorf("unescapeMountPath(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} diff --git a/internal/imagecache/options_test.go b/internal/imagecache/options_test.go new file mode 100644 index 000000000..63921ff0f --- /dev/null +++ b/internal/imagecache/options_test.go @@ -0,0 +1,69 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +func TestOptionsApply(t *testing.T) { + auth := authn.FromConfig(authn.AuthConfig{Username: "u"}) + s, err := New(t.TempDir(), + WithAuthenticator(auth), + WithLocalhostRegistryReplacement("kind-registry:5000"), + WithPlatform(v1.Platform{OS: "linux", Architecture: "amd64"}), + ) + if err != nil { + t.Fatalf("New: %v", err) + } + if s.authenticator != auth { + t.Errorf("WithAuthenticator not applied") + } + if s.localhostRegistryReplacement != "kind-registry:5000" { + t.Errorf("WithLocalhostRegistryReplacement not applied: %q", s.localhostRegistryReplacement) + } + if s.platform == nil || s.platform.Architecture != "amd64" || s.platform.OS != "linux" { + t.Errorf("WithPlatform not applied: %+v", s.platform) + } +} + +func TestRegistryUsesGCPAuth(t *testing.T) { + tests := []struct { + registry string + want bool + }{ + {"gcr.io", true}, + {"us.gcr.io", true}, + {"eu.gcr.io", true}, + {"pkg.dev", true}, + {"us-docker.pkg.dev", true}, + {"docker.io", false}, + {"index.docker.io", false}, + {"quay.io", false}, + {"notgcr.io", false}, + {"gcr.io.evil.example", false}, + {"pkg.dev.evil.example", false}, + {"kind-registry:5000", false}, + {"", false}, + } + for _, tc := range tests { + if got := registryUsesGCPAuth(tc.registry); got != tc.want { + t.Errorf("registryUsesGCPAuth(%q) = %v, want %v", tc.registry, got, tc.want) + } + } +} diff --git a/internal/imagecache/removeall.go b/internal/imagecache/removeall.go new file mode 100644 index 000000000..58cc5a135 --- /dev/null +++ b/internal/imagecache/removeall.go @@ -0,0 +1,40 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "io/fs" + "os" + "path/filepath" +) + +// RemoveAllWritable removes path and everything under it, first making every +// directory owner-writable so its children can be unlinked. Unpacked image +// trees keep the image's (possibly read-only) directory modes, which atelet +// cannot remove as plain root without CAP_DAC_OVERRIDE — os.RemoveAll alone +// fails there with EACCES. atelet owns these files, so chmod needs no +// capability. +func RemoveAllWritable(path string) error { + // Make dirs traversable/writable top-down (WalkDir visits a directory before + // reading it, so chmod here lets the walk descend into otherwise-unreadable + // dirs). Best-effort: ignore errors and let os.RemoveAll surface real ones. + _ = filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error { + if err == nil && d.IsDir() { + _ = os.Chmod(p, 0o700) + } + return nil + }) + return os.RemoveAll(path) +} diff --git a/internal/imagecache/spec.go b/internal/imagecache/spec.go new file mode 100644 index 000000000..6a78a8939 --- /dev/null +++ b/internal/imagecache/spec.go @@ -0,0 +1,142 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" +) + +// OverlaySpecFileName is the file atelet writes into each container bundle, +// next to config.json, describing how to compose the bundle's rootfs from +// cached layers. Its absence means the bundle's rootfs is a plain directory +// (e.g. one prepared by a pre-imagecache atelet) and needs no mount. +const OverlaySpecFileName = "rootfs-overlay.json" + +// OverlaySpec is the contract between atelet (which cannot mount) and the +// ateom runtimes (which mount the rootfs overlay just before running the +// workload). The overlay's mountpoint, upperdir, and workdir are always the +// bundle-local rootfs/, upper/, and work/ directories — derived from the +// bundle path by the consumer rather than trusted from the file. +type OverlaySpec struct { + Version int `json:"version"` + // Layers are the cached layer directories (each holding its tree under + // fs/), bottom-most layer first — the order the image manifest lists + // them. Consumers reverse this into overlayfs's top-first lowerdir. + Layers []string `json:"layers"` + // ExtraDirs are absolute in-rootfs directories the consumer creates after + // mounting (they land in the actor's private upper): bind-mount targets + // that must exist for the runtime to attach them, e.g. the actor identity + // mount. + ExtraDirs []string `json:"extraDirs,omitempty"` +} + +// WriteSpec writes spec into the bundle at bundlePath. +func WriteSpec(bundlePath string, spec *OverlaySpec) error { + spec.Version = 1 + b, err := json.MarshalIndent(spec, "", " ") + if err != nil { + return fmt.Errorf("while encoding overlay spec: %w", err) + } + if err := os.WriteFile(filepath.Join(bundlePath, OverlaySpecFileName), b, 0o600); err != nil { + return fmt.Errorf("while writing overlay spec: %w", err) + } + return nil +} + +// ReadSpec reads the bundle's overlay spec. It returns (nil, nil) when the +// bundle has none. +func ReadSpec(bundlePath string) (*OverlaySpec, error) { + b, err := os.ReadFile(filepath.Join(bundlePath, OverlaySpecFileName)) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("while reading overlay spec: %w", err) + } + var spec OverlaySpec + if err := json.Unmarshal(b, &spec); err != nil { + return nil, fmt.Errorf("while decoding overlay spec: %w", err) + } + if spec.Version != 1 { + return nil, fmt.Errorf("overlay spec %q has version %d, want 1", filepath.Join(bundlePath, OverlaySpecFileName), spec.Version) + } + return &spec, nil +} + +// overlayMountOptions builds the overlayfs option string: lowerdir is +// top-most layer first (the reverse of the spec's bottom-first order), each +// entry pointing at the layer's fs/ tree. +func overlayMountOptions(layers []string, upper, work string) (string, error) { + lowers := make([]string, 0, len(layers)) + seen := make(map[string]bool, len(layers)) + for _, layer := range slices.Backward(layers) { + p := filepath.Join(layer, layerFSDirName) + // An image may legitimately list the same layer at several positions + // (identical build steps produce identical diffids; SWE-bench images + // do this). The pool stores that layer once, and overlayfs rejects a + // repeated lower directory (its overlapping-layers check fails the + // mount with ELOOP). For identical content only the topmost + // occurrence can affect the merged view, so keep the first one seen + // walking top-first and drop the rest. + if seen[p] { + continue + } + seen[p] = true + // ':' separates lowerdirs and ',' separates mount options; cache paths + // are digest-derived so this never fires, but refuse rather than + // assemble a corrupt option string. + if strings.ContainsAny(p, ":,") { + return "", fmt.Errorf("layer path %q contains overlay option separators", p) + } + lowers = append(lowers, p) + } + if strings.ContainsAny(upper, ":,") || strings.ContainsAny(work, ":,") { + return "", fmt.Errorf("bundle path %q contains overlay option separators", upper) + } + return "lowerdir=" + strings.Join(lowers, ":") + ",upperdir=" + upper + ",workdir=" + work, nil +} + +// createExtraDirs creates the spec's ExtraDirs inside the (mounted) rootfs. +// It uses os.Root so the operation is confined to rootfsPath: a symlink +// planted by the image cannot redirect the write outside the rootfs. +func createExtraDirs(rootfsPath string, extraDirs []string) error { + if len(extraDirs) == 0 { + return nil + } + root, err := os.OpenRoot(rootfsPath) + if err != nil { + return fmt.Errorf("while opening rootfs %q: %w", rootfsPath, err) + } + defer root.Close() + + for _, d := range extraDirs { + rel := strings.TrimPrefix(d, "/") + if rel == "" { + continue + } + if !filepath.IsLocal(rel) { + return fmt.Errorf("extra dir %q escapes the rootfs", d) + } + if err := root.MkdirAll(rel, 0o755); err != nil { + return fmt.Errorf("while creating extra dir %q: %w", rel, err) + } + } + return nil +} diff --git a/internal/imagecache/spec_test.go b/internal/imagecache/spec_test.go new file mode 100644 index 000000000..264f4e3c5 --- /dev/null +++ b/internal/imagecache/spec_test.go @@ -0,0 +1,145 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "os" + "path/filepath" + "slices" + "testing" +) + +func TestOverlaySpecRoundTrip(t *testing.T) { + bundle := t.TempDir() + in := &OverlaySpec{ + Layers: []string{"/cache/layers/sha256/aaa", "/cache/layers/sha256/bbb"}, + ExtraDirs: []string{"/run/ate"}, + } + if err := WriteSpec(bundle, in); err != nil { + t.Fatalf("WriteSpec: %v", err) + } + out, err := ReadSpec(bundle) + if err != nil { + t.Fatalf("ReadSpec: %v", err) + } + if out == nil { + t.Fatalf("ReadSpec returned nil for a bundle with a spec") + } + if !slices.Equal(out.Layers, in.Layers) { + t.Errorf("Layers = %v, want %v", out.Layers, in.Layers) + } + if !slices.Equal(out.ExtraDirs, in.ExtraDirs) { + t.Errorf("ExtraDirs = %v, want %v", out.ExtraDirs, in.ExtraDirs) + } +} + +// A bundle without a spec (e.g. prepared by a pre-imagecache atelet) reads +// as nil-and-no-error: the consumer must leave its rootfs untouched. +func TestReadSpec_Absent(t *testing.T) { + spec, err := ReadSpec(t.TempDir()) + if err != nil { + t.Fatalf("ReadSpec: %v", err) + } + if spec != nil { + t.Errorf("ReadSpec = %+v, want nil for a bundle without a spec", spec) + } +} + +func TestReadSpec_UnknownVersion(t *testing.T) { + bundle := t.TempDir() + if err := os.WriteFile(filepath.Join(bundle, OverlaySpecFileName), []byte(`{"version":99}`), 0o600); err != nil { + t.Fatalf("writing spec: %v", err) + } + if _, err := ReadSpec(bundle); err == nil { + t.Errorf("ReadSpec accepted an unknown spec version") + } +} + +func TestOverlayMountOptions(t *testing.T) { + t.Run("reverses to top-first", func(t *testing.T) { + got, err := overlayMountOptions([]string{"/c/base", "/c/top"}, "/b/upper", "/b/work") + if err != nil { + t.Fatalf("overlayMountOptions: %v", err) + } + want := "lowerdir=/c/top/fs:/c/base/fs,upperdir=/b/upper,workdir=/b/work" + if got != want { + t.Errorf("got %q, want %q", got, want) + } + }) + + // Images can list the same layer at several positions (identical build + // steps → identical diffids; SWE-bench images do). overlayfs rejects a + // repeated lowerdir with ELOOP, so duplicates must collapse to the + // topmost occurrence. Regression test for the "too many levels of + // symbolic links" mount failure. + t.Run("deduplicates repeated layers keeping topmost", func(t *testing.T) { + got, err := overlayMountOptions([]string{"/c/base", "/c/dup", "/c/dup", "/c/top"}, "/b/upper", "/b/work") + if err != nil { + t.Fatalf("overlayMountOptions: %v", err) + } + want := "lowerdir=/c/top/fs:/c/dup/fs:/c/base/fs,upperdir=/b/upper,workdir=/b/work" + if got != want { + t.Errorf("got %q, want %q", got, want) + } + }) + + t.Run("refuses option separators in paths", func(t *testing.T) { + if _, err := overlayMountOptions([]string{"/c/evil:layer"}, "/b/upper", "/b/work"); err == nil { + t.Errorf("expected error for ':' in layer path") + } + }) +} + +func TestCreateExtraDirs(t *testing.T) { + t.Run("creates target inside rootfs", func(t *testing.T) { + root := t.TempDir() + if err := createExtraDirs(root, []string{"/run/ate"}); err != nil { + t.Fatalf("createExtraDirs: %v", err) + } + info, err := os.Stat(filepath.Join(root, "run", "ate")) + if err != nil { + t.Fatalf("extra dir not created in rootfs: %v", err) + } + if !info.IsDir() { + t.Errorf("extra dir must be a directory to host the identity bind mount") + } + }) + + t.Run("refuses symlink escaping the rootfs", func(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + // A malicious image could ship /run as a symlink pointing out of the + // rootfs; os.Root must refuse to follow it. + if err := os.Symlink(outside, filepath.Join(root, "run")); err != nil { + t.Fatalf("planting symlink: %v", err) + } + if err := createExtraDirs(root, []string{"/run/ate"}); err == nil { + t.Errorf("expected error when /run escapes the rootfs, got nil") + } + // Nothing may be created through the escaping symlink. + if entries, err := os.ReadDir(outside); err != nil { + t.Errorf("reading outside dir: %v", err) + } else if len(entries) != 0 { + t.Errorf("write escaped the rootfs: %s is not empty (%d entries)", outside, len(entries)) + } + }) + + t.Run("refuses parent traversal", func(t *testing.T) { + root := t.TempDir() + if err := createExtraDirs(root, []string{"/../escape"}); err == nil { + t.Errorf("expected error for a parent-traversal extra dir, got nil") + } + }) +} diff --git a/internal/imagecache/unpack.go b/internal/imagecache/unpack.go new file mode 100644 index 000000000..81281da4c --- /dev/null +++ b/internal/imagecache/unpack.go @@ -0,0 +1,259 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "archive/tar" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" +) + +const ( + // whiteoutPrefix marks an OCI layer entry that deletes the same-named + // path from lower layers. + whiteoutPrefix = ".wh." + // opaqueMarkerName marks its directory as opaque: lower-layer contents of + // that directory are hidden entirely. + opaqueMarkerName = ".wh..wh..opq" +) + +// whiteoutSet records a layer's whiteout state, captured at unpack time and +// materialized later by FinalizeLayer (which runs with the privileges atelet +// lacks). Paths are clean and relative to the layer's fs/ root. +type whiteoutSet struct { + Version int `json:"version"` + // Whiteouts are paths that must become 0:0 char devices in the lowerdir. + Whiteouts []string `json:"whiteouts,omitempty"` + // Opaques are directories that must carry trusted.overlay.opaque=y. + Opaques []string `json:"opaques,omitempty"` +} + +func readWhiteouts(layerDir string) (*whiteoutSet, error) { + b, err := os.ReadFile(filepath.Join(layerDir, layerWhiteoutsFileName)) + if errors.Is(err, os.ErrNotExist) { + return &whiteoutSet{Version: 1}, nil + } else if err != nil { + return nil, fmt.Errorf("while reading layer whiteouts: %w", err) + } + var wh whiteoutSet + if err := json.Unmarshal(b, &wh); err != nil { + return nil, fmt.Errorf("while decoding layer whiteouts: %w", err) + } + return &wh, nil +} + +func validateTarName(name string) (cleaned string, skip bool, err error) { + if name == "" { + return "", true, nil + } + cleaned = filepath.Clean(name) + if cleaned == "." { + return "", true, nil + } + cleaned = strings.TrimPrefix(cleaned, "/") + if cleaned == "" || cleaned == "." { + return "", true, nil + } + if !filepath.IsLocal(cleaned) { + return "", false, fmt.Errorf("not a local path: %q", name) + } + return cleaned, false, nil +} + +// unpackLayer extracts one uncompressed OCI layer tar into root. Whiteout +// entries (.wh.*) are not written to the tree; they are returned so the +// caller can persist them for later materialization by a privileged process. +// +// Unlike a flattened-image extract, cross-layer "later entry wins" semantics +// are overlayfs's job now; the handling here only needs to cope with +// duplicate entries within a single layer (real ko images repeat directory +// entries). +func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteoutSet, error) { + wh := &whiteoutSet{Version: 1} + + // Directories are created owner-writable during extraction (so their children + // can be written even when the image marks them read-only, e.g. ko ships + // /ko-app as 0555) and their real modes are restored afterwards. This lets + // atelet, running as plain root, unpack arbitrary actor images without + // CAP_DAC_OVERRIDE. Keyed by name so a repeated dir entry's last mode wins. + dirModes := map[string]os.FileMode{} + + tarReader := tar.NewReader(tarData) + for { + hdr, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, fmt.Errorf("in tarReader.Next: %w", err) + } + + name, skip, err := validateTarName(hdr.Name) + if err != nil { + return nil, fmt.Errorf("invalid tar entry: %w", err) + } + if skip { + continue + } + + if base := filepath.Base(name); base == opaqueMarkerName { + if dir := filepath.Dir(name); dir != "." { + wh.Opaques = append(wh.Opaques, dir) + } + continue + } else if strings.HasPrefix(base, whiteoutPrefix+whiteoutPrefix) { + // AUFS bookkeeping entries (.wh..wh.plnk, .wh..wh.aufs, ...); + // nothing to represent in an overlayfs lowerdir. + continue + } else if deleted, ok := strings.CutPrefix(base, whiteoutPrefix); ok { + wh.Whiteouts = append(wh.Whiteouts, filepath.Join(filepath.Dir(name), deleted)) + continue + } + + mode := hdr.FileInfo().Mode().Perm() + + // A layer tar routinely omits entries for parent directories that + // exist in lower layers (e.g. just "etc/nsswitch.conf", with "etc/" + // declared only in the base layer). Unpacking per layer, those + // parents must be created here; overlayfs merges them with the + // lower layers' directories at compose time. + if parent := filepath.Dir(name); parent != "." { + if err := root.MkdirAll(parent, 0o755); err != nil { + return nil, fmt.Errorf("while creating parent directories for %q: %w", name, err) + } + } + + switch hdr.Typeflag { + case tar.TypeReg: // Regular file + // "Later entry wins" within the layer: if any entry exists at the target + // path, remove it first. This ensures that: + // 1. If it's a symlink, we don't write through it (security vulnerability / incorrectness). + // 2. If it's a hardlink, we unlink it instead of truncating the shared inode. + // 3. If it's a directory, we recursively remove it so we can write the file. + if _, err := root.Lstat(name); err == nil { + if err := root.RemoveAll(name); err != nil { + return nil, fmt.Errorf("while replacing existing path at %q before regular file: %w", name, err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("while checking existing path at %q before regular file: %w", name, err) + } + + // Stream directly from tarReader to target file to avoid buffering in memory. + outFile, err := root.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, mode) + if err != nil { + return nil, fmt.Errorf("while creating file %q: %w", name, err) + } + + _, err = io.Copy(outFile, tarReader) + closeErr := outFile.Close() + + if err != nil { + return nil, fmt.Errorf("while writing contents of %q from tar stream: %w", name, err) + } + if closeErr != nil { + return nil, fmt.Errorf("while closing file %q: %w", name, closeErr) + } + + case tar.TypeDir: + // Create owner-writable so children can be written even when the image + // marks the dir read-only; the real mode is restored after extraction + // (see dirModes / the restore pass below). + err := root.Mkdir(name, mode|0o700) + if errors.Is(err, os.ErrExist) { + // OCI layers can repeat a directory entry (real ko images do); the + // existing dir is already owner-writable, so let the later entry's + // mode win at restore time. + } else if err != nil { + return nil, fmt.Errorf("while creating directory=%q, mode=%v: %w", name, mode, err) + } + dirModes[name] = mode + + case tar.TypeSymlink: + // A layer may re-define the same path (e.g. declare /var/run as a dir + // then re-declare it as a symlink). Standard tar-extract semantics are + // "later entry wins": replace any existing entry. + if existing, err := root.Lstat(name); err == nil { + // If it's already the same symlink, skip the unlink+symlink pair. + if existing.Mode()&os.ModeSymlink != 0 { + if cur, rerr := root.Readlink(name); rerr == nil && cur == hdr.Linkname { + continue + } + } + // Root.RemoveAll removes the symlink entry itself; it does NOT + // traverse and remove the directory the symlink points to. + // That's the desired semantic here — replace this path's + // entry without touching whatever the prior symlink targeted. + if err := root.RemoveAll(name); err != nil { + return nil, fmt.Errorf("while replacing existing path at %q before symlink: %w", name, err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("while checking existing path at %q before symlink: %w", name, err) + } + if err := root.Symlink(hdr.Linkname, name); err != nil { + return nil, fmt.Errorf("while creating symlink src=%q target=%q: %w", name, hdr.Linkname, err) + } + + case tar.TypeLink: + linkname, linkSkip, err := validateTarName(hdr.Linkname) + if err != nil { + return nil, fmt.Errorf("invalid hardlink target for %q: %w", name, err) + } + if linkSkip { + return nil, fmt.Errorf("invalid hardlink target for %q: empty", name) + } + // Same "later entry wins" handling as TypeSymlink: replace existing entry. + if _, err := root.Lstat(name); err == nil { + if err := root.RemoveAll(name); err != nil { + return nil, fmt.Errorf("while replacing existing path at %q before hardlink: %w", name, err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("while checking existing path at %q before hardlink: %w", name, err) + } + if err := root.Link(linkname, name); err != nil { + return nil, fmt.Errorf("while creating hardlink src=%q target=%q: %w", name, linkname, err) + } + + default: + tfStr := string([]byte{hdr.Typeflag}) + slog.ErrorContext(ctx, "Unhandled tar entry typeflag", slog.String("typeflag", tfStr), slog.Any("hdr", hdr)) + return nil, fmt.Errorf("unhandled tar entry typeflag %q", tfStr) + } + } + + // Restore the image's intended directory modes now that every child exists. + // Deepest paths first: a child's path is always longer than its parent's, so + // length-descending order guarantees a directory is restored before any of its + // ancestors — restoring a parent to a non-traversable mode then can't block + // restoring its children. + dirs := make([]string, 0, len(dirModes)) + for name := range dirModes { + dirs = append(dirs, name) + } + sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) }) + for _, name := range dirs { + if err := root.Chmod(name, dirModes[name]); err != nil { + return nil, fmt.Errorf("while restoring mode %v on directory %q: %w", dirModes[name], name, err) + } + } + + return wh, nil +} diff --git a/internal/imagecache/unpack_test.go b/internal/imagecache/unpack_test.go new file mode 100644 index 000000000..a37eb5652 --- /dev/null +++ b/internal/imagecache/unpack_test.go @@ -0,0 +1,572 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "archive/tar" + "bytes" + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +type tarEntry struct { + name string + typeflag byte + mode int64 + body string + linkname string +} + +func defaultMode(typeflag byte) int64 { + switch typeflag { + case tar.TypeDir: + return 0o755 + case tar.TypeSymlink: + return 0o777 + default: + return 0o644 + } +} + +func buildTar(t *testing.T, entries []tarEntry) []byte { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, e := range entries { + mode := e.mode + if mode == 0 { + mode = defaultMode(e.typeflag) + } + hdr := &tar.Header{ + Name: e.name, + Typeflag: e.typeflag, + Mode: mode, + Size: int64(len(e.body)), + Linkname: e.linkname, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar.WriteHeader(%+v): %v", hdr, err) + } + if e.body != "" { + if _, err := tw.Write([]byte(e.body)); err != nil { + t.Fatalf("tar.Write(%q): %v", e.name, err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatalf("tar.Close: %v", err) + } + return buf.Bytes() +} + +func unpackInto(t *testing.T, dir string, tarBytes []byte) (*whiteoutSet, error) { + t.Helper() + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatalf("os.OpenRoot(%q): %v", dir, err) + } + defer root.Close() + return unpackLayer(context.Background(), bytes.NewReader(tarBytes), root) +} + +func runUnpack(t *testing.T, entries []tarEntry) (string, *whiteoutSet, error) { + t.Helper() + dir := t.TempDir() + wh, err := unpackInto(t, dir, buildTar(t, entries)) + return dir, wh, err +} + +func TestValidateTarName(t *testing.T) { + tests := []struct { + name string + input string + wantClean string + wantSkip bool + wantErr bool + }{ + {name: "regular file", input: "etc/passwd", wantClean: "etc/passwd"}, + {name: "current dir", input: ".", wantSkip: true}, + {name: "empty", input: "", wantSkip: true}, + {name: "trailing slash", input: "etc/", wantClean: "etc"}, + {name: "absolute path", input: "/etc/passwd", wantClean: "etc/passwd"}, + {name: "double slash absolute", input: "//etc/passwd", wantClean: "etc/passwd"}, + {name: "parent escape", input: "../etc/passwd", wantErr: true}, + {name: "parent only", input: "..", wantErr: true}, + {name: "embedded escape", input: "a/../../escape", wantErr: true}, + {name: "ok with dot segments", input: "./a/./b", wantClean: "a/b"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + gotClean, gotSkip, err := validateTarName(tc.input) + if (err != nil) != tc.wantErr { + t.Fatalf("validateTarName(%q) err = %v, wantErr %v", tc.input, err, tc.wantErr) + } + if err != nil { + return + } + if gotSkip != tc.wantSkip { + t.Errorf("skip = %v, want %v", gotSkip, tc.wantSkip) + } + if gotClean != tc.wantClean { + t.Errorf("clean = %q, want %q", gotClean, tc.wantClean) + } + }) + } +} + +func TestUnpackLayer_HappyPath(t *testing.T) { + entries := []tarEntry{ + {name: ".", typeflag: tar.TypeDir}, + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/hostname", typeflag: tar.TypeReg, body: "demo\n"}, + {name: "bin/", typeflag: tar.TypeDir}, + {name: "bin/sh", typeflag: tar.TypeReg, mode: 0o755, body: "#!/sh\n"}, + {name: "bin/bash", typeflag: tar.TypeLink, linkname: "bin/sh"}, + {name: "etc/host-link", typeflag: tar.TypeSymlink, linkname: "hostname"}, + } + dir, wh, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + if len(wh.Whiteouts) != 0 || len(wh.Opaques) != 0 { + t.Errorf("whiteouts recorded for a layer with none: %+v", wh) + } + + if got, err := os.ReadFile(filepath.Join(dir, "etc/hostname")); err != nil { + t.Errorf("read etc/hostname: %v", err) + } else if string(got) != "demo\n" { + t.Errorf("etc/hostname = %q, want %q", got, "demo\n") + } + + if target, err := os.Readlink(filepath.Join(dir, "etc/host-link")); err != nil { + t.Errorf("readlink etc/host-link: %v", err) + } else if target != "hostname" { + t.Errorf("symlink target = %q, want %q", target, "hostname") + } + + srcInfo, err := os.Stat(filepath.Join(dir, "bin/sh")) + if err != nil { + t.Fatalf("stat bin/sh: %v", err) + } + dstInfo, err := os.Stat(filepath.Join(dir, "bin/bash")) + if err != nil { + t.Fatalf("stat bin/bash: %v", err) + } + if !os.SameFile(srcInfo, dstInfo) { + t.Errorf("bin/bash is not a hardlink to bin/sh") + } +} + +// Whiteout entries must be recorded — never written into the layer tree, +// where a literal .wh. file would leak into the composed rootfs. +func TestUnpackLayer_Whiteouts(t *testing.T) { + entries := []tarEntry{ + {name: "a/", typeflag: tar.TypeDir}, + {name: "a/.wh.deleted", typeflag: tar.TypeReg}, + {name: "b/", typeflag: tar.TypeDir}, + {name: "b/.wh..wh..opq", typeflag: tar.TypeReg}, + {name: ".wh.toplevel", typeflag: tar.TypeReg}, + // AUFS bookkeeping entries carry no overlayfs meaning; silently skipped. + {name: ".wh..wh.plnk", typeflag: tar.TypeReg}, + {name: "a/kept", typeflag: tar.TypeReg, body: "kept"}, + } + dir, wh, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + + if want := []string{"a/deleted", "toplevel"}; !slices.Equal(wh.Whiteouts, want) { + t.Errorf("whiteouts = %v, want %v", wh.Whiteouts, want) + } + if want := []string{"b"}; !slices.Equal(wh.Opaques, want) { + t.Errorf("opaques = %v, want %v", wh.Opaques, want) + } + + // No .wh. artifact may exist in the tree. + for _, p := range []string{"a/.wh.deleted", "b/.wh..wh..opq", ".wh.toplevel", ".wh..wh.plnk"} { + if _, err := os.Lstat(filepath.Join(dir, p)); err == nil { + t.Errorf("whiteout entry %q was written into the layer tree", p) + } + } + if got, _ := os.ReadFile(filepath.Join(dir, "a/kept")); string(got) != "kept" { + t.Errorf("a/kept = %q, want %q", got, "kept") + } +} + +// A layer tar routinely omits parent-directory entries when the parent +// exists only in a lower layer (the flattened-extract path never saw this; +// per-layer unpack must create the parents). Regression test for the +// "etc/nsswitch.conf: no such file or directory" failure on real images. +func TestUnpackLayer_MissingParentDirs(t *testing.T) { + entries := []tarEntry{ + {name: "etc/nsswitch.conf", typeflag: tar.TypeReg, body: "hosts: files dns"}, + {name: "a/b/c/deep", typeflag: tar.TypeReg, body: "deep"}, + {name: "usr/share/doc/", typeflag: tar.TypeDir}, + {name: "lnk/target", typeflag: tar.TypeReg, body: "t"}, + {name: "other/dir/sym", typeflag: tar.TypeSymlink, linkname: "../../lnk/target"}, + {name: "hard/link", typeflag: tar.TypeLink, linkname: "lnk/target"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "etc/nsswitch.conf")); string(got) != "hosts: files dns" { + t.Errorf("etc/nsswitch.conf = %q, want %q", got, "hosts: files dns") + } + if got, _ := os.ReadFile(filepath.Join(dir, "a/b/c/deep")); string(got) != "deep" { + t.Errorf("a/b/c/deep = %q, want %q", got, "deep") + } + if fi, err := os.Stat(filepath.Join(dir, "usr/share/doc")); err != nil || !fi.IsDir() { + t.Errorf("usr/share/doc not created: fi=%v err=%v", fi, err) + } + if got, err := os.Readlink(filepath.Join(dir, "other/dir/sym")); err != nil || got != "../../lnk/target" { + t.Errorf("other/dir/sym = %q (%v), want ../../lnk/target", got, err) + } + srcInfo, err := os.Stat(filepath.Join(dir, "lnk/target")) + if err != nil { + t.Fatalf("stat lnk/target: %v", err) + } + if dstInfo, err := os.Stat(filepath.Join(dir, "hard/link")); err != nil || !os.SameFile(srcInfo, dstInfo) { + t.Errorf("hard/link is not a hardlink to lnk/target (err=%v)", err) + } +} + +func TestUnpackLayer_LaterEntryWins(t *testing.T) { + t.Run("dir then symlink", func(t *testing.T) { + entries := []tarEntry{ + {name: "var/", typeflag: tar.TypeDir}, + {name: "var/run/", typeflag: tar.TypeDir}, + {name: "run/", typeflag: tar.TypeDir}, + {name: "run/sock", typeflag: tar.TypeReg, body: "sock"}, + {name: "var/run", typeflag: tar.TypeSymlink, linkname: "../run"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + fi, err := os.Lstat(filepath.Join(dir, "var/run")) + if err != nil { + t.Fatalf("lstat var/run: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Fatalf("var/run not a symlink, mode = %v", fi.Mode()) + } + if got, _ := os.Readlink(filepath.Join(dir, "var/run")); got != "../run" { + t.Errorf("symlink target = %q, want %q", got, "../run") + } + }) + + t.Run("file overwrite", func(t *testing.T) { + entries := []tarEntry{ + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/conf", typeflag: tar.TypeReg, body: "v1"}, + {name: "etc/conf", typeflag: tar.TypeReg, body: "v2"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "etc/conf")); string(got) != "v2" { + t.Errorf("etc/conf = %q, want %q", got, "v2") + } + }) + + t.Run("symlink retargeted", func(t *testing.T) { + entries := []tarEntry{ + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/x", typeflag: tar.TypeReg, body: "x"}, + {name: "etc/y", typeflag: tar.TypeReg, body: "y"}, + {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, + {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "y"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + if got, _ := os.Readlink(filepath.Join(dir, "etc/link")); got != "y" { + t.Errorf("symlink target = %q, want %q", got, "y") + } + }) + + t.Run("repeated dir entry tolerated", func(t *testing.T) { + entries := []tarEntry{ + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/", typeflag: tar.TypeDir}, + } + if _, _, err := runUnpack(t, entries); err != nil { + t.Errorf("unpackLayer: %v", err) + } + }) + + t.Run("identical symlink redeclaration is a no-op", func(t *testing.T) { + entries := []tarEntry{ + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/x", typeflag: tar.TypeReg, body: "x"}, + {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, + {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + if got, _ := os.Readlink(filepath.Join(dir, "etc/link")); got != "x" { + t.Errorf("symlink target = %q, want %q", got, "x") + } + }) + + t.Run("symlink overwritten by file", func(t *testing.T) { + entries := []tarEntry{ + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/x", typeflag: tar.TypeReg, body: "original"}, + {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "x"}, + {name: "etc/link", typeflag: tar.TypeReg, body: "replacement"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + fi, err := os.Lstat(filepath.Join(dir, "etc/link")) + if err != nil { + t.Fatalf("lstat etc/link: %v", err) + } + if fi.Mode().IsRegular() { + got, err := os.ReadFile(filepath.Join(dir, "etc/link")) + if err != nil { + t.Fatalf("read etc/link: %v", err) + } + if string(got) != "replacement" { + t.Errorf("etc/link content = %q, want %q", got, "replacement") + } + } else { + t.Errorf("etc/link mode is not regular file: %v", fi.Mode()) + } + // Also verify etc/x was NOT overwritten + gotX, err := os.ReadFile(filepath.Join(dir, "etc/x")) + if err != nil { + t.Fatalf("read etc/x: %v", err) + } + if string(gotX) != "original" { + t.Errorf("etc/x content was overwritten to %q", gotX) + } + }) + + t.Run("file overwritten by symlink", func(t *testing.T) { + entries := []tarEntry{ + {name: "etc/", typeflag: tar.TypeDir}, + {name: "etc/link", typeflag: tar.TypeReg, body: "original-file"}, + {name: "etc/link", typeflag: tar.TypeSymlink, linkname: "target-doesnt-exist"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + fi, err := os.Lstat(filepath.Join(dir, "etc/link")) + if err != nil { + t.Fatalf("lstat etc/link: %v", err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Errorf("etc/link mode is not a symlink: %v", fi.Mode()) + } + got, err := os.Readlink(filepath.Join(dir, "etc/link")) + if err != nil { + t.Fatalf("readlink etc/link: %v", err) + } + if got != "target-doesnt-exist" { + t.Errorf("etc/link target = %q, want %q", got, "target-doesnt-exist") + } + }) + + t.Run("hardlink overwritten by file", func(t *testing.T) { + entries := []tarEntry{ + {name: "bin/", typeflag: tar.TypeDir}, + {name: "bin/sh", typeflag: tar.TypeReg, body: "sh-original"}, + {name: "bin/bash", typeflag: tar.TypeLink, linkname: "bin/sh"}, + {name: "bin/bash", typeflag: tar.TypeReg, body: "bash-new"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + gotBash, err := os.ReadFile(filepath.Join(dir, "bin/bash")) + if err != nil { + t.Fatalf("read bin/bash: %v", err) + } + if string(gotBash) != "bash-new" { + t.Errorf("bin/bash content = %q, want %q", gotBash, "bash-new") + } + // Verify bin/sh was NOT modified! + gotSh, err := os.ReadFile(filepath.Join(dir, "bin/sh")) + if err != nil { + t.Fatalf("read bin/sh: %v", err) + } + if string(gotSh) != "sh-original" { + t.Errorf("bin/sh content was overwritten to %q (hardlink was not unlinked)", gotSh) + } + }) +} + +// A read-only directory in the image (e.g. ko ships /ko-app as 0555) must still +// get its child written AND keep the image's mode, so atelet can unpack arbitrary +// actor images as plain root without CAP_DAC_OVERRIDE. RemoveAllWritable must +// then still be able to delete the restored read-only tree. (Meaningful as a +// non-root test run; as root the dir-permission checks are bypassed.) +func TestUnpackLayer_ReadOnlyDir(t *testing.T) { + entries := []tarEntry{ + {name: "ko-app", typeflag: tar.TypeDir, mode: 0o555}, + {name: "ko-app/counter", typeflag: tar.TypeReg, mode: 0o755, body: "bin"}, + } + dir, _, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpack into read-only dir: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(dir, "ko-app/counter")); string(got) != "bin" { + t.Errorf("ko-app/counter = %q, want %q", got, "bin") + } + info, err := os.Stat(filepath.Join(dir, "ko-app")) + if err != nil { + t.Fatalf("stat ko-app: %v", err) + } + if info.Mode().Perm() != 0o555 { + t.Errorf("ko-app mode = %v, want the image's 0555 preserved", info.Mode().Perm()) + } + // atelet must be able to delete the restored read-only tree (this also lets + // t.TempDir's cleanup succeed, which plain os.RemoveAll could not on 0555). + if err := RemoveAllWritable(filepath.Join(dir, "ko-app")); err != nil { + t.Errorf("RemoveAllWritable on restored read-only dir: %v", err) + } +} + +func TestUnpackLayer_PathTraversal(t *testing.T) { + tests := []struct { + name string + entry tarEntry + }{ + {name: "parent prefix", entry: tarEntry{name: "../escape", typeflag: tar.TypeReg, body: "x"}}, + {name: "embedded parent", entry: tarEntry{name: "a/b/../../../escape", typeflag: tar.TypeReg, body: "x"}}, + {name: "parent only", entry: tarEntry{name: "..", typeflag: tar.TypeReg, body: "x"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, err := runUnpack(t, []tarEntry{tc.entry}) + if err == nil { + t.Fatalf("unpackLayer(%q) succeeded, want error", tc.entry.name) + } + if !strings.Contains(err.Error(), "invalid tar entry") { + t.Errorf("error = %q, want it to mention 'invalid tar entry'", err.Error()) + } + }) + } +} + +func TestUnpackLayer_SymlinkEscape(t *testing.T) { + // CVE-2024-24579 / CVE-2020-27833 pattern: a tar declares a symlink + // pointing outside the rootfs, then a later entry writes through it. + parent := t.TempDir() + rootfsDir := filepath.Join(parent, "rootfs") + if err := os.Mkdir(rootfsDir, 0o755); err != nil { + t.Fatalf("mkdir rootfs: %v", err) + } + hostDir := filepath.Join(parent, "host") + if err := os.Mkdir(hostDir, 0o755); err != nil { + t.Fatalf("mkdir host: %v", err) + } + hostFile := filepath.Join(hostDir, "passwd") + if err := os.WriteFile(hostFile, []byte("original"), 0o644); err != nil { + t.Fatalf("write host file: %v", err) + } + + entries := []tarEntry{ + {name: "etc", typeflag: tar.TypeSymlink, linkname: hostDir}, + {name: "etc/passwd", typeflag: tar.TypeReg, body: "OWNED"}, + } + if _, err := unpackInto(t, rootfsDir, buildTar(t, entries)); err == nil { + t.Fatalf("unpackLayer succeeded; expected escape via symlink to be refused") + } + + got, err := os.ReadFile(hostFile) + if err != nil { + t.Fatalf("read host file: %v", err) + } + if string(got) != "original" { + t.Errorf("host file modified to %q -- symlink escape was NOT prevented", got) + } +} + +func TestUnpackLayer_HardlinkEscape(t *testing.T) { + tests := []struct { + name string + entry tarEntry + }{ + {name: "parent target", entry: tarEntry{name: "etc/passwd", typeflag: tar.TypeLink, linkname: "../host/passwd"}}, + {name: "embedded escape target", entry: tarEntry{name: "etc/passwd", typeflag: tar.TypeLink, linkname: "a/../../host"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, err := runUnpack(t, []tarEntry{tc.entry}) + if err == nil { + t.Fatalf("unpackLayer succeeded, want hardlink escape refused") + } + if !strings.Contains(err.Error(), "invalid hardlink target") { + t.Errorf("error = %q, want it to mention 'invalid hardlink target'", err.Error()) + } + }) + } +} + +func TestUnpackLayer_RejectSpecialFiles(t *testing.T) { + tests := []struct { + name string + typeflag byte + }{ + {name: "char device", typeflag: tar.TypeChar}, + {name: "block device", typeflag: tar.TypeBlock}, + {name: "fifo", typeflag: tar.TypeFifo}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, _, err := runUnpack(t, []tarEntry{{name: "weird", typeflag: tc.typeflag}}) + if err == nil { + t.Fatalf("unpackLayer succeeded, want unhandled-typeflag error") + } + if !strings.Contains(err.Error(), "unhandled tar entry typeflag") { + t.Errorf("error = %q, want it to mention 'unhandled tar entry typeflag'", err.Error()) + } + }) + } +} + +func TestUnpackLayer_TruncatedArchive(t *testing.T) { + full := buildTar(t, []tarEntry{ + {name: "ok", typeflag: tar.TypeReg, body: "hello"}, + }) + if len(full) < 64 { + t.Fatalf("buildTar produced suspiciously small output: %d bytes", len(full)) + } + truncated := full[:len(full)-64] + + dir := t.TempDir() + _, err := unpackInto(t, dir, truncated) + if err == nil { + t.Fatalf("unpackLayer on truncated archive succeeded; want error") + } + if !strings.Contains(err.Error(), "in tarReader.Next") && + !strings.Contains(err.Error(), "unexpected EOF") { + t.Errorf("error = %v, want it to surface the underlying tar/copy error", err) + } +} diff --git a/tools/validate-image-cache/README.md b/tools/validate-image-cache/README.md new file mode 100644 index 000000000..62dbdfea6 --- /dev/null +++ b/tools/validate-image-cache/README.md @@ -0,0 +1,102 @@ +# validate-image-cache + +Batch-validates that OCI images can be **pulled, parsed, and unpacked** by +`internal/imagecache` — the atelet-side half of substrate's node-local image +cache. For every ref it exercises the exact production pull path +(`Store.EnsureImage`: registry pull, parallel per-layer streaming unpack, +whiteout capture, record write) and reports per-image results as CSV. + +It does **not** mount overlays or run workloads: the consumer half is +Linux-only and privileged, and is covered by the `bundle_linux_test.go` +unit tests and the e2e suites instead. This tool answers one question at +corpus scale: *"can every image in this repository be loaded into the +cache?"* — the class of failure that only shows up on real images +(layer tars without parent-dir entries, duplicated identical layers, exotic +whiteouts, oversized layer counts, ...). + +## Prerequisites + +- Application-default credentials with read access to the registry + (`gcloud auth application-default login`). gcr.io / pkg.dev registries are + authenticated with the GCP env authenticator, the same mechanism atelet + uses. Note that a repo readable by *your user* (e.g. via a + `domain:google.com` grant) is not necessarily readable by a *service + account* — validate with the identity that production will use. +- Disk: unpacked layers are 2–3× their compressed size. The tool bounds + usage by evicting idle cached layers when the cache volume's free space + drops below `--min-free-gb`, but give it room to breathe (tens of GB + minimum). +- Runs anywhere Go runs, including macOS (unpack is pure file I/O). Caveat: + a case-insensitive filesystem (default macOS APFS) unpacks case-colliding + paths slightly differently than Linux — silently, not as an error. Linux + is the reference environment for a final sign-off run. + +## Generating a refs file + +One image ref per line; digest refs are recommended (they validate cache +hits offline and are immune to tag moves): + +```bash +gcloud artifacts docker images list REPOSITORY \ + --format="value[separator='@'](package,version)" > refs.txt +``` + +## Running + +Random sample of 500 images (reproducible via the seed): + +```bash +go run ./tools/validate-image-cache \ + --refs-file refs.txt \ + --sample 500 --seed 1 \ + --cache-dir "$HOME/imagecache-validate" \ + --out results.csv +``` + +Full corpus, pulled for the platform your nodes actually run: + +```bash +go run ./tools/validate-image-cache \ + --refs-file refs.txt \ + --cache-dir /cache/imagecache \ + --platform linux/amd64 \ + --parallel 4 --min-free-gb 40 \ + --out results.csv +``` + +### Flags + +| Flag | Default | Meaning | +|---|---|---| +| `--refs-file` | (required) | file with one image ref per line | +| `--cache-dir` | (required) | cache root; reused (and cache-hit) across runs | +| `--sample` | 0 (= all) | validate a random sample of N refs | +| `--seed` | 1 | sampling seed; same seed + file ⇒ same sample | +| `--out` | `validate-results.csv` | results CSV, written incrementally | +| `--parallel` | 3 | images validated concurrently (each pulls up to 4 layers in parallel) | +| `--timeout` | 20m | per-image timeout | +| `--min-free-gb` | 150 | evict oldest idle layers below this free-space floor | +| `--platform` | `linux/amd64` | image platform to pull | + +## Output and rerunning + +`results.csv` columns: `ref, digest, layers, seconds, error` (empty error = +pass). Progress logs every 10 images with an ETA; failures are logged +immediately as `FAIL [n/total]` and never stop the run. Exit code is +non-zero if any image failed. + +Reruns are cheap by design: completed layers stay in the cache (and even an +interrupted pull keeps every layer that finished), so re-running the same +sample — or just the failed refs — mostly re-validates from local disk. +Eviction only removes layers idle for 30+ minutes, so in-flight images are +never raced. + +## Scaling out + +The tool is embarrassingly parallel across a refs file: to sweep a large +corpus quickly, split the refs into shards (sorted by image name, so image +families keep their shared base layers on one machine) and run one process +per VM near the registry region, e.g. via a GCE startup script that fetches +a shard from GCS, runs the tool with `--cache-dir` on a local SSD, uploads +its CSV, and powers off. In-region, a c3-standard-4 with a local NVMe SSD +sustains roughly 10–15 images/min at `--parallel 4`. diff --git a/tools/validate-image-cache/main.go b/tools/validate-image-cache/main.go new file mode 100644 index 000000000..65a41632e --- /dev/null +++ b/tools/validate-image-cache/main.go @@ -0,0 +1,263 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// validate-image-cache batch-validates that OCI images can be pulled, +// parsed, and unpacked by internal/imagecache (the atelet-side half of the +// node-local image cache). It exercises Store.EnsureImage — registry pull, +// parallel per-layer streaming unpack, whiteout capture, record write — for +// every ref in a file, and reports per-image results as CSV. +// +// It does NOT mount overlays or run workloads (that half is Linux-only and +// privileged); it answers "can this image be loaded into the cache". +// +// Refs file: one image ref per line (digest refs recommended, e.g. +// us-docker.pkg.dev/proj/repo/img@sha256:...). Generate with: +// +// gcloud artifacts docker images list REPO --format="value[separator='@'](package,version)" +// +// Disk is bounded by evicting the oldest cached layers when the cache +// volume's free space drops below --min-free-gb. Only layers idle for more +// than 30 minutes are evicted, so in-flight images are never raced. +package main + +import ( + "bufio" + "context" + "encoding/csv" + "flag" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/agent-substrate/substrate/internal/imagecache" + v1 "github.com/google/go-containerregistry/pkg/v1" + googlecontainerauth "github.com/google/go-containerregistry/pkg/v1/google" + "golang.org/x/sys/unix" +) + +var ( + refsFile = flag.String("refs-file", "", "File with one image ref per line (required)") + sample = flag.Int("sample", 0, "Validate a random sample of N refs (0 = all)") + seed = flag.Int64("seed", 1, "Seed for reproducible sampling") + cacheDir = flag.String("cache-dir", "", "Cache root (required); reused across runs") + outCSV = flag.String("out", "validate-results.csv", "Results CSV path") + parallel = flag.Int("parallel", 3, "Images validated concurrently (each pulls up to 4 layers in parallel)") + timeout = flag.Duration("timeout", 20*time.Minute, "Per-image timeout") + minFreeGB = flag.Uint64("min-free-gb", 150, "Evict oldest idle cached layers when the cache volume has less free space than this") + platform = flag.String("platform", "linux/amd64", "Image platform to pull") +) + +type result struct { + ref string + digest string + layers int + took time.Duration + errText string +} + +func main() { + flag.Parse() + if *refsFile == "" || *cacheDir == "" { + flag.Usage() + os.Exit(2) + } + ctx := context.Background() + + refs, err := loadRefs(*refsFile) + if err != nil { + log.Fatalf("loading refs: %v", err) + } + if *sample > 0 && *sample < len(refs) { + rand.New(rand.NewSource(*seed)).Shuffle(len(refs), func(i, j int) { refs[i], refs[j] = refs[j], refs[i] }) + refs = refs[:*sample] + } + log.Printf("validating %d images (parallel=%d, cache=%s)", len(refs), *parallel, *cacheDir) + + osName, arch, ok := strings.Cut(*platform, "/") + if !ok { + log.Fatalf("invalid --platform %q", *platform) + } + + auth, err := googlecontainerauth.NewEnvAuthenticator(ctx) + if err != nil { + log.Fatalf("creating GCP authenticator (need application-default credentials): %v", err) + } + + store, err := imagecache.New(*cacheDir, + imagecache.WithAuthenticator(auth), + imagecache.WithPlatform(v1.Platform{OS: osName, Architecture: arch}), + ) + if err != nil { + log.Fatalf("opening cache: %v", err) + } + + outF, err := os.Create(*outCSV) + if err != nil { + log.Fatalf("creating %s: %v", *outCSV, err) + } + defer outF.Close() + csvW := csv.NewWriter(outF) + _ = csvW.Write([]string{"ref", "digest", "layers", "seconds", "error"}) + var csvMu sync.Mutex + + var ( + wg sync.WaitGroup + sem = make(chan struct{}, *parallel) + done, fails int64 + countMu sync.Mutex + ) + tStart := time.Now() + + for _, ref := range refs { + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + + evictIfLow(*cacheDir, *minFreeGB*1e9) + + r := validateOne(ctx, store, ref, *timeout) + + csvMu.Lock() + _ = csvW.Write([]string{r.ref, r.digest, strconv.Itoa(r.layers), fmt.Sprintf("%.1f", r.took.Seconds()), r.errText}) + csvW.Flush() + csvMu.Unlock() + + countMu.Lock() + done++ + if r.errText != "" { + fails++ + log.Printf("FAIL [%d/%d] %s: %s", done, len(refs), r.ref, r.errText) + } else if done%10 == 0 || done == int64(len(refs)) { + elapsed := time.Since(tStart) + eta := time.Duration(float64(elapsed) / float64(done) * float64(int64(len(refs))-done)).Round(time.Minute) + log.Printf("ok [%d/%d] fails=%d elapsed=%s eta=%s (last: %s in %.0fs, %d layers)", + done, len(refs), fails, elapsed.Round(time.Second), eta, shortRef(r.ref), r.took.Seconds(), r.layers) + } + countMu.Unlock() + }) + } + wg.Wait() + + log.Printf("done: %d images, %d failures, %s total; results in %s", done, fails, time.Since(tStart).Round(time.Second), *outCSV) + if fails > 0 { + os.Exit(1) + } +} + +func validateOne(ctx context.Context, store *imagecache.Store, ref string, timeout time.Duration) result { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + t := time.Now() + img, err := store.EnsureImage(ctx, ref) + r := result{ref: ref, took: time.Since(t)} + if err != nil { + r.errText = err.Error() + return r + } + r.digest = img.Digest.String() + r.layers = len(img.LayerDirs) + return r +} + +func loadRefs(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + var refs []string + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1024*1024), 1024*1024) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line != "" { + refs = append(refs, line) + } + } + return refs, sc.Err() +} + +func shortRef(ref string) string { + if i := strings.LastIndex(ref, "/"); i >= 0 { + return ref[i+1:] + } + return ref +} + +// evictIfLow deletes the oldest cached layer trees until the cache volume +// has at least minFree bytes available. Layers touched within the last 30 +// minutes are skipped so an in-flight image's freshly unpacked layers are +// never raced (per-image timeout is shorter than that). +var evictMu sync.Mutex + +func evictIfLow(cacheRoot string, minFree uint64) { + evictMu.Lock() + defer evictMu.Unlock() + + if freeBytes(cacheRoot) >= minFree { + return + } + layersDir := filepath.Join(cacheRoot, "layers", "sha256") + entries, err := os.ReadDir(layersDir) + if err != nil { + return + } + type aged struct { + path string + mod time.Time + } + var candidates []aged + for _, e := range entries { + info, err := e.Info() + if err != nil || time.Since(info.ModTime()) < 30*time.Minute { + continue + } + candidates = append(candidates, aged{filepath.Join(layersDir, e.Name()), info.ModTime()}) + } + sort.Slice(candidates, func(i, j int) bool { return candidates[i].mod.Before(candidates[j].mod) }) + + evicted := 0 + for _, c := range candidates { + if freeBytes(cacheRoot) >= minFree { + break + } + if err := imagecache.RemoveAllWritable(c.path); err == nil { + evicted++ + } + } + // Records referencing evicted layers re-pull only the missing layers, so + // stale manifests are harmless; drop them anyway to keep the dir tidy. + if evicted > 0 { + manifests, _ := filepath.Glob(filepath.Join(cacheRoot, "manifests", "sha256", "*.json")) + for _, m := range manifests { + _ = os.Remove(m) + } + log.Printf("evicted %d idle layers to reclaim disk (free now %.0f GB)", evicted, float64(freeBytes(cacheRoot))/1e9) + } +} + +func freeBytes(path string) uint64 { + var st unix.Statfs_t + if err := unix.Statfs(path, &st); err != nil { + return ^uint64(0) // unknown: don't evict + } + return st.Bavail * uint64(st.Bsize) +} diff --git a/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go b/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go new file mode 100644 index 000000000..85b171907 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go @@ -0,0 +1,104 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httptest provides a method for testing a TLS server a la net/http/httptest. +package httptest + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "net/http" + "net/http/httptest" + "time" +) + +// NewTLSServer returns an httptest server, with an http client that has been configured to +// send all requests to the returned server. The TLS certs are generated for the given domain. +// If you need a transport, Client().Transport is correctly configured. +func NewTLSServer(domain string, handler http.Handler) (*httptest.Server, error) { + s := httptest.NewUnstartedServer(handler) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{ + net.IPv4(127, 0, 0, 1), + net.IPv6loopback, + }, + DNSNames: []string{domain}, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + return nil, err + } + + b, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + pc := &bytes.Buffer{} + if err := pem.Encode(pc, &pem.Block{Type: "CERTIFICATE", Bytes: b}); err != nil { + return nil, err + } + + ek, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return nil, err + } + + pk := &bytes.Buffer{} + if err := pem.Encode(pk, &pem.Block{Type: "EC PRIVATE KEY", Bytes: ek}); err != nil { + return nil, err + } + + c, err := tls.X509KeyPair(pc.Bytes(), pk.Bytes()) + if err != nil { + return nil, err + } + s.TLS = &tls.Config{ + Certificates: []tls.Certificate{c}, + } + s.StartTLS() + + certpool := x509.NewCertPool() + certpool.AddCert(s.Certificate()) + + t := &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + }, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial(s.Listener.Addr().Network(), s.Listener.Addr().String()) + }, + } + s.Client().Transport = t + + return s, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/README.md b/vendor/github.com/google/go-containerregistry/pkg/registry/README.md new file mode 100644 index 000000000..5e58bbcd5 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/README.md @@ -0,0 +1,14 @@ +# `pkg/registry` + +This package implements a Docker v2 registry and the OCI distribution specification. + +It is designed to be used anywhere a low dependency container registry is needed, with an initial focus on tests. + +Its goal is to be standards compliant and its strictness will increase over time. + +This is currently a low flightmiles system. It's likely quite safe to use in tests; If you're using it in production, please let us know how and send us PRs for integration tests. + +Before sending a PR, understand that the expectation of this package is that it remain free of extraneous dependencies. +This means that we expect `pkg/registry` to only have dependencies on Go's standard library, and other packages in `go-containerregistry`. + +You may be asked to change your code to reduce dependencies, and your PR might be rejected if this is deemed impossible. diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go new file mode 100644 index 000000000..5b666ddbf --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go @@ -0,0 +1,540 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net/http" + "path" + "strings" + "sync" + + "github.com/google/go-containerregistry/internal/verify" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Returns whether this url should be handled by the blob handler +// This is complicated because blob is indicated by the trailing path, not the leading path. +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-a-layer +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-a-layer +func isBlob(req *http.Request) bool { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + if len(elem) < 3 { + return false + } + return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" && + elem[len(elem)-2] == "uploads") +} + +// BlobHandler represents a minimal blob storage backend, capable of serving +// blob contents. +type BlobHandler interface { + // Get gets the blob contents, or errNotFound if the blob wasn't found. + Get(ctx context.Context, repo string, h v1.Hash) (io.ReadCloser, error) +} + +// BlobStatHandler is an extension interface representing a blob storage +// backend that can serve metadata about blobs. +type BlobStatHandler interface { + // Stat returns the size of the blob, or errNotFound if the blob wasn't + // found, or RedirectError if the blob can be found elsewhere. + Stat(ctx context.Context, repo string, h v1.Hash) (int64, error) +} + +// BlobPutHandler is an extension interface representing a blob storage backend +// that can write blob contents. +type BlobPutHandler interface { + // Put puts the blob contents. + // + // The contents will be verified against the expected size and digest + // as the contents are read, and an error will be returned if these + // don't match. Implementations should return that error, or a wrapper + // around that error, to return the correct error when these don't match. + Put(ctx context.Context, repo string, h v1.Hash, rc io.ReadCloser) error +} + +// BlobDeleteHandler is an extension interface representing a blob storage +// backend that can delete blob contents. +type BlobDeleteHandler interface { + // Delete the blob contents. + Delete(ctx context.Context, repo string, h v1.Hash) error +} + +// RedirectError represents a signal that the blob handler doesn't have the blob +// contents, but that those contents are at another location which registry +// clients should redirect to. +type RedirectError struct { + // Location is the location to find the contents. + Location string + + // Code is the HTTP redirect status code to return to clients. + Code int +} + +type bytesCloser struct { + *bytes.Reader +} + +func (r *bytesCloser) Close() error { + return nil +} + +func (e RedirectError) Error() string { return fmt.Sprintf("redirecting (%d): %s", e.Code, e.Location) } + +// ErrNotFound represents an error locating the blob. +var ErrNotFound = errors.New("not found") + +type memHandler struct { + m map[string][]byte + lock sync.Mutex +} + +func NewInMemoryBlobHandler() BlobHandler { return &memHandler{m: map[string][]byte{}} } + +func (m *memHandler) Stat(_ context.Context, _ string, h v1.Hash) (int64, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[h.String()] + if !found { + return 0, ErrNotFound + } + return int64(len(b)), nil +} + +func (m *memHandler) Get(_ context.Context, _ string, h v1.Hash) (io.ReadCloser, error) { + m.lock.Lock() + defer m.lock.Unlock() + + b, found := m.m[h.String()] + if !found { + return nil, ErrNotFound + } + return &bytesCloser{bytes.NewReader(b)}, nil +} + +func (m *memHandler) Put(_ context.Context, _ string, h v1.Hash, rc io.ReadCloser) error { + m.lock.Lock() + defer m.lock.Unlock() + + defer rc.Close() + all, err := io.ReadAll(rc) + if err != nil { + return err + } + m.m[h.String()] = all + return nil +} + +func (m *memHandler) Delete(_ context.Context, _ string, h v1.Hash) error { + m.lock.Lock() + defer m.lock.Unlock() + + if _, found := m.m[h.String()]; !found { + return ErrNotFound + } + + delete(m.m, h.String()) + return nil +} + +// blobs +type blobs struct { + blobHandler BlobHandler + + // Each upload gets a unique id that writes occur to until finalized. + uploads map[string][]byte + lock sync.Mutex + log *log.Logger +} + +func (b *blobs) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + // Must have a path of form /v2/{name}/blobs/{upload,sha256:} + if len(elem) < 4 { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "blobs must be attached to a repo", + } + } + target := elem[len(elem)-1] + service := elem[len(elem)-2] + digest := req.URL.Query().Get("digest") + contentRange := req.Header.Get("Content-Range") + rangeHeader := req.Header.Get("Range") + + repo := req.URL.Host + path.Join(elem[1:len(elem)-2]...) + + switch req.Method { + case http.MethodHead: + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + var size int64 + if bsh, ok := b.blobHandler.(BlobStatHandler); ok { + size, err = bsh.Stat(req.Context(), repo, h) + if errors.Is(err, ErrNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr RedirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + } else { + rc, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, ErrNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr RedirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + defer rc.Close() + size, err = io.Copy(io.Discard, rc) + if err != nil { + return regErrInternal(err) + } + } + + resp.Header().Set("Content-Length", fmt.Sprint(size)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusOK) + return nil + + case http.MethodGet: + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + var size int64 + var r io.Reader + if bsh, ok := b.blobHandler.(BlobStatHandler); ok { + size, err = bsh.Stat(req.Context(), repo, h) + if errors.Is(err, ErrNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr RedirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + return regErrInternal(err) + } + + rc, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, ErrNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr RedirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + + return regErrInternal(err) + } + + defer rc.Close() + r = rc + + } else { + tmp, err := b.blobHandler.Get(req.Context(), repo, h) + if errors.Is(err, ErrNotFound) { + return regErrBlobUnknown + } else if err != nil { + var rerr RedirectError + if errors.As(err, &rerr) { + http.Redirect(resp, req, rerr.Location, rerr.Code) + return nil + } + + return regErrInternal(err) + } + defer tmp.Close() + var buf bytes.Buffer + io.Copy(&buf, tmp) + size = int64(buf.Len()) + r = &buf + } + + if rangeHeader != "" { + start, end := int64(0), int64(0) + if _, err := fmt.Sscanf(rangeHeader, "bytes=%d-%d", &start, &end); err != nil { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UNKNOWN", + Message: "We don't understand your Range", + } + } + + n := (end + 1) - start + if ra, ok := r.(io.ReaderAt); ok { + if end+1 > size { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UNKNOWN", + Message: fmt.Sprintf("range end %d > %d size", end+1, size), + } + } + r = io.NewSectionReader(ra, start, n) + } else { + if _, err := io.CopyN(io.Discard, r, start); err != nil { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UNKNOWN", + Message: fmt.Sprintf("Failed to discard %d bytes", start), + } + } + + r = io.LimitReader(r, n) + } + + resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size)) + resp.Header().Set("Content-Length", fmt.Sprint(n)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusPartialContent) + } else { + resp.Header().Set("Content-Length", fmt.Sprint(size)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusOK) + } + + io.Copy(resp, r) + return nil + + case http.MethodPost: + bph, ok := b.blobHandler.(BlobPutHandler) + if !ok { + return regErrUnsupported + } + + // It is weird that this is "target" instead of "service", but + // that's how the index math works out above. + if target != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("POST to /blobs must be followed by /uploads, got %s", target), + } + } + + if digest != "" { + h, err := v1.NewHash(digest) + if err != nil { + return regErrDigestInvalid + } + + vrc, err := verify.ReadCloser(req.Body, req.ContentLength, h) + if err != nil { + return regErrInternal(err) + } + defer vrc.Close() + + if err = bph.Put(req.Context(), repo, h, vrc); err != nil { + if errors.As(err, &verify.Error{}) { + log.Printf("Digest mismatch: %v", err) + return regErrDigestMismatch + } + return regErrInternal(err) + } + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + } + + id := fmt.Sprint(rand.Int63()) + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id)) + resp.Header().Set("Range", "0-0") + resp.WriteHeader(http.StatusAccepted) + return nil + + case http.MethodPatch: + if service != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("PATCH to /blobs must be followed by /uploads, got %s", service), + } + } + + if contentRange != "" { + start, end := 0, 0 + if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "We don't understand your Content-Range", + } + } + b.lock.Lock() + defer b.lock.Unlock() + if start != len(b.uploads[target]) { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "Your content range doesn't match what we have", + } + } + l := bytes.NewBuffer(b.uploads[target]) + io.Copy(l, req.Body) + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + // OCI Distribution spec §10.5 requires 202 Accepted for chunk uploads. + resp.WriteHeader(http.StatusAccepted) + return nil + } + + b.lock.Lock() + defer b.lock.Unlock() + if _, ok := b.uploads[target]; ok { + return ®Error{ + Status: http.StatusBadRequest, + Code: "BLOB_UPLOAD_INVALID", + Message: "Stream uploads after first write are not allowed", + } + } + + l := &bytes.Buffer{} + io.Copy(l, req.Body) + + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + // OCI Distribution spec §10.5 requires 202 Accepted for chunk uploads. + resp.WriteHeader(http.StatusAccepted) + return nil + + case http.MethodPut: + bph, ok := b.blobHandler.(BlobPutHandler) + if !ok { + return regErrUnsupported + } + + if service != "uploads" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: fmt.Sprintf("PUT to /blobs must be followed by /uploads, got %s", service), + } + } + + if digest == "" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest not specified", + } + } + + b.lock.Lock() + defer b.lock.Unlock() + + h, err := v1.NewHash(digest) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + + defer req.Body.Close() + in := io.NopCloser(io.MultiReader(bytes.NewBuffer(b.uploads[target]), req.Body)) + + size := int64(verify.SizeUnknown) + if req.ContentLength > 0 { + size = int64(len(b.uploads[target])) + req.ContentLength + } + + vrc, err := verify.ReadCloser(in, size, h) + if err != nil { + return regErrInternal(err) + } + defer vrc.Close() + + if err := bph.Put(req.Context(), repo, h, vrc); err != nil { + if errors.As(err, &verify.Error{}) { + log.Printf("Digest mismatch: %v", err) + return regErrDigestMismatch + } + return regErrInternal(err) + } + + delete(b.uploads, target) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.WriteHeader(http.StatusCreated) + return nil + + case http.MethodDelete: + bdh, ok := b.blobHandler.(BlobDeleteHandler) + if !ok { + return regErrUnsupported + } + + h, err := v1.NewHash(target) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", + } + } + if err := bdh.Delete(req.Context(), repo, h); err != nil { + return regErrInternal(err) + } + resp.WriteHeader(http.StatusAccepted) + return nil + + default: + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/blobs_disk.go b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs_disk.go new file mode 100644 index 000000000..fe1c7c789 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs_disk.go @@ -0,0 +1,84 @@ +// Copyright 2023 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +type diskHandler struct { + dir string +} + +func NewDiskBlobHandler(dir string) BlobHandler { return &diskHandler{dir: dir} } + +func (m *diskHandler) blobHashPath(h v1.Hash) string { + return filepath.Join(m.dir, h.Algorithm, h.Hex) +} + +func (m *diskHandler) Stat(_ context.Context, _ string, h v1.Hash) (int64, error) { + f, err := os.Open(m.blobHashPath(h)) + if errors.Is(err, os.ErrNotExist) { + return 0, ErrNotFound + } else if err != nil { + return 0, err + } + defer f.Close() + + got, size, err := v1.SHA256(f) + if err != nil { + return 0, err + } + if got != h { + return 0, fmt.Errorf("%w: blob %s has digest %s", ErrNotFound, h, got) + } + return size, nil +} + +func (m *diskHandler) Get(_ context.Context, _ string, h v1.Hash) (io.ReadCloser, error) { + return os.Open(m.blobHashPath(h)) +} + +func (m *diskHandler) Put(_ context.Context, _ string, h v1.Hash, rc io.ReadCloser) error { + // Put the temp file in the same directory to avoid cross-device problems + // during the os.Rename. The filenames cannot conflict. + f, err := os.CreateTemp(m.dir, "upload-*") + if err != nil { + return err + } + + if err := func() error { + defer f.Close() + _, err := io.Copy(f, rc) + return err + }(); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(m.dir, h.Algorithm), os.ModePerm); err != nil { + return err + } + return os.Rename(f.Name(), m.blobHashPath(h)) +} + +func (m *diskHandler) Delete(_ context.Context, _ string, h v1.Hash) error { + return os.Remove(m.blobHashPath(h)) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/error.go b/vendor/github.com/google/go-containerregistry/pkg/registry/error.go new file mode 100644 index 000000000..f8e126dac --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/error.go @@ -0,0 +1,79 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "encoding/json" + "net/http" +) + +type regError struct { + Status int + Code string + Message string +} + +func (r *regError) Write(resp http.ResponseWriter) error { + resp.WriteHeader(r.Status) + + type err struct { + Code string `json:"code"` + Message string `json:"message"` + } + type wrap struct { + Errors []err `json:"errors"` + } + return json.NewEncoder(resp).Encode(wrap{ + Errors: []err{ + { + Code: r.Code, + Message: r.Message, + }, + }, + }) +} + +// regErrInternal returns an internal server error. +func regErrInternal(err error) *regError { + return ®Error{ + Status: http.StatusInternalServerError, + Code: "INTERNAL_SERVER_ERROR", + Message: err.Error(), + } +} + +var regErrBlobUnknown = ®Error{ + Status: http.StatusNotFound, + Code: "BLOB_UNKNOWN", + Message: "Unknown blob", +} + +var regErrUnsupported = ®Error{ + Status: http.StatusMethodNotAllowed, + Code: "UNSUPPORTED", + Message: "Unsupported operation", +} + +var regErrDigestMismatch = ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest does not match contents", +} + +var regErrDigestInvalid = ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "invalid digest", +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go new file mode 100644 index 000000000..db8a8dc69 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go @@ -0,0 +1,444 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "sort" + "strconv" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type catalog struct { + Repos []string `json:"repositories"` +} + +type listTags struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + +type manifest struct { + contentType string + blob []byte +} + +type manifests struct { + // maps repo -> manifest tag/digest -> manifest + manifests map[string]map[string]manifest + lock sync.RWMutex + log *log.Logger +} + +func isManifest(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "manifests" +} + +func isTags(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "tags" +} + +func isCatalog(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 2 { + return false + } + + return elems[len(elems)-1] == "_catalog" +} + +// Returns whether this url should be handled by the referrers handler +func isReferrers(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "referrers" +} + +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-an-image-manifest +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-an-image +func (m *manifests) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + target := elem[len(elem)-1] + repo := strings.Join(elem[1:len(elem)-2], "/") + + switch req.Method { + case http.MethodGet: + m.lock.RLock() + defer m.lock.RUnlock() + + c, ok := m.manifests[repo] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + m, ok := c[target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + + h, _, _ := v1.SHA256(bytes.NewReader(m.blob)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.Header().Set("Content-Type", m.contentType) + resp.Header().Set("Content-Length", fmt.Sprint(len(m.blob))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader(m.blob)) + return nil + + case http.MethodHead: + m.lock.RLock() + defer m.lock.RUnlock() + + if _, ok := m.manifests[repo]; !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + m, ok := m.manifests[repo][target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + + h, _, _ := v1.SHA256(bytes.NewReader(m.blob)) + resp.Header().Set("Docker-Content-Digest", h.String()) + resp.Header().Set("Content-Type", m.contentType) + resp.Header().Set("Content-Length", fmt.Sprint(len(m.blob))) + resp.WriteHeader(http.StatusOK) + return nil + + case http.MethodPut: + b := &bytes.Buffer{} + io.Copy(b, req.Body) + h, _, _ := v1.SHA256(bytes.NewReader(b.Bytes())) + digest := h.String() + mf := manifest{ + blob: b.Bytes(), + contentType: req.Header.Get("Content-Type"), + } + + // If the manifest is a manifest list, check that the manifest + // list's constituent manifests are already uploaded. + // This isn't strictly required by the registry API, but some + // registries require this. + if types.MediaType(mf.contentType).IsIndex() { + if err := func() *regError { + m.lock.RLock() + defer m.lock.RUnlock() + + im, err := v1.ParseIndexManifest(b) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "MANIFEST_INVALID", + Message: err.Error(), + } + } + for _, desc := range im.Manifests { + if !desc.MediaType.IsDistributable() { + continue + } + if desc.MediaType.IsIndex() || desc.MediaType.IsImage() { + if _, found := m.manifests[repo][desc.Digest.String()]; !found { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: fmt.Sprintf("Sub-manifest %q not found", desc.Digest), + } + } + } else { + // TODO: Probably want to do an existence check for blobs. + m.log.Printf("TODO: Check blobs for %q", desc.Digest) + } + } + return nil + }(); err != nil { + return err + } + } + + m.lock.Lock() + defer m.lock.Unlock() + + if _, ok := m.manifests[repo]; !ok { + m.manifests[repo] = make(map[string]manifest, 2) + } + + // Allow future references by target (tag) and immutable digest. + // See https://docs.docker.com/engine/reference/commandline/pull/#pull-an-image-by-digest-immutable-identifier. + m.manifests[repo][digest] = mf + m.manifests[repo][target] = mf + resp.Header().Set("Docker-Content-Digest", digest) + resp.WriteHeader(http.StatusCreated) + return nil + + case http.MethodDelete: + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + _, ok := m.manifests[repo][target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + + delete(m.manifests[repo], target) + resp.WriteHeader(http.StatusAccepted) + return nil + + default: + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } +} + +func (m *manifests) handleTags(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + repo := strings.Join(elem[1:len(elem)-2], "/") + + if req.Method == "GET" { + m.lock.RLock() + defer m.lock.RUnlock() + + c, ok := m.manifests[repo] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + var tags []string + for tag := range c { + if !strings.Contains(tag, "sha256:") { + tags = append(tags, tag) + } + } + sort.Strings(tags) + + // https://github.com/opencontainers/distribution-spec/blob/b505e9cc53ec499edbd9c1be32298388921bb705/detail.md#tags-paginated + // Offset using last query parameter. + if last := req.URL.Query().Get("last"); last != "" { + for i, t := range tags { + if t > last { + tags = tags[i:] + break + } + } + } + + // Limit using n query parameter. + if ns := req.URL.Query().Get("n"); ns != "" { + if n, err := strconv.Atoi(ns); err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "BAD_REQUEST", + Message: fmt.Sprintf("parsing n: %v", err), + } + } else if n < len(tags) { + tags = tags[:n] + } + } + + tagsToList := listTags{ + Name: repo, + Tags: tags, + } + + msg, _ := json.Marshal(tagsToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} + +func (m *manifests) handleCatalog(resp http.ResponseWriter, req *http.Request) *regError { + query := req.URL.Query() + nStr := query.Get("n") + n := 10000 + if nStr != "" { + n, _ = strconv.Atoi(nStr) + } + + if req.Method == "GET" { + m.lock.RLock() + defer m.lock.RUnlock() + + var repos []string + countRepos := 0 + // TODO: implement pagination + for key := range m.manifests { + if countRepos >= n { + break + } + countRepos++ + + repos = append(repos, key) + } + + repositoriesToList := catalog{ + Repos: repos, + } + + msg, _ := json.Marshal(repositoriesToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} + +// TODO: implement handling of artifactType querystring +func (m *manifests) handleReferrers(resp http.ResponseWriter, req *http.Request) *regError { + // Ensure this is a GET request + if req.Method != "GET" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } + + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + target := elem[len(elem)-1] + repo := strings.Join(elem[1:len(elem)-2], "/") + + // Validate that incoming target is a valid digest + if _, err := v1.NewHash(target); err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "UNSUPPORTED", + Message: "Target must be a valid digest", + } + } + + m.lock.RLock() + defer m.lock.RUnlock() + + digestToManifestMap, repoExists := m.manifests[repo] + if !repoExists { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + im := v1.IndexManifest{ + SchemaVersion: 2, + MediaType: types.OCIImageIndex, + Manifests: []v1.Descriptor{}, + } + for digest, manifest := range digestToManifestMap { + h, err := v1.NewHash(digest) + if err != nil { + continue + } + var refPointer struct { + Subject *v1.Descriptor `json:"subject"` + } + json.Unmarshal(manifest.blob, &refPointer) + if refPointer.Subject == nil { + continue + } + referenceDigest := refPointer.Subject.Digest + if referenceDigest.String() != target { + continue + } + // At this point, we know the current digest references the target + var imageAsArtifact struct { + Config struct { + MediaType string `json:"mediaType"` + } `json:"config"` + } + json.Unmarshal(manifest.blob, &imageAsArtifact) + im.Manifests = append(im.Manifests, v1.Descriptor{ + MediaType: types.MediaType(manifest.contentType), + Size: int64(len(manifest.blob)), + Digest: h, + ArtifactType: imageAsArtifact.Config.MediaType, + }) + } + msg, _ := json.Marshal(&im) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.Header().Set("Content-Type", string(types.OCIImageIndex)) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go b/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go new file mode 100644 index 000000000..2f8fd1127 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go @@ -0,0 +1,144 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package registry implements a docker V2 registry and the OCI distribution specification. +// +// It is designed to be used anywhere a low dependency container registry is needed, with an +// initial focus on tests. +// +// Its goal is to be standards compliant and its strictness will increase over time. +// +// This is currently a low flightmiles system. It's likely quite safe to use in tests; If you're using it +// in production, please let us know how and send us CL's for integration tests. +package registry + +import ( + "fmt" + "log" + "math/rand" + "net/http" + "os" +) + +type registry struct { + log *log.Logger + blobs blobs + manifests manifests + referrersEnabled bool + warnings map[float64]string +} + +// https://docs.docker.com/registry/spec/api/#api-version-check +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#api-version-check +func (r *registry) v2(resp http.ResponseWriter, req *http.Request) *regError { + if r.warnings != nil { + rnd := rand.Float64() + for prob, msg := range r.warnings { + if prob > rnd { + resp.Header().Add("Warning", fmt.Sprintf(`299 - "%s"`, msg)) + } + } + } + + if isBlob(req) { + return r.blobs.handle(resp, req) + } + if isManifest(req) { + return r.manifests.handle(resp, req) + } + if isTags(req) { + return r.manifests.handleTags(resp, req) + } + if isCatalog(req) { + return r.manifests.handleCatalog(resp, req) + } + if r.referrersEnabled && isReferrers(req) { + return r.manifests.handleReferrers(resp, req) + } + resp.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + if req.URL.Path != "/v2/" && req.URL.Path != "/v2" { + return ®Error{ + Status: http.StatusNotFound, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } + resp.WriteHeader(200) + return nil +} + +func (r *registry) root(resp http.ResponseWriter, req *http.Request) { + if rerr := r.v2(resp, req); rerr != nil { + r.log.Printf("%s %s %d %s %s", req.Method, req.URL, rerr.Status, rerr.Code, rerr.Message) + rerr.Write(resp) + return + } + r.log.Printf("%s %s", req.Method, req.URL) +} + +// New returns a handler which implements the docker registry protocol. +// It should be registered at the site root. +func New(opts ...Option) http.Handler { + r := ®istry{ + log: log.New(os.Stderr, "", log.LstdFlags), + blobs: blobs{ + blobHandler: &memHandler{m: map[string][]byte{}}, + uploads: map[string][]byte{}, + log: log.New(os.Stderr, "", log.LstdFlags), + }, + manifests: manifests{ + manifests: map[string]map[string]manifest{}, + log: log.New(os.Stderr, "", log.LstdFlags), + }, + } + for _, o := range opts { + o(r) + } + return http.HandlerFunc(r.root) +} + +// Option describes the available options +// for creating the registry. +type Option func(r *registry) + +// Logger overrides the logger used to record requests to the registry. +func Logger(l *log.Logger) Option { + return func(r *registry) { + r.log = l + r.manifests.log = l + r.blobs.log = l + } +} + +// WithReferrersSupport enables the referrers API endpoint (OCI 1.1+) +func WithReferrersSupport(enabled bool) Option { + return func(r *registry) { + r.referrersEnabled = enabled + } +} + +func WithWarning(prob float64, msg string) Option { + return func(r *registry) { + if r.warnings == nil { + r.warnings = map[float64]string{} + } + r.warnings[prob] = msg + } +} + +func WithBlobHandler(h BlobHandler) Option { + return func(r *registry) { + r.blobs.blobHandler = h + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go b/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go new file mode 100644 index 000000000..cb2644e61 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "net/http/httptest" + + ggcrtest "github.com/google/go-containerregistry/internal/httptest" +) + +// TLS returns an httptest server, with an http client that has been configured to +// send all requests to the returned server. The TLS certs are generated for the given domain +// which should correspond to the domain the image is stored in. +// If you need a transport, Client().Transport is correctly configured. +func TLS(domain string) (*httptest.Server, error) { + return ggcrtest.NewTLSServer(domain, New()) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 540501410..17a1f4008 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -452,6 +452,7 @@ github.com/google/go-cmp/cmp/internal/value github.com/google/go-containerregistry/internal/and github.com/google/go-containerregistry/internal/compression github.com/google/go-containerregistry/internal/gzip +github.com/google/go-containerregistry/internal/httptest github.com/google/go-containerregistry/internal/limit github.com/google/go-containerregistry/internal/redact github.com/google/go-containerregistry/internal/retry @@ -462,6 +463,7 @@ github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/compression github.com/google/go-containerregistry/pkg/logs github.com/google/go-containerregistry/pkg/name +github.com/google/go-containerregistry/pkg/registry github.com/google/go-containerregistry/pkg/v1 github.com/google/go-containerregistry/pkg/v1/empty github.com/google/go-containerregistry/pkg/v1/google From 92048b10f2fab356be44fc14e5ae882ef443e2a1 Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 14:41:58 -0700 Subject: [PATCH 2/8] validate-image-cache: make eviction idle window configurable The fixed 30-minute idle guard made nothing evictable while a fast corpus filled a small disk (32-VM hicard sweep: local SSDs filled in ~12 minutes, then every remaining image failed ENOSPC). Expose it as --evict-idle. --- .gitignore | 3 +++ tools/validate-image-cache/README.md | 7 +++++-- tools/validate-image-cache/main.go | 11 +++++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 9ee8bf831..a89337985 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ Thumbs.db # Local worktrees /.worktrees/ + +# Stray local build outputs (go build ./tools/... without -o) +/validate-image-cache diff --git a/tools/validate-image-cache/README.md b/tools/validate-image-cache/README.md index 62dbdfea6..54f28f81d 100644 --- a/tools/validate-image-cache/README.md +++ b/tools/validate-image-cache/README.md @@ -76,6 +76,7 @@ go run ./tools/validate-image-cache \ | `--parallel` | 3 | images validated concurrently (each pulls up to 4 layers in parallel) | | `--timeout` | 20m | per-image timeout | | `--min-free-gb` | 150 | evict oldest idle layers below this free-space floor | +| `--evict-idle` | 10m | only evict layers idle at least this long; must be far below disk-fill time on small disks | | `--platform` | `linux/amd64` | image platform to pull | ## Output and rerunning @@ -88,8 +89,10 @@ non-zero if any image failed. Reruns are cheap by design: completed layers stay in the cache (and even an interrupted pull keeps every layer that finished), so re-running the same sample — or just the failed refs — mostly re-validates from local disk. -Eviction only removes layers idle for 30+ minutes, so in-flight images are -never raced. +Eviction only removes layers idle for at least `--evict-idle`, so in-flight +images are not raced; on a small disk with high throughput, set it well +below the time the corpus needs to fill the disk, or nothing will be +evictable while it fills. ## Scaling out diff --git a/tools/validate-image-cache/main.go b/tools/validate-image-cache/main.go index 65a41632e..3800611d9 100644 --- a/tools/validate-image-cache/main.go +++ b/tools/validate-image-cache/main.go @@ -62,6 +62,7 @@ var ( parallel = flag.Int("parallel", 3, "Images validated concurrently (each pulls up to 4 layers in parallel)") timeout = flag.Duration("timeout", 20*time.Minute, "Per-image timeout") minFreeGB = flag.Uint64("min-free-gb", 150, "Evict oldest idle cached layers when the cache volume has less free space than this") + evictIdle = flag.Duration("evict-idle", 10*time.Minute, "Only evict layers idle for at least this long (must exceed the time any in-flight image needs a just-unpacked layer; small disks + high throughput need small values)") platform = flag.String("platform", "linux/amd64", "Image platform to pull") ) @@ -203,9 +204,11 @@ func shortRef(ref string) string { } // evictIfLow deletes the oldest cached layer trees until the cache volume -// has at least minFree bytes available. Layers touched within the last 30 -// minutes are skipped so an in-flight image's freshly unpacked layers are -// never raced (per-image timeout is shorter than that). +// has at least minFree bytes available. Layers touched within the last +// --evict-idle are skipped so an in-flight image's freshly unpacked layers +// are not raced away mid-validation. NOTE: if the corpus unpacks faster +// than the idle window elapses on a small disk, nothing is evictable while +// the disk fills — size --evict-idle well below disk-fill time. var evictMu sync.Mutex func evictIfLow(cacheRoot string, minFree uint64) { @@ -227,7 +230,7 @@ func evictIfLow(cacheRoot string, minFree uint64) { var candidates []aged for _, e := range entries { info, err := e.Info() - if err != nil || time.Since(info.ModTime()) < 30*time.Minute { + if err != nil || time.Since(info.ModTime()) < *evictIdle { continue } candidates = append(candidates, aged{filepath.Join(layersDir, e.Name()), info.ModTime()}) From 91a2f0d6e0c802532e0be0dadbc829f9a7d93d91 Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 19:31:08 -0700 Subject: [PATCH 3/8] gvisor: detach bundle rootfs overlays on Run/Restore failure paths Unmount runs before cleanupActorNetworkOrExit, which exits the process on failure and would otherwise skip the overlay detach. --- cmd/ateom-gvisor/main.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index affedf08e..4d245dc6f 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -194,6 +194,15 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload } defer func() { if retErr != nil { + // Detach any bundle rootfs overlays a partially-completed setup + // mounted, mirroring the post-checkpoint cleanup — otherwise they + // linger in this namespace until atelet wipes the bundle dirs. + // Run before the network cleanup, which exits the process on + // failure and would skip this. + if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(req.GetActorUid())); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Run failure", + "actorUID", req.GetActorUid(), "err", err) + } s.cleanupActorNetworkOrExit(ctx, "Failed to clean up actor network after Run failure") } }() @@ -387,6 +396,11 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore } defer func() { if retErr != nil { + // Same overlay detach as the Run-failure path above. + if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(req.GetActorUid())); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", + "actorUID", req.GetActorUid(), "err", err) + } s.cleanupActorNetworkOrExit(ctx, "Failed to clean up actor network after Restore failure") } }() From d5787a95b2030ff61e42c15aa7169b8f85532f3f Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 19:33:29 -0700 Subject: [PATCH 4/8] microvm: detach bundle rootfs overlays on Run/Restore failure paths --- cmd/ateom-microvm/restore.go | 6 ++++++ cmd/ateom-microvm/run.go | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index b8778e254..3a98860e1 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -29,6 +29,7 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/ch" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" "google.golang.org/grpc/codes" @@ -121,6 +122,11 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore if cleanupErr := s.cleanupActorNetwork(ctx); cleanupErr != nil { slog.WarnContext(ctx, "Failed to clean up actor network after Restore failure", slog.Any("err", cleanupErr)) } + // Detach any bundle rootfs overlays mounted by buildActorContainers + // before the failure, mirroring teardownActor's cleanup. + if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(actorUID)); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Restore failure", slog.Any("err", err)) + } } }() netDevs, err := ch.SnapshotNetDevices(restoreDir) diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index c519debb2..7c323e787 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -228,6 +228,11 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload if cleanupErr := s.cleanupActorNetwork(ctx); cleanupErr != nil { slog.WarnContext(ctx, "Failed to clean up actor network after Run failure", slog.Any("err", cleanupErr)) } + // Detach any bundle rootfs overlays mounted by buildActorContainers + // before the failure, mirroring teardownActor's cleanup. + if err := imagecache.UnmountAllUnder(ateompath.OCIBundleDir(actorUID)); err != nil { + slog.WarnContext(ctx, "Failed to unmount bundle rootfs overlays after Run failure", slog.Any("err", err)) + } } }() From 77026523509bc8c9aa2c4efbd13ea0feb5d6485f Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 19:55:00 -0700 Subject: [PATCH 5/8] imagecache: mount rootfs overlays via the new mount API to lift the layer cap mount(2) copies its option string through a single page, which caps digest-derived lowerdir chains (~114 bytes per layer path) at roughly 34 layers and fails with a bare EINVAL beyond that. Appending lowerdirs one fsconfig(2) call at a time removes the aggregate limit structurally, and failed mounts now carry the kernel's fs-context error log instead of an opaque errno. Adds a 64-layer regression test that asserts the joined paths exceed one page before mounting. Minimum supported kernel: Linux 6.5 (overlayfs "lowerdir+"). All current GKE channels meet it: Stable runs COS 121 LTS (kernel 6.6), Regular and Rapid run COS 125/129 (kernel 6.12). --- internal/imagecache/README.md | 5 ++ internal/imagecache/bundle_linux.go | 75 ++++++++++++++++++++++-- internal/imagecache/bundle_linux_test.go | 46 +++++++++++++++ internal/imagecache/spec.go | 36 +++++------- internal/imagecache/spec_test.go | 26 +++----- 5 files changed, 144 insertions(+), 44 deletions(-) diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index 0c5855990..f602cbabc 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -118,6 +118,11 @@ staging the virtio-fs lower (micro-VM): legitimately list the same diffID twice — are collapsed to the topmost occurrence, which overlayfs otherwise rejects with `ELOOP`), `upperdir` / `workdir` are the bundle-local dirs, holding this actor's private writes. + The mount uses the new mount API (`fsopen` + one `fsconfig` `lowerdir+` + append per layer) rather than `mount(2)`, whose single-page option-string + cap the digest-derived layer paths would hit at ~34 layers. **Minimum + supported kernel: Linux 6.5** (`lowerdir+`); every current GKE channel + ships ≥ 6.6 (Stable: COS 121 LTS). 3. **ExtraDirs** are created through the mount (landing in the upper), again under `os.Root` confinement. diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go index 31e551ec1..5a950b045 100644 --- a/internal/imagecache/bundle_linux.go +++ b/internal/imagecache/bundle_linux.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "sort" + "strings" "golang.org/x/sys/unix" ) @@ -75,15 +76,81 @@ func SetupBundleRootfs(bundlePath string) error { return createExtraDirs(rootfs, spec.ExtraDirs) } - opts, err := overlayMountOptions(spec.Layers, upper, work) + if err := mountOverlay(rootfs, overlayLowerDirs(spec.Layers), upper, work); err != nil { + return fmt.Errorf("while mounting overlay rootfs at %q: %w", rootfs, err) + } + + return createExtraDirs(rootfs, spec.ExtraDirs) +} + +// mountOverlay attaches an overlay of lowers (top-most first) with the given +// upper/work dirs at mountpoint, using the new mount API rather than +// mount(2): appending lowerdirs one fsconfig(2) call at a time sidesteps +// mount(2)'s single-page option-string cap, which digest-derived layer paths +// (~114 bytes each) would hit at roughly 34 layers. +// +// Minimum supported kernel: Linux 6.5, where overlayfs gained the +// incremental "lowerdir+" option. Every current GKE channel is at or above +// it (Stable runs COS 121 LTS on kernel 6.6; Regular and Rapid run COS +// 125/129 on 6.12). +func mountOverlay(mountpoint string, lowers []string, upper, work string) error { + fsfd, err := unix.Fsopen("overlay", unix.FSOPEN_CLOEXEC) if err != nil { + return fmt.Errorf("while opening overlay fs context: %w", err) + } + defer unix.Close(fsfd) + + set := func(key, val string) error { + if err := unix.FsconfigSetString(fsfd, key, val); err != nil { + return fmt.Errorf("while setting overlay %s=%q: %w%s", key, val, err, fsContextLog(fsfd)) + } + return nil + } + for _, lower := range lowers { + if err := set("lowerdir+", lower); err != nil { + return err + } + } + if err := set("upperdir", upper); err != nil { return err } - if err := unix.Mount("overlay", rootfs, "overlay", 0, opts); err != nil { - return fmt.Errorf("while mounting overlay rootfs at %q (%s): %w", rootfs, opts, err) + if err := set("workdir", work); err != nil { + return err } - return createExtraDirs(rootfs, spec.ExtraDirs) + if err := unix.FsconfigCreate(fsfd); err != nil { + return fmt.Errorf("while creating overlay superblock: %w%s", err, fsContextLog(fsfd)) + } + mfd, err := unix.Fsmount(fsfd, unix.FSMOUNT_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("while creating overlay mount object: %w%s", err, fsContextLog(fsfd)) + } + defer unix.Close(mfd) + if err := unix.MoveMount(mfd, "", unix.AT_FDCWD, mountpoint, unix.MOVE_MOUNT_F_EMPTY_PATH); err != nil { + return fmt.Errorf("while attaching overlay at %q: %w", mountpoint, err) + } + return nil +} + +// fsContextLog drains the human-readable message log the kernel queues on an +// fs context fd (one "e/w/i "-prefixed message per read, ENODATA when empty) +// and renders it for appending to an error. mount(2) had no equivalent — a +// failed overlay mount was a bare errno; here the kernel says which option +// it rejected and why. +func fsContextLog(fsfd int) string { + var msgs []string + buf := make([]byte, 1024) + for range 8 { + n, err := unix.Read(fsfd, buf) + if err != nil || n <= 0 { + break + } + msgs = append(msgs, strings.TrimSpace(string(buf[:n]))) + } + if len(msgs) == 0 { + return "" + } + return " (kernel: " + strings.Join(msgs, "; ") + ")" } // FinalizeLayer materializes the whiteout state recorded at unpack time: diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 88851b86f..4ea7d3559 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -18,6 +18,7 @@ package imagecache import ( "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -208,3 +209,48 @@ func TestSetupBundleRootfs_MountAndUnmount(t *testing.T) { t.Errorf("rootfs still shows layer content after unmount") } } + +// Regression test for the mount(2) single-page option-string cap: a lowerdir +// chain whose joined paths exceed one page (~34 digest-derived layers) used +// to fail with a bare EINVAL. The fsconfig lowerdir+ path has no aggregate +// limit. Needs CAP_SYS_ADMIN. +func TestSetupBundleRootfs_ManyLayers(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (mount/unmount)") + } + // Digest-length dir names so each path matches production length (~114 + // bytes); 64 of them comfortably exceed the page that motivated this. + pool := filepath.Join(t.TempDir(), "sha256") + const n = 64 + layers := make([]string, n) + joined := 0 + for i := range layers { + layers[i] = filepath.Join(pool, fmt.Sprintf("%064d", i)) + writeLayer(t, layers[i], map[string]string{fmt.Sprintf("from-layer-%02d.txt", i): "x"}, nil) + joined += len(layers[i]) + len("/fs") + 1 + } + if pageSize := os.Getpagesize(); joined <= pageSize { + t.Fatalf("test layers join to %d bytes, not exceeding the %d-byte page this test guards against", joined, pageSize) + } + + bundle := t.TempDir() + if err := WriteSpec(bundle, &OverlaySpec{Layers: layers}); err != nil { + t.Fatalf("WriteSpec: %v", err) + } + if err := SetupBundleRootfs(bundle); err != nil { + t.Fatalf("SetupBundleRootfs with %d layers: %v", n, err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + // Bottom-most and top-most layers are both visible in the merged view. + for _, i := range []int{0, n - 1} { + p := filepath.Join(bundle, "rootfs", fmt.Sprintf("from-layer-%02d.txt", i)) + if _, err := os.Stat(p); err != nil { + t.Errorf("layer %d content missing from merged rootfs: %v", i, err) + } + } + + if err := UnmountAllUnder(bundle); err != nil { + t.Fatalf("UnmountAllUnder: %v", err) + } +} diff --git a/internal/imagecache/spec.go b/internal/imagecache/spec.go index 6a78a8939..9a0b9571a 100644 --- a/internal/imagecache/spec.go +++ b/internal/imagecache/spec.go @@ -80,37 +80,31 @@ func ReadSpec(bundlePath string) (*OverlaySpec, error) { return &spec, nil } -// overlayMountOptions builds the overlayfs option string: lowerdir is -// top-most layer first (the reverse of the spec's bottom-first order), each -// entry pointing at the layer's fs/ tree. -func overlayMountOptions(layers []string, upper, work string) (string, error) { +// overlayLowerDirs returns the overlayfs lowerdir paths for the spec's +// layers: top-most layer first (the reverse of the spec's bottom-first +// order), each pointing at the layer's fs/ tree. +// +// An image may legitimately list the same layer at several positions +// (identical build steps produce identical diffids; SWE-bench images do +// this). The pool stores that layer once, and overlayfs rejects a repeated +// lower directory (its overlapping-layers check fails the mount with ELOOP). +// For identical content only the topmost occurrence can affect the merged +// view, so keep the first one seen walking top-first and drop the rest. +// +// Each path is handed to the kernel in its own fsconfig(2) "lowerdir+" call, +// so no separator escaping or aggregate option-string length cap applies. +func overlayLowerDirs(layers []string) []string { lowers := make([]string, 0, len(layers)) seen := make(map[string]bool, len(layers)) for _, layer := range slices.Backward(layers) { p := filepath.Join(layer, layerFSDirName) - // An image may legitimately list the same layer at several positions - // (identical build steps produce identical diffids; SWE-bench images - // do this). The pool stores that layer once, and overlayfs rejects a - // repeated lower directory (its overlapping-layers check fails the - // mount with ELOOP). For identical content only the topmost - // occurrence can affect the merged view, so keep the first one seen - // walking top-first and drop the rest. if seen[p] { continue } seen[p] = true - // ':' separates lowerdirs and ',' separates mount options; cache paths - // are digest-derived so this never fires, but refuse rather than - // assemble a corrupt option string. - if strings.ContainsAny(p, ":,") { - return "", fmt.Errorf("layer path %q contains overlay option separators", p) - } lowers = append(lowers, p) } - if strings.ContainsAny(upper, ":,") || strings.ContainsAny(work, ":,") { - return "", fmt.Errorf("bundle path %q contains overlay option separators", upper) - } - return "lowerdir=" + strings.Join(lowers, ":") + ",upperdir=" + upper + ",workdir=" + work, nil + return lowers } // createExtraDirs creates the spec's ExtraDirs inside the (mounted) rootfs. diff --git a/internal/imagecache/spec_test.go b/internal/imagecache/spec_test.go index 264f4e3c5..8397f563a 100644 --- a/internal/imagecache/spec_test.go +++ b/internal/imagecache/spec_test.go @@ -67,14 +67,11 @@ func TestReadSpec_UnknownVersion(t *testing.T) { } } -func TestOverlayMountOptions(t *testing.T) { +func TestOverlayLowerDirs(t *testing.T) { t.Run("reverses to top-first", func(t *testing.T) { - got, err := overlayMountOptions([]string{"/c/base", "/c/top"}, "/b/upper", "/b/work") - if err != nil { - t.Fatalf("overlayMountOptions: %v", err) - } - want := "lowerdir=/c/top/fs:/c/base/fs,upperdir=/b/upper,workdir=/b/work" - if got != want { + got := overlayLowerDirs([]string{"/c/base", "/c/top"}) + want := []string{"/c/top/fs", "/c/base/fs"} + if !slices.Equal(got, want) { t.Errorf("got %q, want %q", got, want) } }) @@ -85,21 +82,12 @@ func TestOverlayMountOptions(t *testing.T) { // topmost occurrence. Regression test for the "too many levels of // symbolic links" mount failure. t.Run("deduplicates repeated layers keeping topmost", func(t *testing.T) { - got, err := overlayMountOptions([]string{"/c/base", "/c/dup", "/c/dup", "/c/top"}, "/b/upper", "/b/work") - if err != nil { - t.Fatalf("overlayMountOptions: %v", err) - } - want := "lowerdir=/c/top/fs:/c/dup/fs:/c/base/fs,upperdir=/b/upper,workdir=/b/work" - if got != want { + got := overlayLowerDirs([]string{"/c/base", "/c/dup", "/c/dup", "/c/top"}) + want := []string{"/c/top/fs", "/c/dup/fs", "/c/base/fs"} + if !slices.Equal(got, want) { t.Errorf("got %q, want %q", got, want) } }) - - t.Run("refuses option separators in paths", func(t *testing.T) { - if _, err := overlayMountOptions([]string{"/c/evil:layer"}, "/b/upper", "/b/work"); err == nil { - t.Errorf("expected error for ':' in layer path") - } - }) } func TestCreateExtraDirs(t *testing.T) { From f7970cf90fd63e50f4729aed92b80257021646a7 Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 20:06:10 -0700 Subject: [PATCH 6/8] imagecache: sweep orphaned manifest record temp files at startup Startup recovery swept layer unpack temp dirs but not writeRecord's "..json.tmp-*" files left by a crash between create and rename. Addresses PR #467 review feedback. --- internal/imagecache/imagecache.go | 23 ++++++++++++++++++++--- internal/imagecache/imagecache_test.go | 14 ++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index 1d589bd1e..f0fdf1458 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -192,9 +192,10 @@ func (s *Store) recordPath(digest v1.Hash) string { return filepath.Join(s.root, "manifests", digest.Algorithm, digest.Hex+".json") } -// sweepTempDirs removes unpack temp dirs orphaned by a crash. A layer dir -// without the temp prefix is always complete (it was moved into place with a -// single rename), so this is the only recovery the pool needs. +// sweepTempDirs removes unpack temp dirs and manifest-record temp files +// orphaned by a crash. A layer dir without the temp prefix and a record +// without a leading dot are always complete (both are moved into place with +// a single rename), so this is the only recovery the pool needs. func (s *Store) sweepTempDirs() error { entries, err := os.ReadDir(s.layersDir()) if err != nil { @@ -209,6 +210,22 @@ func (s *Store) sweepTempDirs() error { return fmt.Errorf("while sweeping orphaned layer temp dir %q: %w", p, err) } } + + // writeRecord's temp files are "..json.tmp-"; finished records + // are ".json", so a leading dot alone identifies an orphan. + records, err := os.ReadDir(s.manifestsDir()) + if err != nil { + return fmt.Errorf("while listing manifest records: %w", err) + } + for _, e := range records { + if !strings.HasPrefix(e.Name(), ".") { + continue + } + p := filepath.Join(s.manifestsDir(), e.Name()) + if err := os.Remove(p); err != nil { + return fmt.Errorf("while sweeping orphaned manifest temp file %q: %w", p, err) + } + } return nil } diff --git a/internal/imagecache/imagecache_test.go b/internal/imagecache/imagecache_test.go index 2bb19a98c..f11748775 100644 --- a/internal/imagecache/imagecache_test.go +++ b/internal/imagecache/imagecache_test.go @@ -219,12 +219,26 @@ func TestNew_RecoveryAndVersioning(t *testing.T) { if err := os.MkdirAll(orphan, 0o700); err != nil { t.Fatalf("planting orphan: %v", err) } + recOrphan := filepath.Join(s.manifestsDir(), ".deadbeef.json.tmp-456") + if err := os.WriteFile(recOrphan, []byte("{"), 0o600); err != nil { + t.Fatalf("planting record orphan: %v", err) + } + record := filepath.Join(s.manifestsDir(), "deadbeef.json") + if err := os.WriteFile(record, []byte("{}"), 0o600); err != nil { + t.Fatalf("planting record: %v", err) + } if _, err := New(root); err != nil { t.Fatalf("New(recovery): %v", err) } if _, err := os.Stat(orphan); !os.IsNotExist(err) { t.Errorf("orphaned temp dir survived recovery") } + if _, err := os.Stat(recOrphan); !os.IsNotExist(err) { + t.Errorf("orphaned manifest temp file survived recovery") + } + if _, err := os.Stat(record); err != nil { + t.Errorf("completed manifest record swept by recovery: %v", err) + } if err := os.WriteFile(filepath.Join(root, versionFileName), []byte("99\n"), 0o600); err != nil { t.Fatalf("writing version marker: %v", err) From 94fb9efed54990609cb1114ff8ad8dca2927d855 Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 20:21:35 -0700 Subject: [PATCH 7/8] imagecache: repair implicit parent-dir metadata in the composed rootfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer tars omit parents that exist in lower layers; unpack fabricates them (root:root 0755) and overlayfs takes merged dir attrs from the top-most layer containing the dir — so a fabricated parent shadowed real lower-layer metadata (/tmp's 1777, /root's 0700). Record implicit dirs in the layer metadata at unpack, and at compose repair the merged view from the top-most non-implicit provider in the image's chain; the chown/chmod copy-ups land in the bundle's private upper, never in the shared pool. Residual gaps (mtimes, xattrs, implicit-everywhere dirs) documented in the README. Addresses PR #467 review feedback (phantom parent dirs shadowing lower- layer directory metadata). --- internal/imagecache/README.md | 13 ++ internal/imagecache/bundle_linux.go | 16 ++- internal/imagecache/bundle_linux_test.go | 41 ++++++ internal/imagecache/implicitdirs.go | 176 +++++++++++++++++++++++ internal/imagecache/implicitdirs_test.go | 157 ++++++++++++++++++++ internal/imagecache/unpack.go | 62 +++++++- internal/imagecache/unpack_test.go | 30 ++++ 7 files changed, 491 insertions(+), 4 deletions(-) create mode 100644 internal/imagecache/implicitdirs.go create mode 100644 internal/imagecache/implicitdirs_test.go diff --git a/internal/imagecache/README.md b/internal/imagecache/README.md index f602cbabc..1b2b7118f 100644 --- a/internal/imagecache/README.md +++ b/internal/imagecache/README.md @@ -125,6 +125,19 @@ staging the virtio-fs lower (micro-VM): ships ≥ 6.6 (Stable: COS 121 LTS). 3. **ExtraDirs** are created through the mount (landing in the upper), again under `os.Root` confinement. +4. **Implicit-parent metadata repair.** A layer tar routinely omits entries + for parent directories that exist only in lower layers; unpack fabricates + them (root:root 0755) and records them as `implicitDirs` in the layer + metadata. Because overlayfs takes a merged directory's attributes from + the **top-most** layer containing it, such a fabricated dir would shadow + the real metadata a lower layer declared (`/tmp` losing its 1777 sticky + bit, `/root` opening from 0700 to 0755). At compose time the consumer + resolves each shadowed dir's true mode/ownership from the top-most + *non-implicit* layer in this image's chain and applies it through the + mount — the copy-up lands in the actor's private upper; the shared pool + is never modified. Residual gaps: directory mtimes and xattrs are not + repaired, and a dir implicit in *every* layer of the chain keeps the + fabricated attrs. A bundle without a spec file is left untouched (compatibility with bundles prepared by a pre-imagecache atelet). A zero-layer spec composes an empty diff --git a/internal/imagecache/bundle_linux.go b/internal/imagecache/bundle_linux.go index 5a950b045..dc15a1460 100644 --- a/internal/imagecache/bundle_linux.go +++ b/internal/imagecache/bundle_linux.go @@ -80,7 +80,21 @@ func SetupBundleRootfs(bundlePath string) error { return fmt.Errorf("while mounting overlay rootfs at %q: %w", rootfs, err) } - return createExtraDirs(rootfs, spec.ExtraDirs) + if err := createExtraDirs(rootfs, spec.ExtraDirs); err != nil { + return err + } + + // Repair merged directory metadata shadowed by implicitly-created parent + // dirs (see implicitdirs.go); the chmod/chowns copy up into this bundle's + // private upper, never into the shared pool. + fixups, err := resolveImplicitDirFixups(spec.Layers) + if err != nil { + return fmt.Errorf("while resolving implicit dir metadata: %w", err) + } + if err := applyDirFixups(rootfs, fixups); err != nil { + return fmt.Errorf("while repairing implicit dir metadata: %w", err) + } + return nil } // mountOverlay attaches an overlay of lowers (top-most first) with the given diff --git a/internal/imagecache/bundle_linux_test.go b/internal/imagecache/bundle_linux_test.go index 4ea7d3559..00fafae58 100644 --- a/internal/imagecache/bundle_linux_test.go +++ b/internal/imagecache/bundle_linux_test.go @@ -168,6 +168,47 @@ func TestSetupBundleRootfs_ZeroLayers(t *testing.T) { } } +// Implicit-parent metadata repair through a real overlay: the base declares +// a 0700 dir, the top layer created it implicitly (0755 in its tree), and +// after compose the merged view must show 0700 — copied up into the +// bundle's upper, with the shared layer trees untouched. Needs root. +func TestSetupBundleRootfs_ImplicitDirMetadataRepair(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("needs root (mount/unmount)") + } + base := t.TempDir() + writeLayer(t, base, map[string]string{"secret/keep.txt": "k"}, nil) + if err := os.Chmod(filepath.Join(base, layerFSDirName, "secret"), 0o700); err != nil { + t.Fatal(err) + } + top := t.TempDir() + writeLayer(t, top, map[string]string{"secret/new.txt": "n"}, &whiteoutSet{ImplicitDirs: []string{"secret"}}) + + bundle := t.TempDir() + if err := WriteSpec(bundle, &OverlaySpec{Layers: []string{base, top}}); err != nil { + t.Fatalf("WriteSpec: %v", err) + } + if err := SetupBundleRootfs(bundle); err != nil { + t.Fatalf("SetupBundleRootfs: %v", err) + } + t.Cleanup(func() { _ = UnmountAllUnder(bundle) }) + + fi, err := os.Lstat(filepath.Join(bundle, "rootfs", "secret")) + if err != nil { + t.Fatalf("stat merged dir: %v", err) + } + if fi.Mode().Perm() != 0o700 { + t.Errorf("merged secret mode = %v, want 0700 from the declaring base layer", fi.Mode().Perm()) + } + // The repair must land in the bundle upper, not the shared pool. + if fi, err := os.Lstat(filepath.Join(top, layerFSDirName, "secret")); err != nil || fi.Mode().Perm() != 0o755 { + t.Errorf("shared top layer tree was modified: %v %v", fi, err) + } + if _, err := os.Lstat(filepath.Join(bundle, "upper", "secret")); err != nil { + t.Errorf("repair did not copy up into the bundle upper: %v", err) + } +} + // Full overlay mount + UnmountAllUnder round trip; needs CAP_SYS_ADMIN. func TestSetupBundleRootfs_MountAndUnmount(t *testing.T) { if os.Geteuid() != 0 { diff --git a/internal/imagecache/implicitdirs.go b/internal/imagecache/implicitdirs.go new file mode 100644 index 000000000..db0c06306 --- /dev/null +++ b/internal/imagecache/implicitdirs.go @@ -0,0 +1,176 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Repair of implicit-parent-directory metadata in the composed rootfs. +// +// In overlayfs, the TOP-most layer containing a directory supplies the +// merged directory's attributes. When an upper layer's tar omitted a parent +// entry, the pool holds a fabricated root:root 0755 directory there +// (recorded as ImplicitDirs at unpack time), which would shadow the real +// metadata a lower layer declared — turning /tmp's 1777 into 0755, or +// /root's 0700 into 0755. containerd avoids this structurally by applying +// each layer through the mounted parent chain; this cache unpacks layers +// standalone (atelet cannot mount), so the repair happens at compose time +// instead: after the overlay is mounted, chmod/chown the affected +// directories to the attrs of the top-most NON-implicit provider in this +// image's chain. The writes copy up into the actor's private upper — the +// shared layer pool is never modified, so images with different chains +// sharing these layers are unaffected. +// +// Residual (documented) gaps: directory mtimes and xattrs are not repaired, +// and a directory implicit in every layer of the chain keeps the fabricated +// root:root 0755. + +package imagecache + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "syscall" +) + +// dirFixup is one merged-view repair: set Path (rootfs-relative) to Mode and, +// when owner info was available, UID/GID. +type dirFixup struct { + Path string + Mode os.FileMode // permission bits + setuid/setgid/sticky + UID, GID int + HasOwner bool +} + +// resolveImplicitDirFixups computes the repairs needed for an image whose +// (bottom-first) layer chain is layers: for every directory whose merged +// attributes would come from a layer that only created it implicitly, find +// the top-most layer below it that declared the directory for real and take +// its attributes. Layers whose metadata predates ImplicitDirs recording +// contribute no fixups (nothing recorded ⇒ nothing repaired), so old cached +// layers stay compatible. +func resolveImplicitDirFixups(layers []string) ([]dirFixup, error) { + implicitAt := make([]map[string]bool, len(layers)) + union := map[string]bool{} + for i, layerDir := range layers { + meta, err := readWhiteouts(layerDir) + if err != nil { + return nil, err + } + set := make(map[string]bool, len(meta.ImplicitDirs)) + for _, p := range meta.ImplicitDirs { + rel, skip, err := validateTarName(p) + if err != nil { + return nil, fmt.Errorf("invalid implicit dir path in %q: %w", layerDir, err) + } + if skip { + continue + } + set[rel] = true + union[rel] = true + } + implicitAt[i] = set + } + if len(union) == 0 { + return nil, nil + } + + paths := make([]string, 0, len(union)) + for p := range union { + paths = append(paths, p) + } + sort.Strings(paths) + + statDir := func(layerIdx int, rel string) (os.FileInfo, bool) { + fi, err := os.Lstat(filepath.Join(layers[layerIdx], layerFSDirName, rel)) + if err != nil || !fi.IsDir() { + return nil, false + } + return fi, true + } + + var fixups []dirFixup + for _, p := range paths { + // The merged view takes the dir's attrs from the top-most layer + // containing it; only repair when that provider is implicit. + top := -1 + for i := range slices.Backward(layers) { + if _, ok := statDir(i, p); ok { + top = i + break + } + } + if top < 0 || !implicitAt[top][p] { + continue + } + for j := top - 1; j >= 0; j-- { + if implicitAt[j][p] { + continue + } + fi, ok := statDir(j, p) + if !ok { + continue + } + f := dirFixup{ + Path: p, + Mode: fi.Mode().Perm() | fi.Mode()&(os.ModeSetuid|os.ModeSetgid|os.ModeSticky), + } + if st, ok := fi.Sys().(*syscall.Stat_t); ok { + f.UID, f.GID, f.HasOwner = int(st.Uid), int(st.Gid), true + } + fixups = append(fixups, f) + break + } + } + return fixups, nil +} + +// applyDirFixups applies the repairs through the (mounted) rootfs at +// rootfsPath; on an overlay each write copies the directory's metadata up +// into the bundle's private upper. Paths hidden in the merged view (removed +// by a whiteout, or shadowed by a non-directory) are skipped. os.Root +// confines the writes to the rootfs. +func applyDirFixups(rootfsPath string, fixups []dirFixup) error { + if len(fixups) == 0 { + return nil + } + root, err := os.OpenRoot(rootfsPath) + if err != nil { + return fmt.Errorf("while opening rootfs %q: %w", rootfsPath, err) + } + defer root.Close() + + for _, f := range fixups { + fi, err := root.Lstat(f.Path) + if errors.Is(err, os.ErrNotExist) { + continue + } else if err != nil { + return fmt.Errorf("while checking %q: %w", f.Path, err) + } + if !fi.IsDir() { + continue + } + // Chown first: chown can clear setuid/setgid bits, so the chmod + // must come after it. + if f.HasOwner { + if err := root.Chown(f.Path, f.UID, f.GID); err != nil { + return fmt.Errorf("while restoring owner of %q: %w", f.Path, err) + } + } + if err := root.Chmod(f.Path, f.Mode); err != nil { + return fmt.Errorf("while restoring mode of %q: %w", f.Path, err) + } + } + return nil +} diff --git a/internal/imagecache/implicitdirs_test.go b/internal/imagecache/implicitdirs_test.go new file mode 100644 index 000000000..b89a31452 --- /dev/null +++ b/internal/imagecache/implicitdirs_test.go @@ -0,0 +1,157 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package imagecache + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// makeLayer materializes a fake pool layer: dirs maps rel path -> mode for +// real directory entries; implicit lists the ImplicitDirs metadata. +func makeLayer(t *testing.T, dirs map[string]os.FileMode, implicit []string) string { + t.Helper() + layer := t.TempDir() + fs := filepath.Join(layer, layerFSDirName) + if err := os.Mkdir(fs, 0o755); err != nil { + t.Fatal(err) + } + for rel, mode := range dirs { + p := filepath.Join(fs, rel) + if err := os.MkdirAll(p, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatal(err) + } + } + if implicit != nil { + b, err := json.Marshal(&whiteoutSet{Version: 1, ImplicitDirs: implicit}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(layer, layerWhiteoutsFileName), b, 0o600); err != nil { + t.Fatal(err) + } + } + return layer +} + +// The canonical shadowing case: the base declares tmp as 1777, an upper +// layer created it implicitly (0755) — the merged view must be repaired to +// the base's attrs. +func TestResolveAndApplyImplicitDirFixups(t *testing.T) { + base := makeLayer(t, map[string]os.FileMode{"tmp": 0o777 | os.ModeSticky}, nil) + upper := makeLayer(t, map[string]os.FileMode{"tmp": 0o755}, []string{"tmp"}) + + fixups, err := resolveImplicitDirFixups([]string{base, upper}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(fixups) != 1 || fixups[0].Path != "tmp" { + t.Fatalf("fixups = %+v, want exactly one for \"tmp\"", fixups) + } + if fixups[0].Mode != 0o777|os.ModeSticky { + t.Errorf("fixup mode = %v, want 1777 (sticky preserved)", fixups[0].Mode) + } + if !fixups[0].HasOwner { + t.Errorf("fixup carries no owner info") + } + + // Apply against a stand-in for the merged rootfs (a plain dir here; the + // overlay copy-up behavior is covered by the root-gated test). + rootfs := t.TempDir() + if err := os.Mkdir(filepath.Join(rootfs, "tmp"), 0o755); err != nil { + t.Fatal(err) + } + if err := applyDirFixups(rootfs, fixups); err != nil { + t.Fatalf("apply: %v", err) + } + fi, err := os.Lstat(filepath.Join(rootfs, "tmp")) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm() | fi.Mode()&os.ModeSticky; got != 0o777|os.ModeSticky { + t.Errorf("merged tmp mode = %v, want 1777", got) + } +} + +func TestResolveImplicitDirFixups_NoRepairCases(t *testing.T) { + t.Run("topmost provider declared the dir", func(t *testing.T) { + // Lower created it implicitly, but the top layer declares it: the + // merged attrs already come from a real declaration. + lower := makeLayer(t, map[string]os.FileMode{"d": 0o755}, []string{"d"}) + top := makeLayer(t, map[string]os.FileMode{"d": 0o700}, nil) + fixups, err := resolveImplicitDirFixups([]string{lower, top}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(fixups) != 0 { + t.Errorf("fixups = %+v, want none", fixups) + } + }) + + t.Run("implicit in every layer", func(t *testing.T) { + a := makeLayer(t, map[string]os.FileMode{"d": 0o755}, []string{"d"}) + b := makeLayer(t, map[string]os.FileMode{"d": 0o755}, []string{"d"}) + fixups, err := resolveImplicitDirFixups([]string{a, b}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(fixups) != 0 { + t.Errorf("fixups = %+v, want none (no authoritative attrs exist)", fixups) + } + }) + + t.Run("no metadata at all (pre-recording layers)", func(t *testing.T) { + a := makeLayer(t, map[string]os.FileMode{"d": 0o700}, nil) + b := makeLayer(t, map[string]os.FileMode{"d": 0o755}, nil) + fixups, err := resolveImplicitDirFixups([]string{a, b}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if len(fixups) != 0 { + t.Errorf("fixups = %+v, want none", fixups) + } + }) + + t.Run("escaping metadata path is rejected", func(t *testing.T) { + a := makeLayer(t, nil, []string{"../escape"}) + if _, err := resolveImplicitDirFixups([]string{a}); err == nil { + t.Errorf("resolve accepted an escaping implicit dir path") + } + }) +} + +// applyDirFixups must skip paths that are hidden or shadowed in the merged +// view rather than failing or following the shadow. +func TestApplyDirFixups_SkipsShadowedPaths(t *testing.T) { + rootfs := t.TempDir() + if err := os.WriteFile(filepath.Join(rootfs, "afile"), nil, 0o644); err != nil { + t.Fatal(err) + } + fixups := []dirFixup{ + {Path: "missing", Mode: 0o700}, + {Path: "afile", Mode: 0o700}, + } + if err := applyDirFixups(rootfs, fixups); err != nil { + t.Fatalf("apply: %v", err) + } + if fi, _ := os.Lstat(filepath.Join(rootfs, "afile")); fi.Mode().Perm() != 0o644 { + t.Errorf("non-directory was chmodded: %v", fi.Mode()) + } +} diff --git a/internal/imagecache/unpack.go b/internal/imagecache/unpack.go index 81281da4c..3d581bd80 100644 --- a/internal/imagecache/unpack.go +++ b/internal/imagecache/unpack.go @@ -37,15 +37,24 @@ const ( opaqueMarkerName = ".wh..wh..opq" ) -// whiteoutSet records a layer's whiteout state, captured at unpack time and -// materialized later by FinalizeLayer (which runs with the privileges atelet -// lacks). Paths are clean and relative to the layer's fs/ root. +// whiteoutSet records per-layer metadata captured at unpack time that the +// (privileged) consumer needs at compose time: whiteout state materialized +// by FinalizeLayer, and the directories this layer only created implicitly. +// Paths are clean and relative to the layer's fs/ root. type whiteoutSet struct { Version int `json:"version"` // Whiteouts are paths that must become 0:0 char devices in the lowerdir. Whiteouts []string `json:"whiteouts,omitempty"` // Opaques are directories that must carry trusted.overlay.opaque=y. Opaques []string `json:"opaques,omitempty"` + // ImplicitDirs are directories the layer tar never declared but that + // exist in the tree because a child entry (or a whiteout materialized at + // finalize) needed a parent. Their root:root 0755 attrs are fabricated; + // in the composed overlay the top-most layer containing a directory + // supplies its metadata, so an implicit dir here would shadow the real + // attrs a lower layer declared (e.g. /tmp's 1777). SetupBundleRootfs + // repairs the merged view from these records (see resolveImplicitDirFixups). + ImplicitDirs []string `json:"implicitDirs,omitempty"` } func readWhiteouts(layerDir string) (*whiteoutSet, error) { @@ -98,6 +107,21 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo // CAP_DAC_OVERRIDE. Keyed by name so a repeated dir entry's last mode wins. dirModes := map[string]os.FileMode{} + // Ancestors an entry needed vs. directories the tar declared: the + // difference is recorded as ImplicitDirs (attrs fabricated, see the + // whiteoutSet field doc). Whiteout/opaque markers count too — their + // parents are created by FinalizeLayer's MkdirAll with the same + // fabricated attrs. + declared := map[string]bool{} + implicit := map[string]bool{} + markAncestors := func(name string) { + for p := filepath.Dir(name); p != "."; p = filepath.Dir(p) { + if !declared[p] { + implicit[p] = true + } + } + } + tarReader := tar.NewReader(tarData) for { hdr, err := tarReader.Next() @@ -119,6 +143,7 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo if dir := filepath.Dir(name); dir != "." { wh.Opaques = append(wh.Opaques, dir) } + markAncestors(name) continue } else if strings.HasPrefix(base, whiteoutPrefix+whiteoutPrefix) { // AUFS bookkeeping entries (.wh..wh.plnk, .wh..wh.aufs, ...); @@ -126,6 +151,7 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo continue } else if deleted, ok := strings.CutPrefix(base, whiteoutPrefix); ok { wh.Whiteouts = append(wh.Whiteouts, filepath.Join(filepath.Dir(name), deleted)) + markAncestors(name) continue } @@ -136,6 +162,7 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo // declared only in the base layer). Unpacking per layer, those // parents must be created here; overlayfs merges them with the // lower layers' directories at compose time. + markAncestors(name) if parent := filepath.Dir(name); parent != "." { if err := root.MkdirAll(parent, 0o755); err != nil { return nil, fmt.Errorf("while creating parent directories for %q: %w", name, err) @@ -186,6 +213,8 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo return nil, fmt.Errorf("while creating directory=%q, mode=%v: %w", name, mode, err) } dirModes[name] = mode + declared[name] = true + delete(implicit, name) case tar.TypeSymlink: // A layer may re-define the same path (e.g. declare /var/run as a dir @@ -255,5 +284,32 @@ func unpackLayer(ctx context.Context, tarData io.Reader, root *os.Root) (*whiteo } } + // Keep only implicit candidates that survive in the tree as directories + // ("later entry wins" may have replaced one with a file or symlink), plus + // the ones FinalizeLayer will create for whiteout/opaque materialization + // (they may not exist yet). Sorted for deterministic metadata. + finalizeDirs := map[string]bool{} + for _, w := range wh.Whiteouts { + for p := filepath.Dir(w); p != "."; p = filepath.Dir(p) { + finalizeDirs[p] = true + } + } + for _, o := range wh.Opaques { + for p := o; p != "."; p = filepath.Dir(p) { + finalizeDirs[p] = true + } + } + for p := range implicit { + if declared[p] { + continue + } + if fi, err := root.Lstat(p); err == nil && fi.IsDir() { + wh.ImplicitDirs = append(wh.ImplicitDirs, p) + } else if finalizeDirs[p] { + wh.ImplicitDirs = append(wh.ImplicitDirs, p) + } + } + sort.Strings(wh.ImplicitDirs) + return wh, nil } diff --git a/internal/imagecache/unpack_test.go b/internal/imagecache/unpack_test.go index a37eb5652..338b21097 100644 --- a/internal/imagecache/unpack_test.go +++ b/internal/imagecache/unpack_test.go @@ -247,6 +247,36 @@ func TestUnpackLayer_MissingParentDirs(t *testing.T) { } } +// Implicitly-created parent dirs must be recorded (their attrs are +// fabricated and can shadow lower-layer metadata in the merged view); +// declared dirs must not be, and candidates replaced by non-dirs drop out. +func TestUnpackLayer_ImplicitDirRecording(t *testing.T) { + entries := []tarEntry{ + {name: "declared/", typeflag: tar.TypeDir}, + {name: "declared/file", typeflag: tar.TypeReg, body: "x"}, + {name: "etc/nsswitch.conf", typeflag: tar.TypeReg, body: "hosts: files"}, + {name: "a/b/c/deep", typeflag: tar.TypeReg, body: "d"}, + // Parent exists only for a whiteout marker: created at finalize, but + // must be recorded now. + {name: "gone/.wh.victim", typeflag: tar.TypeReg}, + // Implicitly created, then declared later: the declaration wins. + {name: "late/child", typeflag: tar.TypeReg, body: "c"}, + {name: "late/", typeflag: tar.TypeDir, mode: 0o700}, + // Implicit parent whose subtree is later replaced by a file: + // no longer a dir, must not be recorded. + {name: "swap/inner", typeflag: tar.TypeReg, body: "i"}, + {name: "swap", typeflag: tar.TypeReg, body: "now a file"}, + } + _, wh, err := runUnpack(t, entries) + if err != nil { + t.Fatalf("unpackLayer: %v", err) + } + want := []string{"a", "a/b", "a/b/c", "etc", "gone"} + if !slices.Equal(wh.ImplicitDirs, want) { + t.Errorf("ImplicitDirs = %v, want %v", wh.ImplicitDirs, want) + } +} + func TestUnpackLayer_LaterEntryWins(t *testing.T) { t.Run("dir then symlink", func(t *testing.T) { entries := []tarEntry{ From 1c7a77bec71a50907556c6132c50ba5779b84fe8 Mon Sep 17 00:00:00 2001 From: dberkov Date: Mon, 20 Jul 2026 22:54:02 -0700 Subject: [PATCH 8/8] imagecache: generalize duplicate-layer comment wording --- internal/imagecache/spec.go | 7 ++++--- internal/imagecache/spec_test.go | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/imagecache/spec.go b/internal/imagecache/spec.go index 9a0b9571a..5c85883ed 100644 --- a/internal/imagecache/spec.go +++ b/internal/imagecache/spec.go @@ -85,9 +85,10 @@ func ReadSpec(bundlePath string) (*OverlaySpec, error) { // order), each pointing at the layer's fs/ tree. // // An image may legitimately list the same layer at several positions -// (identical build steps produce identical diffids; SWE-bench images do -// this). The pool stores that layer once, and overlayfs rejects a repeated -// lower directory (its overlapping-layers check fails the mount with ELOOP). +// (repeated identical build steps produce identical diffids; real images +// in the wild do this). The pool stores that layer once, and overlayfs +// rejects a repeated lower directory (its overlapping-layers check fails +// the mount with ELOOP). // For identical content only the topmost occurrence can affect the merged // view, so keep the first one seen walking top-first and drop the rest. // diff --git a/internal/imagecache/spec_test.go b/internal/imagecache/spec_test.go index 8397f563a..2fa776e3a 100644 --- a/internal/imagecache/spec_test.go +++ b/internal/imagecache/spec_test.go @@ -76,8 +76,8 @@ func TestOverlayLowerDirs(t *testing.T) { } }) - // Images can list the same layer at several positions (identical build - // steps → identical diffids; SWE-bench images do). overlayfs rejects a + // Images can list the same layer at several positions (repeated + // identical build steps produce identical diffids). overlayfs rejects a // repeated lowerdir with ELOOP, so duplicates must collapse to the // topmost occurrence. Regression test for the "too many levels of // symbolic links" mount failure.