diff --git a/sandboxexec/sandbox/BUILD b/sandboxexec/sandbox/BUILD index b66faec694..9a5d792f19 100644 --- a/sandboxexec/sandbox/BUILD +++ b/sandboxexec/sandbox/BUILD @@ -10,6 +10,7 @@ go_library( srcs = [ "oci.go", "sandbox.go", + "storage.go", ], visibility = ["//:__subpackages__"], deps = ["@com_github_opencontainers_runtime_spec//specs-go:go_default_library"], @@ -20,6 +21,7 @@ go_test( srcs = [ "oci_test.go", "sandbox_test.go", + "storage_test.go", ], data = ["//runsc"], tags = ["nogotsan"], diff --git a/sandboxexec/sandbox/oci.go b/sandboxexec/sandbox/oci.go index 7bc5f77bb1..06f3dacdb6 100644 --- a/sandboxexec/sandbox/oci.go +++ b/sandboxexec/sandbox/oci.go @@ -23,10 +23,19 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" ) -// NewBundle creates a temporary OCI bundle on the fly. -func NewBundle(sandboxID string, runscRuntimeDir string, enableNetworking bool, mounts []Mount) (string, error) { +// BundleConfig holds configuration for creating an OCI bundle. +type BundleConfig struct { + ID string + RuntimeDir string + EnableNetworking bool + Mounts []Mount + Annotations map[string]string +} + +// NewBundle creates a temporary OCI bundle on the fly with the given configuration. +func NewBundle(cfg BundleConfig) (string, error) { // Create a bundle directory for the sandbox. - bundleDir := filepath.Join(runscRuntimeDir, sandboxID) + bundleDir := filepath.Join(cfg.RuntimeDir, cfg.ID) rootfsDir := filepath.Join(bundleDir, "rootfs") if err := os.MkdirAll(rootfsDir, 0755); err != nil { @@ -44,17 +53,16 @@ func NewBundle(sandboxID string, runscRuntimeDir string, enableNetworking bool, if os.Geteuid() != 0 { namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.UserNamespace}) } - if enableNetworking { + if cfg.EnableNetworking { namespaces = append(namespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace}) } spec := &specs.Spec{ - Version: "1.0.0", + Version: "1.0.0", + Annotations: cfg.Annotations, Root: &specs.Root{ - Path: "rootfs", - // The root filesystem is read-only for now. We can add support for - // writable rootfs later if needed. - Readonly: true, + Path: "rootfs", + Readonly: false, }, Process: &specs.Process{ Terminal: false, @@ -103,7 +111,7 @@ func NewBundle(sandboxID string, runscRuntimeDir string, enableNetworking bool, // Add custom mounts. Custom mounts overriding default host mounts create duplicate OCI // entries. The later entry overrides the earlier one, as expected by OCI specs. - for _, m := range mounts { + for _, m := range cfg.Mounts { switch m.Type { case MountTypeBind: opts := []string{"rbind"} diff --git a/sandboxexec/sandbox/oci_test.go b/sandboxexec/sandbox/oci_test.go index 80a24b143b..72bd0524f1 100644 --- a/sandboxexec/sandbox/oci_test.go +++ b/sandboxexec/sandbox/oci_test.go @@ -31,18 +31,20 @@ func TestNewBundle(t *testing.T) { tempDir := t.TempDir() sandboxID := "test-sandbox" - bundleDir, err := sandbox.NewBundle(sandboxID, tempDir, enableNetworking, nil) + bundleDir, err := sandbox.NewBundle(sandbox.BundleConfig{ + ID: sandboxID, + RuntimeDir: tempDir, + EnableNetworking: enableNetworking, + }) if err != nil { t.Fatalf("NewBundle(enableNet=%v) failed: %v", enableNetworking, err) } defer os.RemoveAll(bundleDir) - expectedBundleDir := filepath.Join(tempDir, sandboxID) if bundleDir != expectedBundleDir { t.Fatalf("NewBundle(%v, %v) = %q, want %q", sandboxID, tempDir, bundleDir, expectedBundleDir) } - // Verify config.json was created and contains valid OCI spec. configPath := filepath.Join(bundleDir, "config.json") configFile, err := os.Open(configPath) if err != nil { @@ -61,6 +63,9 @@ func TestNewBundle(t *testing.T) { if spec.Root == nil || spec.Root.Path != "rootfs" { t.Errorf("spec.Root.Path is not 'rootfs', got: %+v", spec.Root) } + if spec.Root.Readonly { + t.Errorf("spec.Root.Readonly is true, want false") + } if spec.Linux == nil { t.Fatalf("spec.Linux is nil") } @@ -114,7 +119,11 @@ func TestNewBundleNormalization(t *testing.T) { }, } - bundleDir, err := sandbox.NewBundle(sandboxID, tempDir, false, mounts) + bundleDir, err := sandbox.NewBundle(sandbox.BundleConfig{ + ID: sandboxID, + RuntimeDir: tempDir, + Mounts: mounts, + }) if err != nil { t.Fatalf("NewBundle failed: %v", err) } @@ -132,7 +141,6 @@ func TestNewBundleNormalization(t *testing.T) { t.Fatalf("failed to decode config.json: %v", err) } - // Verify our custom mounts are present and normalized var foundBind, foundTmpfs bool for _, m := range spec.Mounts { if m.Type == "bind" && m.Destination == "/mnt/foo/bar" { @@ -153,3 +161,37 @@ func TestNewBundleNormalization(t *testing.T) { t.Errorf("failed to find normalized tmpfs mount '/mnt'") } } + +func TestNewBundleWithAnnotations(t *testing.T) { + tempDir := t.TempDir() + sandboxID := "test-sandbox-annotations" + annotations := map[string]string{ + "dev.gvisor.tar.rootfs.upper": "/tmp/test.tar", + } + + bundleDir, err := sandbox.NewBundle(sandbox.BundleConfig{ + ID: sandboxID, + RuntimeDir: tempDir, + Annotations: annotations, + }) + if err != nil { + t.Fatalf("NewBundle failed: %v", err) + } + defer os.RemoveAll(bundleDir) + + configPath := filepath.Join(bundleDir, "config.json") + configFile, err := os.Open(configPath) + if err != nil { + t.Fatalf("failed to open config.json: %v", err) + } + defer configFile.Close() + + var spec specs.Spec + if err := json.NewDecoder(configFile).Decode(&spec); err != nil { + t.Fatalf("failed to decode config.json: %v", err) + } + + if val, ok := spec.Annotations["dev.gvisor.tar.rootfs.upper"]; !ok || val != "/tmp/test.tar" { + t.Errorf("expected annotation 'dev.gvisor.tar.rootfs.upper' with value '/tmp/test.tar', got spec.Annotations: %+v", spec.Annotations) + } +} diff --git a/sandboxexec/sandbox/sandbox.go b/sandboxexec/sandbox/sandbox.go index b9e56bb194..8c199171c8 100644 --- a/sandboxexec/sandbox/sandbox.go +++ b/sandboxexec/sandbox/sandbox.go @@ -19,12 +19,14 @@ package sandbox import ( "bytes" "context" + "encoding/json" "fmt" "io" "math/rand" "os" "os/exec" "path/filepath" + "time" ) // Options holds the configuration for a Sandbox. @@ -33,6 +35,7 @@ type Options struct { id string enableNetworking bool mounts []Mount + snapshot *Snapshot } // Option configures the Options struct. @@ -99,6 +102,15 @@ func WithTmpfsMount(destination string) Option { } } +// WithSnapshot configures the sandbox to restore state from the given snapshot. +// The sandbox automatically reads the snapshot metadata to determine if it is a +// full Checkpoint/Restore, Filesystem snapshot, or Rootfs Tar snapshot. +func WithSnapshot(snapshot *Snapshot) Option { + return func(o *Options) { + o.snapshot = snapshot + } +} + // Sandbox represents a running gVisor sandbox where applications // run inside. type Sandbox struct { @@ -171,7 +183,75 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) { return nil, fmt.Errorf("sandbox state directory has incorrect permissions: got %v, want %v", fi.Mode().Perm(), os.FileMode(0700)) } - bundleDir, err := NewBundle(options.id, runDir, options.enableNetworking, options.mounts) + var annotations map[string]string + var globalFlags []string + var runFlags []string + var isCheckpointRestore bool + var checkpointRestoreDir string + + if options.snapshot != nil { + store := options.snapshot.Storage + snapshotID := options.snapshot.ID + if store == nil { + return nil, fmt.Errorf("no snapshot storage configured for restore") + } + + // Fetch metadata.json from store. + metaReader, err := store.GetReader(ctx, snapshotID, MetadataAsset) + if err != nil { + return nil, fmt.Errorf("failed to read snapshot metadata: %w", err) + } + defer metaReader.Close() + + var meta SnapshotMetadata + if err := json.NewDecoder(metaReader).Decode(&meta); err != nil { + return nil, fmt.Errorf("failed to parse snapshot metadata: %w", err) + } + + // Perform restore based on type. + switch meta.Type { + case RootfsTarSnapshot: + tarPath, err := readRootfsTar(ctx, snapshotID, store) + if err != nil { + return nil, err + } + defer os.Remove(tarPath) + + annotations = map[string]string{ + "dev.gvisor.tar.rootfs.upper": tarPath, + } + globalFlags = append(globalFlags, "--allow-rootfs-tar-annotation") + + case FilesystemSnapshot: + fsRestoreDir := filepath.Join(stateDir, "fs-restore") + if err := os.MkdirAll(fsRestoreDir, 0700); err != nil { + return nil, err + } + // TODO: List assets in store and download all filesystem image assets to fsRestoreDir. + runFlags = append(runFlags, fmt.Sprintf("--fs-restore-image-path=%s", fsRestoreDir)) + + case CheckpointRestore: + var err error + checkpointRestoreDir, err = os.MkdirTemp("", "sandbox-restore-*") + if err != nil { + return nil, fmt.Errorf("failed to create temp restore directory: %w", err) + } + defer os.RemoveAll(checkpointRestoreDir) + + if err := restoreCheckpoint(ctx, snapshotID, store, checkpointRestoreDir); err != nil { + return nil, err + } + isCheckpointRestore = true + } + } + + bundleDir, err := NewBundle(BundleConfig{ + ID: options.id, + RuntimeDir: runDir, + EnableNetworking: options.enableNetworking, + Mounts: options.mounts, + Annotations: annotations, + }) if err != nil { return nil, fmt.Errorf("failed to create OCI bundle: %v", err) } @@ -183,8 +263,7 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) { rootState: stateDir, } - // Launch the sandbox in detached mode via os/exec, we use `runsc run` here - // as a shortcut for `runsc create` and `runsc start`. + // Launch the sandbox in detached mode via os/exec. args := []string{"--root", sb.rootState} if os.Geteuid() != 0 { args = append(args, "--ignore-cgroups") @@ -192,9 +271,16 @@ func New(ctx context.Context, opts ...Option) (*Sandbox, error) { if !options.enableNetworking { args = append(args, "--network=none") } - args = append(args, "run", "--bundle", sb.bundleDir, "--detach", sb.id) + args = append(args, globalFlags...) + + if isCheckpointRestore { + args = append(args, "restore", "--bundle", sb.bundleDir, "--image-path", checkpointRestoreDir, "--detach", sb.id) + } else { + args = append(args, "run") + args = append(args, runFlags...) + args = append(args, "--bundle", sb.bundleDir, "--detach", sb.id) + } cmd := exec.CommandContext(ctx, sb.runscPath, args...) - if err := cmd.Run(); err != nil { return nil, fmt.Errorf("failed to create sandbox via subprocess: %v", err) } @@ -235,6 +321,10 @@ func (s *Sandbox) Close(ctx context.Context) error { return fmt.Errorf("failed to clean up sandbox bundle directory: %v", err) } + if err := os.RemoveAll(s.rootState); err != nil { + return fmt.Errorf("failed to clean up sandbox state directory: %v", err) + } + return nil } @@ -242,3 +332,238 @@ func (s *Sandbox) Close(ctx context.Context) error { func (s *Sandbox) Bundle() string { return s.bundleDir } + +// SnapshotOptions holds configuration for taking a snapshot. +type SnapshotOptions struct { + LeaveRunning bool +} + +// SnapshotOption configures SnapshotOptions. +type SnapshotOption func(*SnapshotOptions) + +// WithLeaveRunning keeps the sandbox running after taking the snapshot. +func WithLeaveRunning(leaveRunning bool) SnapshotOption { + return func(o *SnapshotOptions) { + o.LeaveRunning = leaveRunning + } +} + +func newSnapshotID() SnapshotID { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + panic(fmt.Sprintf("failed to generate random bytes for snapshot ID: %v", err)) + } + return SnapshotID(fmt.Sprintf("snap-%x", b)) +} + +// Snapshot serializes and saves the sandbox state to storage, returning the generated snapshot. +// Depending on the snapshotType, it will perform a full Checkpoint, a Filesystem Snapshot, or a Rootfs Tar Snapshot. +// It also automatically generates and writes "metadata.json" into the storage. +func (s *Sandbox) Snapshot(ctx context.Context, snapshotType SnapshotType, storage SnapshotStorage, opts ...SnapshotOption) (*Snapshot, error) { + options := SnapshotOptions{ + LeaveRunning: false, // Default is false. + } + for _, o := range opts { + o(&options) + } + + snapshotID := newSnapshotID() + + switch snapshotType { + case RootfsTarSnapshot: + if err := s.snapshotRootfsTar(ctx, snapshotID, storage); err != nil { + return nil, err + } + + case FilesystemSnapshot: + // TODO: Run `runsc fscheckpoint --image-path= [--leave-running] `. + // TODO: Walk `` and upload each file to storage. + + case CheckpointRestore: + if err := s.checkpoint(ctx, snapshotID, storage, options); err != nil { + return nil, err + } + } + + meta := SnapshotMetadata{ + Type: snapshotType, + CreatedAt: time.Now().UTC().Format(time.RFC3339), + } + + metaWriter, err := storage.PutWriter(ctx, snapshotID, MetadataAsset) + if err != nil { + return nil, fmt.Errorf("failed to create metadata.json in storage: %w", err) + } + defer metaWriter.Close() + + if err := json.NewEncoder(metaWriter).Encode(&meta); err != nil { + return nil, fmt.Errorf("failed to write metadata.json to storage: %w", err) + } + + return &Snapshot{ + ID: snapshotID, + Storage: storage, + }, nil +} + +func (s *Sandbox) snapshotRootfsTar(ctx context.Context, snapshotID SnapshotID, storage SnapshotStorage) error { + tarFile, err := os.CreateTemp(os.TempDir(), "rootfs-*.tar") + if err != nil { + return fmt.Errorf("failed to create temp tar file: %w", err) + } + tarPath := tarFile.Name() + tarFile.Close() + defer os.Remove(tarPath) + + cmd := exec.CommandContext(ctx, s.runscPath, "--root", s.rootState, "tar", "rootfs-upper", "--file", tarPath, s.id) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("runsc tar failed: %v (stderr: %q)", err, stderr.String()) + } + + localFile, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("failed to open temp tar file: %w", err) + } + defer localFile.Close() + + storageWriter, err := storage.PutWriter(ctx, snapshotID, RootfsAsset) + if err != nil { + return fmt.Errorf("failed to create storage writer: %w", err) + } + defer storageWriter.Close() + + if _, err := io.Copy(storageWriter, localFile); err != nil { + return fmt.Errorf("failed to upload rootfs tar: %w", err) + } + return nil +} + +func (s *Sandbox) checkpoint(ctx context.Context, snapshotID SnapshotID, storage SnapshotStorage, options SnapshotOptions) error { + checkpointDir, err := os.MkdirTemp("", "sandbox-checkpoint-*") + if err != nil { + return fmt.Errorf("failed to create temp checkpoint directory: %w", err) + } + defer os.RemoveAll(checkpointDir) + + args := []string{"--root", s.rootState, "checkpoint", "--image-path", checkpointDir} + if options.LeaveRunning { + args = append(args, "--leave-running") + } + args = append(args, s.id) + + cmd := exec.CommandContext(ctx, s.runscPath, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("runsc checkpoint failed: %v (stderr: %q)", err, stderr.String()) + } + + // Walk checkpointDir and upload files to storage. + err = filepath.Walk(checkpointDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + relPath, err := filepath.Rel(checkpointDir, path) + if err != nil { + return err + } + + localFile, err := os.Open(path) + if err != nil { + return fmt.Errorf("failed to open local checkpoint file: %w", err) + } + defer localFile.Close() + + storageWriter, err := storage.PutWriter(ctx, snapshotID, Asset(relPath)) + if err != nil { + return fmt.Errorf("failed to create storage writer for %q: %w", relPath, err) + } + defer storageWriter.Close() + + if _, err := io.Copy(storageWriter, localFile); err != nil { + return fmt.Errorf("failed to upload checkpoint file %q: %w", relPath, err) + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to upload checkpoint assets: %w", err) + } + return nil +} + +func readRootfsTar(ctx context.Context, snapshotID SnapshotID, store SnapshotStorage) (string, error) { + tarFile, err := os.CreateTemp(os.TempDir(), "rootfs-*.tar") + if err != nil { + return "", fmt.Errorf("failed to create temp tar file: %w", err) + } + tarPath := tarFile.Name() + defer tarFile.Close() + + cleanup := true + defer func() { + if cleanup { + os.Remove(tarPath) + } + }() + + storageReader, err := store.GetReader(ctx, snapshotID, RootfsAsset) + if err != nil { + return "", fmt.Errorf("failed to get rootfs reader from storage: %w", err) + } + defer storageReader.Close() + + if _, err := io.Copy(tarFile, storageReader); err != nil { + return "", fmt.Errorf("failed to download rootfs asset: %w", err) + } + + cleanup = false + return tarPath, nil +} + +func restoreCheckpoint(ctx context.Context, snapshotID SnapshotID, store SnapshotStorage, destDir string) error { + if err := os.MkdirAll(destDir, 0700); err != nil { + return err + } + assets, err := store.ListAssets(ctx, snapshotID) + if err != nil { + return fmt.Errorf("failed to list checkpoint assets: %w", err) + } + + for _, asset := range assets { + if asset == MetadataAsset { + continue + } + err := func() error { + assetReader, err := store.GetReader(ctx, snapshotID, asset) + if err != nil { + return fmt.Errorf("failed to get reader for asset %q: %w", asset, err) + } + defer assetReader.Close() + + localPath := filepath.Join(destDir, string(asset)) + if err := os.MkdirAll(filepath.Dir(localPath), 0700); err != nil { + return fmt.Errorf("failed to create directory for asset %q: %w", asset, err) + } + + localFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return fmt.Errorf("failed to create local file for asset %q: %w", asset, err) + } + defer localFile.Close() + + if _, err := io.Copy(localFile, assetReader); err != nil { + return fmt.Errorf("failed to download asset %q: %w", asset, err) + } + return nil + }() + if err != nil { + return err + } + } + return nil +} diff --git a/sandboxexec/sandbox/sandbox_test.go b/sandboxexec/sandbox/sandbox_test.go index 94e19992c1..e1d2ad55c5 100644 --- a/sandboxexec/sandbox/sandbox_test.go +++ b/sandboxexec/sandbox/sandbox_test.go @@ -203,3 +203,316 @@ func TestCustomBindMountWrite(t *testing.T) { t.Errorf("ReadFile(%q) = %q, want %q", hostFile, got, want) } } + +func TestRootfsTarSnapshot(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + storageDir := filepath.Join(tempDir, "storage") + if err := os.MkdirAll(storageDir, 0700); err != nil { + t.Fatalf("failed to create storage dir: %v", err) + } + storage, err := sandbox.NewFilesystemStorage(storageDir) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + runtimeDirA := filepath.Join(tempDir, "runtime-a") + enableNetworking := os.Geteuid() == 0 + sbA, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirA), + sandbox.WithNetworking(enableNetworking), + ) + if err != nil { + t.Fatalf("failed to start sandbox A: %v", err) + } + defer sbA.Close(ctx) + + _, _, err = sbA.Exec(ctx, "sh", "-c", "echo 'hello' > /test.txt") + if err != nil { + t.Fatalf("failed to create file in sandbox A: %v", err) + } + + snapshot, err := sbA.Snapshot(ctx, sandbox.RootfsTarSnapshot, storage) + if err != nil { + t.Fatalf("failed to take snapshot: %v", err) + } + + runtimeDirB := filepath.Join(tempDir, "runtime-b") + sbB, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirB), + sandbox.WithNetworking(enableNetworking), + sandbox.WithSnapshot(snapshot), + ) + if err != nil { + t.Fatalf("failed to start sandbox B: %v", err) + } + defer sbB.Close(ctx) + + outB, _, err := sbB.Exec(ctx, "cat", "/test.txt") + if err != nil { + t.Fatalf("failed to cat file in sandbox B: %v", err) + } + if strings.TrimSpace(outB) != "hello" { + t.Errorf("unexpected content in B: got %q, want %q", outB, "hello") + } +} + +func TestNoSnapshotStorageError(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + runtimeDir := filepath.Join(tempDir, "runtime") + enableNetworking := os.Geteuid() == 0 + _, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDir), + sandbox.WithNetworking(enableNetworking), + sandbox.WithSnapshot(&sandbox.Snapshot{ID: "some-snapshot-id"}), + ) + if err == nil { + t.Fatalf("expected error when starting sandbox with SnapshotID but no SnapshotStore or default storage configured, got nil") + } + expectedErrSubstr := "no snapshot storage configured for restore" + if !strings.Contains(err.Error(), expectedErrSubstr) { + t.Errorf("unexpected error: %v, want it to contain %q", err, expectedErrSubstr) + } +} + +// TestCheckpointRestore verifies that the sandbox state can be saved to a +// checkpoint and restored later. +// +// We use a tmpfs mount to verify memory restoration. Because each Exec call +// runs in a new process, we cannot use process-local state (like environment +// variables) to verify restoration across Exec calls. Instead, we use tmpfs, +// which is an in-memory filesystem managed by the sandbox sentry. Files in +// tmpfs reside entirely in the sandbox's memory, so verifying they survive +// restore confirms that the sandbox memory state was correctly restored. +func TestCheckpointRestore(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + storageDir := filepath.Join(tempDir, "storage") + if err := os.MkdirAll(storageDir, 0700); err != nil { + t.Fatalf("failed to create storage dir: %v", err) + } + storage, err := sandbox.NewFilesystemStorage(storageDir) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + runtimeDirA := filepath.Join(tempDir, "runtime-a") + enableNetworking := os.Geteuid() == 0 + sbA, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirA), + sandbox.WithNetworking(enableNetworking), + sandbox.WithTmpfsMount("/mnt/scratch"), + ) + if err != nil { + t.Fatalf("failed to start sandbox A: %v", err) + } + defer sbA.Close(ctx) + + // Write to tmpfs (in-memory filesystem) + _, _, err = sbA.Exec(ctx, "sh", "-c", "echo 'memory-file-value' > /mnt/scratch/state.txt") + if err != nil { + t.Fatalf("failed to write to tmpfs in A: %v", err) + } + + // Take a checkpoint snapshot. + snapshot, err := sbA.Snapshot(ctx, sandbox.CheckpointRestore, storage) + if err != nil { + t.Fatalf("failed to take checkpoint: %v", err) + } + + runtimeDirB := filepath.Join(tempDir, "runtime-b") + sbB, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirB), + sandbox.WithNetworking(enableNetworking), + sandbox.WithSnapshot(snapshot), + sandbox.WithTmpfsMount("/mnt/scratch"), // Must match A + ) + if err != nil { + t.Fatalf("failed to restore sandbox B: %v", err) + } + defer sbB.Close(ctx) + + // Verify tmpfs state is restored. + outB, _, err := sbB.Exec(ctx, "cat", "/mnt/scratch/state.txt") + if err != nil { + t.Fatalf("failed to read tmpfs in B: %v", err) + } + if got, want := strings.TrimSpace(outB), "memory-file-value"; got != want { + t.Fatalf("tmpfs content in B = %q, want %q", got, want) + } +} + +func TestCheckpointRestoreMultiple(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + storageDir := filepath.Join(tempDir, "storage") + if err := os.MkdirAll(storageDir, 0700); err != nil { + t.Fatalf("failed to create storage dir: %v", err) + } + storage, err := sandbox.NewFilesystemStorage(storageDir) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + runtimeDirA := filepath.Join(tempDir, "runtime-a") + enableNetworking := os.Geteuid() == 0 + sbA, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirA), + sandbox.WithNetworking(enableNetworking), + sandbox.WithTmpfsMount("/mnt/scratch"), + ) + if err != nil { + t.Fatalf("failed to start sandbox A: %v", err) + } + defer sbA.Close(ctx) + + _, _, err = sbA.Exec(ctx, "sh", "-c", "echo 'val1' > /mnt/scratch/file1.txt") + if err != nil { + t.Fatalf("failed to write file1 in A: %v", err) + } + + // Checkpoint at sandbox A for snapshot1. + snapshot1, err := sbA.Snapshot(ctx, sandbox.CheckpointRestore, storage) + if err != nil { + t.Fatalf("failed to take checkpoint 1: %v", err) + } + + // Restore at sandbox B from snapshot1. + runtimeDirB := filepath.Join(tempDir, "runtime-b") + sbB, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirB), + sandbox.WithNetworking(enableNetworking), + sandbox.WithSnapshot(snapshot1), + sandbox.WithTmpfsMount("/mnt/scratch"), + ) + if err != nil { + t.Fatalf("failed to restore sandbox B: %v", err) + } + defer sbB.Close(ctx) + + outB, _, err := sbB.Exec(ctx, "cat", "/mnt/scratch/file1.txt") + if err != nil { + t.Fatalf("failed to read file1 in B: %v", err) + } + if got, want := strings.TrimSpace(outB), "val1"; got != want { + t.Fatalf("file1 content in B = %q, want %q", got, want) + } + + _, _, err = sbB.Exec(ctx, "sh", "-c", "echo 'val2' > /mnt/scratch/file2.txt") + if err != nil { + t.Fatalf("failed to write file2 in B: %v", err) + } + + // Checkpoint at sandbox B for snapshot2. + snapshot2, err := sbB.Snapshot(ctx, sandbox.CheckpointRestore, storage) + if err != nil { + t.Fatalf("failed to take checkpoint 2: %v", err) + } + + // Restore sandbox C from snapshot2 + runtimeDirC := filepath.Join(tempDir, "runtime-c") + sbC, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirC), + sandbox.WithNetworking(enableNetworking), + sandbox.WithSnapshot(snapshot2), + sandbox.WithTmpfsMount("/mnt/scratch"), + ) + if err != nil { + t.Fatalf("failed to restore sandbox C: %v", err) + } + defer sbC.Close(ctx) + + // Verify file1 and file2 in C + outC1, _, err := sbC.Exec(ctx, "cat", "/mnt/scratch/file1.txt") + if err != nil { + t.Fatalf("failed to read file1 in C: %v", err) + } + if got, want := strings.TrimSpace(outC1), "val1"; got != want { + t.Fatalf("file1 content in C = %q, want %q", got, want) + } + + outC2, _, err := sbC.Exec(ctx, "cat", "/mnt/scratch/file2.txt") + if err != nil { + t.Fatalf("failed to read file2 in C: %v", err) + } + if got, want := strings.TrimSpace(outC2), "val2"; got != want { + t.Fatalf("file2 content in C = %q, want %q", got, want) + } +} + +func TestCheckpointRestoreCorrupted(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + + storageDir := filepath.Join(tempDir, "storage") + if err := os.MkdirAll(storageDir, 0700); err != nil { + t.Fatalf("failed to create storage dir: %v", err) + } + storage, err := sandbox.NewFilesystemStorage(storageDir) + if err != nil { + t.Fatalf("failed to create storage: %v", err) + } + + runtimeDirA := filepath.Join(tempDir, "runtime-a") + enableNetworking := os.Geteuid() == 0 + sbA, err := sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirA), + sandbox.WithNetworking(enableNetworking), + sandbox.WithTmpfsMount("/mnt/scratch"), + ) + if err != nil { + t.Fatalf("failed to start sandbox A: %v", err) + } + defer sbA.Close(ctx) + + _, _, err = sbA.Exec(ctx, "sh", "-c", "echo 'hello' > /mnt/scratch/state.txt") + if err != nil { + t.Fatalf("failed to write to tmpfs in A: %v", err) + } + + snapshot, err := sbA.Snapshot(ctx, sandbox.CheckpointRestore, storage) + if err != nil { + t.Fatalf("failed to take checkpoint: %v", err) + } + + // List assets and delete one that is NOT metadata.json. + assets, err := storage.ListAssets(ctx, snapshot.ID) + if err != nil { + t.Fatalf("failed to list assets: %v", err) + } + corrupted := false + for _, asset := range assets { + if asset != sandbox.MetadataAsset { + // Overwrite the asset with empty content to corrupt it. + writer, err := storage.PutWriter(ctx, snapshot.ID, asset) + if err != nil { + t.Fatalf("failed to get writer to corrupt asset %q: %v", asset, err) + } + if err := writer.Close(); err != nil { + t.Fatalf("failed to close writer for corrupted asset %q: %v", asset, err) + } + corrupted = true + break + } + } + if !corrupted { + t.Fatalf("no asset found to corrupt other than metadata.json") + } + + // Try to restore, it should fail. + runtimeDirB := filepath.Join(tempDir, "runtime-b") + _, err = sandbox.New(ctx, + sandbox.WithRuntimeDir(runtimeDirB), + sandbox.WithNetworking(enableNetworking), + sandbox.WithSnapshot(snapshot), + sandbox.WithTmpfsMount("/mnt/scratch"), + ) + if err == nil { + t.Fatalf("expected restore to fail with corrupted snapshot, got nil") + } +} diff --git a/sandboxexec/sandbox/storage.go b/sandboxexec/sandbox/storage.go new file mode 100644 index 0000000000..ecbef9c668 --- /dev/null +++ b/sandboxexec/sandbox/storage.go @@ -0,0 +1,185 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +// SnapshotType defines the type of snapshot. +type SnapshotType string + +const ( + // CheckpointRestore represents a full process state checkpoint and restore. + CheckpointRestore SnapshotType = "CheckpointRestore" + + // FilesystemSnapshot represents a snapshot of the container's filesystems. + FilesystemSnapshot SnapshotType = "FilesystemSnapshot" + + // RootfsTarSnapshot represents a tar file snapshot of rootfs changes. + RootfsTarSnapshot SnapshotType = "RootfsTarSnapshot" +) + +// SnapshotID defines the type for snapshot IDs. +type SnapshotID string + +// SnapshotMetadata stores the metadata of a snapshot. +type SnapshotMetadata struct { + Type SnapshotType `json:"type"` + CreatedAt string `json:"created_at"` +} + +// Snapshot groups SnapshotID and SnapshotStorage together. +type Snapshot struct { + ID SnapshotID + Storage SnapshotStorage +} + +// Asset defines the type for snapshot asset names. +type Asset string + +const ( + // MetadataAsset is the name of the metadata file. + MetadataAsset Asset = "metadata.json" + // RootfsAsset is the name of the rootfs tarball (if using RootfsTarSnapshot). + RootfsAsset Asset = "rootfs.tar" + // CheckpointAsset is the main checkpoint state file. + CheckpointAsset Asset = "checkpoint.img" + // PagesAsset is the memory pages file. + PagesAsset Asset = "pages.img" + // PagesMetaAsset is the memory pages metadata file. + PagesMetaAsset Asset = "pages_meta.img" +) + +// SnapshotStorage defines a pluggable storage interface for snapshots. +type SnapshotStorage interface { + // PutWriter returns a WriteCloser to write a file asset of a snapshot. + PutWriter(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.WriteCloser, error) + + // GetReader returns a ReadCloser to read a file asset of a snapshot. + GetReader(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.ReadCloser, error) + + // Delete deletes all assets associated with a snapshot ID. + Delete(ctx context.Context, snapshotID SnapshotID) error + + // List returns all snapshot IDs known to this storage. + List(ctx context.Context) ([]SnapshotID, error) + + // Lookup verifies that the snapshot ID exists in this storage and returns a Snapshot. + Lookup(ctx context.Context, snapshotID SnapshotID) (*Snapshot, error) + + // ListAssets returns all asset names associated with a snapshot ID. + ListAssets(ctx context.Context, snapshotID SnapshotID) ([]Asset, error) +} + +// ErrSnapshotNotFound is returned when the snapshot ID is not found. +var ErrSnapshotNotFound = errors.New("snapshot not found") + +// FilesystemStorage implements SnapshotStorage using a local directory. +type FilesystemStorage struct { + rootDir string +} + +// NewFilesystemStorage creates a new FilesystemStorage at the given root directory. +// The root directory must already exist. +func NewFilesystemStorage(rootDir string) (*FilesystemStorage, error) { + fi, err := os.Stat(rootDir) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("root directory %q is not a directory", rootDir) + } + return &FilesystemStorage{rootDir: rootDir}, nil +} + +// PutWriter returns a WriteCloser to write a file asset of a snapshot. +func (f *FilesystemStorage) PutWriter(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.WriteCloser, error) { + path := filepath.Join(f.rootDir, string(snapshotID), string(assetName)) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + return nil, err + } + return os.Create(path) +} + +// GetReader returns a ReadCloser to read a file asset of a snapshot. +func (f *FilesystemStorage) GetReader(ctx context.Context, snapshotID SnapshotID, assetName Asset) (io.ReadCloser, error) { + path := filepath.Join(f.rootDir, string(snapshotID), string(assetName)) + return os.Open(path) +} + +// Delete deletes all assets associated with a snapshot ID. +func (f *FilesystemStorage) Delete(ctx context.Context, snapshotID SnapshotID) error { + path := filepath.Join(f.rootDir, string(snapshotID)) + return os.RemoveAll(path) +} + +// List returns all snapshot IDs known to this storage. +func (f *FilesystemStorage) List(ctx context.Context) ([]SnapshotID, error) { + entries, err := os.ReadDir(f.rootDir) + if err != nil { + return nil, err + } + ids := make([]SnapshotID, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + ids = append(ids, SnapshotID(entry.Name())) + } + } + return ids, nil +} + +// Lookup returns a Snapshot by a given snapshot ID. +func (f *FilesystemStorage) Lookup(ctx context.Context, snapshotID SnapshotID) (*Snapshot, error) { + dir := filepath.Join(f.rootDir, string(snapshotID)) + fi, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrSnapshotNotFound + } + return nil, err + } + if !fi.IsDir() { + return nil, fmt.Errorf("snapshot path %q is not a directory", dir) + } + return &Snapshot{ + ID: snapshotID, + Storage: f, + }, nil +} + +// ListAssets returns all asset names associated with a snapshot ID. +func (f *FilesystemStorage) ListAssets(ctx context.Context, snapshotID SnapshotID) ([]Asset, error) { + dir := filepath.Join(f.rootDir, string(snapshotID)) + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrSnapshotNotFound + } + return nil, err + } + assets := make([]Asset, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + assets = append(assets, Asset(entry.Name())) + } + } + return assets, nil +} diff --git a/sandboxexec/sandbox/storage_test.go b/sandboxexec/sandbox/storage_test.go new file mode 100644 index 0000000000..3aac0655c7 --- /dev/null +++ b/sandboxexec/sandbox/storage_test.go @@ -0,0 +1,241 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox_test + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "gvisor.dev/gvisor/sandboxexec/sandbox" +) + +func createTestSnapshot(ctx context.Context, t *testing.T, storage sandbox.SnapshotStorage, id sandbox.SnapshotID) { + t.Helper() + writer, err := storage.PutWriter(ctx, id, sandbox.MetadataAsset) + if err != nil { + t.Fatalf("failed to create test metadata: %v", err) + } + defer writer.Close() + meta := sandbox.SnapshotMetadata{ + Type: sandbox.CheckpointRestore, + CreatedAt: "2026-05-15T12:00:00Z", + } + if err := json.NewEncoder(writer).Encode(&meta); err != nil { + t.Fatalf("failed to encode metadata: %v", err) + } + + checkpointWriter, err := storage.PutWriter(ctx, id, sandbox.CheckpointAsset) + if err != nil { + t.Fatalf("failed to create test checkpoint: %v", err) + } + defer checkpointWriter.Close() + if _, err := checkpointWriter.Write([]byte("dummy checkpoint data")); err != nil { + t.Fatalf("failed to write dummy checkpoint data: %v", err) + } +} + +func TestFilesystemStorage(t *testing.T) { + ctx := context.Background() + + t.Run("Put and Read", func(t *testing.T) { + tempDir := t.TempDir() + storage, err := sandbox.NewFilesystemStorage(tempDir) + if err != nil { + t.Fatalf("failed to create FilesystemStorage: %v", err) + } + + snapshotID := sandbox.SnapshotID("test-snapshot-123") + createTestSnapshot(ctx, t, storage, snapshotID) + + // Verify metadata. + reader, err := storage.GetReader(ctx, snapshotID, sandbox.MetadataAsset) + if err != nil { + t.Fatalf("GetReader failed: %v", err) + } + defer reader.Close() + + var readMeta sandbox.SnapshotMetadata + if err := json.NewDecoder(reader).Decode(&readMeta); err != nil { + t.Fatalf("failed to decode read metadata: %v", err) + } + + if readMeta.Type != sandbox.CheckpointRestore { + t.Errorf("readMeta.Type = %v, want %v", readMeta.Type, sandbox.CheckpointRestore) + } + if readMeta.CreatedAt != "2026-05-15T12:00:00Z" { + t.Errorf("readMeta.CreatedAt = %q, want %q", readMeta.CreatedAt, "2026-05-15T12:00:00Z") + } + + // Verify content + reader2, err := storage.GetReader(ctx, snapshotID, sandbox.CheckpointAsset) + if err != nil { + t.Fatalf("GetReader failed: %v", err) + } + defer reader2.Close() + + gotContent, err := io.ReadAll(reader2) + if err != nil { + t.Fatalf("failed to read fake content: %v", err) + } + if string(gotContent) != "dummy checkpoint data" { + t.Errorf("got content %q, want %q", string(gotContent), "dummy checkpoint data") + } + }) + + t.Run("List", func(t *testing.T) { + tempDir := t.TempDir() + storage, err := sandbox.NewFilesystemStorage(tempDir) + if err != nil { + t.Fatalf("failed to create FilesystemStorage: %v", err) + } + + snapshots, err := storage.List(ctx) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(snapshots) != 0 { + t.Errorf("expected empty list, got %v", snapshots) + } + + snapshotID := sandbox.SnapshotID("test-snapshot-123") + createTestSnapshot(ctx, t, storage, snapshotID) + + snapshots, err = storage.List(ctx) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(snapshots) != 1 || snapshots[0] != snapshotID { + t.Errorf("got snapshots %v, want [%s]", snapshots, snapshotID) + } + }) + + t.Run("ListAssets", func(t *testing.T) { + tempDir := t.TempDir() + storage, err := sandbox.NewFilesystemStorage(tempDir) + if err != nil { + t.Fatalf("failed to create FilesystemStorage: %v", err) + } + + snapshotID := sandbox.SnapshotID("test-snapshot-123") + createTestSnapshot(ctx, t, storage, snapshotID) + + assets, err := storage.ListAssets(ctx, snapshotID) + if err != nil { + t.Fatalf("ListAssets failed: %v", err) + } + + // createTestSnapshot creates MetadataAsset and CheckpointAsset. + expectedAssets := map[sandbox.Asset]bool{ + sandbox.MetadataAsset: true, + sandbox.CheckpointAsset: true, + } + + if len(assets) != len(expectedAssets) { + t.Errorf("got %d assets, want %d", len(assets), len(expectedAssets)) + } + + for _, asset := range assets { + if !expectedAssets[asset] { + t.Errorf("unexpected asset: %v", asset) + } + } + }) + + t.Run("Lookup", func(t *testing.T) { + tempDir := t.TempDir() + storage, err := sandbox.NewFilesystemStorage(tempDir) + if err != nil { + t.Fatalf("failed to create FilesystemStorage: %v", err) + } + + snapshotID := sandbox.SnapshotID("test-snapshot-123") + + // Lookup non-existent + _, err = storage.Lookup(ctx, snapshotID) + if !errors.Is(err, sandbox.ErrSnapshotNotFound) { + t.Errorf("Lookup on non-existent snapshot returned error %v, want %v", err, sandbox.ErrSnapshotNotFound) + } + + createTestSnapshot(ctx, t, storage, snapshotID) + + // Lookup existent + snap, err := storage.Lookup(ctx, snapshotID) + if err != nil { + t.Fatalf("Lookup failed: %v", err) + } + if snap.ID != snapshotID { + t.Errorf("Lookup returned snapshot with wrong ID: got %s, want %s", snap.ID, snapshotID) + } + if snap.Storage != storage { + t.Errorf("Lookup returned snapshot with wrong storage") + } + }) + + t.Run("Delete", func(t *testing.T) { + tempDir := t.TempDir() + storage, err := sandbox.NewFilesystemStorage(tempDir) + if err != nil { + t.Fatalf("failed to create FilesystemStorage: %v", err) + } + + snapshotID := sandbox.SnapshotID("test-snapshot-123") + createTestSnapshot(ctx, t, storage, snapshotID) + + if err := storage.Delete(ctx, snapshotID); err != nil { + t.Fatalf("Delete failed: %v", err) + } + + _, err = storage.Lookup(ctx, snapshotID) + if !errors.Is(err, sandbox.ErrSnapshotNotFound) { + t.Errorf("Lookup after delete returned error %v, want %v", err, sandbox.ErrSnapshotNotFound) + } + }) +} + +func TestNewFilesystemStorage(t *testing.T) { + tempDir := t.TempDir() + filePath := filepath.Join(tempDir, "file.txt") + if err := os.WriteFile(filePath, []byte("hello"), 0644); err != nil { + t.Fatalf("failed to create file: %v", err) + } + + tests := []struct { + name string + rootDir string + }{ + { + name: "non-existent directory", + rootDir: filepath.Join(tempDir, "does-not-exist"), + }, + { + name: "path is a file", + rootDir: filePath, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := sandbox.NewFilesystemStorage(tc.rootDir) + if err == nil { + t.Errorf("NewFilesystemStorage(%q) succeeded, want error", tc.rootDir) + } + }) + } +}