diff --git a/g3doc/user_guide/tutorials/docker-in-gvisor.md b/g3doc/user_guide/tutorials/docker-in-gvisor.md index 0092ad252c..cf50ceaf0f 100644 --- a/g3doc/user_guide/tutorials/docker-in-gvisor.md +++ b/g3doc/user_guide/tutorials/docker-in-gvisor.md @@ -6,41 +6,79 @@ applications. In gVisor, all basic docker commands should function as expected. The host network driver and the bridge network driver are tested and supported. +### Supported Docker Versions + + + + + + + + + + + + + + + + + + + + + + + + + + +
Docker VersionSupport StatusRequired gVisor Configuration
Docker v27Supported +
    +
  • --net-raw
    • +
+
Docker v28Supported +
    +
  • --net-raw
    • +
  • --allow-packet-socket-write
    • +
+
Docker v29Supported +
    +
  • --net-raw
    • +
  • --allow-packet-socket-write
    • +
  • Either tmpfs mount at /var/lib/docker OR --feature containerd-snapshotter=false
    • +
+
+ ### Limitations -- Since version 29, docker defaults to using the - [containerd image store](https://docs.docker.com/engine/storage/containerd/) - as its default storage backend. For this to work inside gVisor, we need to - mount a tmpfs at `/var/lib/docker` because gVisor currently only allows - tmpfs mounts as upper layers of an overlay filesystem. See - [images/basic/docker/start-dockerd.sh](https://github.com/google/gvisor/blob/master/images/basic/docker/start-dockerd.sh) - for reference. Alternately, one can chose to avoid the new storage backend - altogether by launching dockerd with `--feature - containerd-snapshotter=false`. - `dockerd` inside gVisor needs to be executed with flags `--iptables=false --ip6tables=false` and additional network setup is needed, check [images/basic/docker/start-dockerd.sh](https://github.com/google/gvisor/blob/master/images/basic/docker/start-dockerd.sh) - for reference. -- With iptables disabled, `docker run --expose=` does not expose the port; if - a nested container needs to expose ports, inside gVisor use `docker run - --network=host`. + for reference. With iptables disabled, `docker run --expose=` does not + expose the port; if a nested container needs to expose ports, inside gVisor + use `docker run --network=host`. + +### Configuration Reference -### NOTE on runsc setup +To run Docker inside gVisor, you need to configure `runsc` and potentially the +container storage depending on the Docker version. -To run docker within gvisor, runsc must be enabled to allow raw sockets. This is -not the default, `--net-raw` must be passed to runsc. +#### runsc Flags (in `/etc/docker/daemon.json`) -In addition, Docker versions 28 and beyond need the ability to write to -AF_PACKET sockets. This is because dockerd sends unsolicited ARP/NA requests -when bringing up interfaces. To allow this, the `--allow-packet-socket-write` is -also to be supplied (the default behavior is to disallow writes to AF_PACKET -sockets). +To update your `/etc/docker/daemon.json` with the necessary `runtimeArgs`: -To use the following tutorial, that means having the following runtimes -configuration in `/etc/docker/daemon.json`: +* **`--net-raw`** (Required for all Docker versions) + * Enables raw sockets inside the sandbox. This is required for Docker's + networking to function. +* **`--allow-packet-socket-write`** (Required for Docker v28 and later) + * Allows the sandbox to write to `AF_PACKET` sockets. This is necessary + because `dockerd` (v28+) sends unsolicited ARP/NA requests when bringing + up network interfaces. + * > **Note:** `--allow-packet-socket-write` allows sandboxed code to craft + arbitrary network packets. -> **Note:** `--allow-packet-socket-write` allows sandboxed code to craft -> arbitrary packets. It is only needed for Docker versions 28 and beyond. +##### Example `/etc/docker/daemon.json` ```json { @@ -56,9 +94,31 @@ configuration in `/etc/docker/daemon.json`: } ``` -If you have an existing entry for `runsc`, likely created by `runsc install`, -then edit the entry and add the `"runtimeArgs"` key and value to the existing -entry. +If you have an existing entry for `runsc` (likely created by `runsc install`), +edit the entry to add the `"runtimeArgs"` key and value. + +#### Docker Storage Backend + +Since version 29, Docker Engine defaults to using the containerd image store, +which uses `overlayfs` for container root filesystems. + +In nested container and sandbox environments where `/var/lib/docker` is already +on an `overlayfs` mount, attempting to mount another `overlayfs` on top will +fail because Linux does not permit overlay-on-overlay mounts (and gVisor +currently requires `tmpfs` as an upper layer for `overlayfs`). + +To run Docker in gVisor, you must use one of the following workarounds: + +* **Mount `tmpfs` at `/var/lib/docker` (Recommended):** Mount a `tmpfs` + filesystem at `/var/lib/docker` before starting the Docker daemon inside the + sandbox so `overlayfs` can use `tmpfs` as its backing store. +* **Disable containerd image store:** Launch `dockerd` with `--feature + containerd-snapshotter=false` to fall back to classic storage drivers. + +See +[images/basic/docker/start-dockerd.sh](https://github.com/google/gvisor/blob/master/images/basic/docker/start-dockerd.sh) +as a reference implementation of how to handle these configurations at container +startup. ## How to run Docker in a gVisor container diff --git a/images/basic/docker/Dockerfile b/images/basic/docker/Dockerfile index 27bc8a00ba..797a7552e7 100644 --- a/images/basic/docker/Dockerfile +++ b/images/basic/docker/Dockerfile @@ -1,6 +1,4 @@ -FROM ubuntu:24.04 - -ARG DOCKER_VERSION="5:28.5.2-1~ubuntu.24.04~noble" +FROM ubuntu:latest ENV DEBIAN_FRONTEND="noninteractive" RUN apt-get update && apt-get -y install ca-certificates curl @@ -11,8 +9,8 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list && \ apt-get update && \ apt-get install -qqy \ - docker-ce=${DOCKER_VERSION} \ - docker-ce-cli=${DOCKER_VERSION} \ + docker-ce \ + docker-ce-cli \ containerd.io \ docker-buildx-plugin \ docker-compose-plugin \ diff --git a/pkg/sentry/fsimpl/tmpfs/BUILD b/pkg/sentry/fsimpl/tmpfs/BUILD index 9b35b60145..a622d95248 100644 --- a/pkg/sentry/fsimpl/tmpfs/BUILD +++ b/pkg/sentry/fsimpl/tmpfs/BUILD @@ -172,6 +172,7 @@ go_test( "//pkg/context", "//pkg/errors/linuxerr", "//pkg/fspath", + "//pkg/hostarch", "//pkg/sentry/contexttest", "//pkg/sentry/fsimpl/lock", "//pkg/sentry/kernel/auth", diff --git a/pkg/sentry/fsimpl/tmpfs/tmpfs.go b/pkg/sentry/fsimpl/tmpfs/tmpfs.go index 911463d570..454b1b19a3 100644 --- a/pkg/sentry/fsimpl/tmpfs/tmpfs.go +++ b/pkg/sentry/fsimpl/tmpfs/tmpfs.go @@ -256,109 +256,128 @@ func (fstype FilesystemType) GetFilesystem(ctx context.Context, vfsObj *vfs.Virt rootMode = 01777 } - mopts := vfs.GenericParseMountOptions(opts.Data) - var printedOpts []string - maxSizeInPages := getDefaultSizeLimit(disableDefaultSizeLimit) / hostarch.PageSize - maxSizeStr, ok := mopts["size"] - if ok { - delete(mopts, "size") - maxSizeInBytes, _, err := parseSize(maxSizeStr) - if err != nil { - ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: parseSize() failed: %v", err) - return nil, nil, linuxerr.EINVAL - } + maxInodes := getDefaultInodeLimit(disableDefaultSizeLimit) + rootKUID := creds.EffectiveKUID + rootKGID := creds.EffectiveKGID - var printedSize uint64 - if maxSizeInBytes > 0 { - // If size > 0, convert size in bytes to nearest Page Size - // bytes as Linux allocates memory in terms of Page size. - maxSizeInPages, ok = hostarch.ToPagesRoundUp(maxSizeInBytes) - if !ok { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: Pages RoundUp Overflow error: %v", ok) + printedOptsMap := make(map[string]string) + + for _, opt := range vfs.GenericParseMountOptionsOrdered(opts.Data) { + key := opt.Key + value := opt.Value + + switch key { + case "size": + maxSizeInBytes, _, err := parseSize(value) + if err != nil { + ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: parseSize() failed for size: %v", err) return nil, nil, linuxerr.EINVAL } - printedSize = (maxSizeInPages*hostarch.PageSize + 1023) / 1024 - } else { - // size = 0 is a special case: no size limit - maxSizeInPages = math.MaxInt64 - printedSize = 0 - } + var printedSize uint64 + if maxSizeInBytes > 0 { + var ok bool + maxSizeInPages, ok = hostarch.ToPagesRoundUp(maxSizeInBytes) + if !ok { + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: Pages RoundUp Overflow error") + return nil, nil, linuxerr.EINVAL + } + printedSize = (maxSizeInPages*hostarch.PageSize + 1023) / 1024 + } else { + maxSizeInPages = math.MaxInt64 + printedSize = 0 + } + printedOptsMap["size"] = fmt.Sprintf("size=%vk", printedSize) - printedOpts = append(printedOpts, fmt.Sprintf("size=%vk", printedSize)) - } - maxInodes := getDefaultInodeLimit(disableDefaultSizeLimit) - maxInodesStr, ok := mopts["nr_inodes"] - if ok { - delete(mopts, "nr_inodes") - var err error - var percentageSpecified bool - maxInodes, percentageSpecified, err = parseSize(maxInodesStr) - if percentageSpecified { - // Percentage not allowed for specifying inode limit - return nil, nil, linuxerr.EINVAL - } - if err != nil { - ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: parseSize() failed: %v", err) - return nil, nil, linuxerr.EINVAL - } + case "nr_blocks": + maxBlocks, percentageSpecified, err := parseSize(value) + if percentageSpecified { + ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: percentage not allowed for nr_blocks") + return nil, nil, linuxerr.EINVAL + } + if err != nil { + ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: parseSize() failed for nr_blocks: %v", err) + return nil, nil, linuxerr.EINVAL + } + maxSizeInPages = maxBlocks + printedSize := (maxSizeInPages*hostarch.PageSize + 1023) / 1024 + printedOptsMap["size"] = fmt.Sprintf("size=%vk", printedSize) + + case "nr_inodes": + var err error + var percentageSpecified bool + maxInodes, percentageSpecified, err = parseSize(value) + if percentageSpecified { + ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: percentage not allowed for nr_inodes") + return nil, nil, linuxerr.EINVAL + } + if err != nil { + ctx.Debugf("tmpfs.FilesystemType.GetFilesystem: parseSize() failed for nr_inodes: %v", err) + return nil, nil, linuxerr.EINVAL + } + printedOptsMap["nr_inodes"] = fmt.Sprintf("nr_inodes=%v", maxInodes) - printedOpts = append(printedOpts, fmt.Sprintf("nr_inodes=%v", maxInodes)) - } - modeStr, ok := mopts["mode"] - if ok { - delete(mopts, "mode") - mode, err := strconv.ParseUint(modeStr, 8, 32) - if err != nil { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: invalid mode: %q", modeStr) - return nil, nil, linuxerr.EINVAL - } - rootMode = linux.FileMode(mode & 07777) + case "mode": + mode, err := strconv.ParseUint(value, 8, 32) + if err != nil { + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: invalid mode: %q", value) + return nil, nil, linuxerr.EINVAL + } + rootMode = linux.FileMode(mode & 07777) + printedOptsMap["mode"] = fmt.Sprintf("mode=%s", value) - printedOpts = append(printedOpts, fmt.Sprintf("mode=%s", modeStr)) - } - rootKUID := creds.EffectiveKUID - uidStr, ok := mopts["uid"] - if ok { - delete(mopts, "uid") - uid, err := strconv.ParseUint(uidStr, 10, 32) - if err != nil { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: invalid uid: %q", uidStr) - return nil, nil, linuxerr.EINVAL - } - kuid := creds.UserNamespace.MapToKUID(auth.UID(uid)) - if !kuid.Ok() { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: unmapped uid: %d", uid) - return nil, nil, linuxerr.EINVAL - } - rootKUID = kuid + case "uid": + uid, err := strconv.ParseUint(value, 10, 32) + if err != nil { + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: invalid uid: %q", value) + return nil, nil, linuxerr.EINVAL + } + kuid := creds.UserNamespace.MapToKUID(auth.UID(uid)) + if !kuid.Ok() { + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: unmapped uid: %d", uid) + return nil, nil, linuxerr.EINVAL + } + rootKUID = kuid + printedOptsMap["uid"] = fmt.Sprintf("uid=%s", value) - printedOpts = append(printedOpts, fmt.Sprintf("uid=%s", uidStr)) - } - rootKGID := creds.EffectiveKGID - gidStr, ok := mopts["gid"] - if ok { - delete(mopts, "gid") - gid, err := strconv.ParseUint(gidStr, 10, 32) - if err != nil { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: invalid gid: %q", gidStr) - return nil, nil, linuxerr.EINVAL - } - kgid := creds.UserNamespace.MapToKGID(auth.GID(gid)) - if !kgid.Ok() { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: unmapped gid: %d", gid) + case "gid": + gid, err := strconv.ParseUint(value, 10, 32) + if err != nil { + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: invalid gid: %q", value) + return nil, nil, linuxerr.EINVAL + } + kgid := creds.UserNamespace.MapToKGID(auth.GID(gid)) + if !kgid.Ok() { + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: unmapped gid: %d", gid) + return nil, nil, linuxerr.EINVAL + } + rootKGID = kgid + printedOptsMap["gid"] = fmt.Sprintf("gid=%s", value) + + case "noswap": + // Accept, but ignore, noswap. + + default: + ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: unknown option: %s", key) return nil, nil, linuxerr.EINVAL } - rootKGID = kgid - - printedOpts = append(printedOpts, fmt.Sprintf("gid=%s", gidStr)) } - // Accept, but ignore, noswap - delete(mopts, "noswap") - if len(mopts) != 0 { - ctx.Warningf("tmpfs.FilesystemType.GetFilesystem: unknown options: %v", mopts) - return nil, nil, linuxerr.EINVAL + var printedOpts []string + if val, ok := printedOptsMap["size"]; ok { + printedOpts = append(printedOpts, val) + } + if val, ok := printedOptsMap["nr_inodes"]; ok { + printedOpts = append(printedOpts, val) + } + if val, ok := printedOptsMap["mode"]; ok { + printedOpts = append(printedOpts, val) + } + if val, ok := printedOptsMap["uid"]; ok { + printedOpts = append(printedOpts, val) + } + if val, ok := printedOptsMap["gid"]; ok { + printedOpts = append(printedOpts, val) } devMinor, err := vfsObj.GetAnonBlockDevMinor() diff --git a/pkg/sentry/fsimpl/tmpfs/tmpfs_test.go b/pkg/sentry/fsimpl/tmpfs/tmpfs_test.go index ca4e8b2384..7cf0def06e 100644 --- a/pkg/sentry/fsimpl/tmpfs/tmpfs_test.go +++ b/pkg/sentry/fsimpl/tmpfs/tmpfs_test.go @@ -21,9 +21,13 @@ import ( "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/atomicbitops" "gvisor.dev/gvisor/pkg/context" + "gvisor.dev/gvisor/pkg/errors/linuxerr" "gvisor.dev/gvisor/pkg/fspath" + "gvisor.dev/gvisor/pkg/hostarch" + "gvisor.dev/gvisor/pkg/sentry/contexttest" "gvisor.dev/gvisor/pkg/sentry/kernel/auth" "gvisor.dev/gvisor/pkg/sentry/vfs" + "gvisor.dev/gvisor/pkg/usermem" ) // nextFileID is used to generate unique file names. @@ -196,3 +200,167 @@ func TestParseSize(t *testing.T) { }) } } + +func TestMountOptions(t *testing.T) { + ctx := contexttest.Context(t) + creds := auth.CredentialsFromContext(ctx) + + // Ensure totalHostMem is set for default size calculation. + SetTotalHostMem(4096 * 1024 * 1024) // 4 GiB + + tests := []struct { + name string + opts string + wantBlocks uint64 + wantErr bool + }{ + { + name: "default size", + opts: "", + wantBlocks: (totalHostMem / 2) / hostarch.PageSize, + wantErr: false, + }, + { + name: "size 1M", + opts: "size=1M", + wantBlocks: (1024 * 1024) / hostarch.PageSize, + wantErr: false, + }, + { + name: "nr_blocks 10", + opts: "nr_blocks=10", + wantBlocks: 10, + wantErr: false, + }, + { + name: "nr_blocks 10, size 1M (nr_blocks wins if last)", + opts: "size=1M,nr_blocks=10", + wantBlocks: 10, + wantErr: false, + }, + { + name: "nr_blocks 10, size 1M (size wins if last)", + opts: "nr_blocks=10,size=1M", + wantBlocks: (1024 * 1024) / hostarch.PageSize, + wantErr: false, + }, + { + name: "nr_blocks percentage fails", + opts: "nr_blocks=10%", + wantErr: true, + }, + { + name: "invalid nr_blocks fails", + opts: "nr_blocks=invalid", + wantErr: true, + }, + { + name: "unknown option fails", + opts: "unknown=1", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + vfsObj := &vfs.VirtualFilesystem{} + if err := vfsObj.Init(ctx); err != nil { + t.Fatalf("VFS init: %v", err) + } + vfsObj.MustRegisterFilesystemType("tmpfs", FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ + AllowUserMount: true, + }) + + mntns, err := vfsObj.NewMountNamespace(ctx, creds, "", "tmpfs", &vfs.MountOptions{ + GetFilesystemOptions: vfs.GetFilesystemOptions{ + Data: tt.opts, + }, + }, nil) + if tt.wantErr { + if err == nil { + mntns.DecRef(ctx) + t.Errorf("expected mount to fail for opts %q", tt.opts) + } + return + } + if err != nil { + t.Fatalf("mount failed for opts %q: %v", tt.opts, err) + } + defer mntns.DecRef(ctx) + + root := mntns.Root(ctx) + defer root.DecRef(ctx) + + statfs, err := vfsObj.StatFSAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + }) + if err != nil { + t.Fatalf("StatFS failed: %v", err) + } + + if statfs.Blocks != tt.wantBlocks { + t.Errorf("got blocks %d, want %d", statfs.Blocks, tt.wantBlocks) + } + }) + } +} + +func TestMountOptionsEnforcement(t *testing.T) { + ctx := contexttest.Context(t) + creds := auth.CredentialsFromContext(ctx) + + vfsObj := &vfs.VirtualFilesystem{} + if err := vfsObj.Init(ctx); err != nil { + t.Fatalf("VFS init: %v", err) + } + + vfsObj.MustRegisterFilesystemType("tmpfs", FilesystemType{}, &vfs.RegisterFilesystemTypeOptions{ + AllowUserMount: true, + }) + + // Mount with nr_blocks=1 (which means max 1 page = 4096 bytes). + mntns, err := vfsObj.NewMountNamespace(ctx, creds, "", "tmpfs", &vfs.MountOptions{ + GetFilesystemOptions: vfs.GetFilesystemOptions{ + Data: "nr_blocks=1", + }, + }, nil) + if err != nil { + t.Fatalf("failed to create tmpfs mount: %v", err) + } + root := mntns.Root(ctx) + defer root.DecRef(ctx) + defer mntns.DecRef(ctx) + + // Create a file. + filename := "test-file" + fd, err := vfsObj.OpenAt(ctx, creds, &vfs.PathOperation{ + Root: root, + Start: root, + Path: fspath.Parse(filename), + }, &vfs.OpenOptions{ + Flags: linux.O_RDWR | linux.O_CREAT | linux.O_EXCL, + Mode: 0644, + }) + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + defer fd.DecRef(ctx) + + // Write 1 page of data. + data1 := make([]byte, hostarch.PageSize) + n1, err := fd.Write(ctx, usermem.BytesIOSequence(data1), vfs.WriteOptions{}) + if err != nil { + t.Fatalf("first write failed: %v", err) + } + if n1 != int64(len(data1)) { + t.Fatalf("first write short: got %d, want %d", n1, len(data1)) + } + + // Write 1 more byte (should fail with ENOSPC because we are at the limit). + data2 := []byte{'a'} + _, err = fd.Write(ctx, usermem.BytesIOSequence(data2), vfs.WriteOptions{}) + if !linuxerr.Equals(linuxerr.ENOSPC, err) { + t.Fatalf("second write got err %v, want ENOSPC", err) + } +} diff --git a/pkg/sentry/vfs/filesystem_impl_util.go b/pkg/sentry/vfs/filesystem_impl_util.go index b16c41fa5b..aa72496983 100644 --- a/pkg/sentry/vfs/filesystem_impl_util.go +++ b/pkg/sentry/vfs/filesystem_impl_util.go @@ -54,3 +54,26 @@ func GenericStatFS(fsMagic uint64) linux.Statfs { NameLength: linux.NAME_MAX, } } + +// KeyValuePair represents a key-value pair of mount options. +type KeyValuePair struct { + Key string + Value string +} + +// GenericParseMountOptionsOrdered parses a comma-separated list of options +// of the form "key" or "key=value", preserving the original order. +func GenericParseMountOptionsOrdered(str string) []KeyValuePair { + var options []KeyValuePair + for _, opt := range strings.Split(str, ",") { + if len(opt) > 0 { + res := strings.SplitN(opt, "=", 2) + if len(res) == 2 { + options = append(options, KeyValuePair{Key: res[0], Value: res[1]}) + } else { + options = append(options, KeyValuePair{Key: opt, Value: ""}) + } + } + } + return options +} diff --git a/test/image/image_test.go b/test/image/image_test.go index ad3878aac7..398fce72f5 100644 --- a/test/image/image_test.go +++ b/test/image/image_test.go @@ -563,14 +563,13 @@ func startDockerdInGvisor(ctx context.Context, t *testing.T, overlay bool) *dock } // Wait for the docker daemon. - for i := 0; i < 10; i++ { + cb := backoff.NewConstantBackOff(5 * time.Second) + err := backoff.Retry(func() error { _, err := d.Exec(ctx, dockerutil.ExecOpts{}, "docker", "info") - if err != nil { - t.Logf("docker exec failed: %v", err) - time.Sleep(5 * time.Second) - continue - } - break + return err + }, backoff.WithMaxRetries(cb, 10)) + if err != nil { + t.Fatalf("docker daemon failed to start: %v", err) } return d } @@ -671,15 +670,19 @@ func testDockerExec(ctx context.Context, t *testing.T, d *dockerutil.Container, } }() - for i := 0; i < 10; i++ { + cb := backoff.NewConstantBackOff(5 * time.Second) + err = backoff.Retry(func() error { inspectOutput, err := dockerInGvisorExecOutput(ctx, d, []string{"docker", "container", "inspect", containerName}) if err != nil { - t.Fatalf("docker exec failed: %v", err) + return err } - if strings.Contains(inspectOutput, "\"Status\": \"running\"") { - break + if !strings.Contains(inspectOutput, "\"Status\": \"running\"") { + return fmt.Errorf("container %s is not running yet", containerName) } - time.Sleep(5 * time.Second) + return nil + }, backoff.WithMaxRetries(cb, 10)) + if err != nil { + t.Fatalf("container failed to start: %v", err) } execCmd := []string{"docker", "exec"}