support runsc container-less sandbox mode

pkg/shim/v1/runsc/BUILD

@@ -38,6 +38,7 @@ go_library(
         "@com_github_containerd_containerd_api//events:go_default_library",
         "@com_github_containerd_containerd_api//runtime/sandbox/v1:go_default_library",
         "@com_github_containerd_containerd_api//runtime/task/v2:go_default_library",
+        "@com_github_containerd_containerd_api//types:go_default_library",
         "@com_github_containerd_containerd_api//types/runc/options:go_default_library",
         "@com_github_containerd_containerd_api//types/task:go_default_library",
         "@com_github_containerd_containerd_v2//cmd/containerd-shim-runc-v2/process:go_default_library",
@@ -59,6 +60,7 @@ go_library(
         "@com_github_containerd_typeurl_v2//:go_default_library",
         "@com_github_opencontainers_runtime_spec//specs-go:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@org_golang_google_protobuf//types/known/anypb:go_default_library",
         "@org_golang_x_sys//unix:go_default_library",
     ],
 )

pkg/shim/v1/runsc/container.go

@@ -28,6 +28,7 @@ import (
 	cgroupsv2stats "github.com/containerd/cgroups/v3/cgroup2/stats"
 	"github.com/containerd/console"
 	task "github.com/containerd/containerd/api/runtime/task/v2"
+	types "github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/v2/core/mount"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/stdio"
@@ -37,6 +38,7 @@ import (
 	"github.com/containerd/log"
 	typeurl "github.com/containerd/typeurl/v2"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/protobuf/types/known/anypb"
 	"gvisor.dev/gvisor/pkg/cleanup"
 	"gvisor.dev/gvisor/pkg/shim/v1/extension"
 	"gvisor.dev/gvisor/pkg/shim/v1/proc"
@@ -71,16 +73,30 @@ type Container struct {
 	cgroup CgroupMode
 }
 
+// ContainerConfig contains configuration for creating a container.
+type ContainerConfig struct {
+	ID                 string
+	Bundle             string
+	Rootfs             []*types.Mount
+	Options            *anypb.Any
+	Terminal           bool
+	Stdin              string
+	Stdout             string
+	Stderr             string
+	FSRestoreImagePath string
+	FSRestoreDirect    bool
+}
+
 // NewContainer returns a new runsc container
-func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTaskRequest, FSRestoreImagePath string, FSRestoreDirect bool) (*Container, error) {
+func NewContainer(ctx context.Context, platform stdio.Platform, conf *ContainerConfig) (*Container, error) {
 	ns, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("create namespace: %w", err)
 	}
 	var opts Options
-	if r.Options != nil {
+	if conf.Options != nil {
 		runtimeOptions := &runtimeoptions.Options{}
-		if err := typeurl.UnmarshalTo(r.Options, runtimeOptions); err != nil {
+		if err := typeurl.UnmarshalTo(conf.Options, runtimeOptions); err != nil {
 			return nil, fmt.Errorf("unmarshal runtime options: %w", err)
 		}
 
@@ -101,7 +117,7 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 		logrus.SetLevel(lvl)
 	}
 	if len(opts.LogPath) != 0 {
-		logPath := runsccmd.FormatShimLogPath(opts.LogPath, r.ID)
+		logPath := runsccmd.FormatShimLogPath(opts.LogPath, conf.ID)
 		if err := os.MkdirAll(filepath.Dir(logPath), 0777); err != nil {
 			return nil, fmt.Errorf("failed to create log dir: %w", err)
 		}
@@ -116,13 +132,13 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 		log.L.Debugf("***************************")
 		log.L.Debugf("Args: %s", os.Args)
 		log.L.Debugf("PID: %d", os.Getpid())
-		log.L.Debugf("ID: %s", r.ID)
+		log.L.Debugf("ID: %s", conf.ID)
 		log.L.Debugf("Options: %+v", opts)
-		log.L.Debugf("Bundle: %s", r.Bundle)
-		log.L.Debugf("Terminal: %t", r.Terminal)
-		log.L.Debugf("stdin: %s", r.Stdin)
-		log.L.Debugf("stdout: %s", r.Stdout)
-		log.L.Debugf("stderr: %s", r.Stderr)
+		log.L.Debugf("Bundle: %s", conf.Bundle)
+		log.L.Debugf("Terminal: %t", conf.Terminal)
+		log.L.Debugf("stdin: %s", conf.Stdin)
+		log.L.Debugf("stdout: %s", conf.Stdout)
+		log.L.Debugf("stderr: %s", conf.Stderr)
 		log.L.Debugf("***************************")
 		if log.L.Logger.IsLevelEnabled(logrus.DebugLevel) {
 			setDebugSigHandler()
@@ -132,10 +148,10 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 	// Save state before any action is taken to ensure Cleanup() will have all
 	// the information it needs to undo the operations.
 	st := State{
-		Rootfs:  filepath.Join(r.Bundle, "rootfs"),
+		Rootfs:  filepath.Join(conf.Bundle, "rootfs"),
 		Options: opts,
 	}
-	if err := st.Save(r.Bundle); err != nil {
+	if err := st.Save(conf.Bundle); err != nil {
 		return nil, err
 	}
 
@@ -145,7 +161,7 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 
 	// Convert from types.Mount to proc.Mount.
 	var mounts []proc.Mount
-	for _, m := range r.Rootfs {
+	for _, m := range conf.Rootfs {
 		mounts = append(mounts, proc.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
@@ -174,19 +190,19 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 	}
 
 	config := &proc.CreateConfig{
-		ID:                 r.ID,
-		Bundle:             r.Bundle,
+		ID:                 conf.ID,
+		Bundle:             conf.Bundle,
 		Runtime:            opts.BinaryName,
 		Rootfs:             mounts,
-		Terminal:           r.Terminal,
-		Stdin:              r.Stdin,
-		Stdout:             r.Stdout,
-		Stderr:             r.Stderr,
-		FSRestoreImagePath: FSRestoreImagePath,
-		FSRestoreDirect:    FSRestoreDirect,
+		Terminal:           conf.Terminal,
+		Stdin:              conf.Stdin,
+		Stdout:             conf.Stdout,
+		Stderr:             conf.Stderr,
+		FSRestoreImagePath: conf.FSRestoreImagePath,
+		FSRestoreDirect:    conf.FSRestoreDirect,
 	}
 
-	process, err := newInit(filepath.Join(r.Bundle, "work"), ns, platform, config, &opts, st.Rootfs)
+	process, err := newInit(filepath.Join(conf.Bundle, "work"), ns, platform, config, &opts, st.Rootfs)
 	if err != nil {
 		return nil, err
 	}
@@ -202,8 +218,8 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 	// Success
 	cu.Release()
 	c := Container{
-		ID:        r.ID,
-		Bundle:    r.Bundle,
+		ID:        conf.ID,
+		Bundle:    conf.Bundle,
 		task:      process,
 		cgroup:    cgroupMode,
 		processes: make(map[string]extension.Process),

pkg/shim/v1/runsc/service.go

@@ -188,7 +188,18 @@ func (s *runscService) CreateWithFSRestore(ctx context.Context, rfs *extension.C
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	c, err := NewContainer(ctx, s.platform, rfs.Create, rfs.Conf.ImagePath, rfs.Conf.Direct)
+	c, err := NewContainer(ctx, s.platform, &ContainerConfig{
+		ID:                 rfs.Create.ID,
+		Bundle:             rfs.Create.Bundle,
+		Rootfs:             rfs.Create.Rootfs,
+		Options:            rfs.Create.Options,
+		Terminal:           rfs.Create.Terminal,
+		Stdin:              rfs.Create.Stdin,
+		Stdout:             rfs.Create.Stdout,
+		Stderr:             rfs.Create.Stderr,
+		FSRestoreImagePath: rfs.Conf.ImagePath,
+		FSRestoreDirect:    rfs.Conf.Direct,
+	})
 	if err != nil {
 		return nil, err
 	}

runsc/boot/controller.go

@@ -235,7 +235,10 @@ func (c *controller) registerHandlers() {
 	c.srv.Register(c.manager)
 	c.srv.Register(&control.Cgroups{Kernel: l.k})
 	c.srv.Register(&control.Fs{Kernel: l.k})
-	c.srv.Register(&control.Lifecycle{Kernel: l.k})
+	c.srv.Register(&control.Lifecycle{
+		Kernel:     l.k,
+		ShutdownCh: l.sandboxShutdownCh,
+	})
 	c.srv.Register(&control.Logging{})
 	c.srv.Register(&control.Proc{Kernel: l.k})
 	c.srv.Register(&control.State{Kernel: l.k})

runsc/boot/loader.go

@@ -254,6 +254,9 @@ type Loader struct {
 
 	hostTHP HostTHP
 
+	// sandboxShutdownCh is closed when the sandbox is requested to shut down.
+	sandboxShutdownCh chan struct{}
+
 	// mu guards the fields below.
 	mu sync.Mutex
 
@@ -542,6 +545,7 @@ func New(args Args) (*Loader, error) {
 		saveCheckpointGofer:   args.SaveCheckpointGofer,
 		fsSaveFDs:             args.FSSaveFDs,
 		fsSaveCheckpointGofer: args.FSSaveCheckpointGofer,
+		sandboxShutdownCh:     make(chan struct{}),
 	}
 
 	if args.NumCPU == 0 {
@@ -1131,68 +1135,72 @@ func (l *Loader) run() error {
 			return err
 		}
 
-		// Create the root container init task. It will begin running
-		// when the kernel is started.
-		var (
-			tg  *kernel.ThreadGroup
-			err error
-		)
-		tg, ep.tty, err = l.createContainerProcess(&l.root)
-		if err != nil {
-			return err
-		}
-
-		if seccheck.Global.Enabled(seccheck.PointContainerStart) {
-			evt := pb.Start{
-				Id:       l.sandboxID,
-				Cwd:      l.root.spec.Process.Cwd,
-				Args:     l.root.spec.Process.Args,
-				Terminal: l.root.spec.Process.Terminal,
-			}
-			fields := seccheck.Global.GetFieldSet(seccheck.PointContainerStart)
-			if fields.Local.Contains(seccheck.FieldContainerStartEnv) {
-				evt.Env = l.root.spec.Process.Env
+		if !l.root.conf.Sandbox {
+			// Create the root container init task. It will begin running
+			// when the kernel is started.
+			var (
+				tg  *kernel.ThreadGroup
+				err error
+			)
+			tg, ep.tty, err = l.createContainerProcess(&l.root)
+			if err != nil {
+				return err
 			}
-			if !fields.Context.Empty() {
-				evt.ContextData = &pb.ContextData{}
-				kernel.LoadSeccheckData(tg.Leader(), fields.Context, evt.ContextData)
+
+			if seccheck.Global.Enabled(seccheck.PointContainerStart) {
+				evt := pb.Start{
+					Id:       l.sandboxID,
+					Cwd:      l.root.spec.Process.Cwd,
+					Args:     l.root.spec.Process.Args,
+					Terminal: l.root.spec.Process.Terminal,
+				}
+				fields := seccheck.Global.GetFieldSet(seccheck.PointContainerStart)
+				if fields.Local.Contains(seccheck.FieldContainerStartEnv) {
+					evt.Env = l.root.spec.Process.Env
+				}
+				if !fields.Context.Empty() {
+					evt.ContextData = &pb.ContextData{}
+					kernel.LoadSeccheckData(tg.Leader(), fields.Context, evt.ContextData)
+				}
+				_ = seccheck.Global.SentToSinks(func(c seccheck.Sink) error {
+					return c.ContainerStart(context.Background(), fields, &evt)
+				})
 			}
-			_ = seccheck.Global.SentToSinks(func(c seccheck.Sink) error {
-				return c.ContainerStart(context.Background(), fields, &evt)
-			})
 		}
 	case restoringUnstarted:
 		// If we are restoring, we do not want to create a process.
 	default:
 		return fmt.Errorf("Loader.Run() called in unexpected state=%s", l.state)
 	}
 
-	ep.tg = l.k.GlobalInit()
-	if ns, ok := specutils.GetNS(specs.PIDNamespace, l.root.spec); ok {
-		ep.pidnsPath = ns.Path
-	}
-
-	// Handle signals by forwarding them to the root container process
-	// (except for panic signal, which should cause a panic).
-	l.stopSignalForwarding = sighandling.StartSignalForwarding(func(sig linux.Signal) {
-		// Panic signal should cause a panic.
-		if l.root.conf.PanicSignal != -1 && sig == linux.Signal(l.root.conf.PanicSignal) {
-			panic("Signal-induced panic")
+	if !l.root.conf.Sandbox {
+		ep.tg = l.k.GlobalInit()
+		if ns, ok := specutils.GetNS(specs.PIDNamespace, l.root.spec); ok {
+			ep.pidnsPath = ns.Path
 		}
 
-		// Otherwise forward to root container.
-		deliveryMode := DeliverToProcess
-		if l.root.spec.Process.Terminal {
-			// Since we are running with a console, we should forward the signal to
-			// the foreground process group so that job control signals like ^C can
-			// be handled properly.
-			deliveryMode = DeliverToForegroundProcessGroup
-		}
-		log.Infof("Received external signal %d, mode: %s", sig, deliveryMode)
-		if err := l.signal(l.sandboxID, 0, int32(sig), deliveryMode); err != nil {
-			log.Warningf("error sending signal %d to container %q: %s", sig, l.sandboxID, err)
-		}
-	})
+		// Handle signals by forwarding them to the root container process
+		// (except for panic signal, which should cause a panic).
+		l.stopSignalForwarding = sighandling.StartSignalForwarding(func(sig linux.Signal) {
+			// Panic signal should cause a panic.
+			if l.root.conf.PanicSignal != -1 && sig == linux.Signal(l.root.conf.PanicSignal) {
+				panic("Signal-induced panic")
+			}
+
+			// Otherwise forward to root container.
+			deliveryMode := DeliverToProcess
+			if l.root.spec.Process.Terminal {
+				// Since we are running with a console, we should forward the signal to
+				// the foreground process group so that job control signals like ^C can
+				// be handled properly.
+				deliveryMode = DeliverToForegroundProcessGroup
+			}
+			log.Infof("Received external signal %d, mode: %s", sig, deliveryMode)
+			if err := l.signal(l.sandboxID, 0, int32(sig), deliveryMode); err != nil {
+				log.Warningf("error sending signal %d to container %q: %s", sig, l.sandboxID, err)
+			}
+		})
+	}
 
 	log.Infof("Process should have started...")
 	l.watchdog.Start()
@@ -1717,6 +1725,13 @@ func (l *Loader) WaitForStartSignal() {
 
 // WaitExit waits for the root container to exit, and returns its exit status.
 func (l *Loader) WaitExit() linux.WaitStatus {
+	if l.root.conf.Sandbox {
+		<-l.sandboxShutdownCh
+		l.k.Kill(linux.WaitStatus(0))
+		l.k.WaitExited()
+		return linux.WaitStatus(0)
+	}
+
 	// Wait for container.
 	l.k.WaitExited()
 

runsc/cmd/create.go

@@ -92,6 +92,10 @@ func (c *Create) FetchSpec(conf *config.Config, f *flag.FlagSet) (string, *specs
 		return "", nil, fmt.Errorf("a container id is required")
 	}
 	cid := f.Arg(0)
+	if conf.Sandbox {
+		c.spec = specutils.DefaultSandboxSpec(cid)
+		return cid, c.spec, nil
+	}
 	if c.spec != nil {
 		return cid, c.spec, nil
 	}
@@ -115,7 +119,7 @@ func (c *Create) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcom
 
 	conf := args[0].(*config.Config)
 
-	if conf.Rootless {
+	if conf.Rootless && !conf.Sandbox {
 		return util.Errorf("Rootless mode not supported with %q", c.Name())
 	}
 

runsc/config/config.go

@@ -99,6 +99,10 @@ type Config struct {
 	// will be backed by application memory.
 	Overlay bool `flag:"overlay"`
 
+	// Sandbox indicates that we are creating/starting a sandbox without a
+	// root container.
+	Sandbox bool `flag:"sandbox"`
+
 	// Overlay2 holds configuration about wrapping mounts in overlayfs.
 	// DO NOT call it directly, use GetOverlay2() instead.
 	Overlay2 Overlay2 `flag:"overlay2"`

runsc/config/flags.go

@@ -110,6 +110,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.String("profile-mutex", "", "collects a mutex profile to this file path for the duration of the container execution. Requires -profile=true.")
 	flagSet.String("trace", "", "collects a Go runtime execution trace to this file path for the duration of the container execution.")
 	flagSet.Bool("rootless", false, "it allows the sandbox to be started with a user that is not root. Sandbox and Gofer processes may run with same privileges as current user.")
+	flagSet.Bool("sandbox", false, "create/start a sandbox without a root container")
 	flagSet.Var(leakModePtr(refs.NoLeakChecking), "ref-leak-mode", "sets reference leak check mode: disabled (default), log-names, log-traces.")
 	flagSet.Bool("cpu-num-from-quota", true, "set cpu number to cpu quota (least integer greater or equal to quota value, but not less than 2)")
 	flagSet.Bool(flagOCISeccomp, false, "Enables loading OCI seccomp filters inside the sandbox.")