From 00160b93c79e563748dea3892360ff321df0edf5 Mon Sep 17 00:00:00 2001 From: Etienne Perot Date: Mon, 13 Jul 2026 19:44:17 -0700 Subject: [PATCH] Shave ~150ms off gVisor startup by making host membarrier lazy + async. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Host membarrier support lookup and registation used to run as part of `init` for the `hostmm` package. Due to `runsc` being a monolithic binary, this executes unconditionally several times during the sandbox startup process. This change skips this process for all `runsc` invocations that don't need it by moving it to the Platform constructors, and makes initialization itself async because the `MEMBARRIER_CMD_PRIVATE_EXPEDITED` probe can take tens of milliseconds by itself. Benchmarks: ``` # Plain Docker, Systrap in a VM, running just # `docker run --runtime=runsc --rm alpine true`: │ before │ after │ │ sec │ sec vs base │ DockerRunBinTrue 836.5m ± 4% 674.6m ± 4% -19.35% (p=0.000 n=16) # Kubernetes with containerd, measuring time-to-first-log latency # minus Kubernetes scheduling overhead: │ before │ after │ │ totalruntimeinit-min-sec │ totalruntimeinit-min-sec vs base │ StartUp 1083.7m ± 2% 943.9m ± 3% -12.91% (p=0.000 n=16) │ before │ after │ │ before │ after │ │ totalruntimeinit-p50-sec │ totalruntimeinit-p50-sec vs base │ StartUp 1083.7m ± 2% 943.9m ± 3% -12.91% (p=0.000 n=16) │ before │ after │ │ totalruntimeinit-p95-sec │ totalruntimeinit-p95-sec vs base │ StartUp 1083.7m ± 2% 943.9m ± 3% -12.91% (p=0.000 n=16) ``` PiperOrigin-RevId: 947382582 --- pkg/sentry/hostmm/membarrier.go | 106 ++++++++++++++++--------- pkg/sentry/platform/kvm/BUILD | 1 + pkg/sentry/platform/kvm/kvm.go | 8 +- pkg/sentry/platform/platform.go | 44 +++++----- pkg/sentry/platform/ptrace/BUILD | 1 + pkg/sentry/platform/ptrace/ptrace.go | 7 +- pkg/sentry/platform/slimvm/BUILD | 1 + pkg/sentry/platform/slimvm/slimvm.go | 7 +- pkg/sentry/platform/systrap/BUILD | 1 + pkg/sentry/platform/systrap/systrap.go | 8 +- 10 files changed, 115 insertions(+), 69 deletions(-) diff --git a/pkg/sentry/hostmm/membarrier.go b/pkg/sentry/hostmm/membarrier.go index e4e61a3981..e164fd8c31 100644 --- a/pkg/sentry/hostmm/membarrier.go +++ b/pkg/sentry/hostmm/membarrier.go @@ -20,41 +20,26 @@ import ( "gvisor.dev/gvisor/pkg/log" ) -var ( - haveMembarrierGlobal = false - haveMembarrierPrivateExpedited = false -) +// HostMemBarrier provides access to the host membarrier(2) operations that the +// calling process has been verified to support. It is obtained from Probe. +type HostMemBarrier struct { + // global is whether the host supports MEMBARRIER_CMD_GLOBAL. + global bool -func init() { - supported, _, e := unix.RawSyscall(unix.SYS_MEMBARRIER, linux.MEMBARRIER_CMD_QUERY, 0 /* flags */, 0 /* unused */) - if e != 0 { - if e != unix.ENOSYS { - log.Warningf("membarrier(MEMBARRIER_CMD_QUERY) failed: %s", e.Error()) - } - return - } - // We don't use MEMBARRIER_CMD_GLOBAL_EXPEDITED because this sends IPIs to - // all CPUs running tasks that have previously invoked - // MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED, which presents a DOS risk. - // (MEMBARRIER_CMD_GLOBAL is synchronize_rcu(), i.e. it waits for an RCU - // grace period to elapse without bothering other CPUs. - // MEMBARRIER_CMD_PRIVATE_EXPEDITED sends IPIs only to CPUs running tasks - // sharing the caller's MM.) - if supported&linux.MEMBARRIER_CMD_GLOBAL != 0 { - haveMembarrierGlobal = true - } - if req := uintptr(linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED | linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED); supported&req == req { - if _, _, e := unix.RawSyscall(unix.SYS_MEMBARRIER, linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0 /* flags */, 0 /* unused */); e != 0 { - log.Warningf("membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED) failed: %s", e.Error()) - } else { - haveMembarrierPrivateExpedited = true - } - } + // privateExpedited is true if we registered for + // `MEMBARRIER_CMD_PRIVATE_EXPEDITED`. + privateExpedited bool } // HaveGlobalMemoryBarrier returns true if GlobalMemoryBarrier is supported. -func HaveGlobalMemoryBarrier() bool { - return haveMembarrierGlobal +func (h HostMemBarrier) HaveGlobalMemoryBarrier() bool { + return h.global +} + +// HaveProcessMemoryBarrier returns true if ProcessMemoryBarrier is supported +// and registration succeeded. +func (h HostMemBarrier) HaveProcessMemoryBarrier() bool { + return h.privateExpedited } // GlobalMemoryBarrier blocks until "all running threads [in the host OS] have @@ -63,26 +48,71 @@ func HaveGlobalMemoryBarrier() bool { // as for membarrier(2). // // Preconditions: HaveGlobalMemoryBarrier() == true. -func GlobalMemoryBarrier() error { +func (h HostMemBarrier) GlobalMemoryBarrier() error { + if !h.global { + panic("hostmm: GlobalMemoryBarrier called, but host does not support it") + } if _, _, e := unix.Syscall(unix.SYS_MEMBARRIER, linux.MEMBARRIER_CMD_GLOBAL, 0 /* flags */, 0 /* unused */); e != 0 { return e } return nil } -// HaveProcessMemoryBarrier returns true if ProcessMemoryBarrier is supported. -func HaveProcessMemoryBarrier() bool { - return haveMembarrierPrivateExpedited -} - // ProcessMemoryBarrier is equivalent to GlobalMemoryBarrier, but only // synchronizes with threads sharing a virtual address space (from the host OS' // perspective) with the calling thread. // // Preconditions: HaveProcessMemoryBarrier() == true. -func ProcessMemoryBarrier() error { +func (h HostMemBarrier) ProcessMemoryBarrier() error { + if !h.privateExpedited { + panic("hostmm: ProcessMemoryBarrier called unexpectedly") + } if _, _, e := unix.RawSyscall(unix.SYS_MEMBARRIER, linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0 /* flags */, 0 /* unused */); e != 0 { return e } return nil } + +// Probe asynchronously determines host `membarrier(2)` support and, if +// `probePrivateExpedited` is true and the host supports it, registers for +// MEMBARRIER_CMD_PRIVATE_EXPEDITED. +// Returns a channel on which the resulting HostMemBarrier is sent back. +// This is meant to be used in Platform implementation constructors, which +// mus run before seccomp filters installation. +// Runs in the background to allow further platform initialization to proceed +// while we determine this, which can take tens of milliseconds. +func Probe(probePrivateExpedited bool) <-chan HostMemBarrier { + ch := make(chan HostMemBarrier, 1) + go func() { + ch <- probe(probePrivateExpedited) + close(ch) + }() + return ch +} + +func probe(probePrivateExpedited bool) HostMemBarrier { + var mb HostMemBarrier + supported, _, e := unix.RawSyscall(unix.SYS_MEMBARRIER, linux.MEMBARRIER_CMD_QUERY, 0 /* flags */, 0 /* unused */) + if e != 0 { + if e != unix.ENOSYS { + log.Warningf("membarrier(MEMBARRIER_CMD_QUERY) failed: %s", e.Error()) + } + return mb + } + if supported&linux.MEMBARRIER_CMD_GLOBAL != 0 { + mb.global = true + } + if !probePrivateExpedited { + return mb + } + // Registering a process for private-expedited membarrier blocks on an RCU + // grace period (tens of ms), so only do it if needed. + if req := uintptr(linux.MEMBARRIER_CMD_PRIVATE_EXPEDITED | linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED); supported&req == req { + if _, _, e := unix.RawSyscall(unix.SYS_MEMBARRIER, linux.MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0 /* flags */, 0 /* unused */); e != 0 { + log.Warningf("membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED) failed: %s", e.Error()) + } else { + mb.privateExpedited = true + } + } + return mb +} diff --git a/pkg/sentry/platform/kvm/BUILD b/pkg/sentry/platform/kvm/BUILD index e38d21efd0..e89cc68071 100644 --- a/pkg/sentry/platform/kvm/BUILD +++ b/pkg/sentry/platform/kvm/BUILD @@ -91,6 +91,7 @@ go_library( "//pkg/seccomp", "//pkg/sentry/arch", "//pkg/sentry/arch/fpu", + "//pkg/sentry/hostmm", "//pkg/sentry/memmap", "//pkg/sentry/platform", "//pkg/sentry/platform/interrupt", diff --git a/pkg/sentry/platform/kvm/kvm.go b/pkg/sentry/platform/kvm/kvm.go index d9cce0cd66..e1afc67f06 100644 --- a/pkg/sentry/platform/kvm/kvm.go +++ b/pkg/sentry/platform/kvm/kvm.go @@ -25,6 +25,7 @@ import ( "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/ring0/pagetables" + "gvisor.dev/gvisor/pkg/sentry/hostmm" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sync" ) @@ -93,7 +94,7 @@ func New(deviceFile *fd.FD, config Config) (*KVM, error) { if hostarch.PageSize != 4096 { return nil, fmt.Errorf("KVM platform does not support %dK page size", hostarch.PageSize/1024) } - + mbCh := hostmm.Probe(true) fd := deviceFile.FD() // Ensure global initialization is done. @@ -128,10 +129,9 @@ func New(deviceFile *fd.FD, config Config) (*KVM, error) { if err != nil { return nil, err } - - // All set. return &KVM{ - machine: machine, + UseHostProcessMemoryBarrier: platform.UseHostProcessMemoryBarrier{MemBarrier: <-mbCh}, + machine: machine, }, nil } diff --git a/pkg/sentry/platform/platform.go b/pkg/sentry/platform/platform.go index d99903fbc7..507e271154 100644 --- a/pkg/sentry/platform/platform.go +++ b/pkg/sentry/platform/platform.go @@ -151,40 +151,44 @@ func (NoCPUNumbers) PreemptCPU(int32) error { } // UseHostGlobalMemoryBarrier implements Platform.HaveGlobalMemoryBarrier and -// Platform.GlobalMemoryBarrier by invoking equivalent functionality on the -// host. -type UseHostGlobalMemoryBarrier struct{} +// Platform.GlobalMemoryBarrier by invoking the host global memory barrier. +// Platforms must populate `MemBarrier` from `<-hostmm.Probe(false)`. +type UseHostGlobalMemoryBarrier struct { + MemBarrier hostmm.HostMemBarrier +} // HaveGlobalMemoryBarrier implements Platform.HaveGlobalMemoryBarrier. -func (UseHostGlobalMemoryBarrier) HaveGlobalMemoryBarrier() bool { - return hostmm.HaveGlobalMemoryBarrier() +func (u UseHostGlobalMemoryBarrier) HaveGlobalMemoryBarrier() bool { + return u.MemBarrier.HaveGlobalMemoryBarrier() } // GlobalMemoryBarrier implements Platform.GlobalMemoryBarrier. -func (UseHostGlobalMemoryBarrier) GlobalMemoryBarrier() error { - return hostmm.GlobalMemoryBarrier() +func (u UseHostGlobalMemoryBarrier) GlobalMemoryBarrier() error { + return u.MemBarrier.GlobalMemoryBarrier() } // UseHostProcessMemoryBarrier implements Platform.HaveGlobalMemoryBarrier and -// Platform.GlobalMemoryBarrier by invoking a process-local memory barrier. -// This is faster than UseHostGlobalMemoryBarrier, but is only appropriate for -// platforms for which application code executes while using the sentry's -// mm_struct. -type UseHostProcessMemoryBarrier struct{} +// Platform.GlobalMemoryBarrier by preferring a process-local (private-expedited) +// memory barrier, falling back to host global otherwise. +// This is faster than UseHostGlobalMemoryBarrier, but only appropriate for +// platforms for which application code executes while using the +// sentry's mm_struct. +// Platforms must populate `MemBarrier` from `<-hostmm.Probe(true)`. +type UseHostProcessMemoryBarrier struct { + MemBarrier hostmm.HostMemBarrier +} // HaveGlobalMemoryBarrier implements Platform.HaveGlobalMemoryBarrier. -func (UseHostProcessMemoryBarrier) HaveGlobalMemoryBarrier() bool { - // Fall back to a global memory barrier if a process-local one isn't - // available. - return hostmm.HaveProcessMemoryBarrier() || hostmm.HaveGlobalMemoryBarrier() +func (u UseHostProcessMemoryBarrier) HaveGlobalMemoryBarrier() bool { + return u.MemBarrier.HaveProcessMemoryBarrier() || u.MemBarrier.HaveGlobalMemoryBarrier() } // GlobalMemoryBarrier implements Platform.GlobalMemoryBarrier. -func (UseHostProcessMemoryBarrier) GlobalMemoryBarrier() error { - if hostmm.HaveProcessMemoryBarrier() { - return hostmm.ProcessMemoryBarrier() +func (u UseHostProcessMemoryBarrier) GlobalMemoryBarrier() error { + if u.MemBarrier.HaveProcessMemoryBarrier() { + return u.MemBarrier.ProcessMemoryBarrier() } - return hostmm.GlobalMemoryBarrier() + return u.MemBarrier.GlobalMemoryBarrier() } // MemoryManager represents an abstraction above the platform address space diff --git a/pkg/sentry/platform/ptrace/BUILD b/pkg/sentry/platform/ptrace/BUILD index 38606790e8..17ca8d6080 100644 --- a/pkg/sentry/platform/ptrace/BUILD +++ b/pkg/sentry/platform/ptrace/BUILD @@ -38,6 +38,7 @@ go_library( "//pkg/seccomp", "//pkg/sentry/arch", "//pkg/sentry/arch/fpu", + "//pkg/sentry/hostmm", "//pkg/sentry/memmap", "//pkg/sentry/platform", "//pkg/sentry/platform/interrupt", diff --git a/pkg/sentry/platform/ptrace/ptrace.go b/pkg/sentry/platform/ptrace/ptrace.go index b066544ba8..b9a944bb04 100644 --- a/pkg/sentry/platform/ptrace/ptrace.go +++ b/pkg/sentry/platform/ptrace/ptrace.go @@ -52,6 +52,7 @@ import ( "gvisor.dev/gvisor/pkg/fd" "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/sentry/arch" + "gvisor.dev/gvisor/pkg/sentry/hostmm" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/platform/interrupt" "gvisor.dev/gvisor/pkg/sync" @@ -233,6 +234,7 @@ type PTrace struct { // New returns a new ptrace-based implementation of the platform interface. func New() (*PTrace, error) { + mbCh := hostmm.Probe(false) stubInitialized.Do(func() { // Initialize the stub. stubInit() @@ -248,8 +250,9 @@ func New() (*PTrace, error) { // Set the master on the globalPool. globalPool.master = master }) - - return &PTrace{}, nil + return &PTrace{ + UseHostGlobalMemoryBarrier: platform.UseHostGlobalMemoryBarrier{MemBarrier: <-mbCh}, + }, nil } // SupportsAddressSpaceIO implements platform.Platform.SupportsAddressSpaceIO. diff --git a/pkg/sentry/platform/slimvm/BUILD b/pkg/sentry/platform/slimvm/BUILD index 6718c9586e..008394a46d 100644 --- a/pkg/sentry/platform/slimvm/BUILD +++ b/pkg/sentry/platform/slimvm/BUILD @@ -55,6 +55,7 @@ go_library( "//pkg/seccomp", "//pkg/sentry/arch", "//pkg/sentry/arch/fpu", + "//pkg/sentry/hostmm", "//pkg/sentry/memmap", "//pkg/sentry/platform", "//pkg/sentry/platform/interrupt", diff --git a/pkg/sentry/platform/slimvm/slimvm.go b/pkg/sentry/platform/slimvm/slimvm.go index a132171a25..f851aa94ac 100644 --- a/pkg/sentry/platform/slimvm/slimvm.go +++ b/pkg/sentry/platform/slimvm/slimvm.go @@ -26,6 +26,7 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/ring0" "gvisor.dev/gvisor/pkg/ring0/pagetables" + "gvisor.dev/gvisor/pkg/sentry/hostmm" "gvisor.dev/gvisor/pkg/sentry/platform" ) @@ -65,9 +66,9 @@ func OpenDevice(devicePath string) (*fd.FD, error) { // New returns a new SlimVM-based implementation of the platform interface. func New(deviceFile *fd.FD, sandboxID string, applicationCores int) (*SlimVM, error) { + mbCh := hostmm.Probe(true) slimvmFile = deviceFile slimvmFD = uintptr(slimvmFile.FD()) - // Ensure global initialization is done. globalOnce.Do(func() { updateGlobalOnce(int(slimvmFD)) @@ -85,9 +86,9 @@ func New(deviceFile *fd.FD, sandboxID string, applicationCores int) (*SlimVM, er return nil, err } - // All set. return &SlimVM{ - machine: machine, + UseHostProcessMemoryBarrier: platform.UseHostProcessMemoryBarrier{MemBarrier: <-mbCh}, + machine: machine, }, nil } diff --git a/pkg/sentry/platform/systrap/BUILD b/pkg/sentry/platform/systrap/BUILD index bd2553c4ba..7aeffda1d0 100644 --- a/pkg/sentry/platform/systrap/BUILD +++ b/pkg/sentry/platform/systrap/BUILD @@ -93,6 +93,7 @@ go_library( "//pkg/seccomp", "//pkg/seccomp/precompiledseccomp", "//pkg/sentry/arch", + "//pkg/sentry/hostmm", "//pkg/sentry/memmap", "//pkg/sentry/pgalloc", "//pkg/sentry/platform", diff --git a/pkg/sentry/platform/systrap/systrap.go b/pkg/sentry/platform/systrap/systrap.go index c80f9d4265..351edddc75 100644 --- a/pkg/sentry/platform/systrap/systrap.go +++ b/pkg/sentry/platform/systrap/systrap.go @@ -61,6 +61,7 @@ import ( "gvisor.dev/gvisor/pkg/hostarch" "gvisor.dev/gvisor/pkg/memutil" "gvisor.dev/gvisor/pkg/sentry/arch" + "gvisor.dev/gvisor/pkg/sentry/hostmm" "gvisor.dev/gvisor/pkg/sentry/pgalloc" "gvisor.dev/gvisor/pkg/sentry/platform" "gvisor.dev/gvisor/pkg/sentry/platform/interrupt" @@ -271,7 +272,7 @@ func (*Systrap) MinUserAddress() hostarch.Addr { // New returns a new seccomp-based implementation of the platform interface. func New(opts platform.Options) (*Systrap, error) { - + mbCh := hostmm.Probe(false) if !disableSyscallPatching { disableSyscallPatching = opts.DisableSyscallPatching } @@ -334,7 +335,10 @@ func New(opts platform.Options) (*Systrap, error) { }) } - return &Systrap{memoryFile: mf}, nil + return &Systrap{ + UseHostGlobalMemoryBarrier: platform.UseHostGlobalMemoryBarrier{MemBarrier: <-mbCh}, + memoryFile: mf, + }, nil } // SupportsAddressSpaceIO implements platform.Platform.SupportsAddressSpaceIO.