From f41356ed42427aa24098004d7db671d57aff2c0e Mon Sep 17 00:00:00 2001 From: Luis Capelo Date: Wed, 8 Jul 2026 00:38:45 +0000 Subject: [PATCH] Add CUDA GPU coredump support to nvproxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the five missing `ioctls` required to generate [CUDA GPU coredumps](https://docs.nvidia.com/cuda/cuda-gdb/index.html#gpu-core-dump-support) (`CUDA_ENABLE_COREDUMP_ON_EXCEPTION=1`) under nvproxy, all on the `GT200_DEBUGGER` (`0x83de`) class: - `NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG` (`0x83de0307`) control command — enables MMU debug mode during exception handling. - `NV83DE_CTRL_CMD_DEBUG_READ_MEMORY` (`0x83de0315`) control command, with its `NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS` struct — reads a block of memory from the debugged context into an embedded pointer (`buffer`), which requires a custom handler. Note that the driver passes `RMAPI_PARAM_COPY_FLAGS_DISABLE_MAX_SIZE_CHECK` for this control (see `src/nvidia/src/kernel/rmapi/embedded_param_copy.c`), so unlike most embedded pointers the buffer is *not* bounded by `RMAPI_PARAM_COPY_MAX_PARAMS_SIZE`; the handler mirrors this. - `NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT` (`0x83de0317`) control command — suspends the faulting SM context so its state can be captured. - `NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT` (`0x83de0318`) control command — resumes the context after capture. No params. - `NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG` (`0x83de031f`) control command — enables error-barrier debug mode (Volta+), a prerequisite for capturing SM exception state. All are simple pass-through controls except `DEBUG_READ_MEMORY`. They are gated on `compute,utility`, consistent with the existing `NV83DE` debug controls (`SET_EXCEPTION_MASK`, `READ_ALL_SM_ERROR_STATES`, `CLEAR_ALL_SM_ERROR_STATES`). Coredump generation additionally uses `NV2080_CTRL_CMD_GPU_EXEC_REG_OPS` (already supported), so the feature requires running with `--nvproxy-allowed-driver-capabilities=all,profiling`. Tested on an A10G with a faulting Triton kernel: the generated `.nvcudmp` matches native (runc) byte-for-byte in size, and `cuda-gdb` fully decodes it (`CUDA Exception: Warp Illegal Address`, faulting PC/kernel/warp/lane and backtrace). Without this change, coredump generation silently produces an empty dump (`No exception was found on any device`). --- pkg/abi/nvgpu/ctrl.go | 18 ++++++++ pkg/sentry/devices/nvproxy/frontend_unsafe.go | 45 +++++++++++++++++++ pkg/sentry/devices/nvproxy/version.go | 10 +++++ 3 files changed, 73 insertions(+) diff --git a/pkg/abi/nvgpu/ctrl.go b/pkg/abi/nvgpu/ctrl.go index c718b3c4cd..ff802fbdab 100644 --- a/pkg/abi/nvgpu/ctrl.go +++ b/pkg/abi/nvgpu/ctrl.go @@ -791,11 +791,29 @@ type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS struct { // From src/common/sdk/nvidia/inc/ctrl/ctrl83de/ctrl83dedebug.h: const ( + NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG = 0x83de0307 NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK = 0x83de0309 NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES = 0x83de030c NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES = 0x83de0310 + NV83DE_CTRL_CMD_DEBUG_READ_MEMORY = 0x83de0315 + NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT = 0x83de0317 + NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT = 0x83de0318 + NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG = 0x83de031f ) +// NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS is the params type for +// NV83DE_CTRL_CMD_DEBUG_READ_MEMORY, from +// src/common/sdk/nvidia/inc/ctrl/ctrl83de/ctrl83dedebug.h. +// +// +marshal +type NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS struct { + _ structs.HostLayout + HMemory Handle + Length uint32 + Offset uint64 + Buffer P64 +} + // From src/common/sdk/nvidia/inc/ctrl/ctrlc36f.h: const ( NVC36F_CTRL_GET_CLASS_ENGINEID = 0xc36f0101 diff --git a/pkg/sentry/devices/nvproxy/frontend_unsafe.go b/pkg/sentry/devices/nvproxy/frontend_unsafe.go index ca5f315bd1..ad1bd045c1 100644 --- a/pkg/sentry/devices/nvproxy/frontend_unsafe.go +++ b/pkg/sentry/devices/nvproxy/frontend_unsafe.go @@ -188,6 +188,51 @@ func ctrlGpuExecRegOps(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54_PARAMET return n, nil } +// ctrlDebugReadMemory implements NV83DE_CTRL_CMD_DEBUG_READ_MEMORY, whose +// parameters contain an embedded pointer (Buffer) that the driver fills with +// Length bytes read from the debugged context. Compare +// src/nvidia/src/kernel/rmapi/embedded_param_copy.c:embeddedParamCopyIn(); +// note that the driver passes RMAPI_PARAM_COPY_FLAGS_DISABLE_MAX_SIZE_CHECK +// for this control, so Buffer's size is not limited by +// RMAPI_PARAM_COPY_MAX_PARAMS_SIZE, and the driver allocates a kernel buffer +// of the same size that we allocate here. +func ctrlDebugReadMemory(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54_PARAMETERS) (uintptr, error) { + var ctrlParams nvgpu.NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS + if ctrlParams.SizeBytes() != int(ioctlParams.ParamsSize) { + return 0, linuxerr.EINVAL + } + if _, err := ctrlParams.CopyIn(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return 0, err + } + var buffer []byte + if ctrlParams.Buffer != 0 && ctrlParams.Length != 0 { + buffer = make([]byte, ctrlParams.Length) + } + + origBuffer := ctrlParams.Buffer + ctrlParams.Buffer = 0 + if len(buffer) != 0 { + ctrlParams.Buffer = p64FromPtr(unsafe.Pointer(&buffer[0])) + } + n, err := rmControlInvoke(fi, ioctlParams, &ctrlParams) + ctrlParams.Buffer = origBuffer + if err != nil { + return n, err + } + // The driver copies the buffer out unconditionally (even if the control + // returned a non-OK status); see + // src/nvidia/src/kernel/rmapi/embedded_param_copy.c:embeddedParamCopyOut(). + if len(buffer) != 0 { + if _, err := fi.t.CopyOutBytes(addrFromP64(origBuffer), buffer); err != nil { + return n, err + } + } + if _, err := ctrlParams.CopyOut(fi.t, addrFromP64(ioctlParams.Params)); err != nil { + return n, err + } + return n, nil +} + func ctrlDevGRGetCapsInvoke(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54_PARAMETERS, ctrlParams *nvgpu.NV0080_CTRL_GET_CAPS_PARAMS, capsTbl []byte) (uintptr, error) { origCapsTbl := ctrlParams.CapsTbl ctrlParams.CapsTbl = p64FromPtr(unsafe.Pointer(&capsTbl[0])) diff --git a/pkg/sentry/devices/nvproxy/version.go b/pkg/sentry/devices/nvproxy/version.go index 6c4047ab43..febc0a3db4 100644 --- a/pkg/sentry/devices/nvproxy/version.go +++ b/pkg/sentry/devices/nvproxy/version.go @@ -326,9 +326,14 @@ func Init() { nvgpu.NV2080_CTRL_CMD_FB_GET_INFO: ctrlHandler(ctrlIoctlHasInfoList[nvgpu.NvxxxCtrlXxxGetInfoParams], nvconf.CapGraphics), nvgpu.NV503C_CTRL_CMD_REGISTER_VIDMEM: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV503C_CTRL_CMD_UNREGISTER_VIDMEM: ctrlHandler(rmControlSimple, compUtil), + nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES: ctrlHandler(rmControlSimple, compUtil), + nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_MEMORY: ctrlHandler(ctrlDebugReadMemory, compUtil), + nvgpu.NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT: ctrlHandler(rmControlSimple, compUtil), + nvgpu.NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT: ctrlHandler(rmControlSimple, compUtil), + nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV906F_CTRL_GET_CLASS_ENGINEID: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV906F_CTRL_CMD_RESET_CHANNEL: ctrlHandler(rmControlSimple, compUtil), nvgpu.NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE: ctrlHandler(rmControlSimple, nvconf.CapGraphics), @@ -640,9 +645,14 @@ func Init() { nvgpu.NV2080_CTRL_CMD_FB_GET_INFO: ioctlInfoWithStructName("NV2080_CTRL_CMD_FB_GET_INFO", nvgpu.NvxxxCtrlXxxGetInfoParams{}, "NV2080_CTRL_FB_GET_INFO_PARAMS"), nvgpu.NV503C_CTRL_CMD_REGISTER_VIDMEM: simpleIoctlInfo("NV503C_CTRL_CMD_REGISTER_VIDMEM", "NV503C_CTRL_REGISTER_VIDMEM_PARAMS"), nvgpu.NV503C_CTRL_CMD_UNREGISTER_VIDMEM: simpleIoctlInfo("NV503C_CTRL_CMD_UNREGISTER_VIDMEM", "NV503C_CTRL_UNREGISTER_VIDMEM_PARAMS"), + nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG", "NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS"), nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK", "NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS"), nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES", "NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS"), nvgpu.NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES", "NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS"), + nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_MEMORY: ioctlInfo("NV83DE_CTRL_CMD_DEBUG_READ_MEMORY", nvgpu.NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS{}), + nvgpu.NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT", "NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT_PARAMS"), + nvgpu.NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT"), // No params. + nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG", "NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS"), nvgpu.NV906F_CTRL_GET_CLASS_ENGINEID: simpleIoctlInfo("NV906F_CTRL_GET_CLASS_ENGINEID", "NV906F_CTRL_GET_CLASS_ENGINEID_PARAMS"), nvgpu.NV906F_CTRL_CMD_RESET_CHANNEL: simpleIoctlInfo("NV906F_CTRL_CMD_RESET_CHANNEL", "NV906F_CTRL_CMD_RESET_CHANNEL_PARAMS"), nvgpu.NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE: simpleIoctlInfo("NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE", "NV9096_CTRL_GET_ZBC_CLEAR_TABLE_SIZE_PARAMS"),