Skip to content

Add CUDA GPU coredump support to nvproxy #13644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luiscape wants to merge 1 commit into
base: master
Choose a base branch
from
+73 −0
Open

Add CUDA GPU coredump support to nvproxy #13644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions pkg/abi/nvgpu/ctrl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -791,11 +791,29 @@ type NV503C_CTRL_REGISTER_VA_SPACE_PARAMS struct {

// From src/common/sdk/nvidia/inc/ctrl/ctrl83de/ctrl83dedebug.h:
const (
NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG = 0x83de0307
NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK = 0x83de0309
NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES = 0x83de030c
NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES = 0x83de0310
NV83DE_CTRL_CMD_DEBUG_READ_MEMORY = 0x83de0315
NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT = 0x83de0317
NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT = 0x83de0318
NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG = 0x83de031f
)

// NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS is the params type for
// NV83DE_CTRL_CMD_DEBUG_READ_MEMORY, from
// src/common/sdk/nvidia/inc/ctrl/ctrl83de/ctrl83dedebug.h.
//
// +marshal
type NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS struct {
_ structs.HostLayout
HMemory Handle
Length uint32
Offset uint64
Buffer P64
}

// From src/common/sdk/nvidia/inc/ctrl/ctrlc36f.h:
const (
NVC36F_CTRL_GET_CLASS_ENGINEID = 0xc36f0101
Expand Down
45 changes: 45 additions & 0 deletions pkg/sentry/devices/nvproxy/frontend_unsafe.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,51 @@ func ctrlGpuExecRegOps(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54_PARAMET
return n, nil
}

// ctrlDebugReadMemory implements NV83DE_CTRL_CMD_DEBUG_READ_MEMORY, whose
// parameters contain an embedded pointer (Buffer) that the driver fills with
// Length bytes read from the debugged context. Compare
// src/nvidia/src/kernel/rmapi/embedded_param_copy.c:embeddedParamCopyIn();
// note that the driver passes RMAPI_PARAM_COPY_FLAGS_DISABLE_MAX_SIZE_CHECK
// for this control, so Buffer's size is not limited by
// RMAPI_PARAM_COPY_MAX_PARAMS_SIZE, and the driver allocates a kernel buffer
// of the same size that we allocate here.
func ctrlDebugReadMemory(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54_PARAMETERS) (uintptr, error) {
var ctrlParams nvgpu.NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS
if ctrlParams.SizeBytes() != int(ioctlParams.ParamsSize) {
return 0, linuxerr.EINVAL
}
if _, err := ctrlParams.CopyIn(fi.t, addrFromP64(ioctlParams.Params)); err != nil {
return 0, err
}
var buffer []byte
if ctrlParams.Buffer != 0 && ctrlParams.Length != 0 {
buffer = make([]byte, ctrlParams.Length)
}

origBuffer := ctrlParams.Buffer
ctrlParams.Buffer = 0
if len(buffer) != 0 {
ctrlParams.Buffer = p64FromPtr(unsafe.Pointer(&buffer[0]))
}
n, err := rmControlInvoke(fi, ioctlParams, &ctrlParams)
ctrlParams.Buffer = origBuffer
if err != nil {
return n, err
}
// The driver copies the buffer out unconditionally (even if the control
// returned a non-OK status); see
// src/nvidia/src/kernel/rmapi/embedded_param_copy.c:embeddedParamCopyOut().
if len(buffer) != 0 {
if _, err := fi.t.CopyOutBytes(addrFromP64(origBuffer), buffer); err != nil {
return n, err
}
}
if _, err := ctrlParams.CopyOut(fi.t, addrFromP64(ioctlParams.Params)); err != nil {
return n, err
}
return n, nil
}

func ctrlDevGRGetCapsInvoke(fi *frontendIoctlState, ioctlParams *nvgpu.NVOS54_PARAMETERS, ctrlParams *nvgpu.NV0080_CTRL_GET_CAPS_PARAMS, capsTbl []byte) (uintptr, error) {
origCapsTbl := ctrlParams.CapsTbl
ctrlParams.CapsTbl = p64FromPtr(unsafe.Pointer(&capsTbl[0]))
Expand Down
10 changes: 10 additions & 0 deletions pkg/sentry/devices/nvproxy/version.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,14 @@ func Init() {
nvgpu.NV2080_CTRL_CMD_FB_GET_INFO: ctrlHandler(ctrlIoctlHasInfoList[nvgpu.NvxxxCtrlXxxGetInfoParams], nvconf.CapGraphics),
nvgpu.NV503C_CTRL_CMD_REGISTER_VIDMEM: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV503C_CTRL_CMD_UNREGISTER_VIDMEM: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_MEMORY: ctrlHandler(ctrlDebugReadMemory, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV906F_CTRL_GET_CLASS_ENGINEID: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV906F_CTRL_CMD_RESET_CHANNEL: ctrlHandler(rmControlSimple, compUtil),
nvgpu.NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE: ctrlHandler(rmControlSimple, nvconf.CapGraphics),
Expand Down Expand Up @@ -640,9 +645,14 @@ func Init() {
nvgpu.NV2080_CTRL_CMD_FB_GET_INFO: ioctlInfoWithStructName("NV2080_CTRL_CMD_FB_GET_INFO", nvgpu.NvxxxCtrlXxxGetInfoParams{}, "NV2080_CTRL_FB_GET_INFO_PARAMS"),
nvgpu.NV503C_CTRL_CMD_REGISTER_VIDMEM: simpleIoctlInfo("NV503C_CTRL_CMD_REGISTER_VIDMEM", "NV503C_CTRL_REGISTER_VIDMEM_PARAMS"),
nvgpu.NV503C_CTRL_CMD_UNREGISTER_VIDMEM: simpleIoctlInfo("NV503C_CTRL_CMD_UNREGISTER_VIDMEM", "NV503C_CTRL_UNREGISTER_VIDMEM_PARAMS"),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SET_MODE_MMU_DEBUG", "NV83DE_CTRL_DEBUG_SET_MODE_MMU_DEBUG_PARAMS"),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SET_EXCEPTION_MASK", "NV83DE_CTRL_DEBUG_SET_EXCEPTION_MASK_PARAMS"),
nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_READ_ALL_SM_ERROR_STATES", "NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS"),
nvgpu.NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_CLEAR_ALL_SM_ERROR_STATES", "NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS"),
nvgpu.NV83DE_CTRL_CMD_DEBUG_READ_MEMORY: ioctlInfo("NV83DE_CTRL_CMD_DEBUG_READ_MEMORY", nvgpu.NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS{}),
nvgpu.NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT", "NV83DE_CTRL_CMD_DEBUG_SUSPEND_CONTEXT_PARAMS"),
nvgpu.NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_RESUME_CONTEXT"), // No params.
nvgpu.NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG: simpleIoctlInfo("NV83DE_CTRL_CMD_DEBUG_SET_MODE_ERRBAR_DEBUG", "NV83DE_CTRL_DEBUG_SET_MODE_ERRBAR_DEBUG_PARAMS"),
nvgpu.NV906F_CTRL_GET_CLASS_ENGINEID: simpleIoctlInfo("NV906F_CTRL_GET_CLASS_ENGINEID", "NV906F_CTRL_GET_CLASS_ENGINEID_PARAMS"),
nvgpu.NV906F_CTRL_CMD_RESET_CHANNEL: simpleIoctlInfo("NV906F_CTRL_CMD_RESET_CHANNEL", "NV906F_CTRL_CMD_RESET_CHANNEL_PARAMS"),
nvgpu.NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE: simpleIoctlInfo("NV9096_CTRL_CMD_GET_ZBC_CLEAR_TABLE_SIZE", "NV9096_CTRL_GET_ZBC_CLEAR_TABLE_SIZE_PARAMS"),
Expand Down
Loading