Skip to content

Add ECC support for Nvidia professional GPUs #486

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LH-and-FPGA wants to merge 1 commit into
base: master
Choose a base branch
from
+55 −3
Open

Add ECC support for Nvidia professional GPUs #486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/nvtop/extract_gpuinfo_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ enum gpuinfo_dynamic_info_valid {
gpuinfo_power_draw_max_valid,
gpuinfo_effective_load_rate_valid,
gpuinfo_multi_instance_mode_valid,
gpuinfo_ecc_corrected_valid,
gpuinfo_ecc_uncorrected_valid,
gpuinfo_dynamic_info_count,
};

Expand All @@ -123,6 +125,8 @@ struct gpuinfo_dynamic_info {
unsigned long long total_memory; // Total memory (bytes)
unsigned long long free_memory; // Unallocated memory (bytes)
unsigned long long used_memory; // Allocated memory (bytes)
unsigned long long ecc_corrected; // Total correctable ECC errors
unsigned long long ecc_uncorrected; // Total uncorrected ECC errors
unsigned int pcie_link_gen; // PCIe link generation used
unsigned int pcie_link_width; // PCIe line width used
unsigned int pcie_rx; // PCIe throughput in KB/s
Expand Down
2 changes: 2 additions & 0 deletions include/nvtop/interface_internal_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ struct device_window {
WINDOW *fan_speed;
WINDOW *temperature;
WINDOW *power_info;
WINDOW *ecc_info;
WINDOW *gpu_clock_info;
WINDOW *mem_clock_info;
WINDOW *pcie_info;
Expand Down Expand Up @@ -148,6 +149,7 @@ enum device_field {
device_fan_speed,
device_temperature,
device_power,
device_ecc,
device_pcie,
device_clock,
device_mem_clock,
Expand Down
22 changes: 22 additions & 0 deletions src/extract_gpuinfo_nvidia.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,25 @@ typedef enum {
NVML_TEMPERATURE_GPU = 0,
} nvmlTemperatureSensors_t;

typedef enum {
NVML_MEMORY_ERROR_TYPE_CORRECTED = 0,
NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1
} nvmlMemoryErrorType_t;

typedef enum {
NVML_VOLATILE_ECC = 0,
NVML_AGGREGATE_ECC = 1
} nvmlEccCounterType_t;

static nvmlReturn_t (*nvmlDeviceGetTemperature)(nvmlDevice_t device, nvmlTemperatureSensors_t sensorType,
unsigned int *temp);

static nvmlReturn_t (*nvmlDeviceGetPowerUsage)(nvmlDevice_t device, unsigned int *power);

static nvmlReturn_t (*nvmlDeviceGetEnforcedPowerLimit)(nvmlDevice_t device, unsigned int *limit);

static nvmlReturn_t (*nvmlDeviceGetTotalEccErrors)(nvmlDevice_t device, nvmlMemoryErrorType_t errorType, nvmlEccCounterType_t counterType, unsigned long long *eccCounts);

static nvmlReturn_t (*nvmlDeviceGetEncoderUtilization)(nvmlDevice_t device, unsigned int *utilization,
unsigned int *samplingPeriodUs);

Expand Down Expand Up @@ -465,6 +477,7 @@ static bool gpuinfo_nvidia_init(void) {
(nvmlReturn_t(*)(nvmlDevice_t, unsigned int *, void *))nvmlDeviceGetMPSComputeRunningProcesses_v2;
nvmlDeviceGetMPSComputeRunningProcesses[3] =
(nvmlReturn_t(*)(nvmlDevice_t, unsigned int *, void *))nvmlDeviceGetMPSComputeRunningProcesses_v3;
nvmlDeviceGetTotalEccErrors = dlsym(libnvidia_ml_handle, "nvmlDeviceGetTotalEccErrors");

// These ones might not be available
nvmlDeviceGetProcessUtilization = dlsym(libnvidia_ml_handle, "nvmlDeviceGetProcessUtilization");
Expand Down Expand Up @@ -741,6 +754,15 @@ static void gpuinfo_nvidia_refresh_dynamic_info(struct gpu_info *_gpu_info) {
if (last_nvml_return_status == NVML_SUCCESS)
SET_VALID(gpuinfo_power_draw_max_valid, dynamic_info->valid);

if (nvmlDeviceGetTotalEccErrors) {
unsigned long long ecc_count;
last_nvml_return_status = nvmlDeviceGetTotalEccErrors(device, NVML_MEMORY_ERROR_TYPE_CORRECTED, NVML_VOLATILE_ECC, &ecc_count);
if (last_nvml_return_status == NVML_SUCCESS)
SET_GPUINFO_DYNAMIC(dynamic_info, ecc_corrected, ecc_count);
last_nvml_return_status = nvmlDeviceGetTotalEccErrors(device, NVML_MEMORY_ERROR_TYPE_UNCORRECTED, NVML_VOLATILE_ECC, &ecc_count);
if (last_nvml_return_status == NVML_SUCCESS)
SET_GPUINFO_DYNAMIC(dynamic_info, ecc_uncorrected, ecc_count);
}
// MIG mode
if (nvmlDeviceGetMigMode) {
unsigned currentMode, pendingMode;
Expand Down
30 changes: 27 additions & 3 deletions src/interface.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@

static unsigned int sizeof_device_field[device_field_count] = {
[device_name] = 11, [device_fan_speed] = 11, [device_temperature] = 10, [device_power] = 15,
[device_clock] = 11, [device_mem_clock] = 12, [device_pcie] = 46, [device_shadercores] = 7,
[device_l2features] = 11, [device_execengines] = 11,
[device_ecc] = 16, [device_clock] = 11, [device_mem_clock] = 12, [device_pcie] = 46,
[device_shadercores] = 7, [device_l2features] = 11, [device_execengines] = 11,
};

static unsigned int sizeof_process_field[process_field_count] = {
Expand Down Expand Up @@ -94,6 +94,13 @@ static void alloc_device_window(unsigned int start_row, unsigned int start_col,
sizeof_device_field[device_temperature] + sizeof_device_field[device_fan_speed]);
if (dwin->power_info == NULL)
goto alloc_error;
dwin->ecc_info =
newwin(1, sizeof_device_field[device_ecc], start_row + 1,
start_col + spacer * 5 + sizeof_device_field[device_clock] + sizeof_device_field[device_mem_clock] +
sizeof_device_field[device_temperature] + sizeof_device_field[device_fan_speed] +
sizeof_device_field[device_power]);
if (dwin->ecc_info == NULL)
goto alloc_error;

// Line 3 = GPU used | MEM used | Encoder | Decoder

Expand Down Expand Up @@ -199,6 +206,7 @@ static void free_device_windows(struct device_window *dwin) {
delwin(dwin->gpu_clock_info);
delwin(dwin->mem_clock_info);
delwin(dwin->power_info);
delwin(dwin->ecc_info);
delwin(dwin->temperature);
delwin(dwin->fan_speed);
delwin(dwin->pcie_info);
Expand Down Expand Up @@ -350,7 +358,7 @@ static unsigned device_length(void) {
return max(sizeof_device_field[device_name] + sizeof_device_field[device_pcie] + 1,
sizeof_device_field[device_clock] + sizeof_device_field[device_mem_clock] +
sizeof_device_field[device_temperature] + sizeof_device_field[device_fan_speed] +
sizeof_device_field[device_power] + 5);
sizeof_device_field[device_power] + sizeof_device_field[device_ecc] + 6);
}

static pid_t nvtop_pid;
Expand Down Expand Up @@ -830,6 +838,22 @@ static void draw_devices(struct list_head *devices, struct nvtop_interface *inte
mvwchgat(dev->power_info, 0, 0, 3, 0, cyan_color, NULL);
wnoutrefresh(dev->power_info);

// ECC errors (professional/datacenter cards only; hidden otherwise)
werase(dev->ecc_info);
if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_corrected) ||
GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_uncorrected)) {
unsigned long long corrected =
GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_corrected) ? device->dynamic_info.ecc_corrected : 0;
unsigned long long uncorrected =
GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_uncorrected) ? device->dynamic_info.ecc_uncorrected : 0;
mvwprintw(dev->ecc_info, 0, 0, " ECC %llu/%llu", corrected, uncorrected);
mvwchgat(dev->ecc_info, 0, 2, 3, 0, cyan_color, NULL);
// Highlight a non-zero uncorrected count in red: it signals a hardware fault
if (uncorrected > 0)
mvwchgat(dev->ecc_info, 0, 6, -1, 0, red_color, NULL);
}
wnoutrefresh(dev->ecc_info);

// PICe throughput
werase(dev->pcie_info);
if (device->static_info.integrated_graphics) {
Expand Down