diff --git a/include/nvtop/extract_gpuinfo_common.h b/include/nvtop/extract_gpuinfo_common.h index 9e4d1c9..e2d070f 100644 --- a/include/nvtop/extract_gpuinfo_common.h +++ b/include/nvtop/extract_gpuinfo_common.h @@ -107,6 +107,8 @@ enum gpuinfo_dynamic_info_valid { gpuinfo_power_draw_max_valid, gpuinfo_effective_load_rate_valid, gpuinfo_multi_instance_mode_valid, + gpuinfo_ecc_corrected_valid, + gpuinfo_ecc_uncorrected_valid, gpuinfo_dynamic_info_count, }; @@ -123,6 +125,8 @@ struct gpuinfo_dynamic_info { unsigned long long total_memory; // Total memory (bytes) unsigned long long free_memory; // Unallocated memory (bytes) unsigned long long used_memory; // Allocated memory (bytes) + unsigned long long ecc_corrected; // Total correctable ECC errors + unsigned long long ecc_uncorrected; // Total uncorrected ECC errors unsigned int pcie_link_gen; // PCIe link generation used unsigned int pcie_link_width; // PCIe line width used unsigned int pcie_rx; // PCIe throughput in KB/s diff --git a/include/nvtop/interface_internal_common.h b/include/nvtop/interface_internal_common.h index aec93d3..ad29eab 100644 --- a/include/nvtop/interface_internal_common.h +++ b/include/nvtop/interface_internal_common.h @@ -67,6 +67,7 @@ struct device_window { WINDOW *fan_speed; WINDOW *temperature; WINDOW *power_info; + WINDOW *ecc_info; WINDOW *gpu_clock_info; WINDOW *mem_clock_info; WINDOW *pcie_info; @@ -148,6 +149,7 @@ enum device_field { device_fan_speed, device_temperature, device_power, + device_ecc, device_pcie, device_clock, device_mem_clock, diff --git a/src/extract_gpuinfo_nvidia.c b/src/extract_gpuinfo_nvidia.c index 33670a6..0417ab6 100644 --- a/src/extract_gpuinfo_nvidia.c +++ b/src/extract_gpuinfo_nvidia.c @@ -138,6 +138,16 @@ typedef enum { NVML_TEMPERATURE_GPU = 0, } nvmlTemperatureSensors_t; +typedef enum { + NVML_MEMORY_ERROR_TYPE_CORRECTED = 0, + NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1 +} nvmlMemoryErrorType_t; + +typedef enum { + NVML_VOLATILE_ECC = 0, + NVML_AGGREGATE_ECC = 1 +} nvmlEccCounterType_t; + static nvmlReturn_t (*nvmlDeviceGetTemperature)(nvmlDevice_t device, nvmlTemperatureSensors_t sensorType, unsigned int *temp); @@ -145,6 +155,8 @@ static nvmlReturn_t (*nvmlDeviceGetPowerUsage)(nvmlDevice_t device, unsigned int static nvmlReturn_t (*nvmlDeviceGetEnforcedPowerLimit)(nvmlDevice_t device, unsigned int *limit); +static nvmlReturn_t (*nvmlDeviceGetTotalEccErrors)(nvmlDevice_t device, nvmlMemoryErrorType_t errorType, nvmlEccCounterType_t counterType, unsigned long long *eccCounts); + static nvmlReturn_t (*nvmlDeviceGetEncoderUtilization)(nvmlDevice_t device, unsigned int *utilization, unsigned int *samplingPeriodUs); @@ -465,6 +477,7 @@ static bool gpuinfo_nvidia_init(void) { (nvmlReturn_t(*)(nvmlDevice_t, unsigned int *, void *))nvmlDeviceGetMPSComputeRunningProcesses_v2; nvmlDeviceGetMPSComputeRunningProcesses[3] = (nvmlReturn_t(*)(nvmlDevice_t, unsigned int *, void *))nvmlDeviceGetMPSComputeRunningProcesses_v3; + nvmlDeviceGetTotalEccErrors = dlsym(libnvidia_ml_handle, "nvmlDeviceGetTotalEccErrors"); // These ones might not be available nvmlDeviceGetProcessUtilization = dlsym(libnvidia_ml_handle, "nvmlDeviceGetProcessUtilization"); @@ -741,6 +754,15 @@ static void gpuinfo_nvidia_refresh_dynamic_info(struct gpu_info *_gpu_info) { if (last_nvml_return_status == NVML_SUCCESS) SET_VALID(gpuinfo_power_draw_max_valid, dynamic_info->valid); + if (nvmlDeviceGetTotalEccErrors) { + unsigned long long ecc_count; + last_nvml_return_status = nvmlDeviceGetTotalEccErrors(device, NVML_MEMORY_ERROR_TYPE_CORRECTED, NVML_VOLATILE_ECC, &ecc_count); + if (last_nvml_return_status == NVML_SUCCESS) + SET_GPUINFO_DYNAMIC(dynamic_info, ecc_corrected, ecc_count); + last_nvml_return_status = nvmlDeviceGetTotalEccErrors(device, NVML_MEMORY_ERROR_TYPE_UNCORRECTED, NVML_VOLATILE_ECC, &ecc_count); + if (last_nvml_return_status == NVML_SUCCESS) + SET_GPUINFO_DYNAMIC(dynamic_info, ecc_uncorrected, ecc_count); + } // MIG mode if (nvmlDeviceGetMigMode) { unsigned currentMode, pendingMode; diff --git a/src/interface.c b/src/interface.c index ae23199..8159a2d 100644 --- a/src/interface.c +++ b/src/interface.c @@ -44,8 +44,8 @@ static unsigned int sizeof_device_field[device_field_count] = { [device_name] = 11, [device_fan_speed] = 11, [device_temperature] = 10, [device_power] = 15, - [device_clock] = 11, [device_mem_clock] = 12, [device_pcie] = 46, [device_shadercores] = 7, - [device_l2features] = 11, [device_execengines] = 11, + [device_ecc] = 16, [device_clock] = 11, [device_mem_clock] = 12, [device_pcie] = 46, + [device_shadercores] = 7, [device_l2features] = 11, [device_execengines] = 11, }; static unsigned int sizeof_process_field[process_field_count] = { @@ -94,6 +94,13 @@ static void alloc_device_window(unsigned int start_row, unsigned int start_col, sizeof_device_field[device_temperature] + sizeof_device_field[device_fan_speed]); if (dwin->power_info == NULL) goto alloc_error; + dwin->ecc_info = + newwin(1, sizeof_device_field[device_ecc], start_row + 1, + start_col + spacer * 5 + sizeof_device_field[device_clock] + sizeof_device_field[device_mem_clock] + + sizeof_device_field[device_temperature] + sizeof_device_field[device_fan_speed] + + sizeof_device_field[device_power]); + if (dwin->ecc_info == NULL) + goto alloc_error; // Line 3 = GPU used | MEM used | Encoder | Decoder @@ -199,6 +206,7 @@ static void free_device_windows(struct device_window *dwin) { delwin(dwin->gpu_clock_info); delwin(dwin->mem_clock_info); delwin(dwin->power_info); + delwin(dwin->ecc_info); delwin(dwin->temperature); delwin(dwin->fan_speed); delwin(dwin->pcie_info); @@ -350,7 +358,7 @@ static unsigned device_length(void) { return max(sizeof_device_field[device_name] + sizeof_device_field[device_pcie] + 1, sizeof_device_field[device_clock] + sizeof_device_field[device_mem_clock] + sizeof_device_field[device_temperature] + sizeof_device_field[device_fan_speed] + - sizeof_device_field[device_power] + 5); + sizeof_device_field[device_power] + sizeof_device_field[device_ecc] + 6); } static pid_t nvtop_pid; @@ -830,6 +838,22 @@ static void draw_devices(struct list_head *devices, struct nvtop_interface *inte mvwchgat(dev->power_info, 0, 0, 3, 0, cyan_color, NULL); wnoutrefresh(dev->power_info); + // ECC errors (professional/datacenter cards only; hidden otherwise) + werase(dev->ecc_info); + if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_corrected) || + GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_uncorrected)) { + unsigned long long corrected = + GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_corrected) ? device->dynamic_info.ecc_corrected : 0; + unsigned long long uncorrected = + GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, ecc_uncorrected) ? device->dynamic_info.ecc_uncorrected : 0; + mvwprintw(dev->ecc_info, 0, 0, " ECC %llu/%llu", corrected, uncorrected); + mvwchgat(dev->ecc_info, 0, 2, 3, 0, cyan_color, NULL); + // Highlight a non-zero uncorrected count in red: it signals a hardware fault + if (uncorrected > 0) + mvwchgat(dev->ecc_info, 0, 6, -1, 0, red_color, NULL); + } + wnoutrefresh(dev->ecc_info); + // PICe throughput werase(dev->pcie_info); if (device->static_info.integrated_graphics) {