From e5bfcda9bd5c2dd5d27be11edf22d8fe86641257 Mon Sep 17 00:00:00 2001 From: Aditi Gaur Date: Tue, 14 Jul 2026 15:35:16 -0700 Subject: [PATCH 1/2] Add jetpack version checks --- healthagent/reporter.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/healthagent/reporter.py b/healthagent/reporter.py index 791b429..f0579fb 100644 --- a/healthagent/reporter.py +++ b/healthagent/reporter.py @@ -6,6 +6,7 @@ from typing import Any, Dict import logging import os +import subprocess from healthagent.scheduler import Scheduler log = logging.getLogger('healthagent') @@ -142,6 +143,8 @@ def view(self, cli_exclude: bool = True) -> Dict[str, Any]: class Reporter: + JETPACK_VERSION_MINIMUM = "8.8.0" + JETPACK_VERSION_GHR = "8.10.0" def __init__(self, name: str = None): """ name: name of the health report (optional) @@ -149,16 +152,50 @@ def __init__(self, name: str = None): self.jetpack = "/opt/cycle/jetpack/bin/jetpack" self.store = {} self.publish_cc = os.getenv("PUBLISH_CC", 'true').lower() == 'true' + self.enable_ghr = True if not os.path.exists(self.jetpack): log.info(f"{self.jetpack} not found") self.publish_cc = False + self.enable_ghr = False + elif self.publish_cc: + version = self._get_jetpack_version() + if version is None or version < self._parse_version(self.JETPACK_VERSION_MINIMUM): + self.publish_cc = False + self.enable_ghr = False + log.warning(f"Jetpack version too old or unknown, disabling CC publishing and GHR") + elif version < self._parse_version(self.JETPACK_VERSION_GHR): + self.enable_ghr = False + log.info(f"Jetpack version < {self.JETPACK_VERSION_GHR}, disabling GHR") if not self.publish_cc: - log.info(f"Disabling publishing reports to CC") + log.info("Disabling publishing reports to CC.") + self.enable_ghr = False + if not self.enable_ghr: + log.info("Disabling Guest Health Reporting") if name: self.store[name] = HealthReport() + + @staticmethod + def _parse_version(version_str: str) -> tuple: + """Parse '8.10.0-3822' → (8, 10, 0)""" + base = version_str.strip().split('-')[0] + return tuple(int(x) for x in base.split('.')) + + def _get_jetpack_version(self) -> tuple | None: + + try: + result = subprocess.run( + [self.jetpack, '--version'], + capture_output=True, text=True, timeout=10 + ) + if result.returncode == 0: + return self._parse_version(result.stdout) + except Exception as e: + log.warning(f"Failed to determine jetpack version: {e}") + return None + @classmethod def load_reporter_obj(cls, old) -> 'Reporter': From fed3a0ebe1eae7cd5bdcfd55ca1318cce466434d Mon Sep 17 00:00:00 2001 From: Aditi Gaur Date: Mon, 20 Jul 2026 16:32:04 -0700 Subject: [PATCH 2/2] Add GHR for certain error types. --- healthagent/bindings.py | 53 +++++++++++++++++++++++++++++++++++--- healthagent/ghr.py | 33 ++++++++++++++++++++++++ healthagent/gpu.py | 57 +++++++++++++++++++++++++++++++++++++++++ healthagent/network.py | 26 ++++++++++++++++--- healthagent/reporter.py | 9 ++++++- 5 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 healthagent/ghr.py diff --git a/healthagent/bindings.py b/healthagent/bindings.py index c55a7d3..1988307 100644 --- a/healthagent/bindings.py +++ b/healthagent/bindings.py @@ -6,6 +6,7 @@ import asyncio from pathlib import Path from healthagent.reporter import HealthStatus +from healthagent.ghr import GHRCategory import logging DCGM_VERSION = os.getenv("DCGM_VERSION") @@ -75,6 +76,16 @@ def _resolve_error_codes_with_notes(*pairs): codes[code] = f"Healthagent Note: {note}" return codes +def _resolve_ghr_codes(*pairs): + """Resolve (DCGM_FR_name, GHRCategory) pairs to {int_code: GHRCategory} dict, + skipping any that don't exist in the installed dcgm_errors module.""" + codes = {} + for name, ghr in pairs: + code = getattr(dcgm_errors, name, None) + if code is not None: + codes[code] = ghr + return codes + # System/reference fields — needed by non-watch code (XID handling, gpu_count_check, # display config, detail lookups). Not user-configurable thresholds. # (alias, dcgm_fields attribute name) @@ -144,6 +155,42 @@ class DcgmGpuNotFound(Exception): fields = _Fields + XID_GHR_MAP = { + 48: GHRCategory.XID48_DOUBLE_BIT_ECC, + 79: GHRCategory.XID79_FALLEN_OFF_BUS, + 93: GHRCategory.INFOROM_CORRUPTION, + 94: GHRCategory.XID94_CONTAINED_ECC, + 95: GHRCategory.XID95_UNCONTAINED_ECC, + 60: GHRCategory.DCGMI_THERMAL, + 61: GHRCategory.DCGMI_THERMAL, + 62: GHRCategory.DCGMI_THERMAL, + } + + ERROR_GHR_MAP = _resolve_ghr_codes( + # Health watch errors + ("DCGM_FR_NVLINK_DOWN", GHRCategory.NVLINK), + ("DCGM_FR_NVLINK_ERROR_CRITICAL", GHRCategory.NVLINK), + ("DCGM_FR_NVSWITCH_FATAL_ERROR", GHRCategory.NVLINK), + ("DCGM_FR_FIELD_THRESHOLD_DBL", GHRCategory.DCGMI_THERMAL), + # Diagnostic errors + ("DCGM_FR_CUDA_DBE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_MEMORY_MISMATCH", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_FAULTY_MEMORY", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_L1TAG_MISCOMPARE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_BROKEN_P2P_MEMORY_DEVICE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_BROKEN_P2P_WRITER_DEVICE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_BROKEN_P2P_NVLINK_WRITER_DEVICE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_BROKEN_P2P_NVLINK_MEMORY_DEVICE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_BROKEN_P2P_PCIE_MEMORY_DEVICE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_BROKEN_P2P_PCIE_WRITER_DEVICE", GHRCategory.DCGM_DIAG_FAILURE), + ("DCGM_FR_FABRIC_MANAGER_TRAINING_ERROR", GHRCategory.DCGM_DIAG_FAILURE), + ) + + FIELD_GHR_MAP = { + "DCGM_FI_DEV_ROW_REMAP_FAILURE": GHRCategory.ROW_REMAP_FAILURE, + "DCGM_FI_DEV_ECC_DBE_VOL_TOTAL": GHRCategory.XID48_DOUBLE_BIT_ECC, + } + ENTITY_GROUP_NAMES = { dcgm_fields.DCGM_FE_NONE: "None", dcgm_fields.DCGM_FE_GPU: "GPU", @@ -167,12 +214,12 @@ def get_fields(cls): "DCGM_FR_NVSWITCH_FATAL_ERROR", "DCGM_FR_FAULTY_MEMORY", "DCGM_FR_FIELD_VIOLATION", - "DCGM_FR_FABRIC_PROBE_STATE" + "DCGM_FR_FABRIC_PROBE_STATE", + "DCGM_FR_FIELD_THRESHOLD_DBL", ) HEALTH_WARNINGS = _resolve_error_codes( "DCGM_FR_PCI_REPLAY_RATE", - "DCGM_FR_CORRUPT_INFOROM", "DCGM_FR_NVSWITCH_NON_FATAL_ERROR", "DCGM_FR_NVLINK_SYMBOL_BER_THRESHOLD", "DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD", @@ -512,5 +559,5 @@ def set_policy(cls): return policy @classmethod def get_health_mask(cls): - exclude_mask = dcgm_structs.DCGM_HEALTH_WATCH_THERMAL | dcgm_structs.DCGM_HEALTH_WATCH_POWER + exclude_mask = dcgm_structs.DCGM_HEALTH_WATCH_POWER return dcgm_structs.DCGM_HEALTH_WATCH_ALL & ~exclude_mask \ No newline at end of file diff --git a/healthagent/ghr.py b/healthagent/ghr.py new file mode 100644 index 0000000..34bb817 --- /dev/null +++ b/healthagent/ghr.py @@ -0,0 +1,33 @@ +from enum import Enum + +class GHRCategory(Enum): + # Operational + RESET = "Resource.Hpc.Reset" + REBOOT = "Resource.Hpc.Reboot" + # GPU + MISSING_GPU = "Resource.Hpc.Unhealthy.HpcMissingGpu" + DCGM_DIAG_FAILURE = "Resource.Hpc.Unhealthy.HpcGpuDcgmDiagFailure" + ROW_REMAP_FAILURE = "Resource.Hpc.Unhealthy.HpcRowRemapFailure" + INFOROM_CORRUPTION = "Resource.Hpc.Unhealthy.HpcInforomCorruption" + XID95_UNCONTAINED_ECC = "Resource.Hpc.Unhealthy.XID95UncontainedECCError" + XID94_CONTAINED_ECC = "Resource.Hpc.Unhealthy.XID94ContainedECCError" + XID79_FALLEN_OFF_BUS = "Resource.Hpc.Unhealthy.XID79FallenOffBus" + XID48_DOUBLE_BIT_ECC = "Resource.Hpc.Unhealthy.XID48DoubleBitECC" + UNHEALTHY_GPU_NVIDIASMI = "Resource.Hpc.Unhealthy.UnhealthyGPUNvidiasmi" + NVLINK = "Resource.Hpc.Unhealthy.NvLink" + DCGMI_THERMAL = "Resource.Hpc.Unhealthy.HpcDcgmiThermalReport" + ECC_PAGE_RETIRE_FULL = "Resource.Hpc.Unhealthy.ECCPageRetirementTableFull" + DBE_OVER_LIMIT = "Resource.Hpc.Unhealthy.DBEOverLimit" + GPU_XID_ERROR = "Resource.Hpc.Unhealthy.GpuXIDError" + EROT_FAILURE = "Resource.Hpc.Unhealthy.EROTFailure" + GPU_MEM_BW_FAILURE = "Resource.Hpc.Unhealthy.GPUMemoryBWFailure" + # Network / InfiniBand + MISSING_IB = "Resource.Hpc.Unhealthy.MissingIB" + IB_PERFORMANCE = "Resource.Hpc.Unhealthy.IBPerformance" + IB_PORT_DOWN = "Resource.Hpc.Unhealthy.IBPortDown" + IB_PORT_FLAPPING = "Resource.Hpc.Unhealthy.IBPortFlapping" + # Generic + GENERIC_FAILURE = "Resource.Hpc.Unhealthy.HpcGenericFailure" + MANUAL_INVESTIGATION = "Resource.Hpc.Unhealthy.ManualInvestigation" + # CPU + CPU_PERFORMANCE = "Resource.Hpc.Unhealthy.CPUPerformance" \ No newline at end of file diff --git a/healthagent/gpu.py b/healthagent/gpu.py index 4981c40..008fc27 100644 --- a/healthagent/gpu.py +++ b/healthagent/gpu.py @@ -218,6 +218,7 @@ async def gpu_count_check(self): unique_counts = set(custom_fields.values()) if len(unique_counts) > 1: report.status = HealthStatus.ERROR + report.ghr_category = GHRCategory.MISSING_GPU detail_parts = [f"{source} shows {count}" for source, count in custom_fields.items()] report.details = "GPU count mismatch: " + ", ".join(detail_parts) report.description = "GPU Count Mismatch" @@ -326,8 +327,14 @@ def track_fieldsv2(self): custom_fields[entity_key]["errors" if severity == "error" else "warnings"].append(msg) if severity == "error": custom_fields['error_count'] += 1 + ghr_cat = Wrap.FIELD_GHR_MAP.get(watch["field"]) + if ghr_cat: + custom_fields['_ghr_field_error'] = ghr_cat else: custom_fields['warning_count'] += 1 + ghr_cat = Wrap.FIELD_GHR_MAP.get(watch["field"]) + if ghr_cat and '_ghr_field_any' not in custom_fields: + custom_fields['_ghr_field_any'] = ghr_cat custom_fields['category'].add(watch["category"]) # XID handling — GPU entities only @@ -485,6 +492,48 @@ async def run_background_healthchecks(self): if not isinstance(v, dict) or any(v.values()) } + # Resolve GHR category from XIDs and health incidents + ghr_error = None + ghr_any = None + + # Check health incidents for GHR-mapped error codes + for index in range(0, incident_count): + error_code = group_health.incidents[index].error.code + ghr_cat = Wrap.ERROR_GHR_MAP.get(error_code) + if ghr_cat: + if ghr_any is None: + ghr_any = ghr_cat + if error_code in Wrap.HEALTH_ERRORS and ghr_error is None: + ghr_error = ghr_cat + + # Check XIDs + for gpu_id, xids in self.xid_history.items(): + for xid_num in xids: + if xid_num in self.xid_ignore: + continue + category = Wrap.XID_GHR_MAP.get(xid_num, GHRCategory.GPU_XID_ERROR) + if ghr_any is None: + ghr_any = category + is_error = xid_num not in self.xid_warning + if self.xid_error and xid_num not in self.xid_error: + is_error = False + if is_error and ghr_error is None: + ghr_error = category + + # Check field watches + if '_ghr_field_error' in custom_fields: + if ghr_error is None: + ghr_error = custom_fields.pop('_ghr_field_error') + else: + custom_fields.pop('_ghr_field_error') + if '_ghr_field_any' in custom_fields: + if ghr_any is None: + ghr_any = custom_fields.pop('_ghr_field_any') + else: + custom_fields.pop('_ghr_field_any') + + report.ghr_category = ghr_error or ghr_any + if custom_fields['error_count'] == 0 and custom_fields['warning_count'] == 0: await self.reporter.update_report(name=health_system, report=report) return @@ -641,6 +690,14 @@ def run_active_healthchecksv2(gpu_id: list = None, tests: str = '', params: str report.description = "GPU Diagnostic test errors" report.custom_fields = custom_fields + # Resolve GHR from diag error codes + if response and response.numErrors > 0: + for errIdx in range(response.numErrors): + ghr_cat = Wrap.ERROR_GHR_MAP.get(response.errors[errIdx].code) + if ghr_cat: + report.ghr_category = ghr_cat + break + all_errors = [] all_warnings = [] for key, val in custom_fields.items(): diff --git a/healthagent/network.py b/healthagent/network.py index b56b137..5a310eb 100644 --- a/healthagent/network.py +++ b/healthagent/network.py @@ -8,6 +8,7 @@ from healthagent.healthmodule import HealthModule from healthagent.config import NetworkConfig, ThresholdCheck from healthagent.reporter import Reporter, HealthReport, HealthStatus +from healthagent.ghr import GHRCategory from healthagent.scheduler import Scheduler log = logging.getLogger('healthagent') @@ -49,6 +50,12 @@ class NetworkInterface: class NetworkHealthChecks(HealthModule): + _NET_GHR_MAP = { + ("infiniband", "state"): GHRCategory.IB_PORT_DOWN, + ("infiniband", "phys_state"): GHRCategory.IB_PORT_DOWN, + ("infiniband", "link_downed"): GHRCategory.IB_PORT_FLAPPING, + } + def __init__(self, reporter: Reporter, config: 'NetworkConfig | None' = None): super().__init__(reporter, config or NetworkConfig()) self.config: NetworkConfig = self.config @@ -118,10 +125,12 @@ async def run_network_checks(self): report = HealthReport() custom_fields = {} details = [] + ghr_error = None + ghr_any = None # Derive port-level field names from the IBPort dataclass _ib_port_fields = {f.name for f in fields(IBPort)} - def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None): + def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None, config_key=None): key = (iface_name, field, port_num) max_strikes = check.strikes @@ -162,10 +171,19 @@ def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None custom_fields.setdefault(iface_name, {}).setdefault("errors", []).append(msg) details.append(f"ERROR: {iface_name} - {msg}") report.escalate(HealthStatus.ERROR) + ghr_cat = self._NET_GHR_MAP.get((config_key, field)) + if ghr_cat: + nonlocal ghr_error + ghr_error = ghr_cat else: custom_fields.setdefault(iface_name, {}).setdefault("warnings", []).append(msg) details.append(f"WARNING: {iface_name} - {msg}") report.escalate(HealthStatus.WARNING) + ghr_cat = self._NET_GHR_MAP.get((config_key, field)) + if ghr_cat: + nonlocal ghr_any + if ghr_any is None: + ghr_any = ghr_cat break # error takes precedence over warning # Strike tracking: count OK→ERROR transitions @@ -193,13 +211,13 @@ def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None value = getattr(port, field, None) if value is None: continue - check_field(ni.name, field, check, value, port_num=port_num) + check_field(ni.name, field, check, value, port_num=port_num, config_key=config_key) else: # Interface-level field value = getattr(ni, field, None) if value is None: continue - check_field(ni.name, field, check, value) + check_field(ni.name, field, check, value, config_key=config_key) if ni.ib_device: custom_fields[ni.name]["ib_device"] = { @@ -216,4 +234,6 @@ def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None if report.status != HealthStatus.OK: report.description = "Network health issues detected" + report.ghr_category = ghr_error or ghr_any + await self.reporter.update_report(self.run_network_checks.report_name, report=report) diff --git a/healthagent/reporter.py b/healthagent/reporter.py index f0579fb..4836cea 100644 --- a/healthagent/reporter.py +++ b/healthagent/reporter.py @@ -8,6 +8,7 @@ import os import subprocess from healthagent.scheduler import Scheduler +from healthagent.ghr import GHRCategory log = logging.getLogger('healthagent') @@ -86,6 +87,8 @@ class HealthReport: # attribute in __post_init__. Note: aux_data will not survive an asdict() round-trip; # use deepcopy(report) to preserve it. aux_data: InitVar[dict] = None + # Azure Guest Health Reporting + ghr_category: GHRCategory = None def __eq__(self, other): if not isinstance(other, HealthReport): @@ -284,6 +287,10 @@ async def publish_cc_status(self, name): if report.details is not None: args.extend(['--details', report.details]) + if self.enable_ghr: + if report.ghr_category is not None: + args.extend(['--action-type', 'GHR', '--ghr-category', report.ghr_category.value]) + # Schedule the subprocess task task = Scheduler.subprocess(*args) - await Scheduler.add_task(task) \ No newline at end of file + await Scheduler.add_task(task)