Support guest health reporting

healthagent/bindings.py

@@ -6,6 +6,7 @@
 import asyncio
 from pathlib import Path
 from healthagent.reporter import HealthStatus
+from healthagent.ghr import GHRCategory
 import logging
 
 DCGM_VERSION = os.getenv("DCGM_VERSION")
@@ -75,6 +76,16 @@ def _resolve_error_codes_with_notes(*pairs):
             codes[code] = f"Healthagent Note: {note}"
     return codes
 
+def _resolve_ghr_codes(*pairs):
+    """Resolve (DCGM_FR_name, GHRCategory) pairs to {int_code: GHRCategory} dict,
+    skipping any that don't exist in the installed dcgm_errors module."""
+    codes = {}
+    for name, ghr in pairs:
+        code = getattr(dcgm_errors, name, None)
+        if code is not None:
+            codes[code] = ghr
+    return codes
+
 # System/reference fields — needed by non-watch code (XID handling, gpu_count_check,
 # display config, detail lookups). Not user-configurable thresholds.
 # (alias, dcgm_fields attribute name)
@@ -144,6 +155,42 @@ class DcgmGpuNotFound(Exception):
 
     fields = _Fields
 
+    XID_GHR_MAP = {
+        48: GHRCategory.XID48_DOUBLE_BIT_ECC,
+        79: GHRCategory.XID79_FALLEN_OFF_BUS,
+        93: GHRCategory.INFOROM_CORRUPTION,
+        94: GHRCategory.XID94_CONTAINED_ECC,
+        95: GHRCategory.XID95_UNCONTAINED_ECC,
+        60: GHRCategory.DCGMI_THERMAL,
+        61: GHRCategory.DCGMI_THERMAL,
+        62: GHRCategory.DCGMI_THERMAL,
+    }
+
+    ERROR_GHR_MAP = _resolve_ghr_codes(
+        # Health watch errors
+        ("DCGM_FR_NVLINK_DOWN", GHRCategory.NVLINK),
+        ("DCGM_FR_NVLINK_ERROR_CRITICAL", GHRCategory.NVLINK),
+        ("DCGM_FR_NVSWITCH_FATAL_ERROR", GHRCategory.NVLINK),
+        ("DCGM_FR_FIELD_THRESHOLD_DBL", GHRCategory.DCGMI_THERMAL),
+        # Diagnostic errors
+        ("DCGM_FR_CUDA_DBE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_MEMORY_MISMATCH", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_FAULTY_MEMORY", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_L1TAG_MISCOMPARE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_BROKEN_P2P_MEMORY_DEVICE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_BROKEN_P2P_WRITER_DEVICE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_BROKEN_P2P_NVLINK_WRITER_DEVICE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_BROKEN_P2P_NVLINK_MEMORY_DEVICE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_BROKEN_P2P_PCIE_MEMORY_DEVICE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_BROKEN_P2P_PCIE_WRITER_DEVICE", GHRCategory.DCGM_DIAG_FAILURE),
+        ("DCGM_FR_FABRIC_MANAGER_TRAINING_ERROR", GHRCategory.DCGM_DIAG_FAILURE),
+    )
+
+    FIELD_GHR_MAP = {
+        "DCGM_FI_DEV_ROW_REMAP_FAILURE": GHRCategory.ROW_REMAP_FAILURE,
+        "DCGM_FI_DEV_ECC_DBE_VOL_TOTAL": GHRCategory.XID48_DOUBLE_BIT_ECC,
+    }
+
     ENTITY_GROUP_NAMES = {
         dcgm_fields.DCGM_FE_NONE: "None",
         dcgm_fields.DCGM_FE_GPU: "GPU",
@@ -167,12 +214,12 @@ def get_fields(cls):
         "DCGM_FR_NVSWITCH_FATAL_ERROR",
         "DCGM_FR_FAULTY_MEMORY",
         "DCGM_FR_FIELD_VIOLATION",
-        "DCGM_FR_FABRIC_PROBE_STATE"
+        "DCGM_FR_FABRIC_PROBE_STATE",
+        "DCGM_FR_FIELD_THRESHOLD_DBL",
     )
 
     HEALTH_WARNINGS = _resolve_error_codes(
         "DCGM_FR_PCI_REPLAY_RATE",
-        "DCGM_FR_CORRUPT_INFOROM",
         "DCGM_FR_NVSWITCH_NON_FATAL_ERROR",
         "DCGM_FR_NVLINK_SYMBOL_BER_THRESHOLD",
         "DCGM_FR_NVLINK_EFFECTIVE_BER_THRESHOLD",
@@ -512,5 +559,5 @@ def set_policy(cls):
         return policy
     @classmethod
     def get_health_mask(cls):
-        exclude_mask = dcgm_structs.DCGM_HEALTH_WATCH_THERMAL | dcgm_structs.DCGM_HEALTH_WATCH_POWER
+        exclude_mask = dcgm_structs.DCGM_HEALTH_WATCH_POWER
         return dcgm_structs.DCGM_HEALTH_WATCH_ALL & ~exclude_mask

healthagent/ghr.py

@@ -0,0 +1,33 @@
+from enum import Enum
+
+class GHRCategory(Enum):
+    # Operational
+    RESET = "Resource.Hpc.Reset"
+    REBOOT = "Resource.Hpc.Reboot"
+    # GPU
+    MISSING_GPU = "Resource.Hpc.Unhealthy.HpcMissingGpu"
+    DCGM_DIAG_FAILURE = "Resource.Hpc.Unhealthy.HpcGpuDcgmDiagFailure"
+    ROW_REMAP_FAILURE = "Resource.Hpc.Unhealthy.HpcRowRemapFailure"
+    INFOROM_CORRUPTION = "Resource.Hpc.Unhealthy.HpcInforomCorruption"
+    XID95_UNCONTAINED_ECC = "Resource.Hpc.Unhealthy.XID95UncontainedECCError"
+    XID94_CONTAINED_ECC = "Resource.Hpc.Unhealthy.XID94ContainedECCError"
+    XID79_FALLEN_OFF_BUS = "Resource.Hpc.Unhealthy.XID79FallenOffBus"
+    XID48_DOUBLE_BIT_ECC = "Resource.Hpc.Unhealthy.XID48DoubleBitECC"
+    UNHEALTHY_GPU_NVIDIASMI = "Resource.Hpc.Unhealthy.UnhealthyGPUNvidiasmi"
+    NVLINK = "Resource.Hpc.Unhealthy.NvLink"
+    DCGMI_THERMAL = "Resource.Hpc.Unhealthy.HpcDcgmiThermalReport"
+    ECC_PAGE_RETIRE_FULL = "Resource.Hpc.Unhealthy.ECCPageRetirementTableFull"
+    DBE_OVER_LIMIT = "Resource.Hpc.Unhealthy.DBEOverLimit"
+    GPU_XID_ERROR = "Resource.Hpc.Unhealthy.GpuXIDError"
+    EROT_FAILURE = "Resource.Hpc.Unhealthy.EROTFailure"
+    GPU_MEM_BW_FAILURE = "Resource.Hpc.Unhealthy.GPUMemoryBWFailure"
+    # Network / InfiniBand
+    MISSING_IB = "Resource.Hpc.Unhealthy.MissingIB"
+    IB_PERFORMANCE = "Resource.Hpc.Unhealthy.IBPerformance"
+    IB_PORT_DOWN = "Resource.Hpc.Unhealthy.IBPortDown"
+    IB_PORT_FLAPPING = "Resource.Hpc.Unhealthy.IBPortFlapping"
+    # Generic
+    GENERIC_FAILURE = "Resource.Hpc.Unhealthy.HpcGenericFailure"
+    MANUAL_INVESTIGATION = "Resource.Hpc.Unhealthy.ManualInvestigation"
+    # CPU
+    CPU_PERFORMANCE = "Resource.Hpc.Unhealthy.CPUPerformance"

healthagent/gpu.py

@@ -218,6 +218,7 @@ async def gpu_count_check(self):
         unique_counts = set(custom_fields.values())
         if len(unique_counts) > 1:
             report.status = HealthStatus.ERROR
+            report.ghr_category = GHRCategory.MISSING_GPU
             detail_parts = [f"{source} shows {count}" for source, count in custom_fields.items()]
             report.details = "GPU count mismatch: " + ", ".join(detail_parts)
             report.description = "GPU Count Mismatch"
@@ -326,8 +327,14 @@ def track_fieldsv2(self):
                         custom_fields[entity_key]["errors" if severity == "error" else "warnings"].append(msg)
                         if severity == "error":
                             custom_fields['error_count'] += 1
+                            ghr_cat = Wrap.FIELD_GHR_MAP.get(watch["field"])
+                            if ghr_cat:
+                                custom_fields['_ghr_field_error'] = ghr_cat
                         else:
                             custom_fields['warning_count'] += 1
+                            ghr_cat = Wrap.FIELD_GHR_MAP.get(watch["field"])
+                            if ghr_cat and '_ghr_field_any' not in custom_fields:
+                                custom_fields['_ghr_field_any'] = ghr_cat
                         custom_fields['category'].add(watch["category"])
 
                     # XID handling — GPU entities only
@@ -485,6 +492,48 @@ async def run_background_healthchecks(self):
                     if not isinstance(v, dict) or any(v.values())
                 }
 
+                # Resolve GHR category from XIDs and health incidents
+                ghr_error = None
+                ghr_any = None
+
+                # Check health incidents for GHR-mapped error codes
+                for index in range(0, incident_count):
+                    error_code = group_health.incidents[index].error.code
+                    ghr_cat = Wrap.ERROR_GHR_MAP.get(error_code)
+                    if ghr_cat:
+                        if ghr_any is None:
+                            ghr_any = ghr_cat
+                        if error_code in Wrap.HEALTH_ERRORS and ghr_error is None:
+                            ghr_error = ghr_cat
+
+                # Check XIDs
+                for gpu_id, xids in self.xid_history.items():
+                    for xid_num in xids:
+                        if xid_num in self.xid_ignore:
+                            continue
+                        category = Wrap.XID_GHR_MAP.get(xid_num, GHRCategory.GPU_XID_ERROR)
+                        if ghr_any is None:
+                            ghr_any = category
+                        is_error = xid_num not in self.xid_warning
+                        if self.xid_error and xid_num not in self.xid_error:
+                            is_error = False
+                        if is_error and ghr_error is None:
+                            ghr_error = category
+
+                # Check field watches
+                if '_ghr_field_error' in custom_fields:
+                    if ghr_error is None:
+                        ghr_error = custom_fields.pop('_ghr_field_error')
+                    else:
+                        custom_fields.pop('_ghr_field_error')
+                if '_ghr_field_any' in custom_fields:
+                    if ghr_any is None:
+                        ghr_any = custom_fields.pop('_ghr_field_any')
+                    else:
+                        custom_fields.pop('_ghr_field_any')
+
+                report.ghr_category = ghr_error or ghr_any
+
                 if custom_fields['error_count'] == 0 and custom_fields['warning_count'] == 0:
                     await self.reporter.update_report(name=health_system, report=report)
                     return
@@ -641,6 +690,14 @@ def run_active_healthchecksv2(gpu_id: list = None, tests: str = '', params: str
         report.description = "GPU Diagnostic test errors"
         report.custom_fields = custom_fields
 
+        # Resolve GHR from diag error codes
+        if response and response.numErrors > 0:
+            for errIdx in range(response.numErrors):
+                ghr_cat = Wrap.ERROR_GHR_MAP.get(response.errors[errIdx].code)
+                if ghr_cat:
+                    report.ghr_category = ghr_cat
+                    break
+
         all_errors = []
         all_warnings = []
         for key, val in custom_fields.items():

healthagent/network.py

@@ -8,6 +8,7 @@
 from healthagent.healthmodule import HealthModule
 from healthagent.config import NetworkConfig, ThresholdCheck
 from healthagent.reporter import Reporter, HealthReport, HealthStatus
+from healthagent.ghr import GHRCategory
 from healthagent.scheduler import Scheduler
 
 log = logging.getLogger('healthagent')
@@ -49,6 +50,12 @@ class NetworkInterface:
 
 class NetworkHealthChecks(HealthModule):
 
+    _NET_GHR_MAP = {
+        ("infiniband", "state"): GHRCategory.IB_PORT_DOWN,
+        ("infiniband", "phys_state"): GHRCategory.IB_PORT_DOWN,
+        ("infiniband", "link_downed"): GHRCategory.IB_PORT_FLAPPING,
+    }
+
     def __init__(self, reporter: Reporter, config: 'NetworkConfig | None' = None):
         super().__init__(reporter, config or NetworkConfig())
         self.config: NetworkConfig = self.config
@@ -118,10 +125,12 @@ async def run_network_checks(self):
         report = HealthReport()
         custom_fields = {}
         details = []
+        ghr_error = None
+        ghr_any = None
         # Derive port-level field names from the IBPort dataclass
         _ib_port_fields = {f.name for f in fields(IBPort)}
 
-        def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None):
+        def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None, config_key=None):
             key = (iface_name, field, port_num)
             max_strikes = check.strikes
 
@@ -162,10 +171,19 @@ def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None
                         custom_fields.setdefault(iface_name, {}).setdefault("errors", []).append(msg)
                         details.append(f"ERROR: {iface_name} - {msg}")
                         report.escalate(HealthStatus.ERROR)
+                        ghr_cat = self._NET_GHR_MAP.get((config_key, field))
+                        if ghr_cat:
+                            nonlocal ghr_error
+                            ghr_error = ghr_cat
                     else:
                         custom_fields.setdefault(iface_name, {}).setdefault("warnings", []).append(msg)
                         details.append(f"WARNING: {iface_name} - {msg}")
                         report.escalate(HealthStatus.WARNING)
+                        ghr_cat = self._NET_GHR_MAP.get((config_key, field))
+                        if ghr_cat:
+                            nonlocal ghr_any
+                            if ghr_any is None:
+                                ghr_any = ghr_cat
                     break  # error takes precedence over warning
 
             # Strike tracking: count OK→ERROR transitions
@@ -193,13 +211,13 @@ def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None
                         value = getattr(port, field, None)
                         if value is None:
                             continue
-                        check_field(ni.name, field, check, value, port_num=port_num)
+                        check_field(ni.name, field, check, value, port_num=port_num, config_key=config_key)
                 else:
                     # Interface-level field
                     value = getattr(ni, field, None)
                     if value is None:
                         continue
-                    check_field(ni.name, field, check, value)
+                    check_field(ni.name, field, check, value, config_key=config_key)
 
             if ni.ib_device:
                 custom_fields[ni.name]["ib_device"] = {
@@ -216,4 +234,6 @@ def check_field(iface_name, field, check: 'ThresholdCheck', value, port_num=None
         if report.status != HealthStatus.OK:
             report.description = "Network health issues detected"
 
+        report.ghr_category = ghr_error or ghr_any
+
         await self.reporter.update_report(self.run_network_checks.report_name, report=report)

healthagent/reporter.py

@@ -6,7 +6,9 @@
 from typing import Any, Dict
 import logging
 import os
+import subprocess
 from healthagent.scheduler import Scheduler
+from healthagent.ghr import GHRCategory
 
 log = logging.getLogger('healthagent')
 
@@ -85,6 +87,8 @@ class HealthReport:
     # attribute in __post_init__. Note: aux_data will not survive an asdict() round-trip;
     # use deepcopy(report) to preserve it.
     aux_data: InitVar[dict] = None
+    # Azure Guest Health Reporting
+    ghr_category: GHRCategory = None
 
     def __eq__(self, other):
         if not isinstance(other, HealthReport):
@@ -142,23 +146,59 @@ def view(self, cli_exclude: bool = True) -> Dict[str, Any]:
 
 class Reporter:
 
+    JETPACK_VERSION_MINIMUM = "8.8.0"
+    JETPACK_VERSION_GHR = "8.10.0"
     def __init__(self, name: str = None):
         """
         name: name of the health report (optional)
         """
         self.jetpack = "/opt/cycle/jetpack/bin/jetpack"
         self.store = {}
         self.publish_cc = os.getenv("PUBLISH_CC", 'true').lower() == 'true'
+        self.enable_ghr = True
 
         if not os.path.exists(self.jetpack):
             log.info(f"{self.jetpack} not found")
             self.publish_cc = False
+            self.enable_ghr = False
+        elif self.publish_cc:
+            version = self._get_jetpack_version()
+            if version is None or version < self._parse_version(self.JETPACK_VERSION_MINIMUM):
+                self.publish_cc = False
+                self.enable_ghr = False
+                log.warning(f"Jetpack version too old or unknown, disabling CC publishing and GHR")
+            elif version < self._parse_version(self.JETPACK_VERSION_GHR):
+                self.enable_ghr = False
+                log.info(f"Jetpack version < {self.JETPACK_VERSION_GHR}, disabling GHR")
 
         if not self.publish_cc:
-            log.info(f"Disabling publishing reports to CC")
+            log.info("Disabling publishing reports to CC.")
+            self.enable_ghr = False
+        if not self.enable_ghr:
+            log.info("Disabling Guest Health Reporting")
         if name:
             self.store[name] = HealthReport()
 
+
+    @staticmethod
+    def _parse_version(version_str: str) -> tuple:
+        """Parse '8.10.0-3822' → (8, 10, 0)"""
+        base = version_str.strip().split('-')[0]
+        return tuple(int(x) for x in base.split('.'))
+
+    def _get_jetpack_version(self) -> tuple | None:
+
+        try:
+            result = subprocess.run(
+                [self.jetpack, '--version'],
+                capture_output=True, text=True, timeout=10
+            )
+            if result.returncode == 0:
+                return self._parse_version(result.stdout)
+        except Exception as e:
+            log.warning(f"Failed to determine jetpack version: {e}")
+        return None
+
     @classmethod
     def load_reporter_obj(cls, old) -> 'Reporter':
 
@@ -247,6 +287,10 @@ async def publish_cc_status(self, name):
             if report.details is not None:
                 args.extend(['--details', report.details])
 
+            if self.enable_ghr:
+                if report.ghr_category is not None:
+                    args.extend(['--action-type', 'GHR', '--ghr-category', report.ghr_category.value])
+
         # Schedule the subprocess task
         task = Scheduler.subprocess(*args)
-        await Scheduler.add_task(task)
+        await Scheduler.add_task(task)