From f4106f9d8577a892518703474b12ee4b192e9bee Mon Sep 17 00:00:00 2001 From: Amber Xue Date: Thu, 25 Jun 2026 16:26:04 -0700 Subject: [PATCH] feat: mark persistence mode diable as warning level Signed-off-by: Amber Xue --- .../accelerator/nvidia/persistence-mode/component.go | 6 +++++- .../accelerator/nvidia/persistence-mode/component_test.go | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component.go b/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component.go index cdbaedd5..7f5482f1 100644 --- a/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component.go +++ b/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component.go @@ -170,6 +170,7 @@ func (c *component) Check() components.CheckResult { persistenceMode, err := c.getPersistenceModeFunc(uuid, dev) if err != nil { cr.err = err + // deliberately stayed Unhealthy to indicate NVML query errors, GPU-lost, and GPU-requires-reset cr.health = apiv1.HealthStateTypeUnhealthy cr.reason = "error getting persistence mode" @@ -208,7 +209,10 @@ func (c *component) Check() components.CheckResult { } if len(notEnabled) > 0 { - cr.health = apiv1.HealthStateTypeUnhealthy + // Persistence mode being disabled is a configuration issue that does not + // affect running workloads, so it is flagged as a warning (Degraded) + // rather than critical (Unhealthy). + cr.health = apiv1.HealthStateTypeDegraded if len(notEnabled) == len(cr.PersistenceModes) { cr.reason = fmt.Sprintf("all %d GPU(s) disabled persistence mode", len(devs)) } else { diff --git a/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component_test.go b/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component_test.go index 7795d7fb..eead13cc 100644 --- a/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component_test.go +++ b/third_party/fleet-intelligence-sdk/components/accelerator/nvidia/persistence-mode/component_test.go @@ -337,8 +337,8 @@ func TestCheck_MultipleDevices(t *testing.T) { require.True(t, ok, "result should be of type *checkResult") require.NotNil(t, data, "data should not be nil") - // This should be unhealthy since one device has persistence mode supported but not enabled - assert.Equal(t, apiv1.HealthStateTypeUnhealthy, data.health, "data should be marked unhealthy") + // This should be degraded (warning) since one device has persistence mode supported but not enabled + assert.Equal(t, apiv1.HealthStateTypeDegraded, data.health, "data should be marked degraded") assert.Equal(t, "gpu-uuid-456 persistence mode supported but not enabled", data.reason) assert.Len(t, data.PersistenceModes, 2) @@ -734,7 +734,7 @@ func TestPersistenceModeCheck(t *testing.T) { {UUID: "GPU-1", Supported: true, Enabled: true}, {UUID: "GPU-2", Supported: true, Enabled: false}, }, - expectedHealth: apiv1.HealthStateTypeUnhealthy, + expectedHealth: apiv1.HealthStateTypeDegraded, expectedReasonCmp: "GPU-2 persistence mode supported but not enabled", }, { @@ -743,7 +743,7 @@ func TestPersistenceModeCheck(t *testing.T) { {UUID: "GPU-1", Supported: true, Enabled: false}, {UUID: "GPU-2", Supported: true, Enabled: false}, }, - expectedHealth: apiv1.HealthStateTypeUnhealthy, + expectedHealth: apiv1.HealthStateTypeDegraded, expectedReasonCmp: "all 2 GPU(s) disabled persistence mode", }, {