Disable sacct metrics if sacct exists but accounting is disabled

azure-slurm-exporter/exporter/sacct.py

@@ -37,6 +37,9 @@ class Sacct(BaseCollector):
             "153:0": "SIGXFSZ - File size limit",
         }
 
+    ACCOUNTING_STORAGE_TYPE_KEY = "AccountingStorageType"
+    ACCOUNTING_STORAGE_DISABLED_VALUES = {"(null)", "accounting_storage/none"}
+
     def __init__(self, binary_path="/usr/bin/sacct", interval=300, timeout=120, starttime=None):
         self.binary_path = binary_path
         self.interval = interval
@@ -59,6 +62,15 @@ def initialize(self) -> None:
         if not util.is_file_binary(self.binary_path):
             log.error(f"{self.binary_path} is not a file or not executable")
             raise SacctNotAvailException
+
+        storage_type = util.get_scontrol_config_value(
+            self.ACCOUNTING_STORAGE_TYPE_KEY,
+            timeout=min(self.timeout, 10),
+        )
+        if storage_type and storage_type.lower() in self.ACCOUNTING_STORAGE_DISABLED_VALUES:
+            log.warning("Slurm accounting is disabled")
+            raise SacctNotAvailException
+
         disable_created_metrics()
 
     def start(self) -> None:

azure-slurm-exporter/exporter/util.py

@@ -1,5 +1,6 @@
 import os
 import logging
+import subprocess
 
 log = logging.getLogger(__name__)
 
@@ -29,4 +30,38 @@ def validate_port(port_env_var: str, default_port: int) -> int:
         )
         port = default_port
 
-    return port
+    return port
+
+def get_scontrol_config_value(key: str, timeout: int = 10) -> str:
+    """Run "scontrol show config" and return the value for a config key."""
+    try:
+        proc = subprocess.run(
+            ["scontrol", "show", "config"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            check=False,
+            timeout=timeout,
+        )
+    except Exception as e:
+        log.warning("Failed to read scontrol config for %s: %s", key, e)
+        return ""
+
+    if proc.returncode != 0:
+        log.warning(
+            "scontrol show config failed (rc=%s) while reading %s: %s",
+            proc.returncode,
+            key,
+            (proc.stderr or "").strip(),
+        )
+        return ""
+
+    for line in (proc.stdout or "").splitlines():
+        stripped = line.strip()
+        if "=" not in stripped:
+            continue
+        lhs, rhs = stripped.split("=", 1)
+        if lhs.strip() == key:
+            return rhs.strip()
+
+    return ""

azure-slurm-exporter/test/test_sacct.py

@@ -12,10 +12,12 @@ def sacct(self):
         """Create a Sacct collector instance for testing."""
         return Sacct()
 
+    @patch("exporter.util.get_scontrol_config_value")
     @patch("exporter.util.is_file_binary")
-    def test_initialize_valid_binary(self, mock_is_binary, sacct):
+    def test_initialize_valid_binary(self, mock_is_binary, mock_get_config, sacct):
         """Test successful initialization when binary exists."""
         mock_is_binary.return_value = True
+        mock_get_config.return_value = "accounting_storage/slurmdbd"
         sacct.initialize()
 
     @patch("exporter.util.is_file_binary")
@@ -25,14 +27,23 @@ def test_initialize_invalid_binary(self, mock_is_binary, sacct):
         with pytest.raises(SacctNotAvailException):
             sacct.initialize()
 
+    @patch("exporter.util.get_scontrol_config_value")
+    @patch("exporter.util.is_file_binary")
+    def test_initialize_accounting_disabled(self, mock_is_binary, mock_get_config, sacct):
+        """Test initialization fails when scontrol reports accounting storage is disabled."""
+        mock_is_binary.return_value = True
+        mock_get_config.return_value = "(null)"
+        with pytest.raises(SacctNotAvailException):
+            sacct.initialize()
+
 
 class TestSacctParseOutput:
 
     @pytest.fixture
     def sacct(self):
         """Create a Sacct collector instance for testing."""
         sacct = Sacct()
-        with patch("exporter.util.is_file_binary", return_value=True):
+        with patch("exporter.util.is_file_binary", return_value=True), patch("exporter.util.get_scontrol_config_value", return_value=""):
             sacct.initialize()
         return sacct
 
@@ -114,7 +125,7 @@ class TestSacctExportMetrics:
     def sacct(self):
         """Create a Sacct collector instance for testing."""
         sacct = Sacct()
-        with patch("exporter.util.is_file_binary", return_value=True):
+        with patch("exporter.util.is_file_binary", return_value=True), patch("exporter.util.get_scontrol_config_value", return_value=""):
             sacct.initialize()
         return sacct
 
@@ -141,7 +152,7 @@ class TestSacctExitCodeMapping:
     def sacct(self):
         """Create a Sacct collector instance for testing."""
         sacct = Sacct()
-        with patch("exporter.util.is_file_binary", return_value=True):
+        with patch("exporter.util.is_file_binary", return_value=True), patch("exporter.util.get_scontrol_config_value", return_value=""):
             sacct.initialize()
         return sacct
 
@@ -185,7 +196,7 @@ class TestSacctMetricValues:
     def sacct(self):
         """Create a Sacct collector instance for testing."""
         sacct = Sacct()
-        with patch("exporter.util.is_file_binary", return_value=True):
+        with patch("exporter.util.is_file_binary", return_value=True), patch("exporter.util.get_scontrol_config_value", return_value=""):
             sacct.initialize()
         return sacct
 
@@ -269,7 +280,7 @@ async def _sacct_with_one_successful_query() -> Sacct:
         """Create a Sacct instance with one successful query already executed,
         so the counter has a known baseline value of 1 for (node1, batch, completed, '', 0:0)."""
         sacct = Sacct()
-        with patch("exporter.util.is_file_binary", return_value=True):
+        with patch("exporter.util.is_file_binary", return_value=True), patch("exporter.util.get_scontrol_config_value", return_value=""):
             sacct.initialize()
         mock_proc = MagicMock()
         mock_proc.stdout = b"1|job1|node1|1|batch|0:0|0:0|COMPLETED|u1|2024-01-01T10:00:00|2024-01-01T09:00:00|2024-01-01T11:00:00|None\n"