Add slurmrestd (REST API) collection mode to the Slurm integration

slurm/assets/configuration/spec.yaml

@@ -85,6 +85,60 @@ files:
       value:
         type: integer
         example: 1
+    - name: slurm_rest_api_url
+      description: |
+        Base URL of the Slurm REST API (slurmrestd), for example http://slurm-restapi:6820.
+        When set, the check collects metrics over the REST API instead of executing the Slurm
+        CLI binaries. This lets the check run without co-located Slurm binaries or munge/auth
+        access (for example, as a cluster check against a containerized Slurm deployment such
+        as SUNK/Slinky). Leave unset to use the default CLI collection mode. slurmrestd is
+        optional in Slurm, so this mode requires it to be enabled on the cluster.
+      fleet_configurable: true
+      value:
+        type: string
+        display_default: null
+        example: http://slurm-restapi:6820
+    - name: slurm_openapi_version
+      description: |
+        The slurmrestd OpenAPI data_parser version to query (for example v0.0.42). Only used
+        when slurm_rest_api_url is set.
+      fleet_configurable: true
+      value:
+        type: string
+        example: v0.0.42
+    - name: slurm_rest_api_token
+      description: |
+        The Slurm JWT (auth/jwt) used to authenticate to slurmrestd, sent in the
+        X-SLURM-USER-TOKEN header. Only used when slurm_rest_api_url is set. Prefer
+        slurm_rest_api_token_file for tokens that are rotated.
+      fleet_configurable: false
+      value:
+        type: string
+        secret: true
+        example: <SLURM_JWT>
+    - name: slurm_rest_api_token_file
+      description: |
+        Path to a file containing the Slurm JWT used to authenticate to slurmrestd. The file is
+        re-read on every check run so rotated tokens are picked up. Takes precedence over
+        slurm_rest_api_token. Only used when slurm_rest_api_url is set.
+      fleet_configurable: true
+      value:
+        type: string
+        display_default: null
+        example: /etc/datadog-agent/slurm.d/token
+    - name: slurm_rest_api_user
+      description: |
+        Slurm username to send in the X-SLURM-USER-NAME header, in addition to the JWT token.
+        Only needed when slurmrestd sits behind an authenticating proxy that uses one shared
+        privileged token to act on behalf of multiple users (see
+        https://slurm.schedmd.com/jwt.html). Not needed for a JWT generated directly for a
+        single user (e.g. via `scontrol token`), since the token's own `sun` claim already
+        identifies the user in that case. Only used when slurm_rest_api_url is set.
+      fleet_configurable: true
+      value:
+        type: string
+        display_default: null
+        example: datadog
     - name: sinfo_path
       description: Full path to the sinfo binary.
       fleet_configurable: true

slurm/assets/service_checks.json

@@ -1 +1,14 @@
-[]
+[
+    {
+        "agent_version": "7.70.0",
+        "integration": "Slurm",
+        "check": "slurm.rest.can_connect",
+        "statuses": [
+            "ok",
+            "critical"
+        ],
+        "groups": [],
+        "name": "Slurm REST API can connect",
+        "description": "Returns `CRITICAL` if the Agent cannot reach the Slurm REST API (slurmrestd) when `slurm_rest_api_url` is configured, otherwise returns `OK`."
+    }
+]

slurm/changelog.d/24566.added

@@ -0,0 +1 @@
+Add an optional slurmrestd (REST API) collection mode. When `slurm_rest_api_url` is set, the check collects scheduler (`sdiag`), partition, and node (`sinfo`) metrics, including per-node and per-partition GPU counts when `collect_gpu_stats` is enabled, from the Slurm REST API over HTTP with a JWT token instead of executing the Slurm CLI binaries, enabling a no-sidecar deployment (for example as a cluster check) for containerized Slurm such as SUNK/Slinky.

slurm/datadog_checks/slurm/check.py

@@ -33,6 +33,13 @@
     SSHARE_MAP,
     SSHARE_PARAMS,
 )
+from .rest import (
+    SlurmRestAPIClient,
+    iter_node_metrics,
+    iter_partition_gpu_metrics,
+    iter_partition_metrics,
+    iter_sdiag_metrics,
+)
 
 
 def get_subprocess_output(cmd):
@@ -83,6 +90,22 @@ def __init__(self, name, init_config, instances):
         self.gpu_stats = is_affirmative(self.instance.get('collect_gpu_stats', False))
         self.sinfo_collection_level = self.instance.get('sinfo_collection_level', 1)
 
+        # REST (slurmrestd) mode. When slurm_rest_api_url is set, collect over the Slurm REST
+        # API instead of the CLI binaries (see rest.py). Works without co-located binaries,
+        # munge/auth, or a login node -- suitable for a cluster check against containerized
+        # Slurm (SUNK/Slinky). slurmrestd is optional and handled gracefully if unreachable.
+        self.rest_api_url = self.instance.get('slurm_rest_api_url')
+        self.use_rest_api = bool(self.rest_api_url)
+        if self.use_rest_api:
+            self.openapi_version = self.instance.get('slurm_openapi_version', 'v0.0.42')
+            self._rest_token = self.instance.get('slurm_rest_api_token')
+            self._rest_token_file = self.instance.get('slurm_rest_api_token_file')
+            self.rest_client = SlurmRestAPIClient(self.http, self.rest_api_url, self.openapi_version, self.log)
+            # Only needed behind an authenticating proxy that reuses one privileged token for
+            # multiple users (https://slurm.schedmd.com/jwt.html); a direct per-user JWT (e.g.
+            # via `scontrol token`) already identifies the user via its own `sun` claim.
+            self.rest_client.user = self.instance.get('slurm_rest_api_user')
+
         # Binary paths
         self.slurm_binaries_dir = self.init_config.get('slurm_binaries_dir', '/usr/bin/')
 
@@ -133,7 +156,84 @@ def __init__(self, name, init_config, instances):
         self.debug_sacct_stats = is_affirmative(self.instance.get('debug_sacct_stats', False))
         self.debug_scontrol_stats = is_affirmative(self.instance.get('debug_scontrol_stats', False))
 
+    def _get_rest_token(self):
+        if self._rest_token_file:
+            try:
+                with open(self._rest_token_file) as f:
+                    return f.read().strip()
+            except OSError as e:
+                self.log.error("Could not read slurm_rest_api_token_file %s: %s", self._rest_token_file, e)
+                return None
+        return self._rest_token
+
+    def _check_rest(self):
+        base_tags = list(self.tags)
+        # Read the token once per run (rotation-safe without re-reading the file per request).
+        self.rest_client.token = self._get_rest_token()
+
+        connected = self.rest_client.get('ping') is not None
+        self.service_check('rest.can_connect', self.OK if connected else self.CRITICAL, tags=base_tags)
+        if not connected:
+            self.log.warning(
+                "Could not reach slurmrestd at %s. slurmrestd is optional in Slurm; ensure it is "
+                "enabled and the JWT token is valid. Skipping this run.",
+                self.rest_api_url,
+            )
+            return
+
+        if self.collect_sdiag_stats:
+            self.gauge('sdiag.enabled', 1, tags=base_tags)
+            for metric, value, tags in iter_sdiag_metrics(self.rest_client.get('diag')):
+                self.gauge(metric, value, tags=base_tags + tags)
+
+        if self.collect_sinfo_stats:
+            # Mirror CLI semantics: level 1 (default) collects partition metrics; node metrics
+            # require level >= 2.
+            self.gauge('sinfo.partition.enabled', 1, tags=base_tags)
+            for metric, value, tags in iter_partition_metrics(self.rest_client.get('partitions')):
+                self.gauge(metric, value, tags=base_tags + tags)
+
+            # /nodes is needed for node metrics (level >= 2) and, when GPU stats are enabled, for
+            # partition GPU totals (aggregated from nodes, since /partitions has no GPU field).
+            # Fetch it at most once per run and reuse.
+            nodes_payload = None
+            if self.gpu_stats:
+                nodes_payload = self.rest_client.get('nodes')
+                for metric, value, tags in iter_partition_gpu_metrics(nodes_payload):
+                    self.gauge(metric, value, tags=base_tags + tags)
+
+            if self.sinfo_collection_level > 1:
+                self.gauge('sinfo.node.enabled', 1, tags=base_tags)
+                if nodes_payload is None:
+                    nodes_payload = self.rest_client.get('nodes')
+                for metric, value, tags in iter_node_metrics(nodes_payload, collect_gpu=self.gpu_stats):
+                    self.gauge(metric, value, tags=base_tags + tags)
+
+        # These collectors are not (yet) supported over REST. scontrol is node-local OS data not
+        # exposed by slurmrestd; squeue/sacct/sshare REST support is a follow-up. Warn instead of
+        # silently dropping so a CLI user migrating to REST is not surprised by missing data.
+        unsupported = [
+            name
+            for name, enabled in (
+                ('collect_squeue_stats', self.collect_squeue_stats),
+                ('collect_sacct_stats', self.collect_sacct_stats),
+                ('collect_sshare_stats', self.collect_sshare_stats),
+                ('collect_scontrol_stats', self.collect_scontrol_stats),
+            )
+            if enabled
+        ]
+        if unsupported:
+            self.log.warning(
+                "The following collectors are enabled but not supported in REST mode and are "
+                "skipped: %s. scontrol is node-local; squeue/sacct/sshare REST support is a follow-up.",
+                ', '.join(unsupported),
+            )
+
     def check(self, _):
+        if self.use_rest_api:
+            self._check_rest()
+            return
+
         self.collect_metadata()
 
         commands = []

slurm/datadog_checks/slurm/config_models/defaults.py

@@ -112,6 +112,10 @@ def instance_sinfo_path():
     return '/usr/bin/sinfo'
 
 
+def instance_slurm_openapi_version():
+    return 'v0.0.42'
+
+
 def instance_squeue_path():
     return '/usr/bin/squeue'
 

slurm/datadog_checks/slurm/config_models/instance.py

@@ -66,6 +66,11 @@ class InstanceConfig(BaseModel):
     service: Optional[str] = None
     sinfo_collection_level: Optional[int] = None
     sinfo_path: Optional[str] = None
+    slurm_openapi_version: Optional[str] = None
+    slurm_rest_api_token: Optional[str] = None
+    slurm_rest_api_token_file: Optional[str] = None
+    slurm_rest_api_url: Optional[str] = None
+    slurm_rest_api_user: Optional[str] = None
     squeue_path: Optional[str] = None
     sshare_path: Optional[str] = None
     tags: Optional[tuple[str, ...]] = None

slurm/datadog_checks/slurm/data/conf.yaml.example

@@ -75,6 +75,46 @@ instances:
     #
     # sinfo_collection_level: 1
 
+    ## @param slurm_rest_api_url - string - optional
+    ## Base URL of the Slurm REST API (slurmrestd), for example http://slurm-restapi:6820.
+    ## When set, the check collects metrics over the REST API instead of executing the Slurm
+    ## CLI binaries. This lets the check run without co-located Slurm binaries or munge/auth
+    ## access (for example, as a cluster check against a containerized Slurm deployment such
+    ## as SUNK/Slinky). Leave unset to use the default CLI collection mode. slurmrestd is
+    ## optional in Slurm, so this mode requires it to be enabled on the cluster.
+    #
+    # slurm_rest_api_url: http://slurm-restapi:6820
+
+    ## @param slurm_openapi_version - string - optional - default: v0.0.42
+    ## The slurmrestd OpenAPI data_parser version to query (for example v0.0.42). Only used
+    ## when slurm_rest_api_url is set.
+    #
+    # slurm_openapi_version: v0.0.42
+
+    ## @param slurm_rest_api_token - string - optional
+    ## The Slurm JWT (auth/jwt) used to authenticate to slurmrestd, sent in the
+    ## X-SLURM-USER-TOKEN header. Only used when slurm_rest_api_url is set. Prefer
+    ## slurm_rest_api_token_file for tokens that are rotated.
+    #
+    # slurm_rest_api_token: <SLURM_JWT>
+
+    ## @param slurm_rest_api_token_file - string - optional
+    ## Path to a file containing the Slurm JWT used to authenticate to slurmrestd. The file is
+    ## re-read on every check run so rotated tokens are picked up. Takes precedence over
+    ## slurm_rest_api_token. Only used when slurm_rest_api_url is set.
+    #
+    # slurm_rest_api_token_file: /etc/datadog-agent/slurm.d/token
+
+    ## @param slurm_rest_api_user - string - optional
+    ## Slurm username to send in the X-SLURM-USER-NAME header, in addition to the JWT token.
+    ## Only needed when slurmrestd sits behind an authenticating proxy that uses one shared
+    ## privileged token to act on behalf of multiple users (see
+    ## https://slurm.schedmd.com/jwt.html). Not needed for a JWT generated directly for a
+    ## single user (e.g. via `scontrol token`), since the token's own `sun` claim already
+    ## identifies the user in that case. Only used when slurm_rest_api_url is set.
+    #
+    # slurm_rest_api_user: datadog
+
     ## @param sinfo_path - string - optional - default: /usr/bin/sinfo
     ## Full path to the sinfo binary.
     #