diff --git a/README.md b/README.md
index 1082dc3b..b6c338c6 100644
--- a/README.md
+++ b/README.md
@@ -574,14 +574,12 @@ Log can be found under `/var/log/azslurm-exporter.log`
| Metric | Type | Labels | Description |
|---|---|---|---|
| `squeue_partition_jobs_state` | Gauge | `partition`, `state` | Number of jobs in each state per partition |
-| `squeue_job_nodes_allocated` | Gauge | `job_id`, `job_name`, `partition`, `state`, `nodelist`, `start_time` | Nodes allocated to each running job |
**sacct metrics**
| Metric | Type | Labels | Description |
|---|---|---|---|
| `sacct_terminal_jobs_total` | Counter | `partition`, `exit_code`, `reason`, `state`| Cumulative count of completed/failed/cancelled jobs |
-| `sacct_failed_jobs` | Gauge | `jobid`, `jobname`, `nodelist`, `partition`, `exit_code`, `reason`, `state`, `starttime`, `endtime` | Per-job indicator for all non-completed terminal jobs in the query interval (value is always `1`) |
Terminal states tracked: `completed`, `failed`, `cancelled`, `timeout`, `node_fail`, `preempted`, `out_of_memory`, `deadline`, `boot_fail`. Exit codes are mapped to human-readable reasons (e.g. `137:0` → `SIGKILL - Force killed`).
diff --git a/azure-slurm-exporter/dashboards/azslurm-cluster.json b/azure-slurm-exporter/dashboards/azslurm-cluster.json
index f6a4a86f..d7c11a55 100644
--- a/azure-slurm-exporter/dashboards/azslurm-cluster.json
+++ b/azure-slurm-exporter/dashboards/azslurm-cluster.json
@@ -603,16 +603,6 @@
"fixedColor": "orange",
"mode": "fixed"
}
- },
- {
- "id": "links",
- "value": [
- {
- "targetBlank": false,
- "title": "Failed Jobs in last 6 months",
- "url": "/d/cff0f6w0qqwaoa/failed-jobs-dashboard?orgId=1&${__url_time_range}&${partition:queryparam}&${cluster:queryparam}&${instance:queryparam}&${cpu:queryparam}&${gpu:queryparam}&${nodearray:queryparam}"
- }
- ]
}
]
},
diff --git a/azure-slurm-exporter/dashboards/azslurm-details.json b/azure-slurm-exporter/dashboards/azslurm-details.json
index 81099918..065c7d97 100644
--- a/azure-slurm-exporter/dashboards/azslurm-details.json
+++ b/azure-slurm-exporter/dashboards/azslurm-details.json
@@ -16,7 +16,7 @@
}
]
},
- "description": "Slurm Cluster Running Jobs, Failed Jobs, Node Status and Partition Specs Dashboard",
+ "description": "Slurm Cluster Nodelist and Partition Specs Dashboard",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 2,
@@ -66,308 +66,6 @@
"title": "Extra Cluster Details",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "fixedColor": "green",
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
- },
- "filterable": true,
- "inspect": false
- },
- "mappings": [
- {
- "options": {
- "match": "null+nan",
- "result": {
- "index": 0,
- "text": "No jobs are currently running "
- }
- },
- "type": "special"
- }
- ],
- "noValue": "No jobs running within time range",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "id": 1047,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": true,
- "enablePagination": true,
- "fields": "",
- "reducer": [
- "count"
- ],
- "show": false
- },
- "frameIndex": 3,
- "showHeader": true,
- "sortBy": [
- {
- "desc": true,
- "displayName": "Number of Allocated Nodes"
- }
- ]
- },
- "pluginVersion": "11.6.9",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "topk(10,squeue_job_nodes_allocated{cluster=\"$cluster\",partition=~\"$partition\"})",
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Running Jobs between ${__from} and ${__to}",
- "transformations": [
- {
- "id": "timeSeriesTable",
- "options": {}
- },
- {
- "id": "sortBy",
- "options": {
- "fields": {},
- "sort": [
- {
- "field": "Number of Allocated Nodes"
- }
- ]
- }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "__name__": true,
- "cluster": true,
- "instance": true,
- "job": true,
- "microsoft.amwresourceid": true,
- "nodearray": true,
- "physical_host": true,
- "state": true,
- "subscription": true
- },
- "includeByName": {},
- "indexByName": {
- "Trend #A": 14,
- "__name__": 0,
- "cluster": 1,
- "instance": 2,
- "job": 3,
- "job_id": 4,
- "job_name": 5,
- "microsoft.amwresourceid": 6,
- "nodearray": 7,
- "nodelist": 9,
- "partition": 8,
- "physical_host": 10,
- "start_time": 11,
- "state": 12,
- "subscription": 13
- },
- "renameByName": {
- "Trend #A": "Number of Allocated Nodes"
- }
- }
- }
- ],
- "type": "table"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
- },
- "filterable": true,
- "inspect": false,
- "minWidth": 150
- },
- "mappings": [],
- "noValue": "No failed jobs within time range",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "reason"
- },
- "properties": [
- {
- "id": "noValue"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 24,
- "x": 0,
- "y": 14
- },
- "id": 1064,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": true,
- "enablePagination": true,
- "fields": "",
- "reducer": [
- "count"
- ],
- "show": false
- },
- "frameIndex": 1,
- "showHeader": true,
- "sortBy": []
- },
- "pluginVersion": "11.6.9",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sacct_failed_jobs{cluster=\"$cluster\",partition=~\"$partition\",state!=\"completed\"} offset -5m",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Failed Jobs between $__from and $__to",
- "transformations": [
- {
- "id": "timeSeriesTable",
- "options": {
- "B": {
- "stat": "lastNotNull"
- }
- }
- },
- {
- "id": "labelsToFields",
- "options": {
- "keepLabels": [
- "exit_code",
- "partition",
- "state"
- ],
- "mode": "columns"
- }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Trend #B": true,
- "__name__": true,
- "cluster": true,
- "endtime": true,
- "instance": true,
- "job": true,
- "microsoft.amwresourceid": true,
- "nodearray": true,
- "physical_host": true,
- "start_datetime hpc": true,
- "starttime": true,
- "subscription": true
- },
- "includeByName": {},
- "indexByName": {
- "Trend #B": 16,
- "__name__": 0,
- "cluster": 1,
- "endtime": 13,
- "exit_code": 7,
- "instance": 8,
- "job": 9,
- "jobid": 2,
- "jobname": 3,
- "microsoft.amwresourceid": 14,
- "nodearray": 17,
- "nodelist": 5,
- "partition": 4,
- "physical_host": 10,
- "reason": 15,
- "starttime": 12,
- "state": 6,
- "subscription": 11
- },
- "renameByName": {
- "Trend #A": "# of Failed Jobs"
- }
- }
- }
- ],
- "type": "table"
- },
{
"datasource": {
"type": "prometheus",
@@ -639,7 +337,7 @@
"h": 8,
"w": 24,
"x": 0,
- "y": 25
+ "y": 4
},
"id": 1065,
"options": {
@@ -781,7 +479,7 @@
"h": 7,
"w": 24,
"x": 0,
- "y": 33
+ "y": 12
},
"id": 1043,
"options": {
@@ -885,38 +583,12 @@
],
"type": "table"
},
- {
- "fieldConfig": {
- "defaults": {},
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 24,
- "x": 0,
- "y": 40
- },
- "id": 1063,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "**Note:** Failed Jobs panel requires Slurm accounting to be enabled in the cluster. Panel will not populate if accounting is disabled.",
- "mode": "markdown"
- },
- "pluginVersion": "11.6.9",
- "title": "",
- "transparent": true,
- "type": "text"
- },
{
"gridPos": {
"h": 4,
"w": 24,
"x": 0,
- "y": 42
+ "y": 19
},
"id": 1053,
"libraryPanel": {
diff --git a/azure-slurm-exporter/dashboards/failed-jobs.json b/azure-slurm-exporter/dashboards/failed-jobs.json
deleted file mode 100644
index d01cb4f6..00000000
--- a/azure-slurm-exporter/dashboards/failed-jobs.json
+++ /dev/null
@@ -1,366 +0,0 @@
-{
- "annotations": {
- "list": [
- {
- "$$hashKey": "object:1345",
- "builtIn": 1,
- "datasource": {
- "type": "datasource",
- "uid": "grafana"
- },
- "enable": true,
- "hide": true,
- "iconColor": "rgba(0, 211, 255, 1)",
- "name": "Annotations & Alerts",
- "type": "dashboard"
- }
- ]
- },
- "description": "Failed Jobs Overview",
- "editable": true,
- "fiscalYearStartMonth": 0,
- "graphTooltip": 0,
- "id": null,
- "links": [],
- "panels": [
- {
- "gridPos": {
- "h": 3,
- "w": 12,
- "x": 0,
- "y": 0
- },
- "id": 1056,
- "libraryPanel": {
- "name": "Azslurm Dashboard Buttons",
- "uid": "efoq5r3c8tzb4e"
- },
- "title": "",
- "type": "library-panel-ref"
- },
- {
- "gridPos": {
- "h": 3,
- "w": 12,
- "x": 12,
- "y": 0
- },
- "id": 1058,
- "libraryPanel": {
- "name": "Node Dashboard buttons",
- "uid": "cfo8upic84lxce"
- },
- "title": "",
- "type": "library-panel-ref"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 3
- },
- "id": 1047,
- "panels": [],
- "title": "Failed Jobs",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "description": "",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto"
- },
- "filterable": true,
- "inspect": false,
- "minWidth": 150
- },
- "mappings": [],
- "noValue": "No failed jobs within time range",
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "reason"
- },
- "properties": [
- {
- "id": "noValue"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 24,
- "x": 0,
- "y": 4
- },
- "id": 1054,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": true,
- "enablePagination": true,
- "fields": "",
- "reducer": [
- "count"
- ],
- "show": false
- },
- "frameIndex": 1,
- "showHeader": true,
- "sortBy": []
- },
- "pluginVersion": "11.6.9",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sacct_failed_jobs{cluster=\"$cluster\",partition=~\"$partition\",state!=\"completed\"} offset -5m",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Failed Jobs between $__from and $__to",
- "transformations": [
- {
- "id": "timeSeriesTable",
- "options": {
- "B": {
- "stat": "lastNotNull"
- }
- }
- },
- {
- "id": "labelsToFields",
- "options": {
- "keepLabels": [
- "exit_code",
- "partition",
- "state"
- ],
- "mode": "columns"
- }
- },
- {
- "id": "organize",
- "options": {
- "excludeByName": {
- "Trend #B": true,
- "__name__": true,
- "cluster": true,
- "endtime": true,
- "instance": true,
- "job": true,
- "microsoft.amwresourceid": true,
- "nodearray": true,
- "physical_host": true,
- "start_datetime hpc": true,
- "starttime": true,
- "subscription": true
- },
- "includeByName": {},
- "indexByName": {
- "Trend #B": 16,
- "__name__": 0,
- "cluster": 1,
- "endtime": 13,
- "exit_code": 7,
- "instance": 8,
- "job": 9,
- "jobid": 2,
- "jobname": 3,
- "microsoft.amwresourceid": 14,
- "nodearray": 17,
- "nodelist": 5,
- "partition": 4,
- "physical_host": 10,
- "reason": 15,
- "starttime": 12,
- "state": 6,
- "subscription": 11
- },
- "renameByName": {
- "Trend #A": "# of Failed Jobs"
- }
- }
- }
- ],
- "type": "table"
- },
- {
- "fieldConfig": {
- "defaults": {},
- "overrides": []
- },
- "gridPos": {
- "h": 2,
- "w": 16,
- "x": 0,
- "y": 15
- },
- "id": 1055,
- "options": {
- "code": {
- "language": "plaintext",
- "showLineNumbers": false,
- "showMiniMap": false
- },
- "content": "**NOTE**: Failed Jobs dashboard requires Slurm accounting to be enabled in the cluster. Dashboard will not populate if accounting is disabled.",
- "mode": "markdown"
- },
- "pluginVersion": "11.6.9",
- "title": "",
- "transparent": true,
- "type": "text"
- },
- {
- "gridPos": {
- "h": 4,
- "w": 24,
- "x": 0,
- "y": 17
- },
- "id": 1057,
- "libraryPanel": {
- "name": "Azslurm Dashboard README",
- "uid": "bfoq4o6ke718gf"
- },
- "title": "README",
- "type": "library-panel-ref"
- }
- ],
- "preload": false,
- "refresh": "",
- "schemaVersion": 41,
- "tags": [],
- "templating": {
- "list": [
- {
- "current": {
- "text": "cbbe2034-c78b-4e9b-89b4-8b78530247e5",
- "value": "cbbe2034-c78b-4e9b-89b4-8b78530247e5"
- },
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "definition": "label_values(jetpack_cluster_info,subscription)",
- "includeAll": false,
- "label": "Subscription",
- "name": "Subscription",
- "options": [],
- "query": {
- "qryType": 1,
- "query": "label_values(jetpack_cluster_info,subscription)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "type": "query"
- },
- {
- "current": {
- "text": "Managed_Prometheus_ccw-mon-jwna6fuhty7ho",
- "value": "ccw-mon-jwna6fuhty7ho"
- },
- "includeAll": false,
- "label": "Prometheus Data Source",
- "name": "promDatasource",
- "options": [],
- "query": "prometheus",
- "refresh": 1,
- "regex": "",
- "type": "datasource"
- },
- {
- "current": {
- "text": "All",
- "value": [
- "$__all"
- ]
- },
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "definition": "label_values(azslurm_partition_info, partition)",
- "includeAll": true,
- "multi": true,
- "name": "partition",
- "options": [],
- "query": {
- "qryType": 5,
- "query": "label_values(azslurm_partition_info, partition)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "type": "query"
- },
- {
- "current": {
- "text": "azcyclecloudwesteu-rg/u24-rc",
- "value": "azcyclecloudwesteu-rg/u24-rc"
- },
- "datasource": {
- "type": "prometheus",
- "uid": "${promDatasource}"
- },
- "definition": "label_values(jetpack_cluster_info, cluster)",
- "name": "cluster",
- "options": [],
- "query": {
- "qryType": 5,
- "query": "label_values(jetpack_cluster_info, cluster)",
- "refId": "PrometheusVariableQueryEditor-VariableQuery"
- },
- "refresh": 1,
- "regex": "",
- "type": "query"
- }
- ]
- },
- "time": {
- "from": "now-1h",
- "to": "now"
- },
- "timepicker": {},
- "timezone": "",
- "title": "Failed Jobs Dashboard",
- "uid": "cff0f6w0qqwaoa",
- "version": 1
-}
\ No newline at end of file
diff --git a/azure-slurm-exporter/dashboards/library-panels/azslurm-dashboard-buttons.json b/azure-slurm-exporter/dashboards/library-panels/azslurm-dashboard-buttons.json
index 73797c1a..4c72c8bd 100644
--- a/azure-slurm-exporter/dashboards/library-panels/azslurm-dashboard-buttons.json
+++ b/azure-slurm-exporter/dashboards/library-panels/azslurm-dashboard-buttons.json
@@ -16,7 +16,7 @@
"showLineNumbers": false,
"showMiniMap": false
},
- "content": "",
+ "content": "",
"mode": "html"
},
"pluginVersion": "11.6.9"
diff --git a/azure-slurm-exporter/exporter/sacct.py b/azure-slurm-exporter/exporter/sacct.py
index 445b928b..635924a5 100644
--- a/azure-slurm-exporter/exporter/sacct.py
+++ b/azure-slurm-exporter/exporter/sacct.py
@@ -1,10 +1,10 @@
from .exporter import BaseCollector
from collections import namedtuple
-from prometheus_client import Counter, disable_created_metrics, Gauge
+from prometheus_client import Counter, disable_created_metrics
from datetime import datetime
import logging
import exporter.util as util
-from typing import List, Union
+from typing import List
log = logging.getLogger(__name__)
class SacctNotAvailException(Exception):
@@ -68,20 +68,17 @@ def start(self) -> None:
"""
self.launch_task(func=self.sacct_query, interval=self.interval)
- def export_metrics(self) -> List[Union[Counter,Gauge]]:
+ def export_metrics(self) -> List[Counter]:
"""
Return the most recently collected sacct metrics for Prometheus to scrape.
"""
return self.cached_output["sacct_metrics"]
- def parse_output(self, stdout) -> List[Union[Counter,Gauge]]:
+ def parse_output(self, stdout) -> List[Counter]:
"""
Parse sacct command output and increment terminal jobs metric for each job in the output to describe total number of finished jobs by partition,
- exit_code, reason, and state as well as return failed jobs gauge describing each job that failed within the time interval of the query.
+ exit_code, reason, and state.
"""
- sacct_failed_jobs = Gauge("sacct_failed_jobs","Number of failed slurm jobs in an interval",
- ["jobid","jobname","nodelist","partition", "exit_code","reason","state", "starttime","endtime"], registry=None)
-
lines = stdout.decode().strip().splitlines()
log.debug(f"Number of jobs:{len(lines)}")
lines_iter = (line.split("|") for line in lines)
@@ -93,20 +90,7 @@ def parse_output(self, stdout) -> List[Union[Counter,Gauge]]:
reason=reason,
state=row.state.lower()).inc()
- if row.state.lower() != "completed":
- sacct_failed_jobs.labels(
- jobid=row.jobid,
- jobname=row.jobname,
- nodelist=row.nodelist,
- partition=row.partition,
- exit_code=row.exitcode,
- reason=reason,
- state=row.state.lower(),
- starttime=self.starttime,
- endtime=self.endtime).set(1)
-
-
- return [self.sacct_terminal_jobs, sacct_failed_jobs]
+ return [self.sacct_terminal_jobs]
async def sacct_query(self) -> None:
"""
diff --git a/azure-slurm-exporter/exporter/squeue.py b/azure-slurm-exporter/exporter/squeue.py
index a1a80d15..931a8858 100644
--- a/azure-slurm-exporter/exporter/squeue.py
+++ b/azure-slurm-exporter/exporter/squeue.py
@@ -12,8 +12,8 @@ class SqueueNotAvailException(Exception):
class Squeue(BaseCollector):
"""
- A collector that periodically queries SLURM squeue command to gather job state information and node allocation
- data from the job queue and parses the output into Prometheus Gauge metrics.
+ A collector that periodically queries SLURM squeue command to gather job state
+ information from the job queue and parses the output into a Prometheus Gauge metric.
"""
def __init__(self, binary_path="/usr/bin/squeue", interval=60, timeout=30):
self.binary_path = binary_path
@@ -48,34 +48,20 @@ def export_metrics(self) -> List[Gauge]:
def parse_output(self,stdout) -> List[Gauge]:
"""
- Parse the stdout from an squeue command and generates two Prometheus
- Gauge metrics that track the number of jobs currently in the queue in each state per partition
- and track the number of nodes allocated to running jobs in the queue
+ Parse the stdout from an squeue command and generate a Prometheus
+ Gauge metric that tracks the number of jobs currently in each state per partition.
"""
squeue_partition_jobs_state = Gauge(
f"squeue_partition_jobs_state",
f"Number of jobs in a state per partition",
labelnames=['partition','state'], registry=None
)
-
- squeue_job_nodes_allocated = Gauge(
- "squeue_job_nodes_allocated",
- "Number of nodes allocated to a running job",
- labelnames=["job_id", "job_name", "partition", "state", "nodelist", "start_time"], registry=None
- )
# number of jobs per state,partition key
counts = {}
lines = stdout.decode().strip().splitlines()
lines_iter = (line.split("|") for line in lines)
for row in map(self.squeue_output._make, lines_iter):
- if row.state.lower() == "running":
- squeue_job_nodes_allocated.labels(job_id=row.jobid,
- job_name=row.name,
- partition=row.partition,
- state=row.state.lower(),
- nodelist=row.nodelist,
- start_time=row.start_time).set(int(row.nodes))
key = (row.partition, row.state.lower())
counts[key] = counts.get(key, 0) + 1
@@ -83,13 +69,13 @@ def parse_output(self,stdout) -> List[Gauge]:
squeue_partition_jobs_state.labels(partition=partition,
state=state).set(count)
- return [squeue_partition_jobs_state, squeue_job_nodes_allocated]
+ return [squeue_partition_jobs_state]
async def squeue_query(self) -> None:
"""
- Run the squeue query with default options, parse the output to create two Prometheus Gauge metrics
- that represent the current state of each job in the queue as well as the total nodes allocated to
- any running job in the queue, and caches the results
+ Run the squeue query with default options, parse the output to create the
+ Prometheus Gauge metric that represents job counts by partition and state,
+ and cache the results.
"""
args = [self.binary_path]
args.extend(self.default_options)
diff --git a/azure-slurm-exporter/test/test_sacct.py b/azure-slurm-exporter/test/test_sacct.py
index 686a057c..4b8fc821 100644
--- a/azure-slurm-exporter/test/test_sacct.py
+++ b/azure-slurm-exporter/test/test_sacct.py
@@ -2,7 +2,7 @@
from unittest.mock import patch, AsyncMock, MagicMock
from exporter.sacct import Sacct, SacctNotAvailException
from exporter.exporter import CommandFailedException, CommandTimedOutException
-from prometheus_client import Counter, Gauge
+from prometheus_client import Counter
class TestSacctInitialize:
@@ -42,7 +42,7 @@ def test_parse_output_single_completed_job(self, sacct):
sacct.cached_output["sacct_metrics"] = sacct.parse_output(stdout)
samples_by_keys = {}
metrics = sacct.export_metrics()
- assert len(metrics) == 2
+ assert len(metrics) == 1
for metric in metrics:
for family in metric.collect():
@@ -119,12 +119,11 @@ def sacct(self):
return sacct
def test_export_metrics_returns_counter(self, sacct):
- """Test export_metrics returns a Counter and Gauge instance."""
+ """Test export_metrics returns only Counter instances."""
sacct.cached_output["sacct_metrics"] = sacct.parse_output(b"")
metrics = sacct.export_metrics()
- assert len(metrics) == 2
+ assert len(metrics) == 1
assert isinstance(metrics[0], Counter)
- assert isinstance(metrics[1], Gauge)
def test_export_metrics_same_reference(self, sacct):
"""Test export_metrics returns consistent references."""
diff --git a/azure-slurm-exporter/test/test_squeue.py b/azure-slurm-exporter/test/test_squeue.py
index 9aac889c..3c4703b3 100644
--- a/azure-slurm-exporter/test/test_squeue.py
+++ b/azure-slurm-exporter/test/test_squeue.py
@@ -39,6 +39,7 @@ def test_parse_output_running_job(self, squeue):
squeue.cached_output["squeue_metrics"] = squeue.parse_output(stdout)
samples_by_keys = {}
metrics = squeue.export_metrics()
+ assert len(metrics) == 1
# Validate partition job state metric
for metric in metrics:
@@ -48,16 +49,6 @@ def test_parse_output_running_job(self, squeue):
assert "partition" in s.labels
assert "state" in s.labels
samples_by_keys[(s.labels["partition"], s.labels["state"])] = s.value
- elif metric_family.name == "squeue_job_nodes_allocated":
- for s in metric_family.samples:
- assert "partition" in s.labels
- assert "state" in s.labels
- assert "nodelist" in s.labels
- assert "job_id" in s.labels
- assert "job_name" in s.labels
- assert "start_time" in s.labels
- samples_by_keys[(s.labels['nodelist'],s.labels["partition"], s.labels["state"], s.labels["job_id"], s.labels["job_name"], s.labels["start_time"])] = s.value
- assert samples_by_keys.get(("node[1-2]","partition1","running","12345", "job_name", "2024-01-01T00:00:01")) == 2
assert samples_by_keys.get(("partition1","running")) == 1
def test_parse_output_multiple_states(self, squeue):
@@ -103,13 +94,10 @@ def test_parse_output_mixed_states(self, squeue):
# Find the partition jobs state metric
partition_metric = None
- job_nodes_metric = None
for metric in metrics:
for metric_family in metric.collect():
if "partition_jobs_state" in metric_family.name:
partition_metric = metric_family
- elif "job_nodes_allocated" in metric_family.name:
- job_nodes_metric = metric_family
# Validate partition job counts
if partition_metric:
@@ -120,15 +108,6 @@ def test_parse_output_mixed_states(self, squeue):
assert samples_by_key.get(("p1", "running")) == 2
assert samples_by_key.get(("p2", "pending")) == 1
- # Validate node allocations for running jobs
- if job_nodes_metric:
- samples_by_job = {
- s.labels["job_id"]: s.value
- for s in job_nodes_metric.samples
- }
- assert samples_by_job.get("1") == 2
- assert samples_by_job.get("2") == 1
-
class TestSqueueExportMetrics:
@@ -238,17 +217,12 @@ async def test_squeue_query_success(self, mock_run, squeue):
mock_run.return_value = mock_proc
samples_by_key = {}
await squeue.squeue_query()
- assert len(squeue.cached_output["squeue_metrics"]) == 2
+ assert len(squeue.cached_output["squeue_metrics"]) == 1
metric1 = squeue.cached_output["squeue_metrics"][0]
family = metric1.collect()
for s in family[0].samples:
samples_by_key[(s.labels["partition"], s.labels["state"])] = s.value
- metric2 = squeue.cached_output["squeue_metrics"][1]
- family = metric2.collect()
- for s in family[0].samples:
- samples_by_key[(s.labels['nodelist'],s.labels["partition"], s.labels["state"], s.labels["job_id"], s.labels["job_name"], s.labels["start_time"])] = s.value
assert samples_by_key.get(("batch","running")) == 1
- assert samples_by_key.get(("node[1-2]","batch","running","12345","job1","2024-01-01T00:00:01")) == 2
@pytest.mark.asyncio
@patch.object(Squeue, "run_command", new_callable=AsyncMock)
@@ -257,7 +231,7 @@ async def test_squeue_query_caches_result(self, mock_run, squeue):
mock_proc = MagicMock()
mock_proc.stdout = b"12345|job1|2|node[1-2]|batch|RUNNING|2024-01-01T00:00:00|2024-01-01T00:00:01|user1\n"
mock_run.return_value = mock_proc
- mock_gauges = [MagicMock(spec=Gauge), MagicMock(spec=Gauge)]
+ mock_gauges = [MagicMock(spec=Gauge)]
with patch.object(squeue, "parse_output", return_value=mock_gauges):
await squeue.squeue_query()