Provide an API endpoint with statistics usable for monitoring

resallocserver/alembic/versions/9d9d3a7f7c8f_add_pool_statistics.py

@@ -0,0 +1,34 @@
+# pylint: disable=invalid-name
+
+"""Add pool statistics
+
+Revision ID: 9d9d3a7f7c8f
+Revises: d79239ae23a7
+Create Date: 2026-04-21 11:17:44.729847
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9d9d3a7f7c8f'
+down_revision = 'd79239ae23a7'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('pools', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('last_successful_start', sa.Float(), nullable=True, default=None))
+        batch_op.add_column(sa.Column('last_attempt_to_start', sa.Float(), nullable=True, default=None))
+        batch_op.add_column(sa.Column('startup_success_rate', sa.Float(), nullable=True, default=None))
+        batch_op.add_column(sa.Column('startup_time_avg', sa.Float(), nullable=True, default=None))
+
+
+def downgrade():
+    with op.batch_alter_table('pools', schema=None) as batch_op:
+        batch_op.drop_column('startup_time_avg')
+        batch_op.drop_column('startup_success_rate')
+        batch_op.drop_column('last_attempt_to_start')
+        batch_op.drop_column('last_successful_start')

resallocserver/manager.py

@@ -227,6 +227,7 @@ def __init__(self, event, pool, res_id, name=None):
         if name is not None:
             name = "{0}-{1}".format(name, res_id or pool)
         self.event = event
+        self.start_initiated = None
         threading.Thread.__init__(self, name=name)
 
     def job(self):
@@ -239,6 +240,7 @@ def __getattr__(self, attr):
     def run(self):
         self.log = app.log.getChild("worker")
         try:
+            self.start_initiated = time.time()
             self.job()
         except:
             self.log.exception("Worker exception, pool=%s resource=%s",
@@ -363,6 +365,12 @@ def job(self):
         )
 
         with session_scope() as session:
+            # The assumption is, this is the place where we know a resource
+            # successfully started
+            # Another assumption is that `pool.last_start` is when we tried to
+            # start a resource not when we succesfully started it
+            success = not output['status']
+            self.recalculate_statistics(session, success)
             resource.state = RState.ENDED if output['status'] else RState.UP
             resource.data = output['stdout']
             tags = []
@@ -385,6 +393,40 @@ def job(self):
         # Notify manager that it is worth doing re-spin.
         self.event.set()
 
+    def recalculate_statistics(self, session, success):
+        """
+        Re-calculate the pool statistics
+        """
+        dbpool = session.query(models.Pool).get(self.pool.id)
+
+        start_finished = time.time()
+        startup_time = start_finished - self.start_initiated
+
+        # We are going to use exponential moving averages because for
+        # a standard average we would need the have a count how many values
+        # we averaged previously, so that we can now average count+1.
+        # Approximation is good enough here, so we are using the moving average.
+        # The alpha should be between 0.1 and 0.9, the higher the value, the
+        # better it reacts to sudden spikes in failures
+        alpha = 0.5
+
+        if dbpool.startup_time_avg is None:
+            dbpool.startup_time_avg = startup_time
+        else:
+            dbpool.startup_time_avg = \
+                alpha * startup_time + (1 - alpha) * dbpool.startup_time_avg
+
+        if dbpool.startup_success_rate is None:
+            dbpool.startup_success_rate = int(success)
+        else:
+            dbpool.startup_success_rate = \
+                alpha * int(success) + (1 - alpha) * dbpool.startup_success_rate
+
+        dbpool.last_attempt_to_start = start_finished
+        if success:
+            dbpool.last_successful_start = start_finished
+        session.add(dbpool)
+
 
 class CleanUnknownWorker(Worker):
     """

resallocserver/models.py

@@ -39,6 +39,10 @@ class Pool(Base):
     __tablename__ = 'pools'
     name = Column(String, primary_key=True)
     last_start = Column(Float, default=0)
+    last_successful_start = Column(Float, default=None)
+    last_attempt_to_start = Column(Float, default=None)
+    startup_success_rate = Column(Float, default=None)
+    startup_time_avg = Column(Float, default=None)
     cleaning_unknown_resources = Column(DateTime, server_default=func.now())
     max = Column(Integer, default=0)
 

resallocwebui/app.py

@@ -1,5 +1,5 @@
 import yaml
-from flask import Flask, render_template
+from flask import Flask, render_template, jsonify
 from resallocserver.app import session_scope
 from resallocserver.logic import QResources
 from resallocserver import models
@@ -88,5 +88,30 @@ def pools():
     return render_template("pools.html", information=result, summary=summary)
 
 
+@app.route("/api/stats")
+def api_stats():
+    """
+    Provide statistics usable for external monitoring (Nagios/Zabbix/etc)
+    """
+    # This will be a two-dimensional array,
+    # e.g. result["copr_hv_x86_64_01_prod"]["startup_success_rate"]
+    result = {}
+
+    pools_from_config = load_config()
+
+    # Prepare the two-dimensional array, and fill it with zeros
+    with session_scope() as session:
+        for pool in session.query(models.Pool).all():
+            if pool.name not in pools_from_config:
+                continue
+            result[pool.name] = {
+                "startup_success_rate": pool.startup_success_rate,
+                "startup_time_avg": pool.startup_time_avg,
+                "last_successful_start": pool.last_successful_start,
+                "last_attempt_to_start": pool.last_attempt_to_start,
+            }
+    return jsonify(result)
+
+
 if __name__ == '__main__':
     app.run(host="0.0.0.0")