diff --git a/resallocserver/alembic/versions/9d9d3a7f7c8f_add_pool_statistics.py b/resallocserver/alembic/versions/9d9d3a7f7c8f_add_pool_statistics.py new file mode 100644 index 0000000..5babd5e --- /dev/null +++ b/resallocserver/alembic/versions/9d9d3a7f7c8f_add_pool_statistics.py @@ -0,0 +1,34 @@ +# pylint: disable=invalid-name + +"""Add pool statistics + +Revision ID: 9d9d3a7f7c8f +Revises: d79239ae23a7 +Create Date: 2026-04-21 11:17:44.729847 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9d9d3a7f7c8f' +down_revision = 'd79239ae23a7' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('pools', schema=None) as batch_op: + batch_op.add_column(sa.Column('last_successful_start', sa.Float(), nullable=True, default=None)) + batch_op.add_column(sa.Column('last_attempt_to_start', sa.Float(), nullable=True, default=None)) + batch_op.add_column(sa.Column('startup_success_rate', sa.Float(), nullable=True, default=None)) + batch_op.add_column(sa.Column('startup_time_avg', sa.Float(), nullable=True, default=None)) + + +def downgrade(): + with op.batch_alter_table('pools', schema=None) as batch_op: + batch_op.drop_column('startup_time_avg') + batch_op.drop_column('startup_success_rate') + batch_op.drop_column('last_attempt_to_start') + batch_op.drop_column('last_successful_start') diff --git a/resallocserver/manager.py b/resallocserver/manager.py index e3a1d16..ce46443 100644 --- a/resallocserver/manager.py +++ b/resallocserver/manager.py @@ -227,6 +227,7 @@ def __init__(self, event, pool, res_id, name=None): if name is not None: name = "{0}-{1}".format(name, res_id or pool) self.event = event + self.start_initiated = None threading.Thread.__init__(self, name=name) def job(self): @@ -239,6 +240,7 @@ def __getattr__(self, attr): def run(self): self.log = app.log.getChild("worker") try: + self.start_initiated = time.time() self.job() except: self.log.exception("Worker exception, pool=%s resource=%s", @@ -363,6 +365,12 @@ def job(self): ) with session_scope() as session: + # The assumption is, this is the place where we know a resource + # successfully started + # Another assumption is that `pool.last_start` is when we tried to + # start a resource not when we succesfully started it + success = not output['status'] + self.recalculate_statistics(session, success) resource.state = RState.ENDED if output['status'] else RState.UP resource.data = output['stdout'] tags = [] @@ -385,6 +393,40 @@ def job(self): # Notify manager that it is worth doing re-spin. self.event.set() + def recalculate_statistics(self, session, success): + """ + Re-calculate the pool statistics + """ + dbpool = session.query(models.Pool).get(self.pool.id) + + start_finished = time.time() + startup_time = start_finished - self.start_initiated + + # We are going to use exponential moving averages because for + # a standard average we would need the have a count how many values + # we averaged previously, so that we can now average count+1. + # Approximation is good enough here, so we are using the moving average. + # The alpha should be between 0.1 and 0.9, the higher the value, the + # better it reacts to sudden spikes in failures + alpha = 0.5 + + if dbpool.startup_time_avg is None: + dbpool.startup_time_avg = startup_time + else: + dbpool.startup_time_avg = \ + alpha * startup_time + (1 - alpha) * dbpool.startup_time_avg + + if dbpool.startup_success_rate is None: + dbpool.startup_success_rate = int(success) + else: + dbpool.startup_success_rate = \ + alpha * int(success) + (1 - alpha) * dbpool.startup_success_rate + + dbpool.last_attempt_to_start = start_finished + if success: + dbpool.last_successful_start = start_finished + session.add(dbpool) + class CleanUnknownWorker(Worker): """ diff --git a/resallocserver/models.py b/resallocserver/models.py index efb4741..e978957 100644 --- a/resallocserver/models.py +++ b/resallocserver/models.py @@ -39,6 +39,10 @@ class Pool(Base): __tablename__ = 'pools' name = Column(String, primary_key=True) last_start = Column(Float, default=0) + last_successful_start = Column(Float, default=None) + last_attempt_to_start = Column(Float, default=None) + startup_success_rate = Column(Float, default=None) + startup_time_avg = Column(Float, default=None) cleaning_unknown_resources = Column(DateTime, server_default=func.now()) max = Column(Integer, default=0) diff --git a/resallocwebui/app.py b/resallocwebui/app.py index fb417f7..584fc5d 100644 --- a/resallocwebui/app.py +++ b/resallocwebui/app.py @@ -1,5 +1,5 @@ import yaml -from flask import Flask, render_template +from flask import Flask, render_template, jsonify from resallocserver.app import session_scope from resallocserver.logic import QResources from resallocserver import models @@ -88,5 +88,30 @@ def pools(): return render_template("pools.html", information=result, summary=summary) +@app.route("/api/stats") +def api_stats(): + """ + Provide statistics usable for external monitoring (Nagios/Zabbix/etc) + """ + # This will be a two-dimensional array, + # e.g. result["copr_hv_x86_64_01_prod"]["startup_success_rate"] + result = {} + + pools_from_config = load_config() + + # Prepare the two-dimensional array, and fill it with zeros + with session_scope() as session: + for pool in session.query(models.Pool).all(): + if pool.name not in pools_from_config: + continue + result[pool.name] = { + "startup_success_rate": pool.startup_success_rate, + "startup_time_avg": pool.startup_time_avg, + "last_successful_start": pool.last_successful_start, + "last_attempt_to_start": pool.last_attempt_to_start, + } + return jsonify(result) + + if __name__ == '__main__': app.run(host="0.0.0.0")