From 6fcb67eac338c5615ffab189ae9644a35780cce5 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 4 Mar 2026 16:24:03 +0100 Subject: [PATCH 1/3] Configure linters --- .flake8 | 2 + .pylintrc/pylintrc.py | 128 ++++++++++++++++++++++++++++++++++++++++++ pylintrc | 23 +++++++- 3 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 .flake8 create mode 100644 .pylintrc/pylintrc.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..6deafc2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/.pylintrc/pylintrc.py b/.pylintrc/pylintrc.py new file mode 100644 index 0000000..4b55342 --- /dev/null +++ b/.pylintrc/pylintrc.py @@ -0,0 +1,128 @@ +""" +pylint plugin doing resalloc-specific transformation to shut-down specific +errors which we can not easily ignore because the ignore mechanisms don't have +the needed granularity (copy-paste from Copr). +""" + +import os +import subprocess +from astroid import MANAGER, scoped_nodes, nodes, extract_node + + +def register(_linter): + """ required pylint entrypoint """ + + +class Cache: + """ + Some rather expensive checks cached (as a class to avoid using globals). + """ + _gitroot = None + _test_files = { + # Some modules have None in the file argument: + # ipdb> function + # + # ipdb> function.parent + # + # ipdb> function.parent.file is None + # True + None: False, + } + test_paths = { + "tests/", + } + + @classmethod + def gitroot(cls): + """ Obtain the git-root of the current directory, and cache """ + if cls._gitroot: + return cls._gitroot + cls._gitroot = subprocess.check_output(["git", "rev-parse", + "--show-toplevel"]).decode("utf-8").strip() + return cls.gitroot() + + @classmethod + def _slow_is_test_file(cls, test_file): + if not test_file.startswith(cls.gitroot() + os.sep): + return False + relpath = os.path.relpath(test_file, cls.gitroot()) + for test_path in cls.test_paths: + if relpath.startswith(test_path): + return True + return False + + @classmethod + def is_test_file(cls, file_path): + """ + Check if file_path (path relative to the gitroot) is a test-file (per + cls.test_paths configuration). + """ + cached = cls._test_files.get(file_path) + if cached is None: + cls._test_files[file_path] = cls._slow_is_test_file(file_path) + return cls._test_files[file_path] + + +def module_path(node): + """ + Filename where the node (e.g. method) is defined. + """ + while node: + if isinstance(node, scoped_nodes.Module): + return node.file + node = node.parent + return None + + +def add_fake_docs(the_object): + """ + Add fake docs to the specified object so PyLint later doesn't complain about + missing docs. + """ + the_object.doc_node = nodes.Const("fake docs") + the_object.doc = "fake docs" + + +def transform_functions(function): + """ + Transformate some function definitions so pylint doesn't object. + """ + + filename = module_path(function) + + if function.name == 'logger': + for prop in ['debug', 'info', 'warning', 'error', 'exception']: + function.instance_attrs[prop] = extract_node('def {name}(arg): return'.format(name=prop)) + + if function.name in ["upgrade", "downgrade"]: + # ignore missing-function-docstring in migrations + add_fake_docs(function) + + if function.name == "step_impl": + # behave step definition + add_fake_docs(function) + + if Cache.is_test_file(filename): + add_fake_docs(function) + + +def transform_classes(classdef): + """ + Transform Class definitions that don't need to have docstrings. + """ + filename = module_path(classdef) + if Cache.is_test_file(filename): + add_fake_docs(classdef) + + +def transform_modules(moduledef): + """ + Testing modules don't nave to have the doc-strings either. + """ + if Cache.is_test_file(moduledef.file): + add_fake_docs(moduledef) + + +MANAGER.register_transform(scoped_nodes.FunctionDef, transform_functions) +MANAGER.register_transform(scoped_nodes.ClassDef, transform_classes) +MANAGER.register_transform(scoped_nodes.Module, transform_modules) diff --git a/pylintrc b/pylintrc index 67bacd2..9676bc8 100644 --- a/pylintrc +++ b/pylintrc @@ -7,7 +7,11 @@ init-hook= import sys import subprocess gitrootdir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip() - sys.path.insert(0, os.path.join(gitrootdir)) + sys.path.insert(0, gitrootdir) + sys.path.insert(0, os.path.join(gitrootdir, ".pylintrc")) + +# Our own pylint transformations. +load-plugins=pylintrc # Reasoning for class ignore # -------------------------- @@ -15,6 +19,13 @@ init-hook= # Pylint doesn't parse the meta-class to detect the attributes. ignored-classes=RState,TState +# Reasoning for ignored modules +# ----------------------------- +# alembic: +# Pylint entirely fails to parse the module. +# Example error: Module 'alembic.op' has no 'drop_index' member +ignored-modules=alembic,setproctitle + [MESSAGES CONTROL] # Reasoning for wide warning ignore # --------------------------------- @@ -37,3 +48,13 @@ disable=useless-object-inheritance,import-error,too-few-public-methods,consider- [DESIGN] min-public-methods=1 + + +# evision|down_revision|branch_labels|depends_on is for alembic +[BASIC] +const-rgx=^(([A-Z_][A-Z0-9_]*)|(__.*__)|revision|down_revision|branch_labels|depends_on)$ + + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=120 From 40c63144ff62f0cda05d9b4b2c2d00a5c6e0f906 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 4 Mar 2026 11:25:26 +0100 Subject: [PATCH 2/3] Implement cross-pool counters This is to be used for sharing a common IPv6 range across multiple pools. Closes: #179 --- Makefile | 1 + config/pools.yaml | 7 + .../versions/d79239ae23a7_named_counters.py | 39 +++++ resallocserver/manager.py | 87 ++++++--- resallocserver/models.py | 22 +++ shelltests/tests/named_counters.sh | 165 ++++++++++++++++++ tests/test_command.py | 4 +- 7 files changed, 299 insertions(+), 26 deletions(-) create mode 100644 resallocserver/alembic/versions/d79239ae23a7_named_counters.py create mode 100755 shelltests/tests/named_counters.sh diff --git a/Makefile b/Makefile index 332c18c..efd85a8 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ SHELLTEST_OPTIONS := SHELL_TESTS := \ basic.sh \ check.sh \ + named_counters.sh \ ondemand.sh \ reuse.sh diff --git a/config/pools.yaml b/config/pools.yaml index 9eaeee4..271ce3d 100644 --- a/config/pools.yaml +++ b/config/pools.yaml @@ -144,3 +144,10 @@ ## # "reused". The time spent is measured since the resource was taken for the ## # first time (assigned to sandbox). Default is 3600s. ## reuse_max_time: 3600 +## +## # Whenever a new machine is allocated within this pool, the smallest integer values +## # are taken from the following named counters, and exported for the +## # callback scripts as environment variables, for example +## # NAMED_COUNTER_VALUE_foo=10 (considering values 0->9 are already taken). +## #named_counters: +## # - foo diff --git a/resallocserver/alembic/versions/d79239ae23a7_named_counters.py b/resallocserver/alembic/versions/d79239ae23a7_named_counters.py new file mode 100644 index 0000000..0def201 --- /dev/null +++ b/resallocserver/alembic/versions/d79239ae23a7_named_counters.py @@ -0,0 +1,39 @@ +""" +Named Counters added + +Revision ID: d79239ae23a7 +Revises: b50e3f64fc2d +Create Date: 2026-03-04 10:52:27.756502 +""" + +from alembic import op +import sqlalchemy as sa + +revision = 'd79239ae23a7' +down_revision = 'b50e3f64fc2d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'named_counters', + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.Column('value', sa.Integer(), nullable=True), + sa.Column('counter_name', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['resource_id'], ['resources.id'], ), + sa.PrimaryKeyConstraint('counter_name', 'value'), + sa.UniqueConstraint('value', 'counter_name') + ) + with op.batch_alter_table('named_counters', schema=None) as batch_op: + batch_op.create_index(batch_op.f('ix_named_counters_counter_name'), + ['counter_name'], unique=False) + batch_op.create_index(batch_op.f('ix_named_counters_value'), ['value'], + unique=False) + + +def downgrade(): + with op.batch_alter_table('named_counters', schema=None) as batch_op: + batch_op.drop_index(batch_op.f('ix_named_counters_value')) + batch_op.drop_index(batch_op.f('ix_named_counters_counter_name')) + op.drop_table('named_counters') diff --git a/resallocserver/manager.py b/resallocserver/manager.py index b29bcd2..f4d1a30 100644 --- a/resallocserver/manager.py +++ b/resallocserver/manager.py @@ -40,7 +40,7 @@ def command_env(pool_id=None, res_id=None, res_name=None, - id_in_pool=None, data=None): + id_in_pool=None, named_counters=None, data=None): pfx = 'RESALLOC_' env = os.environ.copy() env[pfx + 'ID'] = str(res_id) @@ -49,11 +49,14 @@ def command_env(pool_id=None, res_id=None, res_name=None, env[pfx + 'ID_IN_POOL'] = str(id_in_pool) if data is not None: env[pfx + 'RESOURCE_DATA'] = base64.b64encode(data) + if named_counters is not None: + for key, value in named_counters.items(): + env[pfx + 'NAMED_COUNTER_' + key] = str(value) return env -def run_command(pool_id, res_id, res_name, id_in_pool, command, ltype='alloc', - catch_stdout_bytes=None, data=None, +def run_command(pool_id, res_id, res_name, id_in_pool, named_counters, command, + ltype='alloc', catch_stdout_bytes=None, data=None, catch_stdout_lines_securely=False, timeout=None): """ Run command, and log into according directory (per.app.config). If @@ -61,14 +64,15 @@ def run_command(pool_id, res_id, res_name, id_in_pool, command, ltype='alloc', """ config = app.config logdir = os.path.join(config['logdir'], 'hooks') - return _run_command(app.log, logdir, pool_id, res_id, res_name, id_in_pool, command, ltype, - catch_stdout_bytes, data, catch_stdout_lines_securely, - timeout) + return _run_command(app.log, logdir, pool_id, res_id, res_name, id_in_pool, + named_counters, command, ltype, catch_stdout_bytes, + data, catch_stdout_lines_securely, timeout) -def _run_command(log, logdir, pool_id, res_id, res_name, id_in_pool, command, ltype='alloc', - catch_stdout_bytes=None, data=None, - catch_stdout_lines_securely=False, timeout=None): +def _run_command(log, logdir, pool_id, res_id, res_name, id_in_pool, + named_counters, command, ltype='alloc', + catch_stdout_bytes=None, data=None, + catch_stdout_lines_securely=False, timeout=None): """ Internal variant for _run_command() that is easier to test locally like: In [1] import logging @@ -82,7 +86,8 @@ def _run_command(log, logdir, pool_id, res_id, res_name, id_in_pool, command, lt assert not (timeout and catch_stdout_lines_securely) log.debug("running: " + command) - env = command_env(pool_id, res_id, res_name, id_in_pool, data) + env = command_env(pool_id, res_id, res_name, id_in_pool, named_counters, + data) try: os.mkdir(logdir) except OSError as e: @@ -252,6 +257,9 @@ def close(self): session.add(resource) if resource.id_in_pool_object: session.delete(resource.id_in_pool_object) + for counter in resource.named_counters: + session.delete(counter) + self.event.set() def job(self): @@ -267,6 +275,7 @@ def job(self): session.add(resource) session.flush() id_in_pool = resource.id_in_pool + named_counters = resource.named_counters_dict session.expunge(resource) self.log.info("Terminating %s started", resource.name) @@ -279,6 +288,7 @@ def job(self): resource.id, resource.name, id_in_pool, + named_counters, self.pool.cmd_delete, 'terminate', data=resource.data, @@ -297,11 +307,13 @@ def job(self): resource = session.query(models.Resource).get(self.resource_id) id_in_pool = resource.id_in_pool resource_name = resource.name + named_counters = resource.named_counters_dict session.expunge(resource) self.log.info("Releasing %s", resource_name) out = run_command(self.pool.id, resource.id, resource_name, id_in_pool, - self.pool.cmd_release, "release", data=resource.data) + named_counters, self.pool.cmd_release, "release", + data=resource.data) status = out["status"] with session_scope() as session: @@ -330,6 +342,7 @@ def job(self): with session_scope() as session: resource = session.query(models.Resource).get(self.resource_id) id_in_pool = resource.id_in_pool + named_tags = resource.named_counters_dict session.expunge(resource) self.log.info( @@ -344,6 +357,7 @@ def job(self): resource.id, resource.name, id_in_pool, + named_tags, self.pool.cmd_new, catch_stdout_bytes=512, ) @@ -399,6 +413,7 @@ def job(self): res_id, resource, None, + None, self.pool.cmd_delete, 'terminate', data=None, @@ -415,6 +430,7 @@ def _list_all_resources(self): res_id=0, res_name=None, id_in_pool=None, + named_counters=None, command=self.pool.cmd_list, ltype="list", catch_stdout_bytes=5120, @@ -448,6 +464,7 @@ def loop(self): 'last': item.check_last_time, 'fail': item.check_failed_count, 'id_in_pool': item.id_in_pool, + 'named_counters': item.named_counters_dict, 'data': item.data, } @@ -464,6 +481,7 @@ def loop(self): res_id, data['name'], data['id_in_pool'], + data['named_counters'], pool.cmd_livecheck, 'watch', data=data["data"], @@ -516,6 +534,7 @@ class Pool(object): reuse_opportunity_time = 0 reuse_max_count = 0 reuse_max_time = 3600 + named_counters = [] start_on_demand_this_cycle = 0 @@ -593,6 +612,14 @@ def validate(self, on_demand_tag_set): for tag in on_demand_tag_set: assert tag not in self.tag_set_normal + def _find_smallest_number(self, query, field_name): + ids = {getattr(x, field_name): True for x in query.all()} + try_id = 0 + while True: + if try_id in ids: + try_id += 1 + continue + return try_id def _allocate_pool_id(self, session, resource): # allocate the lowest available pool_id @@ -601,21 +628,29 @@ def _allocate_pool_id(self, session, resource): .filter_by(pool_name=self.name) .order_by(models.IDWithinPool.id) ) - ids = {x.id: True for x in ids_query.all()} - found_id = None - try_id = 0 - while True: - if try_id in ids: - try_id += 1 - continue - - found_id = models.IDWithinPool() - found_id.id = try_id - found_id.pool_name = self.name - found_id.resource_id = resource.id - return found_id + the_id = self._find_smallest_number(ids_query, "id") + found_id = models.IDWithinPool() + found_id.id = the_id + found_id.pool_name = self.name + found_id.resource_id = resource.id + return found_id + def _allocate_named_counter_value(self, counter_name, session, resource): + """ + Allocate the smallest number within COUNTER_NAME counter. + """ + query = ( + session.query(models.NamedCounter) + .filter_by(counter_name=counter_name) + .order_by(models.NamedCounter.value) + ) + found_value = self._find_smallest_number(query, "value") + counter_value = models.NamedCounter() + counter_value.resource = resource + counter_value.value = found_value + counter_value.counter_name = counter_name + return counter_value def allocate(self, event): resource_id = None @@ -629,6 +664,10 @@ def allocate(self, event): pool_id = self._allocate_pool_id(session, resource) session.add(pool_id) + for counter_name in self.named_counters: + session.add(self._allocate_named_counter_value(counter_name, + session, + resource)) session.flush() resource_id = resource.id diff --git a/resallocserver/models.py b/resallocserver/models.py index b09f050..efb4741 100644 --- a/resallocserver/models.py +++ b/resallocserver/models.py @@ -108,6 +108,11 @@ def id_in_pool(self): return self.id_in_pool_object.id return None + @property + def named_counters_dict(self): + """Get counter_name: value dictionary""" + return {i.counter_name: i.value for i in self.named_counters} + def to_dict(self): result = super().to_dict() # Decode resource data for better readability @@ -134,6 +139,23 @@ class IDWithinPool(Base): resource = relationship('Resource', backref=backref('id_in_pool_object', uselist=False)) +class NamedCounter(Base): + """ + Counters that are shared across multiple pools. E.g. for "elastic IPs" + (you have, say, 1024 IPs that can be assigned to VMs from mutliple pools, + each IP has it's own IP, but can be assigned only once). + """ + __tablename__ = 'named_counters' + __table_args__ = ( + UniqueConstraint('value', 'counter_name'), + ) + + resource_id = Column(Integer, ForeignKey('resources.id'), primary_key=True) + counter_name = Column(String, index=True, primary_key=True) + value = Column(Integer, index=True) + resource = relationship('Resource', backref=backref('named_counters')) + + class ResourceTag(Base): __tablename__ = 'resource_tags' id = Column(String, primary_key=True) diff --git a/shelltests/tests/named_counters.sh b/shelltests/tests/named_counters.sh new file mode 100755 index 0000000..91e2f9d --- /dev/null +++ b/shelltests/tests/named_counters.sh @@ -0,0 +1,165 @@ +NORMAL_MAX=5 +NORMAL_PREALLOC=3 +TIME_MAX=4 +TIME_PREALLOC=2 +COUNT_MAX=2 +COUNT_PREALLOC=1 + +MAX=$(( NORMAL_MAX + COUNT_MAX + TIME_MAX )) + +DBNAME=resalloc-test + +: ${DATABASE=sqlite} + +. ./testlib + +cd "$WORKDIR" + +debug "I'm in $PWD" +cleanup_actions=() +cleanup () +{ + debug "cleanup" + set +e + for action in "${cleanup_actions[@]}" + do + eval "$action" + done +} + +trap cleanup EXIT + +mkdir etc +cat > etc/pools.yaml <&2 before; env | grep ^RESALLOC_; echo >&2 after" + cmd_delete: "echo >&2 stderr; echo stdout" + cmd_livecheck: "echo >&2 stderr; echo stdout" + cmd_release: "echo >&2 stderr; echo stdout; echo \$RESALLOC_RESOURCE_DATA | base64 --decode" + livecheck_period: 1 + tags: + - A + reuse_max_count: 3 + # Keep this one running! + reuse_max_time: 60 + named_counters: + - counter_1 +second: + max: 4 + max_prealloc: 1 + cmd_new: "echo >&2 before; env | grep ^RESALLOC_; echo >&2 after" + cmd_delete: "echo >&2 stderr; echo stdout" + cmd_livecheck: "echo >&2 stderr; echo stdout" + cmd_release: "echo >&2 stderr; echo stdout; echo \$RESALLOC_RESOURCE_DATA | base64 --decode" + livecheck_period: 1 + tags: + - B + named_counters: + - counter_1 + - counter_2 +third: + max: 4 + max_prealloc: 1 + cmd_new: "echo >&2 before; env | grep ^RESALLOC_; echo >&2 after" + cmd_delete: "echo >&2 stderr; echo stdout" + cmd_livecheck: "echo >&2 stderr; echo stdout" + cmd_release: "echo >&2 stderr; echo stdout; echo \$RESALLOC_RESOURCE_DATA | base64 --decode" + livecheck_period: 1 + tags: + - C + named_counters: + - counter_1 + - counter_2 + - counter_3 +EOF + +uname=$(id -u -n) +dburl="sqlite:///$WORKDIR/server-sql" +case $DATABASE in + sqlite) ;; + postgresql) + port=${POSTGRESQL_PORT-65432} + host=/tmp + datadir=$WORKDIR/pgdata + info "preparing PostgreSQL server" + postgresql_start "$port" "$datadir" "$host" &>/dev/null + createdb -p "$port" -h "$host" "$DBNAME" + cleanup_actions+=( "pg_ctl stop -D \"$datadir\" -m i >/dev/null" ) + dburl="postgresql://$uname@/$DBNAME?host=$host&port=$port" + ;; + *) false ;; +esac + +cat > etc/server.yaml </dev/null +cleanup_actions+=( "shutdown_server $server_pid" ) + +# Wait for the server to start up. +counter=30 +while ! maint resource-list &>/dev/null; do + counter=$(( counter - 1 )) + test $counter -gt 0 +done + +PREALLOC=3 +info "Wait for $PREALLOC preallocated machines" +counter=30 +while true; do + up=$(maint resource-list --up | wc -l) + counter=$(( counter - 1)) + if test "$up" -eq "$PREALLOC"; then + break + fi + test "$up" -lt "$PREALLOC" || { + maint resource-list --up + error "too many resources" + } + test $counter -gt 0 || fail "some resource did not start" + sleep 1 +done +id1=$(client ticket --tag A) +id2=$(client ticket --tag B) +id3=$(client ticket --tag C) + +set -e + +info "Wait three tickets." +client ticket-wait "$id1" > ticket1 +client ticket-wait "$id2" > ticket2 +client ticket-wait "$id3" > ticket3 + +info "Search for expected counter values." +grep RESALLOC_NAMED_COUNTER_counter_1=0 ticket1 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_2=0 ticket2 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_1=1 ticket2 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_3=0 ticket3 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_2=1 ticket3 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_1=2 ticket3 >/dev/null + +info "Close tickets, wipe existing workers, and take one more ticket." +client ticket-close "$id1" +client ticket-close "$id2" +client ticket-close "$id3" + +sleep 3 +# recyclate resources 4, 5 and 6 +maint resource-delete --all +sleep 3 + +id4=$(client ticket --tag C) +client ticket-wait "$id4" > ticket4 + +info "Check for the right counters" +grep RESALLOC_NAMED_COUNTER_counter_3=0 ticket4 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_2=1 ticket4 >/dev/null +grep RESALLOC_NAMED_COUNTER_counter_1=2 ticket4 >/dev/null diff --git a/tests/test_command.py b/tests/test_command.py index 2f9aced..17c9c03 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -51,8 +51,8 @@ def test_trim(self): "bash -c 'for i in $(seq 100000); do echo stdout; done; " "echo stderr >&2'" ) - res = run_command("pool_trim", 10, "res_10", 1, command, ltype='alloc', - catch_stdout_bytes=capture_bytes) + res = run_command("pool_trim", 10, "res_10", 1, {"foo": 1}, command, + ltype='alloc', catch_stdout_bytes=capture_bytes) assert res["status"] == 0 assert res["stdout"].endswith(b"<< trimmed >>\n") assert res["stdout"].startswith(b"stdout\n") From 31bc370e99225baeccb194b681af28ea3fbbb1a2 Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Wed, 4 Mar 2026 21:45:19 +0100 Subject: [PATCH 3/3] ci: create pr: directory in copr for pull-request builds --- .github/workflows/push-copr-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push-copr-build.yml b/.github/workflows/push-copr-build.yml index b4b9e81..444c493 100644 --- a/.github/workflows/push-copr-build.yml +++ b/.github/workflows/push-copr-build.yml @@ -17,7 +17,7 @@ jobs: - name: Submit the build env: - COPR_PR_WEBHOOK: https://copr.fedorainfracloud.org/webhooks/custom/24190/c23f34c1-e02e-41a0-9435-6c7e4a6e56c0/resalloc/ + COPR_PR_WEBHOOK: https://copr.fedorainfracloud.org/webhooks/custom-dir/praiskup/resalloc-ci:pr:${{github.event.number}}/c23f34c1-e02e-41a0-9435-6c7e4a6e56c0/resalloc/ COPR_PUSH_WEBHOOK: ${{ secrets.COPR_PUSH_WEBHOOK }} run: | curl https://raw.githubusercontent.com/praiskup/copr-ci-tooling/main/copr-gh-actions-submit > submit