From 3f77b176923f6a35dad2e062729653651ee64fe3 Mon Sep 17 00:00:00 2001 From: Kwaizer Date: Mon, 7 Jul 2025 21:52:03 +0100 Subject: [PATCH 1/6] Exclude running some tests for specific packages --- alts/shared/utils/file_utils.py | 24 ++++++++++++ alts/worker/tasks.py | 69 +++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 20 deletions(-) diff --git a/alts/shared/utils/file_utils.py b/alts/shared/utils/file_utils.py index 2e99b6de..022beebc 100644 --- a/alts/shared/utils/file_utils.py +++ b/alts/shared/utils/file_utils.py @@ -1,5 +1,7 @@ import hashlib +import requests + def get_hasher(checksum_type): """ @@ -56,3 +58,25 @@ def feed_hasher(_fd): file_path.seek(0) feed_hasher(file_path) return hasher.hexdigest() + + +def file_url_exists(url): + + """ + Check if a file exists at the specified URL using a HEAD request. + + Parameters + ---------- + url : str + The URL to check. + + Returns + ------- + bool + True if the file exists, False otherwise. + """ + try: + response = requests.head(url) + return response.status_code == 200 + except requests.RequestException as e: + return False diff --git a/alts/worker/tasks.py b/alts/worker/tasks.py index cf045466..f582aba0 100644 --- a/alts/worker/tasks.py +++ b/alts/worker/tasks.py @@ -3,7 +3,6 @@ # created: 2021-04-13 """AlmaLinux Test System package testing tasks running.""" - import logging import traceback import random @@ -25,6 +24,7 @@ from urllib3 import Retry from urllib3.exceptions import TimeoutError +from alts.shared.utils.file_utils import file_url_exists from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT from alts.shared.exceptions import ( InstallPackageError, @@ -120,6 +120,12 @@ def is_success(stage_data_: dict): return False return stage_data_['exit_code'] == 0 + def should_skip_test(package, test_name, skipped_tests_map): + for pkg_pattern, tests in skipped_tests_map.items(): + if package.startswith(pkg_pattern) and test_name in tests: + return True + return False + def set_artifacts_when_stage_has_unexpected_exception( _artifacts: dict, error_message: str, @@ -175,29 +181,52 @@ def set_artifacts_when_stage_has_unexpected_exception( module_name = task_params.get('module_name') module_stream = task_params.get('module_stream') module_version = task_params.get('module_version') + + def get_excluded_packages(): + uri = 'https://git.almalinux.org/almalinux/alts-exclusions/raw/branch/main/skipped_tests.json' + try: + response = requests.get(uri) + response.raise_for_status() + return response.json() + except requests.RequestException: + return {} + try: # Wait a bit to not spawn all environments at once when # a lot of tasks are coming to the machine time.sleep(random.randint(5, 10)) - runner.setup() - runner.run_system_info_commands() - runner.install_package( - package_name, - package_version=package_version, - package_epoch=package_epoch, - module_name=module_name, - module_stream=module_stream, - module_version=module_version, - semi_verbose=True, - ) - if CONFIG.enable_integrity_tests: - runner.run_package_integrity_tests(package_name, package_version) - runner.run_third_party_tests( - package_name, - package_version=package_version, - package_epoch=package_epoch, - ) - runner.uninstall_package(package_name) + + test_steps = [ + ("setup", runner.setup, [], {}), + ("run_system_info_commands", runner.run_system_info_commands, [], {}), + ("install_package", runner.install_package, [ + package_name + ], { + "package_version": package_version, + "package_epoch": package_epoch, + "module_name": module_name, + "module_stream": module_stream, + "module_version": module_version, + "semi_verbose": True, + }), + ("run_package_integrity_tests", runner.run_package_integrity_tests, [ + package_name, package_version + ], {}), + ("run_third_party_tests", runner.run_third_party_tests, [ + package_name + ], { + "package_version": package_version, + "package_epoch": package_epoch, + }), + ("uninstall_package", runner.uninstall_package, [package_name], {}) + ] + packages_to_skip = get_excluded_packages() + for test_name, func, args, kwargs in test_steps: + if test_name == "run_package_integrity_tests" and not CONFIG.enable_integrity_tests: + continue + if should_skip_test(package_name, test_name, packages_to_skip): + continue + func(*args, **kwargs) except VMImageNotFound as exc: logging.exception('Cannot find VM image: %s', exc) except WorkDirPreparationError: From 103c27d8c0522bb4645691d4400e69720d6e7151 Mon Sep 17 00:00:00 2001 From: Kwaizer Date: Thu, 14 Aug 2025 17:45:36 +0100 Subject: [PATCH 2/6] Cache test mapping --- alts/scheduler/scheduling.py | 23 +++++++++++++++++++++-- alts/shared/models.py | 3 +++ alts/shared/utils/file_utils.py | 22 ---------------------- alts/worker/tasks.py | 15 +++------------ requirements/scheduler.txt | 1 + 5 files changed, 28 insertions(+), 36 deletions(-) diff --git a/alts/scheduler/scheduling.py b/alts/scheduler/scheduling.py index a5c81cd8..0b0bbaa4 100644 --- a/alts/scheduler/scheduling.py +++ b/alts/scheduler/scheduling.py @@ -4,7 +4,8 @@ import time import urllib.parse import uuid -from typing import List +from typing import List, Optional +from cachetools import TTLCache import requests @@ -28,6 +29,10 @@ def __init__( self.__graceful_terminate = graceful_terminate self.__celery = celery_app self.__get_result_timeout = get_result_timeout + self.__cached_config = TTLCache( + maxsize=CONFIG.cache_size, + ttl=CONFIG.cache_update_interval, + ) self.logger = logging.getLogger(__file__) def get_available_test_tasks(self) -> List[dict]: @@ -49,6 +54,17 @@ def get_available_test_tasks(self) -> List[dict]: self.logger.exception('Cannot get available test tasks:') return response_as_json + def get_excluded_packages(self) -> Optional[dict]: + if 'excluded_packages' not in self.__cached_config: + uri = f'{CONFIG.excluded_pkgs_url}' + try: + response = requests.get(uri) + response.raise_for_status() + self.__cached_config['excluded_packages'] = response.json() + except requests.RequestException: + return {} + return self.__cached_config.get('excluded_packages', {}) + def schedule_test_task(self, payload: TaskRequestPayload): """ Schedules new tasks in Test System. @@ -130,7 +146,10 @@ def schedule_test_task(self, payload: TaskRequestPayload): task_params['repositories'] = repositories try: run_tests.apply_async( - (task_params,), + ( + task_params, + self.get_excluded_packages(), + ), task_id=task_id, queue=queue_name, ) diff --git a/alts/shared/models.py b/alts/shared/models.py index 0ac6dca1..4a5ce8e9 100644 --- a/alts/shared/models.py +++ b/alts/shared/models.py @@ -389,3 +389,6 @@ class SchedulerConfig(CeleryConfig): working_directory: str = '/srv/alts/scheduler' jwt_secret: str hashing_algorithm: str = 'HS256' + excluded_pkgs_url: str = 'https://git.almalinux.org/almalinux/alts-exclusions/raw/branch/main/skipped_tests.json' + cache_size: str = 1 + cache_update_interval: int = 600 diff --git a/alts/shared/utils/file_utils.py b/alts/shared/utils/file_utils.py index 022beebc..b868231b 100644 --- a/alts/shared/utils/file_utils.py +++ b/alts/shared/utils/file_utils.py @@ -58,25 +58,3 @@ def feed_hasher(_fd): file_path.seek(0) feed_hasher(file_path) return hasher.hexdigest() - - -def file_url_exists(url): - - """ - Check if a file exists at the specified URL using a HEAD request. - - Parameters - ---------- - url : str - The URL to check. - - Returns - ------- - bool - True if the file exists, False otherwise. - """ - try: - response = requests.head(url) - return response.status_code == 200 - except requests.RequestException as e: - return False diff --git a/alts/worker/tasks.py b/alts/worker/tasks.py index f582aba0..18d149c7 100644 --- a/alts/worker/tasks.py +++ b/alts/worker/tasks.py @@ -24,7 +24,6 @@ from urllib3 import Retry from urllib3.exceptions import TimeoutError -from alts.shared.utils.file_utils import file_url_exists from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT from alts.shared.exceptions import ( InstallPackageError, @@ -95,7 +94,7 @@ class RetryableTask(AbortableTask): @celery_app.task(bind=True, base=RetryableTask) -def run_tests(self, task_params: dict): +def run_tests(self, task_params: dict, packages_to_skip: dict): """ Executes a package test in a specified environment. @@ -103,6 +102,8 @@ def run_tests(self, task_params: dict): ---------- task_params : dict Task parameters. + packages_to_skip : dict + Tests mapping. Returns ------- @@ -182,15 +183,6 @@ def set_artifacts_when_stage_has_unexpected_exception( module_stream = task_params.get('module_stream') module_version = task_params.get('module_version') - def get_excluded_packages(): - uri = 'https://git.almalinux.org/almalinux/alts-exclusions/raw/branch/main/skipped_tests.json' - try: - response = requests.get(uri) - response.raise_for_status() - return response.json() - except requests.RequestException: - return {} - try: # Wait a bit to not spawn all environments at once when # a lot of tasks are coming to the machine @@ -220,7 +212,6 @@ def get_excluded_packages(): }), ("uninstall_package", runner.uninstall_package, [package_name], {}) ] - packages_to_skip = get_excluded_packages() for test_name, func, args, kwargs in test_steps: if test_name == "run_package_integrity_tests" and not CONFIG.enable_integrity_tests: continue diff --git a/requirements/scheduler.txt b/requirements/scheduler.txt index 105daf00..16053db8 100644 --- a/requirements/scheduler.txt +++ b/requirements/scheduler.txt @@ -5,3 +5,4 @@ databases[sqlite]==0.8.0 fastapi==0.115.0 pyjwt==2.9.0 uvicorn==0.30.6 +cachetools==6.1.0 From 4d46a0e12550bef859e3dbe920c95e9480cd5785 Mon Sep 17 00:00:00 2001 From: Kwaizer Date: Fri, 15 Aug 2025 12:13:55 +0100 Subject: [PATCH 3/6] Take platforms into account --- alts/scheduler/scheduling.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/alts/scheduler/scheduling.py b/alts/scheduler/scheduling.py index 0b0bbaa4..de88ccc8 100644 --- a/alts/scheduler/scheduling.py +++ b/alts/scheduler/scheduling.py @@ -54,7 +54,7 @@ def get_available_test_tasks(self) -> List[dict]: self.logger.exception('Cannot get available test tasks:') return response_as_json - def get_excluded_packages(self) -> Optional[dict]: + def get_excluded_packages(self, distro: str) -> Optional[dict]: if 'excluded_packages' not in self.__cached_config: uri = f'{CONFIG.excluded_pkgs_url}' try: @@ -63,7 +63,8 @@ def get_excluded_packages(self) -> Optional[dict]: self.__cached_config['excluded_packages'] = response.json() except requests.RequestException: return {} - return self.__cached_config.get('excluded_packages', {}) + pkgs = self.__cached_config.get('excluded_packages', {}) + return pkgs.get(distro, {}) def schedule_test_task(self, payload: TaskRequestPayload): """ @@ -144,11 +145,12 @@ def schedule_test_task(self, payload: TaskRequestPayload): task_params['task_id'] = task_id task_params['runner_type'] = runner_type task_params['repositories'] = repositories + distro = f"{task_params['dist_name']}-{task_params['dist_version']}" try: run_tests.apply_async( ( task_params, - self.get_excluded_packages(), + self.get_excluded_packages(distro), ), task_id=task_id, queue=queue_name, From f4442f34fa44e9bf2294f2ed76012393e43265bf Mon Sep 17 00:00:00 2001 From: Kwaizer Date: Tue, 19 Aug 2025 22:14:28 +0100 Subject: [PATCH 4/6] Adjust alts to send skipped tests for display --- alts/shared/constants.py | 8 ++++++++ alts/worker/tasks.py | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/alts/shared/constants.py b/alts/shared/constants.py index 66ce0ad4..cfee598a 100644 --- a/alts/shared/constants.py +++ b/alts/shared/constants.py @@ -15,6 +15,7 @@ 'DEFAULT_SSH_AUTH_METHODS', 'X32_ARCHITECTURES', 'X64_ARCHITECTURES', + 'TESTS', ] @@ -67,3 +68,10 @@ 'hostbased', 'publickey', ] + +TESTS = { + 'run_system_info_commands': 'system_info', + 'install_package': 'install_package', + 'run_package_integrity_tests': 'package_integrity_tests', + 'uninstall_package': 'uninstall_package', +} diff --git a/alts/worker/tasks.py b/alts/worker/tasks.py index 18d149c7..72ccb3d8 100644 --- a/alts/worker/tasks.py +++ b/alts/worker/tasks.py @@ -24,7 +24,7 @@ from urllib3 import Retry from urllib3.exceptions import TimeoutError -from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT +from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT, TESTS from alts.shared.exceptions import ( InstallPackageError, PackageIntegrityTestsError, @@ -111,6 +111,7 @@ def run_tests(self, task_params: dict, packages_to_skip: dict): Result summary of a test execution. """ aborted = False + summary = defaultdict(dict) def is_success(stage_data_: dict): tap_result = are_tap_tests_success(stage_data_.get('stdout', '')) @@ -212,12 +213,16 @@ def set_artifacts_when_stage_has_unexpected_exception( }), ("uninstall_package", runner.uninstall_package, [package_name], {}) ] + skipped_tests = [] + summary['skipped_tests'] = {} for test_name, func, args, kwargs in test_steps: if test_name == "run_package_integrity_tests" and not CONFIG.enable_integrity_tests: continue if should_skip_test(package_name, test_name, packages_to_skip): + skipped_tests.append(TESTS[test_name]) continue func(*args, **kwargs) + summary['skipped_tests'] = skipped_tests except VMImageNotFound as exc: logging.exception('Cannot find VM image: %s', exc) except WorkDirPreparationError: @@ -258,7 +263,6 @@ def set_artifacts_when_stage_has_unexpected_exception( ) finally: runner.teardown() - summary = defaultdict(dict) if aborted: summary['revoked'] = True else: From 187ab739c7e48c6cc16b2bfdd0c41ce40049e180 Mon Sep 17 00:00:00 2001 From: Kwaizer Date: Fri, 22 Aug 2025 10:57:02 +0100 Subject: [PATCH 5/6] Refactor test steps execution --- alts/scheduler/scheduling.py | 2 +- alts/worker/tasks.py | 60 ++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/alts/scheduler/scheduling.py b/alts/scheduler/scheduling.py index de88ccc8..b14dd497 100644 --- a/alts/scheduler/scheduling.py +++ b/alts/scheduler/scheduling.py @@ -58,7 +58,7 @@ def get_excluded_packages(self, distro: str) -> Optional[dict]: if 'excluded_packages' not in self.__cached_config: uri = f'{CONFIG.excluded_pkgs_url}' try: - response = requests.get(uri) + response = requests.get(uri, timeout=10) response.raise_for_status() self.__cached_config['excluded_packages'] = response.json() except requests.RequestException: diff --git a/alts/worker/tasks.py b/alts/worker/tasks.py index 72ccb3d8..3b3ed67d 100644 --- a/alts/worker/tasks.py +++ b/alts/worker/tasks.py @@ -188,40 +188,40 @@ def set_artifacts_when_stage_has_unexpected_exception( # Wait a bit to not spawn all environments at once when # a lot of tasks are coming to the machine time.sleep(random.randint(5, 10)) - - test_steps = [ - ("setup", runner.setup, [], {}), - ("run_system_info_commands", runner.run_system_info_commands, [], {}), - ("install_package", runner.install_package, [ - package_name - ], { - "package_version": package_version, - "package_epoch": package_epoch, - "module_name": module_name, - "module_stream": module_stream, - "module_version": module_version, - "semi_verbose": True, - }), - ("run_package_integrity_tests", runner.run_package_integrity_tests, [ - package_name, package_version - ], {}), - ("run_third_party_tests", runner.run_third_party_tests, [ - package_name - ], { - "package_version": package_version, - "package_epoch": package_epoch, - }), - ("uninstall_package", runner.uninstall_package, [package_name], {}) - ] skipped_tests = [] summary['skipped_tests'] = {} - for test_name, func, args, kwargs in test_steps: - if test_name == "run_package_integrity_tests" and not CONFIG.enable_integrity_tests: - continue + + def run_or_skip(test_name, func, *args, **kwargs): if should_skip_test(package_name, test_name, packages_to_skip): skipped_tests.append(TESTS[test_name]) - continue - func(*args, **kwargs) + else: + func(*args, **kwargs) + + run_or_skip("setup", runner.setup) + run_or_skip("run_system_info_commands", runner.run_system_info_commands) + run_or_skip( + "install_package", + runner.install_package, + package_name, + package_version=package_version, + package_epoch=package_epoch, + module_name=module_name, + module_stream=module_stream, + module_version=module_version, + semi_verbose=True, + ) + if CONFIG.enable_integrity_tests: + run_or_skip("run_package_integrity_tests", runner.run_package_integrity_tests, package_name, + package_version) + run_or_skip( + "run_third_party_tests", + runner.run_third_party_tests, + package_name, + package_version=package_version, + package_epoch=package_epoch, + ) + run_or_skip("uninstall_package", runner.uninstall_package, package_name) + summary['skipped_tests'] = skipped_tests except VMImageNotFound as exc: logging.exception('Cannot find VM image: %s', exc) From bbbd86fe23748ac8f603806875cf776c7b065e5d Mon Sep 17 00:00:00 2001 From: Kwaizer Date: Sun, 24 Aug 2025 23:36:06 +0100 Subject: [PATCH 6/6] Minor corrections --- README.md | 19 +++++++++++++++++++ alts/scheduler/scheduling.py | 6 +++--- alts/shared/constants.py | 4 ++-- alts/shared/models.py | 6 +++--- alts/shared/utils/file_utils.py | 2 -- alts/worker/tasks.py | 20 ++++++++------------ 6 files changed, 35 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 29121d57..d41c7d20 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,25 @@ For testing purposes, you can add the `--reload` argument to the scheduler launc Both Celery worker and scheduler REST API services need YAML-based configs to function. Config examples are provided in the `configs` folder. +During the task execution some tests may be skipped if it specified in a particular `.json` file which can be found here: https://git.almalinux.org/almalinux/alts-exclusions/src/branch/main/skipped_tests.json + +Example of a file content: +``` +{ + "almalinux-8": { + "libxml2": ["run_package_integrity_tests"], + "curl": ["run_package_integrity_tests"], + "bzip2": ["run_package_integrity_tests"] + }, + "almalinux-9": { + "libxml2": ["run_package_integrity_tests"], + "curl": ["run_package_integrity_tests"] + }, + "ubuntu-24.04": { + "bzip2": ["run_package_integrity_tests"] + } +} +``` Filling options in the config file -- diff --git a/alts/scheduler/scheduling.py b/alts/scheduler/scheduling.py index b14dd497..ad22876a 100644 --- a/alts/scheduler/scheduling.py +++ b/alts/scheduler/scheduling.py @@ -30,8 +30,8 @@ def __init__( self.__celery = celery_app self.__get_result_timeout = get_result_timeout self.__cached_config = TTLCache( - maxsize=CONFIG.cache_size, - ttl=CONFIG.cache_update_interval, + maxsize=CONFIG.excluded_tests_cache_size, + ttl=CONFIG.excluded_tests_cache_update_interval, ) self.logger = logging.getLogger(__file__) @@ -56,7 +56,7 @@ def get_available_test_tasks(self) -> List[dict]: def get_excluded_packages(self, distro: str) -> Optional[dict]: if 'excluded_packages' not in self.__cached_config: - uri = f'{CONFIG.excluded_pkgs_url}' + uri = f'{CONFIG.excluded_tests_url}' try: response = requests.get(uri, timeout=10) response.raise_for_status() diff --git a/alts/shared/constants.py b/alts/shared/constants.py index cfee598a..76c3d742 100644 --- a/alts/shared/constants.py +++ b/alts/shared/constants.py @@ -15,7 +15,7 @@ 'DEFAULT_SSH_AUTH_METHODS', 'X32_ARCHITECTURES', 'X64_ARCHITECTURES', - 'TESTS', + 'TESTS_MAPPING', ] @@ -69,7 +69,7 @@ 'publickey', ] -TESTS = { +TESTS_MAPPING = { 'run_system_info_commands': 'system_info', 'install_package': 'install_package', 'run_package_integrity_tests': 'package_integrity_tests', diff --git a/alts/shared/models.py b/alts/shared/models.py index 4a5ce8e9..18d6cb1f 100644 --- a/alts/shared/models.py +++ b/alts/shared/models.py @@ -389,6 +389,6 @@ class SchedulerConfig(CeleryConfig): working_directory: str = '/srv/alts/scheduler' jwt_secret: str hashing_algorithm: str = 'HS256' - excluded_pkgs_url: str = 'https://git.almalinux.org/almalinux/alts-exclusions/raw/branch/main/skipped_tests.json' - cache_size: str = 1 - cache_update_interval: int = 600 + excluded_tests_url: str = 'https://git.almalinux.org/almalinux/alts-exclusions/raw/branch/main/skipped_tests.json' + excluded_tests_cache_size: str = 1 + excluded_tests_cache_update_interval: int = 600 diff --git a/alts/shared/utils/file_utils.py b/alts/shared/utils/file_utils.py index b868231b..2e99b6de 100644 --- a/alts/shared/utils/file_utils.py +++ b/alts/shared/utils/file_utils.py @@ -1,7 +1,5 @@ import hashlib -import requests - def get_hasher(checksum_type): """ diff --git a/alts/worker/tasks.py b/alts/worker/tasks.py index 3b3ed67d..831f17b0 100644 --- a/alts/worker/tasks.py +++ b/alts/worker/tasks.py @@ -24,7 +24,7 @@ from urllib3 import Retry from urllib3.exceptions import TimeoutError -from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT, TESTS +from alts.shared.constants import API_VERSION, DEFAULT_REQUEST_TIMEOUT, TESTS_MAPPING from alts.shared.exceptions import ( InstallPackageError, PackageIntegrityTestsError, @@ -94,7 +94,7 @@ class RetryableTask(AbortableTask): @celery_app.task(bind=True, base=RetryableTask) -def run_tests(self, task_params: dict, packages_to_skip: dict): +def run_tests(self, task_params: dict, tests_to_skip: dict): """ Executes a package test in a specified environment. @@ -102,7 +102,7 @@ def run_tests(self, task_params: dict, packages_to_skip: dict): ---------- task_params : dict Task parameters. - packages_to_skip : dict + tests_to_skip : dict Tests mapping. Returns @@ -183,22 +183,20 @@ def set_artifacts_when_stage_has_unexpected_exception( module_name = task_params.get('module_name') module_stream = task_params.get('module_stream') module_version = task_params.get('module_version') - try: # Wait a bit to not spawn all environments at once when # a lot of tasks are coming to the machine time.sleep(random.randint(5, 10)) - skipped_tests = [] - summary['skipped_tests'] = {} + summary['skipped_tests'] = [] def run_or_skip(test_name, func, *args, **kwargs): - if should_skip_test(package_name, test_name, packages_to_skip): - skipped_tests.append(TESTS[test_name]) + if should_skip_test(package_name, test_name, tests_to_skip): + summary['skipped_tests'].append(TESTS_MAPPING[test_name]) else: func(*args, **kwargs) - run_or_skip("setup", runner.setup) - run_or_skip("run_system_info_commands", runner.run_system_info_commands) + runner.setup() + runner.run_system_info_commands() run_or_skip( "install_package", runner.install_package, @@ -221,8 +219,6 @@ def run_or_skip(test_name, func, *args, **kwargs): package_epoch=package_epoch, ) run_or_skip("uninstall_package", runner.uninstall_package, package_name) - - summary['skipped_tests'] = skipped_tests except VMImageNotFound as exc: logging.exception('Cannot find VM image: %s', exc) except WorkDirPreparationError: