From 46f5226949ffebcc97ec108d3a11d5595bd0624f Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:14:26 -0400 Subject: [PATCH 1/7] Split configuration loading into static and dynamic modules Move config parsing into elnbuildsync.config with separate static and dynamic loaders, relocate trigger_tag to control, and update daemon and build pipeline callers. Signed-off-by: Stephen Gallagher Co-authored-by: Cursor --- elnbuildsync/batching.py | 2 +- elnbuildsync/config.py | 803 -------------------------------- elnbuildsync/config/__init__.py | 491 +++++++++++++++++++ elnbuildsync/config/dynamic.py | 288 ++++++++++++ elnbuildsync/config/static.py | 254 ++++++++++ elnbuildsync/daemon.py | 62 ++- elnbuildsync/listener.py | 6 +- 7 files changed, 1086 insertions(+), 820 deletions(-) delete mode 100644 elnbuildsync/config.py create mode 100644 elnbuildsync/config/__init__.py create mode 100644 elnbuildsync/config/dynamic.py create mode 100644 elnbuildsync/config/static.py diff --git a/elnbuildsync/batching.py b/elnbuildsync/batching.py index bb73b37..0baecaa 100644 --- a/elnbuildsync/batching.py +++ b/elnbuildsync/batching.py @@ -100,7 +100,7 @@ async def rebuild_from_components(downstream_components): # Fake up a TagMessage for each of these to enqueue into the next batch bsys = kojihelpers.connection.get_buildsys() - src_tag = config.main["koji"]["trigger_tag"] + src_tag = config.control["trigger_tag"] latest_tagged_rawhide_pkgs = await call_koji( bsys.listTagged, src_tag, latest=True, inherit=True ) diff --git a/elnbuildsync/config.py b/elnbuildsync/config.py deleted file mode 100644 index 0f93584..0000000 --- a/elnbuildsync/config.py +++ /dev/null @@ -1,803 +0,0 @@ -# This file is part of ELNBuildSync -# Copyright (C) 2023-2026 Stephen Gallagher - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# SPDX-License-Identifier: GPL-3.0-or-later - - -import git -import json -import logging -import os -import re -import requests.exceptions -import sqlalchemy -from txrequests import Session -import tempfile -import twisted.internet.utils -import yaml - -from tenacity import retry as retry_on_exception, stop_after_delay, wait_exponential -from twisted.internet.threads import deferToThread - -from . import config -from .email import Email - - -# Global logger -logger = logging.getLogger(__name__) - -DEFAULT_CONTENT_RESOLVER = "https://tiny.distro.builders" -DEFAULT_DISTRO_VIEWS = ["eln"] - -# A special Deferred for terminating the program -terminator = None - -# The URL for connecting to the database -db_url = None - -# Configuration options -config_timer = 15 * 60 # 15 minutes -cleanup_timer = 12 * 60 * 60 # 12 hours -task_check_timer = 5 * 60 # 5 minutes -tag_check_timer = 5 * 60 # 5 minutes -task_timeout = 24 * 60 * 60 # 24 hours -tag_timeout = 1 * 60 * 60 # 1 hour -message_batch_timer = 60 # 1 minute -koji_batch = 500 -configuration = None -config_ref = None -distrogitsync = None -dry_run = False -do_untagging = False -retry = 3 -scmurl = None -main = None -comps = None -# If we haven't gotten the repoInit message within 10 minutes, assume we missed it -waitrepo_init_timeout = 10 * 60 - -# The actual generation can take up to 20 minutes -waitrepo_timeout = 20 * 60 - -# Process state -cleanup_processor = None -status_processor = None -tmpdir = None - -# SMTP (see email.py); password set from daemon --smtp-pw-file before load_config -emailer = None -smtp_password = "" - - -class ConfigError(Exception): - pass - - -class UnknownComponentError(ConfigError): - pass - - -class UnknownRefError(ConfigError): - pass - - -def loglevel(val=None): - """Gets or, optionally, sets the logging level of the module. - Standard numeric levels are accepted. - - :param val: The logging level to use, optional - :returns: The current logging level - """ - if val is not None: - try: - logger.setLevel(val) - except ValueError: - logger.warning( - "Invalid log level passed to DistroBuildSync logger: %s", val - ) - except Exception: - logger.exception("Unable to set log level: %s", val) - return logger.getEffectiveLevel() - - -def is_debug(): - """ - Determines if we are in debug logging mode. - - This is useful for enabling/disabling third-party logging such as the - sqlalchemy logger. - """ - return loglevel() <= logging.DEBUG - - -def retries(val=None): - """Gets or, optionally, sets the number of retries for various - operational failures. Typically used for handling dist-git requests. - - :param val: The number of retries to attept, optional - :returns: The current value of retries - """ - global retry - if val is not None: - retry = val - return retry - - -def split_scmurl(url): - """Splits a `link#ref` style URLs into the link and ref parts. While - generic, many code paths in DistroBuildSync expect these to be branch names. - `link` forms are also accepted, in which case the returned `ref` is None. - - It also attempts to extract the namespace and component, where applicable. - These can only be detected if the link matches the standard dist-git - pattern; in other cases the results may be bogus or None. - - :param url: A link#ref style URL, with #ref being optional - :returns: A dictionary with `link`, `ref`, `ns` and `comp` keys - """ - scm = url.split("#", 1) - nscomp = scm[0].split("/") - return { - "link": scm[0], - "ref": scm[1] if len(scm) >= 2 else None, - "ns": nscomp[-2] if nscomp and len(nscomp) >= 2 else None, - "comp": nscomp[-1] if nscomp else None, - } - - -def split_module(comp): - """Splits modules component name into name and stream pair. Expects the - name to be in the `name:stream` format. Defaults to stream=master if the - split fails. - - :param comp: The component name - :returns: Dictionary with name and stream - """ - ms = comp.split(":") - return { - "name": ms[0], - "stream": ms[1] if len(ms) > 1 and ms[1] else "master", - } - - -async def get_config_ref(url): - """Gets the ref for the config SCMURL - - Returns the actual ref for a symbolic ref possibly used in the - config SCMURL. Used by the update function to check whether the - config should be resync'd. - - :param url: Config SCMURL - :returns: Remote ref or None on error - """ - scm = split_scmurl(url) - logger.info(f"Getting config ref for {scm['link']} {scm['ref']}") - - if scm["ref"]: - output = await twisted.internet.utils.getProcessOutput( - executable="/usr/bin/git", - args=("ls-remote", "--branches", scm["link"], scm["ref"]), - errortoo=True, - ) - else: - output = await twisted.internet.utils.getProcessOutput( - executable="/usr/bin/git", - args=("ls-remote", "--branches", scm["link"]), - errortoo=True, - ) - - if not output: - scmref = scm["ref"] - scmlink = scm["link"] - raise UnknownRefError(f"{scmref} not found in {scmlink}") - - return output.split(b"\t", 1)[0] - - -async def update_config(): - global config_ref - - if not scmurl: - logger.info("Config URL not provided.") - return - - logger.critical("Updating configuration") - - try: - ref = await get_config_ref(scmurl) - except UnknownRefError as e: - logger.info(e) - logger.critical( - f"The configuration repository is unavailable, skipping update. Checking again in {config_timer} seconds." - ) - return - - try: - await load_config(config_git_url=scmurl) - config_ref = ref - except ConfigError as e: - logger.info(e) - logger.critical( - f"The configuration is invalid, skipping update. Checking again in {config_timer} seconds." - ) - return - - -async def get_distro_packages( - distro_url, - distro_view=DEFAULT_DISTRO_VIEWS, - arches=None, - which_source=None, -): - """ - Fetches the list of desired sources from Content Resolver - for each of the given 'arches'. - """ - if not arches: - arches = ["aarch64", "ppc64le", "s390x", "x86_64"] - if not which_source: - which_source = ["source", "buildroot-source"] - - packages = dict[str, dict[str, str]]() - - for view in reversed(distro_view): - for this_source in reversed(which_source): - url = ( - "{distro_url}/view-{this_source}-package-name-list--view-{view}.txt" - ).format( - distro_url=distro_url, - this_source=this_source, - view=view, - ) - - logger.debug("downloading {url}".format(url=url)) - - with Session() as session: - r = await session.get(url, allow_redirects=True) - for line in r.text.splitlines(): - packages[line] = { - "view": view, - "source": this_source, - "upstream_name": line, - "downstream_name": line, - } - - # There may be an empty line in the file, ignore it. - packages.pop("", None) - - logger.debug("Found a total of {} packages".format(len(packages))) - - return packages - - -@retry_on_exception( - wait=wait_exponential(), - stop=stop_after_delay(60), - reraise=True, -) -async def get_rawhide_tag(): - """ - Queries Bodhi for the current tag associated with Rawhide - """ - - # Retrieve the list of "pending" (aka development) releases - url = "https://bodhi.fedoraproject.org/releases?state=pending" - with Session() as session: - try: - r = await session.get(url, allow_redirects=True) - r.raise_for_status() - releases = json.loads(r.text) - logger.debug(releases) - except json.decoder.JSONDecodeError as e: - raise ConfigError("Could not parse JSON from Bodhi releases") from e - - except requests.exceptions.HTTPError as e: - raise ConfigError("HTTP Error") from e - - # Get the stable tag corresponding to the rawhide branch - stable_tag = None - for release in releases["releases"]: - # Get the stable tag associated with this release - if release["branch"] == "rawhide": - stable_tag = release["stable_tag"] - break - - # Shouldn't ever happen, but... - if not stable_tag: - raise ConfigError("Unexpectedly received no valid Fedora rawhide release") - - return stable_tag - - -def _parse_open_id_connect(oidc_raw): - """Parse OpenID Connect configuration. Returns None if disabled, else a dict. - Raises ConfigError on invalid or missing required fields. - """ - if oidc_raw is False: - logger.info( - "OpenID Connect explicitly disabled - /trigger endpoint unprotected" - ) - return None - oidc = oidc_raw - default_scopes = [ - "openid", - "profile", - "https://id.fedoraproject.org/scope/groups", - ] - required_fields = [ - "auth_url", - "client_id", - "client_secret", - "token_endpoint", - "admin_groups", - ] - for field in required_fields: - if field not in oidc: - raise ConfigError(f"open_id_connect.{field} missing.") - result = { - "auth_url": str(oidc["auth_url"]), - "client_id": str(oidc["client_id"]), - "client_secret": str(oidc["client_secret"]), - "token_endpoint": str(oidc["token_endpoint"]), - "userinfo_endpoint": str(oidc.get("userinfo_endpoint", "")), - "scopes": list(oidc.get("scopes", default_scopes)), - "admin_groups": list(oidc["admin_groups"]), - } - logger.info( - "OpenID Connect authentication enabled; admin groups: %s", - result["admin_groups"], - ) - return result - - -def _parse_koji(cnf_koji): - """Parse koji configuration. Returns dict with profile, trigger_tag, build_target, stable_tag, scratch_build, fail_fast, and optionally username.""" - if "profile" not in cnf_koji: - raise ConfigError("koji.profile missing.") - result = {"profile": str(cnf_koji["profile"])} - if "trigger_tag" not in cnf_koji: - raise ConfigError("koji.trigger_tag missing.") - result["trigger_tag"] = str(cnf_koji["trigger_tag"]) - if "build_target" not in cnf_koji: - raise ConfigError("koji.build_target missing.") - result["build_target"] = str(cnf_koji["build_target"]) - if "stable_tag" not in cnf_koji: - raise ConfigError("koji.stable_tag missing.") - result["stable_tag"] = str(cnf_koji["stable_tag"]) - if "username" in cnf_koji: - result["username"] = str(cnf_koji["username"]) - if "scratch_build" in cnf_koji: - result["scratch_build"] = bool(cnf_koji["scratch_build"]) - else: - logger.warning( - "Configuration warning: koji.scratch_build not defined, assuming false." - ) - result["scratch_build"] = False - if "fail_fast" in cnf_koji: - result["fail_fast"] = bool(cnf_koji["fail_fast"]) - else: - logger.warning( - "Configuration warning: koji.fail_fast not defined, assuming false." - ) - result["fail_fast"] = False - return result - - -def _parse_bodhi(cnf_bodhi): - """Parse bodhi configuration. Returns dict with batch_size.""" - result = {"batch_size": 0} - if "batch_size" in cnf_bodhi: - try: - result["batch_size"] = int(cnf_bodhi["batch_size"]) - except ValueError: - raise ConfigError("bodhi.batch_size must be an integer") - return result - - -def _parse_email(cnf_email): - """Parse email configuration. Returns None if disabled, else a dict with - smtp_host, smtp_port, smtp_username, from, recipients. - Raises ConfigError on invalid or missing required fields. - """ - if cnf_email is False: - logger.info("Email explicitly disabled") - return None - required = ("smtp_host", "smtp_port", "smtp_username", "from", "recipients") - for key in required: - if key not in cnf_email: - raise ConfigError(f"email.{key} missing.") - try: - port = int(cnf_email["smtp_port"]) - except (TypeError, ValueError): - raise ConfigError("email.smtp_port must be an integer") - recipients = cnf_email["recipients"] - if not isinstance(recipients, list) or len(recipients) == 0: - raise ConfigError("email.recipients must be a non-empty list.") - for r in recipients: - if not isinstance(r, str) or not r: - raise ConfigError("email.recipients must be a list of non-empty strings.") - return { - "smtp_host": str(cnf_email["smtp_host"]), - "smtp_port": port, - "smtp_username": str(cnf_email["smtp_username"]), - "from": str(cnf_email["from"]), - "recipients": [str(x) for x in recipients], - } - - -def _parse_db(cnf_db): - """Parse database configuration. Returns dict with host, port, name, driver, user. - All keys are mandatory. - """ - required = ("host", "port", "name", "driver", "user") - for key in required: - if key not in cnf_db: - raise ConfigError(f"db.{key} missing.") - try: - result = { - "host": str(cnf_db["host"]), - "port": int(cnf_db["port"]), - "name": str(cnf_db["name"]), - "driver": str(cnf_db["driver"]), - "user": str(cnf_db["user"]), - } - except ValueError: - raise ConfigError("db.port must be an integer") - return result - - -def _parse_control(cnf_control): - """Parse control configuration. Returns dict with pause, skip_tag, exclude, ordering, status_interval, etc.""" - result = dict() - for k in ("pause",): - if k in cnf_control: - result[k] = bool(cnf_control[k]) - else: - raise ConfigError(f"control.{k} missing.") - - result["skip_tag"] = set() - if "skip_tag" in cnf_control: - result["skip_tag"].update(cnf_control["skip_tag"]) - - result["exclude"] = set() - if "exclude" in cnf_control: - result["exclude"].update(cnf_control["exclude"]) - - if result["exclude"]: - logger.info( - "Excluding %d component(s).", - len(result["exclude"]), - ) - else: - logger.info("Not excluding any components.") - - result["ordering"] = dict() - if "ordering" in cnf_control: - result["ordering"].update(cnf_control["ordering"]) - - result["status_interval"] = 600 # 10 minutes - if "status_interval" in cnf_control: - val = cnf_control["status_interval"] - if not isinstance(val, int) or val <= 0: - raise ConfigError("control.status_interval must be a positive integer.") - result["status_interval"] = val - - return result - - -async def _parse_components(cnf_components): - """Parse the components block (top-level). Requires at least one of autopackagelist or overrides. - When autopackagelist is present, calls get_distro_packages() and uses the returned dict as comps. - If overrides is present, updates/supplements comps with comps.update() semantics per component. - Returns dict with comps (dict), and autopackagelist (dict or None) for view/content_resolver. - """ - if "autopackagelist" not in cnf_components and "overrides" not in cnf_components: - raise ConfigError( - "At least one of components.autopackagelist or components.overrides must be present." - ) - apl = None - if "autopackagelist" in cnf_components: - apl_raw = cnf_components["autopackagelist"] - if "view" not in apl_raw: - raise ConfigError("components.autopackagelist.view missing.") - if "source" not in apl_raw: - raise ConfigError("components.autopackagelist.source missing.") - apl = { - "view": apl_raw["view"] - if isinstance(apl_raw["view"], list) - else [apl_raw["view"]], - "source": apl_raw["source"] - if isinstance(apl_raw["source"], list) - else [apl_raw["source"]], - "content_resolver": apl_raw.get( - "content_resolver", DEFAULT_CONTENT_RESOLVER - ), - } - downstream_components = await get_distro_packages( - distro_url=apl["content_resolver"], - distro_view=apl["view"], - which_source=apl["source"], - ) - for comp_name, comp_entry in downstream_components.items(): - if not isinstance(comp_entry, dict): - raise ConfigError( - f"components.autopackagelist entry '{comp_name}' must be a dictionary." - ) - if "upstream_name" not in comp_entry: - raise ConfigError( - f"components.autopackagelist entry '{comp_name}' missing upstream_name." - ) - if "downstream_name" not in comp_entry: - raise ConfigError( - f"components.autopackagelist entry '{comp_name}' missing downstream_name." - ) - else: - downstream_components = {} - upstream_components = downstream_components.copy() - - overrides = cnf_components.get("overrides", {}) - if not isinstance(overrides, dict): - raise ConfigError("components.overrides must be a dictionary.") - - for downstream_name, override_options in overrides.items(): - if not isinstance(override_options, dict): - raise ConfigError( - f"components.overrides.{downstream_name} must be a dictionary" - ) - - upstream_name = override_options.get("upstream_name", downstream_name) - - if downstream_name not in downstream_components: - downstream_components[downstream_name] = { - "view": "override", - "source": "override", - "upstream_name": upstream_name, - "downstream_name": downstream_name, - } - downstream_components[downstream_name].update(override_options) - downstream_components[downstream_name].setdefault( - "upstream_name", downstream_name - ) - downstream_components[downstream_name].setdefault( - "downstream_name", downstream_name - ) - upstream_components[upstream_name] = downstream_components[ - downstream_name - ].copy() - - return { - "downstream_components": downstream_components, - "upstream_components": upstream_components, - } - - -def _parse_configuration_block(cnf): - """Parse the full configuration block (no rawhide resolution, no components). - Returns dict n with koji, bodhi, db, open_id_connect, control, email. - """ - if "koji" not in cnf: - raise ConfigError("koji missing.") - n = {"koji": _parse_koji(cnf["koji"])} - - if "bodhi" not in cnf: - raise ConfigError("bodhi missing.") - n["bodhi"] = _parse_bodhi(cnf["bodhi"]) - - if "db" not in cnf: - raise ConfigError("db missing.") - n["db"] = _parse_db(cnf["db"]) - - if "open_id_connect" not in cnf: - raise ConfigError( - "open_id_connect missing. Set open_id_connect: false to disable authentication." - ) - n["open_id_connect"] = _parse_open_id_connect(cnf["open_id_connect"]) - - if "control" not in cnf: - raise ConfigError("control missing.") - n["control"] = _parse_control(cnf["control"]) - - if "email" not in cnf: - raise ConfigError("email missing. Set email: false to disable email.") - n["email"] = _parse_email(cnf["email"]) - - return n - - -# FIXME: This needs even more error checking, e.g. -# - check if blocks are actual dictionaries -# - check if certain values are what we expect -async def load_config(db_pw=None, config_git_url=None, config_file=None): - """Loads or updates the global configuration from the provided URL in - the `link#branch` format. If no branch is provided, assumes `master`. - - The operation is atomic and the function can be safely called to update - the configuration without the danger of clobbering the current one. - - :returns: The configuration dictionary, or None on error - """ - global main - global comps - global scmurl - global db_url - global emailer - - if not (config_git_url or config_file): - raise ValueError("One of 'config_git_url' or 'config_file' must be specified") - - y = None - - with tempfile.TemporaryDirectory(prefix="distrobaker-") as cdir: - if config_git_url: - scmurl = config_git_url - - logger.info(f"Fetching configuration from {scmurl} to {cdir}") - scm = split_scmurl(scmurl) - if scm["ref"] is None: - scm["ref"] = "main" - for attempt in range(retry): - try: - repo = await deferToThread(git.Repo.clone_from, scm["link"], cdir) - await deferToThread(repo.git.checkout, scm["ref"]) - except Exception: - logger.warning( - "Failed to fetch configuration, retrying (#%d).", - attempt + 1, - exc_info=True, - ) - continue - else: - logger.info("Configuration fetched successfully.") - break - else: - raise ConfigError("Failed to fetch configuration, giving up.") - - if os.path.isfile(os.path.join(cdir, "distrobaker.yaml")): - config_file = os.path.join(cdir, "distrobaker.yaml") - else: - raise ConfigError( - "Configuration repository does not contain distrobaker.yaml." - ) - - try: - with open(config_file) as f: - y = await deferToThread(yaml.safe_load, f) - logger.debug(f"{config_file} loaded, processing.") - - except Exception as e: - logger.info(e) - raise ConfigError(f"Could not parse {config_file}.") - - if "configuration" not in y: - raise ConfigError("The required configuration block is missing.") - if "components" not in y: - raise ConfigError("The required components block is missing.") - cnf = y["configuration"] - n = _parse_configuration_block(cnf) - nc = await _parse_components(y["components"]) - logger.info("Found %d component(s).", len(nc["downstream_components"])) - - if n["koji"]["trigger_tag"] == "rawhide": - n["koji"]["trigger_tag"] = await get_rawhide_tag() - logger.info(f"Detected rawhide tag {n['koji']['trigger_tag']}") - - main = n - comps = nc - - # Configure the database credentials - if not db_url: - # Unlike other settings, the DB cannot be changed during a basic - # config file edit. To change DB settings, the process must be - # restarted. - try: - db_config = n["db"] - db_url = sqlalchemy.URL.create( - drivername=db_config["driver"], - host=db_config["host"], - port=db_config["port"], - database=db_config["name"], - username=db_config["user"], - password=db_pw, - ) - - except KeyError as e: - logger.exception(e) - raise ConfigError("Missing database configuration (db block)") - - if main["email"] is not None: - emailer = Email(main["email"], smtp_password) - - -def is_eligible(comp, is_downstream): - # Check whether this component is meaningful to us - if is_downstream: - component_list = config.comps["downstream_components"] - else: - component_list = config.comps["upstream_components"] - if comp not in component_list: - logger.debug( - f"{comp} is not an approved {'downstream' if is_downstream else 'upstream'} component, ignoring" - ) - return False - - for pattern in config.main["control"]["exclude"]: - if re.search(pattern, comp): - logger.debug(f"{comp} is on the exclude list, skipping") - return False - - return True - - -def skip_tag(comp): - for pattern in config.main["control"]["skip_tag"]: - if re.search(pattern, comp): - logger.debug(f"{comp} is on the skip_tag list, building immediately") - return True - return False - - -def get_order(comp): - try: - downstream_name = ensure_downstream_name(comp) - except UnknownComponentError: - # This really shouldn't happen, but in the unlikely event that it - # does, assume it's a downstream component already and continue. - logger.warning(f"Unknown component {comp} in ordering, continuing") - downstream_name = comp - - for pattern in config.main["control"]["ordering"]: - if re.search(pattern, downstream_name): - return config.main["control"]["ordering"][pattern] - - # If we don't have a specific pattern, return a high number (1000) - # so we always build them late in the cycle - return 1000 - - -def is_paused(): - return config.main["control"]["pause"] - - -def get_upstream_name(downstream_component): - try: - return config.comps["downstream_components"][downstream_component][ - "upstream_name" - ] - except KeyError: - raise UnknownComponentError( - f"Downstream component {downstream_component} not found" - ) - - -def get_downstream_name(upstream_component): - try: - return config.comps["upstream_components"][upstream_component][ - "downstream_name" - ] - except KeyError: - raise UnknownComponentError( - f"Upstream component {upstream_component} not found" - ) - - -def ensure_downstream_name(comp): - # Check if the component is in the downstream components list - if comp in config.comps["downstream_components"]: - return comp - - # Otherwise, convert it to the downstream name - # This may raise an UnknownComponentError if the component is not found - return get_downstream_name(comp) diff --git a/elnbuildsync/config/__init__.py b/elnbuildsync/config/__init__.py new file mode 100644 index 0000000..c526d47 --- /dev/null +++ b/elnbuildsync/config/__init__.py @@ -0,0 +1,491 @@ +# This file is part of ELNBuildSync +# Copyright (C) 2023-2026 Stephen Gallagher + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# SPDX-License-Identifier: GPL-3.0-or-later + + +import json +import logging +import re +import requests.exceptions +from txrequests import Session + +from tenacity import retry as retry_on_exception, stop_after_delay, wait_exponential +import twisted.internet.utils + +from . import dynamic as dynamic_config +from . import static as static_config + + +# Global logger +logger = logging.getLogger(__name__) + +DEFAULT_DISTRO_VIEWS = ["eln"] + +# A special Deferred for terminating the program +terminator = None + +# The URL for connecting to the database +db_url = None + +# Configuration options +config_timer = 15 * 60 # 15 minutes +cleanup_timer = 12 * 60 * 60 # 12 hours +task_check_timer = 5 * 60 # 5 minutes +tag_check_timer = 5 * 60 # 5 minutes +task_timeout = 24 * 60 * 60 # 24 hours +tag_timeout = 1 * 60 * 60 # 1 hour +message_batch_timer = 60 # 1 minute +koji_batch = 500 +configuration = None +config_ref = None +distrogitsync = None +dry_run = False +do_untagging = False +retry = 3 +scmurl = None +main = None +control = None +comps = None +# If we haven't gotten the repoInit message within 10 minutes, assume we missed it +waitrepo_init_timeout = 10 * 60 + +# The actual generation can take up to 20 minutes +waitrepo_timeout = 20 * 60 + +# Process state +cleanup_processor = None +status_processor = None +tmpdir = None + +# SMTP (see email.py); password set from daemon --smtp-pw-file before load_config +emailer = None +smtp_password = "" + + +class ConfigError(Exception): + pass + + +class UnknownComponentError(ConfigError): + pass + + +class UnknownRefError(ConfigError): + pass + + +def _parse_static_configuration(cnf): + return static_config._parse_static_configuration(cnf, ConfigError) + + +def _parse_open_id_connect(oidc_raw): + return static_config._parse_open_id_connect(oidc_raw, ConfigError) + + +def _parse_koji(cnf_koji): + return static_config._parse_koji(cnf_koji, ConfigError) + + +def _parse_bodhi(cnf_bodhi): + return static_config._parse_bodhi(cnf_bodhi, ConfigError) + + +def _parse_email(cnf_email): + return static_config._parse_email(cnf_email, ConfigError) + + +def _parse_db(cnf_db): + return static_config._parse_db(cnf_db, ConfigError) + + +def _parse_control(cnf_control): + return dynamic_config._parse_control(cnf_control, ConfigError) + + +async def _parse_components(cnf_components): + return await dynamic_config._parse_components( + cnf_components, get_distro_packages, ConfigError + ) + + +def _parse_configuration_block(cnf): + """Parse static and control sections from a configuration block.""" + n = _parse_static_configuration(cnf) + if "control" not in cnf: + raise ConfigError("control missing.") + n["control"] = _parse_control(cnf["control"]) + return n + + +def loglevel(val=None): + """Gets or, optionally, sets the logging level of the module. + Standard numeric levels are accepted. + + :param val: The logging level to use, optional + :returns: The current logging level + """ + if val is not None: + try: + logger.setLevel(val) + except ValueError: + logger.warning( + "Invalid log level passed to DistroBuildSync logger: %s", val + ) + except Exception: + logger.exception("Unable to set log level: %s", val) + return logger.getEffectiveLevel() + + +def is_debug(): + """ + Determines if we are in debug logging mode. + + This is useful for enabling/disabling third-party logging such as the + sqlalchemy logger. + """ + return loglevel() <= logging.DEBUG + + +def retries(val=None): + """Gets or, optionally, sets the number of retries for various + operational failures. Typically used for handling dist-git requests. + + :param val: The number of retries to attept, optional + :returns: The current value of retries + """ + global retry + if val is not None: + retry = val + return retry + + +def split_scmurl(url): + """Splits a `link#ref` style URLs into the link and ref parts. While + generic, many code paths in DistroBuildSync expect these to be branch names. + `link` forms are also accepted, in which case the returned `ref` is None. + + It also attempts to extract the namespace and component, where applicable. + These can only be detected if the link matches the standard dist-git + pattern; in other cases the results may be bogus or None. + + :param url: A link#ref style URL, with #ref being optional + :returns: A dictionary with `link`, `ref`, `ns` and `comp` keys + """ + scm = url.split("#", 1) + nscomp = scm[0].split("/") + return { + "link": scm[0], + "ref": scm[1] if len(scm) >= 2 else None, + "ns": nscomp[-2] if nscomp and len(nscomp) >= 2 else None, + "comp": nscomp[-1] if nscomp else None, + } + + +def split_module(comp): + """Splits modules component name into name and stream pair. Expects the + name to be in the `name:stream` format. Defaults to stream=master if the + split fails. + + :param comp: The component name + :returns: Dictionary with name and stream + """ + ms = comp.split(":") + return { + "name": ms[0], + "stream": ms[1] if len(ms) > 1 and ms[1] else "master", + } + + +async def get_config_ref(url): + """Gets the ref for the config SCMURL + + Returns the actual ref for a symbolic ref possibly used in the + config SCMURL. Used by the update function to check whether the + config should be resync'd. + + :param url: Config SCMURL + :returns: Remote ref or None on error + """ + scm = split_scmurl(url) + logger.info(f"Getting config ref for {scm['link']} {scm['ref']}") + + if scm["ref"]: + output = await twisted.internet.utils.getProcessOutput( + executable="/usr/bin/git", + args=("ls-remote", "--branches", scm["link"], scm["ref"]), + errortoo=True, + ) + else: + output = await twisted.internet.utils.getProcessOutput( + executable="/usr/bin/git", + args=("ls-remote", "--branches", scm["link"]), + errortoo=True, + ) + + if not output: + scmref = scm["ref"] + scmlink = scm["link"] + raise UnknownRefError(f"{scmref} not found in {scmlink}") + + return output.split(b"\t", 1)[0] + + +async def update_config(): + global config_ref + + if not scmurl: + logger.info("Config URL not provided.") + return + + logger.critical("Updating configuration") + + try: + ref = await get_config_ref(scmurl) + except UnknownRefError as e: + logger.info(e) + logger.critical( + f"The configuration repository is unavailable, skipping update. Checking again in {config_timer} seconds." + ) + return + + try: + await load_dynamic_config(dynamic_config_git_url=scmurl) + config_ref = ref + except ConfigError as e: + logger.info(e) + logger.critical( + f"The configuration is invalid, skipping update. Checking again in {config_timer} seconds." + ) + return + + +async def get_distro_packages( + distro_url, + distro_view=DEFAULT_DISTRO_VIEWS, + arches=None, + which_source=None, +): + """ + Fetches the list of desired sources from Content Resolver + for each of the given 'arches'. + """ + if not arches: + arches = ["aarch64", "ppc64le", "s390x", "x86_64"] + if not which_source: + which_source = ["source", "buildroot-source"] + + packages = dict[str, dict[str, str]]() + + for view in reversed(distro_view): + for this_source in reversed(which_source): + url = ( + "{distro_url}/view-{this_source}-package-name-list--view-{view}.txt" + ).format( + distro_url=distro_url, + this_source=this_source, + view=view, + ) + + logger.debug("downloading {url}".format(url=url)) + + with Session() as session: + r = await session.get(url, allow_redirects=True) + for line in r.text.splitlines(): + packages[line] = { + "view": view, + "source": this_source, + "upstream_name": line, + "downstream_name": line, + } + + # There may be an empty line in the file, ignore it. + packages.pop("", None) + + logger.debug("Found a total of {} packages".format(len(packages))) + + return packages + + +@retry_on_exception( + wait=wait_exponential(), + stop=stop_after_delay(60), + reraise=True, +) +async def get_rawhide_tag(): + """ + Queries Bodhi for the current tag associated with Rawhide + """ + + # Retrieve the list of "pending" (aka development) releases + url = "https://bodhi.fedoraproject.org/releases?state=pending" + with Session() as session: + try: + r = await session.get(url, allow_redirects=True) + r.raise_for_status() + releases = json.loads(r.text) + logger.debug(releases) + except json.decoder.JSONDecodeError as e: + raise ConfigError("Could not parse JSON from Bodhi releases") from e + + except requests.exceptions.HTTPError as e: + raise ConfigError("HTTP Error") from e + + # Get the stable tag corresponding to the rawhide branch + stable_tag = None + for release in releases["releases"]: + # Get the stable tag associated with this release + if release["branch"] == "rawhide": + stable_tag = release["stable_tag"] + break + + # Shouldn't ever happen, but... + if not stable_tag: + raise ConfigError("Unexpectedly received no valid Fedora rawhide release") + + return stable_tag + + +async def load_static_config(static_config_file, db_pw=None): + """Load static configuration from a YAML file.""" + import sys + + await static_config.load_static_config( + static_config_file, + db_pw=db_pw, + config_module=sys.modules[__name__], + ConfigError=ConfigError, + ) + + +async def load_dynamic_config(dynamic_config_git_url=None, dynamic_config_file=None): + """Load dynamic configuration from a file or git URL.""" + import sys + + await dynamic_config.load_dynamic_config( + dynamic_config_git_url=dynamic_config_git_url, + dynamic_config_file=dynamic_config_file, + config_module=sys.modules[__name__], + ConfigError=ConfigError, + split_scmurl=split_scmurl, + get_distro_packages=get_distro_packages, + get_rawhide_tag=get_rawhide_tag, + ) + + +async def load_config( + db_pw=None, + static_config_file=None, + dynamic_config_git_url=None, + dynamic_config_file=None, + *, + config_git_url=None, + config_file=None, +): + """Compatibility wrapper: load static and dynamic configuration. + + Deprecated keyword arguments config_git_url and config_file map to dynamic + sources for tests and transitional callers. + """ + if config_git_url and not dynamic_config_git_url: + dynamic_config_git_url = config_git_url + if config_file and not dynamic_config_file: + dynamic_config_file = config_file + + if static_config_file: + await load_static_config(static_config_file, db_pw=db_pw) + await load_dynamic_config( + dynamic_config_git_url=dynamic_config_git_url, + dynamic_config_file=dynamic_config_file, + ) + + +def is_eligible(comp, is_downstream): + # Check whether this component is meaningful to us + if is_downstream: + component_list = comps["downstream_components"] + else: + component_list = comps["upstream_components"] + if comp not in component_list: + logger.debug( + f"{comp} is not an approved {'downstream' if is_downstream else 'upstream'} component, ignoring" + ) + return False + + for pattern in control["exclude"]: + if re.search(pattern, comp): + logger.debug(f"{comp} is on the exclude list, skipping") + return False + + return True + + +def skip_tag(comp): + for pattern in control["skip_tag"]: + if re.search(pattern, comp): + logger.debug(f"{comp} is on the skip_tag list, building immediately") + return True + return False + + +def get_order(comp): + try: + downstream_name = ensure_downstream_name(comp) + except UnknownComponentError: + # This really shouldn't happen, but in the unlikely event that it + # does, assume it's a downstream component already and continue. + logger.warning(f"Unknown component {comp} in ordering, continuing") + downstream_name = comp + + for pattern in control["ordering"]: + if re.search(pattern, downstream_name): + return control["ordering"][pattern] + + # If we don't have a specific pattern, return a high number (1000) + # so we always build them late in the cycle + return 1000 + + +def is_paused(): + return control["pause"] + + +def get_upstream_name(downstream_component): + try: + return comps["downstream_components"][downstream_component]["upstream_name"] + except KeyError: + raise UnknownComponentError( + f"Downstream component {downstream_component} not found" + ) + + +def get_downstream_name(upstream_component): + try: + return comps["upstream_components"][upstream_component]["downstream_name"] + except KeyError: + raise UnknownComponentError( + f"Upstream component {upstream_component} not found" + ) + + +def ensure_downstream_name(comp): + # Check if the component is in the downstream components list + if comp in comps["downstream_components"]: + return comp + + # Otherwise, convert it to the downstream name + # This may raise an UnknownComponentError if the component is not found + return get_downstream_name(comp) diff --git a/elnbuildsync/config/dynamic.py b/elnbuildsync/config/dynamic.py new file mode 100644 index 0000000..fd89209 --- /dev/null +++ b/elnbuildsync/config/dynamic.py @@ -0,0 +1,288 @@ +# This file is part of ELNBuildSync +# Copyright (C) 2023-2026 Stephen Gallagher + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# SPDX-License-Identifier: GPL-3.0-or-later + + +import git +import logging +import os +import tempfile + +import yaml +from twisted.internet.threads import deferToThread + + +logger = logging.getLogger(__name__) + +DEFAULT_CONTENT_RESOLVER = "https://tiny.distro.builders" + + +def _parse_control(cnf_control, ConfigError): + """Parse control configuration. Returns dict with trigger_tag, pause, skip_tag, + exclude, ordering, status_interval, etc. + """ + result = dict() + for k in ("pause",): + if k in cnf_control: + result[k] = bool(cnf_control[k]) + else: + raise ConfigError(f"control.{k} missing.") + + if "trigger_tag" in cnf_control: + result["trigger_tag"] = str(cnf_control["trigger_tag"]) + else: + raise ConfigError("control.trigger_tag missing.") + + result["skip_tag"] = set() + if "skip_tag" in cnf_control: + result["skip_tag"].update(cnf_control["skip_tag"]) + + result["exclude"] = set() + if "exclude" in cnf_control: + result["exclude"].update(cnf_control["exclude"]) + + if result["exclude"]: + logger.info( + "Excluding %d component(s).", + len(result["exclude"]), + ) + else: + logger.info("Not excluding any components.") + + result["ordering"] = dict() + if "ordering" in cnf_control: + result["ordering"].update(cnf_control["ordering"]) + + result["status_interval"] = 600 # 10 minutes + if "status_interval" in cnf_control: + val = cnf_control["status_interval"] + if not isinstance(val, int) or val <= 0: + raise ConfigError("control.status_interval must be a positive integer.") + result["status_interval"] = val + + return result + + +async def _parse_components(cnf_components, get_distro_packages, ConfigError): + """Parse the components block (top-level). Requires at least one of autopackagelist + or overrides. + """ + if "autopackagelist" not in cnf_components and "overrides" not in cnf_components: + raise ConfigError( + "At least one of components.autopackagelist or components.overrides must be present." + ) + if "autopackagelist" in cnf_components: + apl_raw = cnf_components["autopackagelist"] + if "view" not in apl_raw: + raise ConfigError("components.autopackagelist.view missing.") + if "source" not in apl_raw: + raise ConfigError("components.autopackagelist.source missing.") + apl = { + "view": apl_raw["view"] + if isinstance(apl_raw["view"], list) + else [apl_raw["view"]], + "source": apl_raw["source"] + if isinstance(apl_raw["source"], list) + else [apl_raw["source"]], + "content_resolver": apl_raw.get( + "content_resolver", DEFAULT_CONTENT_RESOLVER + ), + } + downstream_components = await get_distro_packages( + distro_url=apl["content_resolver"], + distro_view=apl["view"], + which_source=apl["source"], + ) + for comp_name, comp_entry in downstream_components.items(): + if not isinstance(comp_entry, dict): + raise ConfigError( + f"components.autopackagelist entry '{comp_name}' must be a dictionary." + ) + if "upstream_name" not in comp_entry: + raise ConfigError( + f"components.autopackagelist entry '{comp_name}' missing upstream_name." + ) + if "downstream_name" not in comp_entry: + raise ConfigError( + f"components.autopackagelist entry '{comp_name}' missing downstream_name." + ) + else: + downstream_components = {} + upstream_components = downstream_components.copy() + + overrides = cnf_components.get("overrides", {}) + if not isinstance(overrides, dict): + raise ConfigError("components.overrides must be a dictionary.") + + for downstream_name, override_options in overrides.items(): + if not isinstance(override_options, dict): + raise ConfigError( + f"components.overrides.{downstream_name} must be a dictionary" + ) + + upstream_name = override_options.get("upstream_name", downstream_name) + + if downstream_name not in downstream_components: + downstream_components[downstream_name] = { + "view": "override", + "source": "override", + "upstream_name": upstream_name, + "downstream_name": downstream_name, + } + downstream_components[downstream_name].update(override_options) + downstream_components[downstream_name].setdefault( + "upstream_name", downstream_name + ) + downstream_components[downstream_name].setdefault( + "downstream_name", downstream_name + ) + upstream_components[upstream_name] = downstream_components[ + downstream_name + ].copy() + + return { + "downstream_components": downstream_components, + "upstream_components": upstream_components, + } + + +async def _load_dynamic_yaml( + y, + get_distro_packages, + get_rawhide_tag, + ConfigError, +): + """Parse loaded dynamic YAML into control and components.""" + if "configuration" not in y: + raise ConfigError("The required configuration block is missing.") + if "components" not in y: + raise ConfigError("The required components block is missing.") + + cnf = y["configuration"] + if "control" not in cnf: + raise ConfigError("control missing.") + + control = _parse_control(cnf["control"], ConfigError) + comps = await _parse_components(y["components"], get_distro_packages, ConfigError) + logger.info("Found %d component(s).", len(comps["downstream_components"])) + + if control["trigger_tag"] == "rawhide": + control["trigger_tag"] = await get_rawhide_tag() + logger.info("Detected rawhide tag %s", control["trigger_tag"]) + + return control, comps + + +async def _fetch_dynamic_config_file( + dynamic_config_git_url, + dynamic_config_file, + split_scmurl, + retry, + ConfigError, +): + """Resolve dynamic config to a local file path, cloning git repos when needed.""" + if not (dynamic_config_git_url or dynamic_config_file): + raise ValueError( + "One of 'dynamic_config_git_url' or 'dynamic_config_file' must be specified" + ) + + if dynamic_config_git_url: + scmurl = dynamic_config_git_url + logger.info("Fetching dynamic configuration from %s", scmurl) + scm = split_scmurl(scmurl) + if scm["ref"] is None: + scm["ref"] = "main" + + with tempfile.TemporaryDirectory(prefix="distrobaker-") as cdir: + for attempt in range(retry): + try: + repo = await deferToThread(git.Repo.clone_from, scm["link"], cdir) + await deferToThread(repo.git.checkout, scm["ref"]) + except Exception: + logger.warning( + "Failed to fetch configuration, retrying (#%d).", + attempt + 1, + exc_info=True, + ) + continue + else: + logger.info("Configuration fetched successfully.") + break + else: + raise ConfigError("Failed to fetch configuration, giving up.") + + config_path = os.path.join(cdir, "distrobaker.yaml") + if not os.path.isfile(config_path): + raise ConfigError( + "Configuration repository does not contain distrobaker.yaml." + ) + + try: + with open(config_path) as f: + y = await deferToThread(yaml.safe_load, f) + logger.debug( + "%s loaded, processing dynamic configuration.", config_path + ) + except Exception as e: + logger.info(e) + raise ConfigError(f"Could not parse {config_path}.") + + return scmurl, y + + try: + with open(dynamic_config_file) as f: + y = await deferToThread(yaml.safe_load, f) + logger.debug( + "%s loaded, processing dynamic configuration.", dynamic_config_file + ) + except Exception as e: + logger.info(e) + raise ConfigError(f"Could not parse {dynamic_config_file}.") + + return None, y + + +async def load_dynamic_config( + dynamic_config_git_url=None, + dynamic_config_file=None, + *, + config_module, + ConfigError, + split_scmurl, + get_distro_packages, + get_rawhide_tag, +): + """Load dynamic configuration from a file or git URL. + + Sets config.control, config.comps, and config.scmurl when loading from git. + """ + scmurl, y = await _fetch_dynamic_config_file( + dynamic_config_git_url, + dynamic_config_file, + split_scmurl, + config_module.retry, + ConfigError, + ) + + if scmurl is not None: + config_module.scmurl = scmurl + + control, comps = await _load_dynamic_yaml( + y, get_distro_packages, get_rawhide_tag, ConfigError + ) + config_module.control = control + config_module.comps = comps diff --git a/elnbuildsync/config/static.py b/elnbuildsync/config/static.py new file mode 100644 index 0000000..83f9204 --- /dev/null +++ b/elnbuildsync/config/static.py @@ -0,0 +1,254 @@ +# This file is part of ELNBuildSync +# Copyright (C) 2023-2026 Stephen Gallagher + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# SPDX-License-Identifier: GPL-3.0-or-later + + +import logging +import os + +import sqlalchemy +import yaml +from twisted.internet.threads import deferToThread + +from ..email import Email + + +logger = logging.getLogger(__name__) + +DEFAULT_CONTENT_RESOLVER = "https://tiny.distro.builders" + + +def _parse_open_id_connect(oidc_raw, ConfigError): + """Parse OpenID Connect configuration. Returns None if disabled, else a dict. + Raises ConfigError on invalid or missing required fields. + """ + if oidc_raw is False: + logger.info( + "OpenID Connect explicitly disabled - /trigger endpoint unprotected" + ) + return None + oidc = oidc_raw + default_scopes = [ + "openid", + "profile", + "https://id.fedoraproject.org/scope/groups", + ] + required_fields = [ + "auth_url", + "client_id", + "client_secret", + "token_endpoint", + "admin_groups", + ] + for field in required_fields: + if field not in oidc: + raise ConfigError(f"open_id_connect.{field} missing.") + result = { + "auth_url": str(oidc["auth_url"]), + "client_id": str(oidc["client_id"]), + "client_secret": str(oidc["client_secret"]), + "token_endpoint": str(oidc["token_endpoint"]), + "userinfo_endpoint": str(oidc.get("userinfo_endpoint", "")), + "scopes": list(oidc.get("scopes", default_scopes)), + "admin_groups": list(oidc["admin_groups"]), + } + logger.info( + "OpenID Connect authentication enabled; admin groups: %s", + result["admin_groups"], + ) + return result + + +def _parse_koji(cnf_koji, ConfigError): + """Parse koji configuration. Returns dict with profile, build_target, stable_tag, + scratch_build, fail_fast, and optionally username. + """ + if "profile" not in cnf_koji: + raise ConfigError("koji.profile missing.") + result = {"profile": str(cnf_koji["profile"])} + if "build_target" not in cnf_koji: + raise ConfigError("koji.build_target missing.") + result["build_target"] = str(cnf_koji["build_target"]) + if "stable_tag" not in cnf_koji: + raise ConfigError("koji.stable_tag missing.") + result["stable_tag"] = str(cnf_koji["stable_tag"]) + if "username" in cnf_koji: + result["username"] = str(cnf_koji["username"]) + if "scratch_build" in cnf_koji: + result["scratch_build"] = bool(cnf_koji["scratch_build"]) + else: + logger.warning( + "Configuration warning: koji.scratch_build not defined, assuming false." + ) + result["scratch_build"] = False + if "fail_fast" in cnf_koji: + result["fail_fast"] = bool(cnf_koji["fail_fast"]) + else: + logger.warning( + "Configuration warning: koji.fail_fast not defined, assuming false." + ) + result["fail_fast"] = False + return result + + +def _parse_bodhi(cnf_bodhi, ConfigError): + """Parse bodhi configuration. Returns dict with batch_size.""" + result = {"batch_size": 0} + if "batch_size" in cnf_bodhi: + try: + result["batch_size"] = int(cnf_bodhi["batch_size"]) + except ValueError: + raise ConfigError("bodhi.batch_size must be an integer") + return result + + +def _parse_email(cnf_email, ConfigError): + """Parse email configuration. Returns None if disabled, else a dict with + smtp_host, smtp_port, smtp_username, from, recipients. + Raises ConfigError on invalid or missing required fields. + """ + if cnf_email is False: + logger.info("Email explicitly disabled") + return None + required = ("smtp_host", "smtp_port", "smtp_username", "from", "recipients") + for key in required: + if key not in cnf_email: + raise ConfigError(f"email.{key} missing.") + try: + port = int(cnf_email["smtp_port"]) + except (TypeError, ValueError): + raise ConfigError("email.smtp_port must be an integer") + recipients = cnf_email["recipients"] + if not isinstance(recipients, list) or len(recipients) == 0: + raise ConfigError("email.recipients must be a non-empty list.") + for r in recipients: + if not isinstance(r, str) or not r: + raise ConfigError("email.recipients must be a list of non-empty strings.") + return { + "smtp_host": str(cnf_email["smtp_host"]), + "smtp_port": port, + "smtp_username": str(cnf_email["smtp_username"]), + "from": str(cnf_email["from"]), + "recipients": [str(x) for x in recipients], + } + + +def _parse_db(cnf_db, ConfigError): + """Parse database configuration. Returns dict with host, port, name, driver, user. + All keys are mandatory. + """ + required = ("host", "port", "name", "driver", "user") + for key in required: + if key not in cnf_db: + raise ConfigError(f"db.{key} missing.") + try: + result = { + "host": str(cnf_db["host"]), + "port": int(cnf_db["port"]), + "name": str(cnf_db["name"]), + "driver": str(cnf_db["driver"]), + "user": str(cnf_db["user"]), + } + except ValueError: + raise ConfigError("db.port must be an integer") + return result + + +def _parse_static_configuration(cnf, ConfigError): + """Parse the static configuration block. + Returns dict with koji, bodhi, db, open_id_connect, email. + """ + if "control" in cnf: + logger.warning( + "Static configuration contains control block; use dynamic config instead." + ) + + if "koji" not in cnf: + raise ConfigError("koji missing.") + n = {"koji": _parse_koji(cnf["koji"], ConfigError)} + + if "bodhi" not in cnf: + raise ConfigError("bodhi missing.") + n["bodhi"] = _parse_bodhi(cnf["bodhi"], ConfigError) + + if "db" not in cnf: + raise ConfigError("db missing.") + n["db"] = _parse_db(cnf["db"], ConfigError) + + if "open_id_connect" not in cnf: + raise ConfigError( + "open_id_connect missing. Set open_id_connect: false to disable authentication." + ) + n["open_id_connect"] = _parse_open_id_connect(cnf["open_id_connect"], ConfigError) + + if "email" not in cnf: + raise ConfigError("email missing. Set email: false to disable email.") + n["email"] = _parse_email(cnf["email"], ConfigError) + + return n + + +async def load_static_config( + static_config_file, + db_pw=None, + *, + config_module, + ConfigError, +): + """Load static configuration from a YAML file. + + Sets config.main, config.db_url (first call only), and config.emailer. + """ + if not static_config_file: + raise ValueError("static_config_file must be specified") + + if not os.path.isfile(static_config_file): + raise ConfigError(f"Could not parse {static_config_file}.") + + try: + with open(static_config_file) as f: + y = await deferToThread(yaml.safe_load, f) + logger.debug("%s loaded, processing static configuration.", static_config_file) + except Exception as e: + logger.info(e) + raise ConfigError(f"Could not parse {static_config_file}.") + + if "configuration" not in y: + raise ConfigError("The required configuration block is missing.") + + n = _parse_static_configuration(y["configuration"], ConfigError) + config_module.main = n + + if not config_module.db_url: + try: + db_config = n["db"] + config_module.db_url = sqlalchemy.URL.create( + drivername=db_config["driver"], + host=db_config["host"], + port=db_config["port"], + database=db_config["name"], + username=db_config["user"], + password=db_pw, + ) + except KeyError as e: + logger.exception(e) + raise ConfigError("Missing database configuration (db block)") + + if n["email"] is not None: + config_module.emailer = Email(n["email"], config_module.smtp_password) + else: + config_module.emailer = None diff --git a/elnbuildsync/daemon.py b/elnbuildsync/daemon.py index a607909..ffdda6c 100644 --- a/elnbuildsync/daemon.py +++ b/elnbuildsync/daemon.py @@ -52,6 +52,8 @@ logger = logging.getLogger(__name__) +DEFAULT_STATIC_CONFIG_FILE = "/etc/elnbuildsync/elnbuildsync.yaml" + def log_filter(record): if record.name.startswith("elnbuildsync"): @@ -63,6 +65,20 @@ def log_filter(record): return False +def _resolve_dynamic_source(dynamic_config_url, dynamic_config_file): + if dynamic_config_file and dynamic_config_url: + raise click.UsageError( + "Only one of --dynamic-config-file or --dynamic-config-url may be set." + ) + if dynamic_config_file: + return None, dynamic_config_file + if dynamic_config_url: + return dynamic_config_url, None + raise click.UsageError( + "One of --dynamic-config-file or --dynamic-config-url is required." + ) + + @click.command() @click.version_option( version=importlib.metadata.version("ELNBuildSync"), @@ -84,8 +100,14 @@ def log_filter(record): type=int, help="How long (in seconds) to wait after the last trigger before starting the batch", ) -@click.option("--config-url", default=None) -@click.option("--config-file", default=None) +@click.option( + "--static-config-file", + default=DEFAULT_STATIC_CONFIG_FILE, + show_default=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option("--dynamic-config-url", default=None) +@click.option("--dynamic-config-file", default=None, type=click.Path(dir_okay=False)) @click.option("--db-pw-file", type=click.File(mode="r"), default="/etc/ebs_db_pw") @click.option( "--smtp-pw-file", @@ -102,8 +124,9 @@ def main( log_level, dry_run, lull_time, - config_url, - config_file, + static_config_file, + dynamic_config_url, + dynamic_config_file, db_pw_file, smtp_pw_file, untagging, @@ -125,16 +148,32 @@ def main( config.do_untagging = untagging config.message_batch_timer = lull_time + dynamic_url, dynamic_file = _resolve_dynamic_source( + dynamic_config_url, dynamic_config_file + ) + logger.debug("Starting Twisted mainloop") return task.react( lambda reactor: Deferred.fromCoroutine( - _main(reactor, db_pw_file, smtp_pw_file, config_url, config_file) + _main( + reactor, + db_pw_file, + smtp_pw_file, + static_config_file, + dynamic_url, + dynamic_file, + ) ) ) async def _main( - reactor, db_pw_file, smtp_pw_file, config_url=None, config_file=None + reactor, + db_pw_file, + smtp_pw_file, + static_config_file, + dynamic_config_url=None, + dynamic_config_file=None, ) -> None: config.terminator = Deferred() with tempfile.TemporaryDirectory(prefix="elnbuildsync-") as cdir: @@ -148,10 +187,11 @@ async def _main( else: config.smtp_password = "" - # Read in the config file try: - await config.load_config( - db_pw, config_git_url=config_url, config_file=config_file + await config.load_static_config(static_config_file, db_pw) + await config.load_dynamic_config( + dynamic_config_git_url=dynamic_config_url, + dynamic_config_file=dynamic_config_file, ) except Exception as e: logger.exception(e) @@ -175,9 +215,7 @@ async def _main( # Schedule periodic status page and run it once at startup config.status_processor = task.LoopingCall(status.create_status_page) - config.status_processor.start( - config.main["control"]["status_interval"], now=True - ) + config.status_processor.start(config.control["status_interval"], now=True) # Schedule periodic cleanup config.cleanup_processor = task.LoopingCall(cleanup.periodic_cleanup) diff --git a/elnbuildsync/listener.py b/elnbuildsync/listener.py index 616a4f0..bff093c 100644 --- a/elnbuildsync/listener.py +++ b/elnbuildsync/listener.py @@ -117,7 +117,7 @@ def _handle_tag(msg): """Handle buildsys.tag messages to trigger rebuilds.""" tag = msg.body["tag"] - if tag == config.main["koji"]["trigger_tag"]: + if tag == config.control["trigger_tag"]: return _handle_trigger_tag(msg) elif tag in state.pending_nvr_tags.keys(): @@ -138,9 +138,7 @@ def _handle_trigger_tag(msg): if batching.running or config.is_paused(): raise Nack() - logger.info( - f"Triggering rebuild on trigger tag {config.main['koji']['trigger_tag']}" - ) + logger.info(f"Triggering rebuild on trigger tag {config.control['trigger_tag']}") # This is a component we care about, so add it to the queue batching.message_batch_processor.reset() From 91d0f222193734da511af283fc21772430927f41 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:14:34 -0400 Subject: [PATCH 2/7] tests: Split config fixtures under tests/etc/local and tests/etc/crc Replace monolithic local_testconfig.yaml and crc_testconfig.yaml with production-aligned static and dynamic YAML files plus moved secrets. Signed-off-by: Stephen Gallagher Co-authored-by: Cursor --- tests/{secrets => etc/crc}/ebs_db_pw | 0 tests/{secrets => etc/crc}/ebs_smtp_pw | 0 tests/etc/crc/elnbuildsync.yaml | 46 ++++++++++++++++++ .../crc/elnbuildsync_dynamic.yaml} | 46 +----------------- tests/etc/local/ebs_db_pw | 1 + tests/etc/local/ebs_smtp_pw | 1 + tests/etc/local/elnbuildsync.yaml | 45 ++++++++++++++++++ .../local/elnbuildsync_dynamic.yaml} | 47 +------------------ 8 files changed, 95 insertions(+), 91 deletions(-) rename tests/{secrets => etc/crc}/ebs_db_pw (100%) rename tests/{secrets => etc/crc}/ebs_smtp_pw (100%) create mode 100644 tests/etc/crc/elnbuildsync.yaml rename tests/{local_testconfig.yaml => etc/crc/elnbuildsync_dynamic.yaml} (74%) create mode 100644 tests/etc/local/ebs_db_pw create mode 100644 tests/etc/local/ebs_smtp_pw create mode 100644 tests/etc/local/elnbuildsync.yaml rename tests/{crc_testconfig.yaml => etc/local/elnbuildsync_dynamic.yaml} (73%) diff --git a/tests/secrets/ebs_db_pw b/tests/etc/crc/ebs_db_pw similarity index 100% rename from tests/secrets/ebs_db_pw rename to tests/etc/crc/ebs_db_pw diff --git a/tests/secrets/ebs_smtp_pw b/tests/etc/crc/ebs_smtp_pw similarity index 100% rename from tests/secrets/ebs_smtp_pw rename to tests/etc/crc/ebs_smtp_pw diff --git a/tests/etc/crc/elnbuildsync.yaml b/tests/etc/crc/elnbuildsync.yaml new file mode 100644 index 0000000..e8f4130 --- /dev/null +++ b/tests/etc/crc/elnbuildsync.yaml @@ -0,0 +1,46 @@ +configuration: + koji: + profile: koji + build_target: eln + stable_tag: eln + scratch_build: true + fail_fast: true + username: eln-buildsync + bodhi: + batch_size: 750 + db: + # Use the sidecar database for testing + host: 127.0.0.1 + port: 5432 + name: elnbuildsync + driver: postgresql+asyncpg + user: elnbuildsync + # OpenID Connect authentication configuration + # Uses Fedora tinystage test OIDC server for development/testing + # See: https://github.com/fedora-infra/tiny-stage + open_id_connect: + # Tinystage Ipsilon Authorization endpoint + auth_url: "https://ipsilon.tinystage.test/idp/openidc/Authorization" + # Client credentials (register with tinystage) + client_id: "YOUR_CLIENT_ID" + client_secret: "YOUR_CLIENT_SECRET" + # Token endpoint for exchanging authorization code + token_endpoint: "https://ipsilon.tinystage.test/idp/openidc/Token" + # UserInfo endpoint for fetching user details and groups + userinfo_endpoint: "https://ipsilon.tinystage.test/idp/openidc/UserInfo" + # OAuth2 scopes to request (groups scope required for authorization) + scopes: + - openid + - profile + - https://id.fedoraproject.org/scope/groups + # Users must be a member of at least one of these groups to access /trigger + admin_groups: + - eln + email: + smtp_host: localhost + smtp_port: 587 + smtp_username: alice + from: elnbuildsync@fedoraproject.org + recipients: + - list1@fedoraproject.org + - list2@redhat.com diff --git a/tests/local_testconfig.yaml b/tests/etc/crc/elnbuildsync_dynamic.yaml similarity index 74% rename from tests/local_testconfig.yaml rename to tests/etc/crc/elnbuildsync_dynamic.yaml index f4e3403..a9788fc 100644 --- a/tests/local_testconfig.yaml +++ b/tests/etc/crc/elnbuildsync_dynamic.yaml @@ -1,42 +1,6 @@ configuration: - koji: - profile: koji - trigger_tag: f40 - build_target: eln - stable_tag: eln - scratch_build: true - fail_fast: true - username: eln-buildsync - bodhi: - batch_size: 750 - db: - host: temp_postgres - port: 5432 - name: elnbuildsync - driver: postgresql+asyncpg - user: elnbuildsync - # OpenID Connect authentication configuration - # Uses Fedora tinystage test OIDC server for development/testing - # See: https://github.com/fedora-infra/tiny-stage - open_id_connect: - # Tinystage Ipsilon Authorization endpoint - auth_url: "https://ipsilon.tinystage.test/idp/openidc/Authorization" - # Client credentials (register with tinystage) - client_id: "YOUR_CLIENT_ID" - client_secret: "YOUR_CLIENT_SECRET" - # Token endpoint for exchanging authorization code - token_endpoint: "https://ipsilon.tinystage.test/idp/openidc/Token" - # UserInfo endpoint for fetching user details and groups - userinfo_endpoint: "https://ipsilon.tinystage.test/idp/openidc/UserInfo" - # OAuth2 scopes to request (groups scope required for authorization) - scopes: - - openid - - profile - - https://id.fedoraproject.org/scope/groups - # Users must be a member of at least one of these groups to access /trigger - admin_groups: - - eln control: + trigger_tag: f40 status_interval: 600 # 10 minutes pause: false skip_tag: @@ -129,14 +93,6 @@ configuration: ^llvm[0-9\.]*$: 0 ^clang[0-9\.]*$: 1 ^fedora-repos.*$: 0 - email: - smtp_host: localhost - smtp_port: 587 - smtp_username: alice - from: elnbuildsync@fedoraproject.org - recipients: - - list1@fedoraproject.org - - list2@redhat.com components: autopackagelist: view: [ eln, eln-extras ] diff --git a/tests/etc/local/ebs_db_pw b/tests/etc/local/ebs_db_pw new file mode 100644 index 0000000..8125fc4 --- /dev/null +++ b/tests/etc/local/ebs_db_pw @@ -0,0 +1 @@ +p@$$w0rd diff --git a/tests/etc/local/ebs_smtp_pw b/tests/etc/local/ebs_smtp_pw new file mode 100644 index 0000000..1a7964e --- /dev/null +++ b/tests/etc/local/ebs_smtp_pw @@ -0,0 +1 @@ +smtppassword diff --git a/tests/etc/local/elnbuildsync.yaml b/tests/etc/local/elnbuildsync.yaml new file mode 100644 index 0000000..d29f6d3 --- /dev/null +++ b/tests/etc/local/elnbuildsync.yaml @@ -0,0 +1,45 @@ +configuration: + koji: + profile: koji + build_target: eln + stable_tag: eln + scratch_build: true + fail_fast: true + username: eln-buildsync + bodhi: + batch_size: 750 + db: + host: temp_postgres + port: 5432 + name: elnbuildsync + driver: postgresql+asyncpg + user: elnbuildsync + # OpenID Connect authentication configuration + # Uses Fedora tinystage test OIDC server for development/testing + # See: https://github.com/fedora-infra/tiny-stage + open_id_connect: + # Tinystage Ipsilon Authorization endpoint + auth_url: "https://ipsilon.tinystage.test/idp/openidc/Authorization" + # Client credentials (register with tinystage) + client_id: "YOUR_CLIENT_ID" + client_secret: "YOUR_CLIENT_SECRET" + # Token endpoint for exchanging authorization code + token_endpoint: "https://ipsilon.tinystage.test/idp/openidc/Token" + # UserInfo endpoint for fetching user details and groups + userinfo_endpoint: "https://ipsilon.tinystage.test/idp/openidc/UserInfo" + # OAuth2 scopes to request (groups scope required for authorization) + scopes: + - openid + - profile + - https://id.fedoraproject.org/scope/groups + # Users must be a member of at least one of these groups to access /trigger + admin_groups: + - eln + email: + smtp_host: localhost + smtp_port: 587 + smtp_username: alice + from: elnbuildsync@fedoraproject.org + recipients: + - list1@fedoraproject.org + - list2@redhat.com diff --git a/tests/crc_testconfig.yaml b/tests/etc/local/elnbuildsync_dynamic.yaml similarity index 73% rename from tests/crc_testconfig.yaml rename to tests/etc/local/elnbuildsync_dynamic.yaml index 42f8a05..a9788fc 100644 --- a/tests/crc_testconfig.yaml +++ b/tests/etc/local/elnbuildsync_dynamic.yaml @@ -1,43 +1,6 @@ configuration: - koji: - profile: koji - trigger_tag: f40 - build_target: eln - stable_tag: eln - scratch_build: true - fail_fast: true - username: eln-buildsync - bodhi: - batch_size: 750 - db: - # Use the sidecar database for testing - host: 127.0.0.1 - port: 5432 - name: elnbuildsync - driver: postgresql+asyncpg - user: elnbuildsync - # OpenID Connect authentication configuration - # Uses Fedora tinystage test OIDC server for development/testing - # See: https://github.com/fedora-infra/tiny-stage - open_id_connect: - # Tinystage Ipsilon Authorization endpoint - auth_url: "https://ipsilon.tinystage.test/idp/openidc/Authorization" - # Client credentials (register with tinystage) - client_id: "YOUR_CLIENT_ID" - client_secret: "YOUR_CLIENT_SECRET" - # Token endpoint for exchanging authorization code - token_endpoint: "https://ipsilon.tinystage.test/idp/openidc/Token" - # UserInfo endpoint for fetching user details and groups - userinfo_endpoint: "https://ipsilon.tinystage.test/idp/openidc/UserInfo" - # OAuth2 scopes to request (groups scope required for authorization) - scopes: - - openid - - profile - - https://id.fedoraproject.org/scope/groups - # Users must be a member of at least one of these groups to access /trigger - admin_groups: - - eln control: + trigger_tag: f40 status_interval: 600 # 10 minutes pause: false skip_tag: @@ -130,14 +93,6 @@ configuration: ^llvm[0-9\.]*$: 0 ^clang[0-9\.]*$: 1 ^fedora-repos.*$: 0 - email: - smtp_host: localhost - smtp_port: 587 - smtp_username: alice - from: elnbuildsync@fedoraproject.org - recipients: - - list1@fedoraproject.org - - list2@redhat.com components: autopackagelist: view: [ eln, eln-extras ] From c647521a66eb3056ebdd3a7e81dbe0835ff0bcd0 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:14:37 -0400 Subject: [PATCH 3/7] tests: Update config parsing tests for static/dynamic split Adjust unit tests for the new config package layout, control.trigger_tag, and separate static/dynamic load paths. Signed-off-by: Stephen Gallagher Co-authored-by: Cursor --- tests/test_parse_config.py | 288 ++++++++++++++++++++----------------- 1 file changed, 153 insertions(+), 135 deletions(-) diff --git a/tests/test_parse_config.py b/tests/test_parse_config.py index c5d6a32..627d1ed 100644 --- a/tests/test_parse_config.py +++ b/tests/test_parse_config.py @@ -38,6 +38,7 @@ _parse_email, _parse_koji, _parse_open_id_connect, + _parse_static_configuration, ensure_downstream_name, get_config_ref, get_order, @@ -46,6 +47,7 @@ is_eligible, is_paused, load_config, + load_dynamic_config, loglevel, retries, skip_tag, @@ -102,13 +104,11 @@ def test_minimal_required_only(self): result = _parse_koji( { "profile": "koji", - "trigger_tag": "f40", "build_target": "eln", "stable_tag": "eln", } ) assert result["profile"] == "koji" - assert result["trigger_tag"] == "f40" assert result["build_target"] == "eln" assert result["stable_tag"] == "eln" assert result["scratch_build"] is False @@ -118,7 +118,6 @@ def test_scratch_build_and_fail_fast_true(self): result = _parse_koji( { "profile": "koji", - "trigger_tag": "f40", "build_target": "eln", "stable_tag": "eln", "scratch_build": True, @@ -126,24 +125,17 @@ def test_scratch_build_and_fail_fast_true(self): } ) assert result["profile"] == "koji" - assert result["trigger_tag"] == "f40" assert result["build_target"] == "eln" assert result["scratch_build"] is True assert result["fail_fast"] is True def test_missing_profile_raises(self): with pytest.raises(ConfigError, match="koji.profile missing"): - _parse_koji( - {"trigger_tag": "f40", "build_target": "eln", "stable_tag": "eln"} - ) - - def test_missing_trigger_tag_raises(self): - with pytest.raises(ConfigError, match="koji.trigger_tag missing"): - _parse_koji({"profile": "koji", "build_target": "eln", "stable_tag": "eln"}) + _parse_koji({"build_target": "eln", "stable_tag": "eln"}) def test_missing_build_target_raises(self): with pytest.raises(ConfigError, match="koji.build_target missing"): - _parse_koji({"profile": "koji", "trigger_tag": "f40", "stable_tag": "eln"}) + _parse_koji({"profile": "koji", "stable_tag": "eln"}) class TestParseBodhi: @@ -203,6 +195,7 @@ def test_missing_user_raises(self): # Minimal valid control config (no db; db is top-level) MINIMAL_CONTROL = { "pause": False, + "trigger_tag": "f40", } MINIMAL_EMAIL = { @@ -240,6 +233,7 @@ class TestParseControl: def test_minimal_required(self): result = _parse_control(MINIMAL_CONTROL) assert result["pause"] is False + assert result["trigger_tag"] == "f40" assert result["skip_tag"] == set() assert result["exclude"] == set() assert result["ordering"] == {} @@ -261,15 +255,20 @@ def test_missing_pause_raises(self): with pytest.raises(ConfigError, match="control.pause missing"): _parse_control({k: v for k, v in MINIMAL_CONTROL.items() if k != "pause"}) + def test_missing_trigger_tag_raises(self): + with pytest.raises(ConfigError, match="control.trigger_tag missing"): + _parse_control( + {k: v for k, v in MINIMAL_CONTROL.items() if k != "trigger_tag"} + ) -def _minimal_cnf(open_id_connect=None): - """Build minimal configuration block for _parse_configuration_block tests.""" + +def _minimal_static_cnf(open_id_connect=None): + """Build minimal static configuration block.""" if open_id_connect is None: open_id_connect = MINIMAL_OIDC return { "koji": { "profile": "koji", - "trigger_tag": "f40", "build_target": "eln", "stable_tag": "eln", "scratch_build": False, @@ -278,28 +277,50 @@ def _minimal_cnf(open_id_connect=None): "bodhi": {"batch_size": 0}, "db": MINIMAL_DB, "open_id_connect": open_id_connect, - "control": MINIMAL_CONTROL, "email": MINIMAL_EMAIL, } -class TestParseConfigurationBlock: +def _minimal_cnf(open_id_connect=None): + """Build minimal configuration block for _parse_configuration_block tests.""" + return { + **_minimal_static_cnf(open_id_connect=open_id_connect), + "control": MINIMAL_CONTROL, + } + + +class TestParseStaticConfiguration: def test_full_valid_cnf_returns_n(self): - cnf = _minimal_cnf() - n = _parse_configuration_block(cnf) + cnf = _minimal_static_cnf() + n = _parse_static_configuration(cnf) assert n["koji"]["profile"] == "koji" - assert n["koji"]["trigger_tag"] == "f40" assert n["koji"]["build_target"] == "eln" assert n["koji"]["stable_tag"] == "eln" assert n["bodhi"]["batch_size"] == 0 assert n["db"]["host"] == "localhost" - assert n["db"]["port"] == 5432 - assert n["db"]["name"] == "testdb" assert n["open_id_connect"] is not None - assert n["open_id_connect"]["auth_url"] == MINIMAL_OIDC["auth_url"] - assert n["control"]["pause"] is False assert n["email"]["smtp_host"] == "localhost" - assert n["email"]["from"] == "elnbuildsync@fedoraproject.org" + + def test_oidc_disabled(self): + cnf = _minimal_static_cnf(open_id_connect=False) + n = _parse_static_configuration(cnf) + assert n["open_id_connect"] is None + + def test_missing_koji_raises(self): + cnf = _minimal_static_cnf() + del cnf["koji"] + with pytest.raises(ConfigError, match="koji missing"): + _parse_static_configuration(cnf) + + +class TestParseConfigurationBlock: + def test_full_valid_cnf_returns_n(self): + cnf = _minimal_cnf() + n = _parse_configuration_block(cnf) + assert n["koji"]["profile"] == "koji" + assert n["koji"]["build_target"] == "eln" + assert n["control"]["trigger_tag"] == "f40" + assert n["control"]["pause"] is False def test_oidc_disabled(self): cnf = _minimal_cnf(open_id_connect=False) @@ -318,12 +339,6 @@ def test_missing_koji_profile_raises(self): with pytest.raises(ConfigError, match="koji.profile missing"): _parse_configuration_block(cnf) - def test_missing_koji_trigger_tag_raises(self): - cnf = _minimal_cnf() - del cnf["koji"]["trigger_tag"] - with pytest.raises(ConfigError, match="koji.trigger_tag missing"): - _parse_configuration_block(cnf) - def test_missing_bodhi_raises(self): cnf = _minimal_cnf() del cnf["bodhi"] @@ -519,12 +534,11 @@ async def test_overrides_child_not_dict_raises(self): await _parse_components({"overrides": {"kernel": "not-a-dict"}}) -# Minimal YAML for load_config integration test (components with overrides only, no autopackagelist) -MINIMAL_LOAD_CONFIG_YAML = """ +# Minimal YAML for load_config integration tests +MINIMAL_STATIC_CONFIG_YAML = """ configuration: koji: profile: koji - trigger_tag: f40 build_target: eln stable_tag: eln scratch_build: false @@ -538,8 +552,6 @@ async def test_overrides_child_not_dict_raises(self): driver: postgresql+asyncpg user: testuser open_id_connect: false - control: - pause: false email: smtp_host: localhost smtp_port: 587 @@ -547,25 +559,49 @@ async def test_overrides_child_not_dict_raises(self): from: elnbuildsync@fedoraproject.org recipients: - list1@fedoraproject.org +""" + +MINIMAL_DYNAMIC_CONFIG_YAML = """ +configuration: + control: + trigger_tag: f40 + pause: false components: overrides: {} """ async def _fake_defer_to_thread(fn, *args, **kwargs): - """Run fn synchronously and return result; used so load_config works under asyncio.""" + """Run fn synchronously and return result; used so config loaders work under asyncio.""" return fn(*args, **kwargs) +def _write_split_config_files(static_yaml, dynamic_yaml): + static_f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) + dynamic_f = tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) + static_f.write(static_yaml) + static_f.close() + dynamic_f.write(dynamic_yaml) + dynamic_f.close() + return static_f.name, dynamic_f.name + + class TestLoadConfig: @pytest.mark.asyncio async def test_load_config_from_file_sets_main_and_comps(self): - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: - f.write(MINIMAL_LOAD_CONFIG_YAML) - path = f.name + static_path, dynamic_path = _write_split_config_files( + MINIMAL_STATIC_CONFIG_YAML, MINIMAL_DYNAMIC_CONFIG_YAML + ) try: - with patch( - "elnbuildsync.config.deferToThread", side_effect=_fake_defer_to_thread + with ( + patch( + "elnbuildsync.config.static.deferToThread", + side_effect=_fake_defer_to_thread, + ), + patch( + "elnbuildsync.config.dynamic.deferToThread", + side_effect=_fake_defer_to_thread, + ), ): with patch( "elnbuildsync.config.get_rawhide_tag", new_callable=AsyncMock @@ -574,34 +610,44 @@ async def test_load_config_from_file_sets_main_and_comps(self): "elnbuildsync.config.get_distro_packages", new_callable=AsyncMock, ) as mock_distro: - await load_config(config_file=path, db_pw="testpw") + await load_config( + static_config_file=static_path, + dynamic_config_file=dynamic_path, + db_pw="testpw", + ) mock_rawhide.assert_not_called() mock_distro.assert_not_called() assert config_mod.main is not None assert config_mod.main["koji"]["profile"] == "koji" - assert config_mod.main["koji"]["trigger_tag"] == "f40" assert config_mod.main["koji"]["build_target"] == "eln" + assert config_mod.control["trigger_tag"] == "f40" assert config_mod.main["bodhi"]["batch_size"] == 0 assert config_mod.main["open_id_connect"] is None assert config_mod.comps is not None - assert "downstream_components" in config_mod.comps - assert "upstream_components" in config_mod.comps assert config_mod.comps["downstream_components"] == {} assert config_mod.comps["upstream_components"] == {} assert config_mod.main["email"]["smtp_host"] == "localhost" assert config_mod.emailer is not None finally: - os.unlink(path) + os.unlink(static_path) + os.unlink(dynamic_path) @pytest.mark.asyncio async def test_load_config_reinstantiates_email_each_load(self): - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: - f.write(MINIMAL_LOAD_CONFIG_YAML) - path = f.name + static_path, dynamic_path = _write_split_config_files( + MINIMAL_STATIC_CONFIG_YAML, MINIMAL_DYNAMIC_CONFIG_YAML + ) try: config_mod.emailer = None - with patch( - "elnbuildsync.config.deferToThread", side_effect=_fake_defer_to_thread + with ( + patch( + "elnbuildsync.config.static.deferToThread", + side_effect=_fake_defer_to_thread, + ), + patch( + "elnbuildsync.config.dynamic.deferToThread", + side_effect=_fake_defer_to_thread, + ), ): with patch( "elnbuildsync.config.get_rawhide_tag", new_callable=AsyncMock @@ -610,25 +656,35 @@ async def test_load_config_reinstantiates_email_each_load(self): "elnbuildsync.config.get_distro_packages", new_callable=AsyncMock, ): - with patch("elnbuildsync.config.Email") as MockEmail: - await load_config(config_file=path, db_pw="testpw") + with patch("elnbuildsync.config.static.Email") as MockEmail: + await load_config( + static_config_file=static_path, + dynamic_config_file=dynamic_path, + db_pw="testpw", + ) assert MockEmail.call_count == 1 - await load_config(config_file=path, db_pw="testpw") + await load_config( + static_config_file=static_path, + dynamic_config_file=dynamic_path, + db_pw="testpw", + ) assert MockEmail.call_count == 2 finally: - os.unlink(path) + os.unlink(static_path) + os.unlink(dynamic_path) @pytest.mark.asyncio - async def test_load_config_trigger_tag_rawhide_resolved_via_bodhi(self): - """When trigger_tag is 'rawhide', load_config calls get_rawhide_tag() which queries Bodhi; we mock the Bodhi HTTP call.""" - yaml_with_rawhide = MINIMAL_LOAD_CONFIG_YAML.replace( + async def test_load_dynamic_config_trigger_tag_rawhide_resolved_via_bodhi(self): + """When trigger_tag is 'rawhide', load_dynamic_config resolves via Bodhi.""" + dynamic_yaml = MINIMAL_DYNAMIC_CONFIG_YAML.replace( "trigger_tag: f40", "trigger_tag: rawhide" ) - with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: - f.write(yaml_with_rawhide) - path = f.name + dynamic_path = tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml", delete=False + ) + dynamic_path.write(dynamic_yaml) + dynamic_path.close() try: - # Mock Bodhi response so get_rawhide_tag() gets rawhide -> f41 without real HTTP bodhi_response = MagicMock() bodhi_response.text = _bodhi_releases_json("f41") bodhi_response.raise_for_status = MagicMock() @@ -639,24 +695,25 @@ async def test_load_config_trigger_tag_rawhide_resolved_via_bodhi(self): mock_session.__exit__ = MagicMock(return_value=False) with patch( - "elnbuildsync.config.deferToThread", side_effect=_fake_defer_to_thread + "elnbuildsync.config.dynamic.deferToThread", + side_effect=_fake_defer_to_thread, ): with patch("elnbuildsync.config.Session", return_value=mock_session): with patch( "elnbuildsync.config.get_distro_packages", new_callable=AsyncMock, ): - await load_config(config_file=path, db_pw="testpw") - assert config_mod.main["koji"]["trigger_tag"] == "f41" + await load_dynamic_config(dynamic_config_file=dynamic_path.name) + assert config_mod.control["trigger_tag"] == "f41" mock_get.assert_called_once() finally: - os.unlink(path) + os.unlink(dynamic_path.name) @pytest.mark.asyncio async def test_load_config_missing_file_raises(self): with pytest.raises(ConfigError, match="Could not parse"): - await load_config( - config_file="/nonexistent/path/distrobaker.yaml", db_pw="" + await load_dynamic_config( + dynamic_config_file="/nonexistent/path/distrobaker.yaml" ) @pytest.mark.asyncio @@ -666,7 +723,7 @@ async def test_load_config_invalid_yaml_raises(self): path = f.name try: with pytest.raises(ConfigError, match="Could not parse"): - await load_config(config_file=path, db_pw="") + await load_dynamic_config(dynamic_config_file=path) finally: os.unlink(path) @@ -674,43 +731,22 @@ async def test_load_config_invalid_yaml_raises(self): async def test_load_config_missing_components_raises(self): yaml_no_components = """ configuration: - koji: - profile: koji - trigger_tag: f40 - build_target: eln - stable_tag: eln - scratch_build: false - fail_fast: false - bodhi: - batch_size: 0 - db: - host: localhost - port: 5432 - name: testdb - driver: postgresql+asyncpg - user: testuser - open_id_connect: false control: + trigger_tag: f40 pause: false - email: - smtp_host: localhost - smtp_port: 587 - smtp_username: alice - from: elnbuildsync@fedoraproject.org - recipients: - - list1@fedoraproject.org """ with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: f.write(yaml_no_components) path = f.name try: with patch( - "elnbuildsync.config.deferToThread", side_effect=_fake_defer_to_thread + "elnbuildsync.config.dynamic.deferToThread", + side_effect=_fake_defer_to_thread, ): with pytest.raises( ConfigError, match="required components block is missing" ): - await load_config(config_file=path, db_pw="testpw") + await load_dynamic_config(dynamic_config_file=path) finally: os.unlink(path) @@ -847,11 +883,9 @@ class TestIsEligible: def test_exclude_pattern_matches_returns_false(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "exclude": {"^kernel$"}, - }, + "exclude": {"^kernel$"}, }, ) monkeypatch.setattr( @@ -868,11 +902,9 @@ def test_exclude_pattern_matches_returns_false(self, monkeypatch): def test_not_excluded_returns_true(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "exclude": set(), - }, + "exclude": set(), }, ) monkeypatch.setattr( @@ -891,11 +923,9 @@ class TestSkipTag: def test_pattern_matches_returns_true(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "skip_tag": {"^kernel$"}, - }, + "skip_tag": {"^kernel$"}, }, ) assert skip_tag("kernel") is True @@ -903,11 +933,9 @@ def test_pattern_matches_returns_true(self, monkeypatch): def test_no_match_returns_false(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "skip_tag": set(), - }, + "skip_tag": set(), }, ) assert skip_tag("ipa") is False @@ -925,11 +953,9 @@ def _comps_with(names): def test_pattern_matches_returns_order(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "ordering": {"^ocaml$": 0}, - }, + "ordering": {"^ocaml$": 0}, }, ) monkeypatch.setattr( @@ -942,11 +968,9 @@ def test_pattern_matches_returns_order(self, monkeypatch): def test_no_pattern_returns_1000(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "ordering": {}, - }, + "ordering": {}, }, ) monkeypatch.setattr( @@ -961,11 +985,9 @@ def test_ordering_uses_downstream_name_when_passed_upstream_component( ): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "ordering": {"^rust$": 5}, - }, + "ordering": {"^rust$": 5}, }, ) monkeypatch.setattr( @@ -986,11 +1008,9 @@ def test_unknown_component_not_in_either_list_matches_ordering_pattern( """Ordering regex applies to the passed name when the component is unknown.""" monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "ordering": {"^ghost$": 42}, - }, + "ordering": {"^ghost$": 42}, }, ) monkeypatch.setattr( @@ -1012,11 +1032,9 @@ def test_unknown_component_not_in_either_list_returns_default_without_pattern( ): monkeypatch.setattr( config_mod, - "main", + "control", { - "control": { - "ordering": {}, - }, + "ordering": {}, }, ) monkeypatch.setattr( @@ -1076,16 +1094,16 @@ class TestIsPaused: def test_paused_true(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", - {"control": {"pause": True}}, + "control", + {"pause": True}, ) assert is_paused() is True def test_paused_false(self, monkeypatch): monkeypatch.setattr( config_mod, - "main", - {"control": {"pause": False}}, + "control", + {"pause": False}, ) assert is_paused() is False From 94291c8c477d69dbcbee7e30561690610fc8eaa0 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:14:38 -0400 Subject: [PATCH 4/7] deploy: Load static and dynamic config from separate sources Update run.sh, local and OpenShift test scripts, and Helm secrets for elnbuildsync.yaml and elnbuildsync_dynamic.yaml. Signed-off-by: Stephen Gallagher Co-authored-by: Cursor --- helm_charts/templates/deployment.yaml | 2 +- helm_charts/templates/ebs_config.yaml | 7 ++- helm_charts/values.yaml | 5 +- run.sh | 81 ++++++++++++++++----------- tests/local_test_daemon.sh | 61 ++++++++++++++------ tests/openshift_test_daemon.sh | 15 +++-- 6 files changed, 112 insertions(+), 59 deletions(-) diff --git a/helm_charts/templates/deployment.yaml b/helm_charts/templates/deployment.yaml index aaa49b8..5975d8b 100644 --- a/helm_charts/templates/deployment.yaml +++ b/helm_charts/templates/deployment.yaml @@ -53,7 +53,7 @@ spec: volumeMounts: - mountPath: /keytab name: keytab - # DB password and optional elnbuildsync_config.yaml + # DB password and optional elnbuildsync.yaml / elnbuildsync_dynamic.yaml - mountPath: /etc/elnbuildsync name: ebs-config - mountPath: /etc/fedora-messaging diff --git a/helm_charts/templates/ebs_config.yaml b/helm_charts/templates/ebs_config.yaml index 5d9be81..b74e772 100644 --- a/helm_charts/templates/ebs_config.yaml +++ b/helm_charts/templates/ebs_config.yaml @@ -10,7 +10,10 @@ data: {{- if .Values.secrets.smtp_password }} ebs_smtp_pw: '{{ .Values.secrets.smtp_password | default "" | b64enc }}' {{- end }} -{{- if .Values.secrets.elnbuildsync_config_file }} - elnbuildsync_config.yaml: '{{ .Values.secrets.elnbuildsync_config_file | default "" | b64enc }}' +{{- if .Values.secrets.elnbuildsync_static_config_file }} + elnbuildsync.yaml: '{{ .Values.secrets.elnbuildsync_static_config_file | default "" | b64enc }}' +{{- end }} +{{- if .Values.secrets.elnbuildsync_dynamic_config_file }} + elnbuildsync_dynamic.yaml: '{{ .Values.secrets.elnbuildsync_dynamic_config_file | default "" | b64enc }}' {{- end }} type: Opaque diff --git a/helm_charts/values.yaml b/helm_charts/values.yaml index 8638190..7798048 100644 --- a/helm_charts/values.yaml +++ b/helm_charts/values.yaml @@ -17,8 +17,9 @@ secrets: fedora_messaging_client_pem: fedora_messaging_client_key: - # Optional: supply with helm --set-file secrets.elnbuildsync_config_file=/path/to/file.yaml - elnbuildsync_config_file: + # Optional: supply with helm --set-file + elnbuildsync_static_config_file: + elnbuildsync_dynamic_config_file: smtp_password: git: diff --git a/run.sh b/run.sh index 61dccd8..7c20b24 100755 --- a/run.sh +++ b/run.sh @@ -20,9 +20,10 @@ # Created by argbash-init v2.11.0 # ARG_OPTIONAL_SINGLE([log-level],[],[Log verbosity],[INFO]) -# ARG_OPTIONAL_SINGLE([config-file],[],[Configuration file]) -# ARG_OPTIONAL_SINGLE([config-url],[],[Configuration Git URL],[https://github.com/fedora-eln/elnbuildsync-config.git]) -# ARG_OPTIONAL_SINGLE([config-branch],[],[Configuration Git branch],[main]) +# ARG_OPTIONAL_SINGLE([static-config-file],[],[Static configuration file],[/etc/elnbuildsync/elnbuildsync.yaml]) +# ARG_OPTIONAL_SINGLE([dynamic-config-file],[],[Dynamic configuration file]) +# ARG_OPTIONAL_SINGLE([dynamic-config-url],[],[Dynamic configuration Git URL],[https://github.com/fedora-eln/elnbuildsync-config.git]) +# ARG_OPTIONAL_SINGLE([dynamic-config-branch],[],[Dynamic configuration Git branch],[main]) # ARG_OPTIONAL_SINGLE([keytab-principal],[],[Keytab principal],[eln-buildsync@FEDORAPROJECT.ORG]) # ARG_OPTIONAL_SINGLE([koji-profile],[],[Koji profile],[koji]) # ARG_POSITIONAL_DOUBLEDASH([]) @@ -56,21 +57,23 @@ _positionals=() _arg_custom=() # THE DEFAULTS INITIALIZATION - OPTIONALS _arg_log_level="INFO" -_arg_config_file= -_arg_config_url="https://github.com/fedora-eln/elnbuildsync-config.git" -_arg_config_branch="main" +_arg_static_config_file="/etc/elnbuildsync/elnbuildsync.yaml" +_arg_dynamic_config_file= +_arg_dynamic_config_url="https://github.com/fedora-eln/elnbuildsync-config.git" +_arg_dynamic_config_branch="main" _arg_keytab_principal="eln-buildsync@FEDORAPROJECT.ORG" _arg_koji_profile="koji" print_help() { - printf 'Usage: %s [--log-level ] [--config-file ] [--config-url ] [--config-branch ] [--keytab-principal ] [--koji-profile ] [-h|--help] [--] [] ... [] ...\n' "$0" + printf 'Usage: %s [--log-level ] [--static-config-file ] [--dynamic-config-file ] [--dynamic-config-url ] [--dynamic-config-branch ] [--keytab-principal ] [--koji-profile ] [-h|--help] [--] [] ... [] ...\n' "$0" printf '\t%s\n' ": Additional arguments to pass to the ELNBuildSync daemon" printf '\t%s\n' "--log-level: Log verbosity (default: 'INFO')" - printf '\t%s\n' "--config-file: Configuration file (no default)" - printf '\t%s\n' "--config-url: Configuration Git URL (default: 'https://github.com/fedora-eln/elnbuildsync-config.git')" - printf '\t%s\n' "--config-branch: Configuration Git branch (default: 'main')" + printf '\t%s\n' "--static-config-file: Static configuration file (default: '/etc/elnbuildsync/elnbuildsync.yaml')" + printf '\t%s\n' "--dynamic-config-file: Dynamic configuration file (no default)" + printf '\t%s\n' "--dynamic-config-url: Dynamic configuration Git URL (default: 'https://github.com/fedora-eln/elnbuildsync-config.git')" + printf '\t%s\n' "--dynamic-config-branch: Dynamic configuration Git branch (default: 'main')" printf '\t%s\n' "--keytab-principal: Keytab principal (default: 'eln-buildsync@FEDORAPROJECT.ORG')" printf '\t%s\n' "--koji-profile: Koji profile (default: 'koji')" printf '\t%s\n' "-h, --help: Prints help" @@ -104,29 +107,37 @@ parse_commandline() --log-level=*) _arg_log_level="${_key##--log-level=}" ;; - --config-file) + --static-config-file) test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 - _arg_config_file="$2" + _arg_static_config_file="$2" shift ;; - --config-file=*) - _arg_config_file="${_key##--config-file=}" + --static-config-file=*) + _arg_static_config_file="${_key##--static-config-file=}" ;; - --config-url) + --dynamic-config-file) test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 - _arg_config_url="$2" + _arg_dynamic_config_file="$2" shift ;; - --config-url=*) - _arg_config_url="${_key##--config-url=}" + --dynamic-config-file=*) + _arg_dynamic_config_file="${_key##--dynamic-config-file=}" ;; - --config-branch) + --dynamic-config-url) test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 - _arg_config_branch="$2" + _arg_dynamic_config_url="$2" shift ;; - --config-branch=*) - _arg_config_branch="${_key##--config-branch=}" + --dynamic-config-url=*) + _arg_dynamic_config_url="${_key##--dynamic-config-url=}" + ;; + --dynamic-config-branch) + test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 + _arg_dynamic_config_branch="$2" + shift + ;; + --dynamic-config-branch=*) + _arg_dynamic_config_branch="${_key##--dynamic-config-branch=}" ;; --keytab-principal) test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 @@ -210,17 +221,22 @@ while true; do koji -p ${_arg_koji_profile} hello && break || sleep 3 done -if [ -f /etc/elnbuildsync/elnbuildsync_config.yaml ]; then - # Check if we have mounted a config file into the container. +STATIC_ARG="--static-config-file /etc/elnbuildsync/elnbuildsync.yaml" +if [ -n "${_arg_static_config_file}" ]; then + STATIC_ARG="--static-config-file ${_arg_static_config_file}" +fi + +if [ -n "${_arg_dynamic_config_file}" ]; then + echo "Using dynamic config file at ${_arg_dynamic_config_file}" + DYNAMIC_ARG="--dynamic-config-file ${_arg_dynamic_config_file}" +elif [ -f /etc/elnbuildsync/elnbuildsync_dynamic.yaml ]; then + # Check if we have mounted a dynamic config file into the container. # This is mostly useful for OpenShift Local testing. - echo "Using config file at /etc/elnbuildsync/elnbuildsync_config.yaml" - CONFIG_ARG="--config-file /etc/elnbuildsync/elnbuildsync_config.yaml" -elif [ -n "${_arg_config_file}" ]; then - echo "Using config file at ${_arg_config_file}" - CONFIG_ARG="--config-file ${_arg_config_file}" + echo "Using dynamic config file at /etc/elnbuildsync/elnbuildsync_dynamic.yaml" + DYNAMIC_ARG="--dynamic-config-file /etc/elnbuildsync/elnbuildsync_dynamic.yaml" else - echo "Using config URL at ${_arg_config_url}#${_arg_config_branch}" - CONFIG_ARG="--config-url ${_arg_config_url}#${_arg_config_branch}" + echo "Using dynamic config URL at ${_arg_dynamic_config_url}#${_arg_dynamic_config_branch}" + DYNAMIC_ARG="--dynamic-config-url ${_arg_dynamic_config_url}#${_arg_dynamic_config_branch}" fi # Check that the DB password file exists @@ -251,7 +267,8 @@ pip install "${SCRIPT_DIR}" export FEDORA_MESSAGING_CONF=/etc/fedora-messaging/config.toml elnbuildsync \ - $CONFIG_ARG \ + $STATIC_ARG \ + $DYNAMIC_ARG \ --log-level ${_arg_log_level} \ --db-pw-file /etc/elnbuildsync/ebs_db_pw \ $SMTP_ARG \ diff --git a/tests/local_test_daemon.sh b/tests/local_test_daemon.sh index b4b4942..76b2b1b 100755 --- a/tests/local_test_daemon.sh +++ b/tests/local_test_daemon.sh @@ -23,7 +23,8 @@ # ARG_OPTIONAL_SINGLE([db-pw-file],[],[Database password file],[tests/ebs_db_pw]) # ARG_OPTIONAL_SINGLE([smtp-pw-file],[],[SMTP password file],[tests/ebs_smtp_pw]) # ARG_OPTIONAL_SINGLE([lull-time],[],[Time to wait after the last trigger before starting the batch],[5]) -# ARG_OPTIONAL_SINGLE([config-file],[],[Configuration file],[tests/local_testconfig.yaml]) +# ARG_OPTIONAL_SINGLE([static-config-file],[],[Static configuration file],[tests/etc/local/elnbuildsync.yaml]) +# ARG_OPTIONAL_SINGLE([dynamic-config-file],[],[Dynamic configuration file],[tests/etc/local/elnbuildsync_dynamic.yaml]) # ARG_OPTIONAL_SINGLE([environment],[],[Environment],[stg]) # ARG_OPTIONAL_BOOLEAN([persistent-db],[],[Use persistent database],[off]) # ARG_OPTIONAL_SINGLE([persistent-db-path],[],[Path to persistent database],[tests/persistent_db]) @@ -59,10 +60,11 @@ _positionals=() _arg_custom=() # THE DEFAULTS INITIALIZATION - OPTIONALS _arg_log_level="INFO" -_arg_db_pw_file="tests/ebs_db_pw" -_arg_smtp_pw_file="tests/ebs_smtp_pw" +_arg_db_pw_file="tests/etc/local/ebs_db_pw" +_arg_smtp_pw_file="tests/etc/local/ebs_smtp_pw" _arg_lull_time="5" -_arg_config_file="tests/local_testconfig.yaml" +_arg_static_config_file="tests/etc/local/elnbuildsync.yaml" +_arg_dynamic_config_file="tests/etc/local/elnbuildsync_dynamic.yaml" _arg_environment="stg" _arg_persistent_db="off" _arg_persistent_db_path="tests/persistent_db" @@ -71,13 +73,14 @@ _arg_build_container="off" print_help() { - printf 'Usage: %s [--log-level ] [--db-pw-file ] [--smtp-pw-file ] [--lull-time ] [--config-file ] [--environment ] [--(no-)persistent-db] [--persistent-db-path ] [--(no-)build-container] [-h|--help] [--] [] ... [] ...\n' "$0" + printf 'Usage: %s [--log-level ] [--db-pw-file ] [--smtp-pw-file ] [--lull-time ] [--static-config-file ] [--dynamic-config-file ] [--environment ] [--(no-)persistent-db] [--persistent-db-path ] [--(no-)build-container] [-h|--help] [--] [] ... [] ...\n' "$0" printf '\t%s\n' ": Additional arguments to pass to the ELNBuildSync daemon" printf '\t%s\n' "--log-level: Log verbosity (default: 'INFO')" - printf '\t%s\n' "--db-pw-file: Database password file (default: 'tests/ebs_db_pw')" - printf '\t%s\n' "--smtp-pw-file: SMTP password file (default: 'tests/ebs_smtp_pw')" + printf '\t%s\n' "--db-pw-file: Database password file (default: 'tests/etc/local/ebs_db_pw')" + printf '\t%s\n' "--smtp-pw-file: SMTP password file (default: 'tests/etc/local/ebs_smtp_pw')" printf '\t%s\n' "--lull-time: Time to wait after the last trigger before starting the batch (default: '5')" - printf '\t%s\n' "--config-file: Configuration file (default: 'tests/local_testconfig.yaml')" + printf '\t%s\n' "--static-config-file: Static configuration file (default: 'tests/etc/local/elnbuildsync.yaml')" + printf '\t%s\n' "--dynamic-config-file: Dynamic configuration file (default: 'tests/etc/local/elnbuildsync_dynamic.yaml')" printf '\t%s\n' "--environment: Environment (default: 'stg')" printf '\t%s\n' "--persistent-db, --no-persistent-db: Use persistent database (off by default)" printf '\t%s\n' "--persistent-db-path: Path to persistent database (default: 'tests/persistent_db')" @@ -137,13 +140,21 @@ parse_commandline() --lull-time=*) _arg_lull_time="${_key##--lull-time=}" ;; - --config-file) + --static-config-file) test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 - _arg_config_file="$2" + _arg_static_config_file="$2" shift ;; - --config-file=*) - _arg_config_file="${_key##--config-file=}" + --static-config-file=*) + _arg_static_config_file="${_key##--static-config-file=}" + ;; + --dynamic-config-file) + test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 + _arg_dynamic_config_file="$2" + shift + ;; + --dynamic-config-file=*) + _arg_dynamic_config_file="${_key##--dynamic-config-file=}" ;; --environment) test $# -lt 2 && die "Missing value for the optional argument '$_key'." 1 @@ -239,7 +250,7 @@ pip install -r requirements.txt pip install --editable $PROJ_DIR function check_db_avail() { - PGPASSWORD=$(cat tests/secrets/ebs_db_pw) \ + PGPASSWORD=$(cat "${_arg_db_pw_file}") \ pg_isready --quiet \ --dbname elnbuildsync \ --username elnbuildsync \ @@ -275,7 +286,7 @@ if [ $db_ready -ne 0 ]; then ${CONTAINER_ENGINE} run --rm --detach \ --publish 5432:5432 \ --network ebs_local_test \ - --volume ${SCRIPT_DIR}/secrets:/run/secrets:Z \ + --volume ${SCRIPT_DIR}/etc/local:/run/secrets:Z \ --name temp_postgres \ --env POSTGRES_PASSWORD_FILE=/run/secrets/ebs_db_pw \ --env POSTGRES_USER=elnbuildsync \ @@ -298,6 +309,22 @@ else export FEDORA_MESSAGING_CONF="$SCRIPT_DIR/fedora-messaging/fedora.toml" fi +DEFAULT_STATIC_CONFIG_FILE="tests/etc/local/elnbuildsync.yaml" +DEFAULT_DYNAMIC_CONFIG_FILE="tests/etc/local/elnbuildsync_dynamic.yaml" +CONTAINER_STATIC_CONFIG="/etc/elnbuildsync/elnbuildsync.yaml" +CONTAINER_DYNAMIC_CONFIG="/etc/elnbuildsync/elnbuildsync_dynamic.yaml" +CUSTOM_MOUNT_ARGS=() + +if [ "${_arg_static_config_file}" != "${DEFAULT_STATIC_CONFIG_FILE}" ]; then + CONTAINER_STATIC_CONFIG="/etc/elnbuildsync/custom/static.yaml" + CUSTOM_MOUNT_ARGS+=(--volume "$(realpath "${_arg_static_config_file}"):${CONTAINER_STATIC_CONFIG}:ro,Z") +fi + +if [ "${_arg_dynamic_config_file}" != "${DEFAULT_DYNAMIC_CONFIG_FILE}" ]; then + CONTAINER_DYNAMIC_CONFIG="/etc/elnbuildsync/custom/dynamic.yaml" + CUSTOM_MOUNT_ARGS+=(--volume "$(realpath "${_arg_dynamic_config_file}"):${CONTAINER_DYNAMIC_CONFIG}:ro,Z") +fi + ${CONTAINER_ENGINE} run --rm --interactive --tty \ --publish 8080:8080 \ --network ebs_local_test \ @@ -306,11 +333,13 @@ ${CONTAINER_ENGINE} run --rm --interactive --tty \ --security-opt label=disable \ --env KRB5CCNAME=KCM:$(id -u) \ --volume /var/run/.heim_org.h5l.kcm-socket:/var/run/.heim_org.h5l.kcm-socket \ - --volume ${SCRIPT_DIR}/secrets:/etc/elnbuildsync:Z \ + --volume ${SCRIPT_DIR}/etc/local:/etc/elnbuildsync:Z \ + "${CUSTOM_MOUNT_ARGS[@]}" \ --volume ${PROJ_DIR}:/tmp:Z \ localhost/elnbuildsync:local_test_daemon \ --log-level "$_arg_log_level" \ - --config-file "$_arg_config_file" \ + --static-config-file "${CONTAINER_STATIC_CONFIG}" \ + --dynamic-config-file "${CONTAINER_DYNAMIC_CONFIG}" \ --lull-time "$_arg_lull_time" \ ${_arg_custom[@]} \ 2>&1 | tee /tmp/elnbuildsync.log diff --git a/tests/openshift_test_daemon.sh b/tests/openshift_test_daemon.sh index 639ff44..8ab7da6 100755 --- a/tests/openshift_test_daemon.sh +++ b/tests/openshift_test_daemon.sh @@ -177,10 +177,12 @@ skopeo copy \ "docker://${FULL_PUSH}" API_SERVER=$(oc whoami --show-server) -CRC_TESTCONFIG="${PROJ_DIR}/tests/crc_testconfig.yaml" -test -r "${CRC_TESTCONFIG}" || die "Cannot read CRC test config: ${CRC_TESTCONFIG}" 1 +CRC_STATIC_CONFIG="${PROJ_DIR}/tests/etc/crc/elnbuildsync.yaml" +CRC_DYNAMIC_CONFIG="${PROJ_DIR}/tests/etc/crc/elnbuildsync_dynamic.yaml" +test -r "${CRC_STATIC_CONFIG}" || die "Cannot read CRC static config: ${CRC_STATIC_CONFIG}" 1 +test -r "${CRC_DYNAMIC_CONFIG}" || die "Cannot read CRC dynamic config: ${CRC_DYNAMIC_CONFIG}" 1 -HELM_DESC="openshift_test_daemon image.tag=${_arg_image_tag} sidecar_database=true release=${RELEASE_NAME} chart_dir=helm_charts values=helm_charts/values.yaml image.repo_from_values=${IMAGE_REPO} image_push=${FULL_PUSH} api=${API_SERVER} config_file=tests/crc_testconfig.yaml" +HELM_DESC="openshift_test_daemon image.tag=${_arg_image_tag} sidecar_database=true release=${RELEASE_NAME} chart_dir=helm_charts values=helm_charts/values.yaml image.repo_from_values=${IMAGE_REPO} image_push=${FULL_PUSH} api=${API_SERVER} static_config=${CRC_STATIC_CONFIG} dynamic_config=${CRC_DYNAMIC_CONFIG}" if [ -n "${_arg_keytab_file}" ]; then HELM_DESC="${HELM_DESC} keytab_file=${_arg_keytab_file}" fi @@ -192,9 +194,10 @@ helm_cmd=( --set "image.tag=${_arg_image_tag}" --values "${CHART_DIR}/values.yaml" --set sidecar_database=true - --set-file secrets.elnbuildsync_config_file="${CRC_TESTCONFIG}" - --set-file secrets.database_password="${PROJ_DIR}/tests/secrets/ebs_db_pw" - --set-file secrets.smtp_password="${PROJ_DIR}/tests/secrets/ebs_smtp_pw" + --set-file secrets.elnbuildsync_static_config_file="${CRC_STATIC_CONFIG}" + --set-file secrets.elnbuildsync_dynamic_config_file="${CRC_DYNAMIC_CONFIG}" + --set-file secrets.database_password="${PROJ_DIR}/tests/etc/crc/ebs_db_pw" + --set-file secrets.smtp_password="${PROJ_DIR}/tests/etc/crc/ebs_smtp_pw" --set-file secrets.fedora_messaging_config="${PROJ_DIR}/tests/fedora-messaging/fedora.toml" --set-file secrets.fedora_messaging_cacert="${PROJ_DIR}/tests/fedora-messaging/cacert.pem" --set-file secrets.fedora_messaging_client_pem="${PROJ_DIR}/tests/fedora-messaging/fedora-cert.pem" From 4bec6916b08f67874404548b886cb1a901b64782 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:14:44 -0400 Subject: [PATCH 5/7] docs: Document static and dynamic configuration layout Signed-off-by: Stephen Gallagher Co-authored-by: Cursor --- README.md | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ca25cdf..b109052 100644 --- a/README.md +++ b/README.md @@ -152,13 +152,18 @@ port **8080** exposes: ### Configuration -Runtime behavior is driven by a YAML file (production: `distrobuildsync-config` -repository; local testing: `tests/local_testconfig.yaml`). Important sections: - -- **`configuration.koji`**: Koji profile, trigger tag, build target, stable - tag, scratch/fail-fast flags. -- **`configuration.control`**: `skip_tag`, `exclude`, `ordering`, pause - flag, status interval. +Runtime behavior is driven by static and dynamic YAML configuration. +Production static settings live in `/etc/elnbuildsync/elnbuildsync.yaml`; +dynamic settings come from the `distrobuildsync-config` git repository or +`/etc/elnbuildsync/elnbuildsync_dynamic.yaml`. Local testing uses +`tests/etc/local/elnbuildsync.yaml` and `tests/etc/local/elnbuildsync_dynamic.yaml`. + +Important sections: + +- **`configuration.koji`**: Koji profile, build target, stable tag, + scratch/fail-fast flags (static). +- **`configuration.control`**: `trigger_tag`, `skip_tag`, `exclude`, `ordering`, + pause flag, status interval (dynamic). - **`configuration.bodhi`**: Maximum builds per Bodhi update (`batch_size`; `0` means no splitting). - **`configuration.db`**: PostgreSQL connection settings. @@ -211,12 +216,13 @@ You also need: koji hello ``` -- **Test secrets** under `tests/secrets/`. At minimum: +- **Test secrets** under `tests/etc/local/` (and `tests/etc/crc/` for + OpenShift testing). At minimum: | File | Purpose | |------|---------| - | `tests/secrets/ebs_db_pw` | PostgreSQL password (one line) | - | `tests/secrets/ebs_smtp_pw` | SMTP password (one line) | + | `tests/etc/local/ebs_db_pw` | PostgreSQL password (one line) | + | `tests/etc/local/ebs_smtp_pw` | SMTP password (one line) | These may be overridden with `--db-pw-file` and `--smtp-pw-file` when calling `tests/local_test_daemon.sh`. @@ -224,7 +230,8 @@ You also need: - **Fedora Messaging certificates** are vendored under `tests/fedora-messaging/` (see `tests/fedora-messaging/README.md`). -Optionally, edit `tests/local_testconfig.yaml` for your environment (Koji +Optionally, edit `tests/etc/local/elnbuildsync.yaml` and +`tests/etc/local/elnbuildsync_dynamic.yaml` for your environment (Koji tags, OIDC client credentials for `/trigger`, package lists). The default file points at tinystage for OIDC; register a client at [tiny-stage](https://github.com/fedora-infra/tiny-stage) if you need @@ -257,8 +264,9 @@ The script: 3. Starts a temporary PostgreSQL 18 container (`temp_postgres`) unless a persistent one is already reachable on port 5432. 4. Runs the EBS container with: - - `tests/local_testconfig.yaml` mounted via `--config-file` - - Secrets mounted at `/etc/elnbuildsync/` + - `tests/etc/local/` mounted at `/etc/elnbuildsync/` (static and dynamic + config plus secrets) + - `--static-config-file` and `--dynamic-config-file` passed to the daemon - Fedora Messaging staging config (`--environment stg`, the default) - Port **8080** published for the web UI and APIs - A short **lull time** of 5 seconds (production uses 60) @@ -318,10 +326,12 @@ day-to-day code changes. ## Production configuration -Production deployments load configuration from the +Production deployments load static configuration from +`/etc/elnbuildsync/elnbuildsync.yaml` and dynamic configuration from the [elnbuildsync-config](https://github.com/fedora-eln/elnbuildsync-config) -git repository (see `run.sh --config-url`), mount database and SMTP secrets at -`/etc/elnbuildsync/`, and use a service keytab for +git repository (see `run.sh --dynamic-config-url`) or +`/etc/elnbuildsync/elnbuildsync_dynamic.yaml`. Database and SMTP secrets +mount at `/etc/elnbuildsync/`, and a service keytab is used for `eln-buildsync@FEDORAPROJECT.ORG`. See `helm_charts/` for Kubernetes resources. From 6f533cd6a2c2b4fdcb384b04eea8dca32d5d2154 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:01:51 -0400 Subject: [PATCH 6/7] config: Add debug logging Signed-off-by: Stephen Gallagher --- elnbuildsync/config/dynamic.py | 40 +++++++++++++++++++++++++++++ elnbuildsync/config/static.py | 47 +++++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/elnbuildsync/config/dynamic.py b/elnbuildsync/config/dynamic.py index fd89209..82abd54 100644 --- a/elnbuildsync/config/dynamic.py +++ b/elnbuildsync/config/dynamic.py @@ -74,6 +74,16 @@ def _parse_control(cnf_control, ConfigError): raise ConfigError("control.status_interval must be a positive integer.") result["status_interval"] = val + logger.debug( + "Parsed control: trigger_tag=%s pause=%s skip_tag=%d exclude=%d " + "ordering=%d status_interval=%s", + result["trigger_tag"], + result["pause"], + len(result["skip_tag"]), + len(result["exclude"]), + len(result["ordering"]), + result["status_interval"], + ) return result @@ -81,6 +91,11 @@ async def _parse_components(cnf_components, get_distro_packages, ConfigError): """Parse the components block (top-level). Requires at least one of autopackagelist or overrides. """ + logger.debug( + "Parsing components (autopackagelist=%s override_count=%d)", + "autopackagelist" in cnf_components, + len(cnf_components.get("overrides", {})), + ) if "autopackagelist" not in cnf_components and "overrides" not in cnf_components: raise ConfigError( "At least one of components.autopackagelist or components.overrides must be present." @@ -102,6 +117,12 @@ async def _parse_components(cnf_components, get_distro_packages, ConfigError): "content_resolver", DEFAULT_CONTENT_RESOLVER ), } + logger.debug( + "Autopackagelist: views=%s sources=%s content_resolver=%s", + apl["view"], + apl["source"], + apl["content_resolver"], + ) downstream_components = await get_distro_packages( distro_url=apl["content_resolver"], distro_view=apl["view"], @@ -154,6 +175,11 @@ async def _parse_components(cnf_components, get_distro_packages, ConfigError): downstream_name ].copy() + logger.debug( + "Parsed components: downstream=%d upstream=%d", + len(downstream_components), + len(upstream_components), + ) return { "downstream_components": downstream_components, "upstream_components": upstream_components, @@ -173,6 +199,12 @@ async def _load_dynamic_yaml( raise ConfigError("The required components block is missing.") cnf = y["configuration"] + static_keys = set(cnf.keys()) - {"control"} + if static_keys: + logger.debug( + "Ignoring static configuration sections in dynamic config: %s", + sorted(static_keys), + ) if "control" not in cnf: raise ConfigError("control missing.") @@ -206,6 +238,9 @@ async def _fetch_dynamic_config_file( scm = split_scmurl(scmurl) if scm["ref"] is None: scm["ref"] = "main" + logger.debug( + "Cloning dynamic config from %s at ref %s", scm["link"], scm["ref"] + ) with tempfile.TemporaryDirectory(prefix="distrobaker-") as cdir: for attempt in range(retry): @@ -286,3 +321,8 @@ async def load_dynamic_config( ) config_module.control = control config_module.comps = comps + logger.debug( + "Dynamic configuration applied: trigger_tag=%s downstream_components=%d", + control["trigger_tag"], + len(comps["downstream_components"]), + ) diff --git a/elnbuildsync/config/static.py b/elnbuildsync/config/static.py index 83f9204..f1fa8d0 100644 --- a/elnbuildsync/config/static.py +++ b/elnbuildsync/config/static.py @@ -40,6 +40,7 @@ def _parse_open_id_connect(oidc_raw, ConfigError): logger.info( "OpenID Connect explicitly disabled - /trigger endpoint unprotected" ) + logger.debug("Parsed open_id_connect: disabled") return None oidc = oidc_raw default_scopes = [ @@ -70,6 +71,13 @@ def _parse_open_id_connect(oidc_raw, ConfigError): "OpenID Connect authentication enabled; admin groups: %s", result["admin_groups"], ) + logger.debug( + "Parsed open_id_connect: auth_url=%s client_id=%s admin_groups=%d scopes=%d", + result["auth_url"], + result["client_id"], + len(result["admin_groups"]), + len(result["scopes"]), + ) return result @@ -102,6 +110,16 @@ def _parse_koji(cnf_koji, ConfigError): "Configuration warning: koji.fail_fast not defined, assuming false." ) result["fail_fast"] = False + logger.debug( + "Parsed koji config: profile=%s build_target=%s stable_tag=%s " + "scratch_build=%s fail_fast=%s username=%s", + result["profile"], + result["build_target"], + result["stable_tag"], + result["scratch_build"], + result["fail_fast"], + result.get("username"), + ) return result @@ -113,6 +131,7 @@ def _parse_bodhi(cnf_bodhi, ConfigError): result["batch_size"] = int(cnf_bodhi["batch_size"]) except ValueError: raise ConfigError("bodhi.batch_size must be an integer") + logger.debug("Parsed bodhi config: batch_size=%s", result["batch_size"]) return result @@ -123,6 +142,7 @@ def _parse_email(cnf_email, ConfigError): """ if cnf_email is False: logger.info("Email explicitly disabled") + logger.debug("Parsed email: disabled") return None required = ("smtp_host", "smtp_port", "smtp_username", "from", "recipients") for key in required: @@ -138,13 +158,22 @@ def _parse_email(cnf_email, ConfigError): for r in recipients: if not isinstance(r, str) or not r: raise ConfigError("email.recipients must be a list of non-empty strings.") - return { + result = { "smtp_host": str(cnf_email["smtp_host"]), "smtp_port": port, "smtp_username": str(cnf_email["smtp_username"]), "from": str(cnf_email["from"]), "recipients": [str(x) for x in recipients], } + logger.debug( + "Parsed email: smtp_host=%s smtp_port=%s smtp_username=%s from=%s recipients=%d", + result["smtp_host"], + result["smtp_port"], + result["smtp_username"], + result["from"], + len(result["recipients"]), + ) + return result def _parse_db(cnf_db, ConfigError): @@ -165,6 +194,14 @@ def _parse_db(cnf_db, ConfigError): } except ValueError: raise ConfigError("db.port must be an integer") + logger.debug( + "Parsed db config: host=%s port=%s name=%s driver=%s user=%s", + result["host"], + result["port"], + result["name"], + result["driver"], + result["user"], + ) return result @@ -172,6 +209,7 @@ def _parse_static_configuration(cnf, ConfigError): """Parse the static configuration block. Returns dict with koji, bodhi, db, open_id_connect, email. """ + logger.debug("Parsing static configuration sections: %s", sorted(cnf.keys())) if "control" in cnf: logger.warning( "Static configuration contains control block; use dynamic config instead." @@ -199,6 +237,7 @@ def _parse_static_configuration(cnf, ConfigError): raise ConfigError("email missing. Set email: false to disable email.") n["email"] = _parse_email(cnf["email"], ConfigError) + logger.debug("Static configuration parsed successfully") return n @@ -232,6 +271,7 @@ async def load_static_config( n = _parse_static_configuration(y["configuration"], ConfigError) config_module.main = n + logger.debug("Static configuration applied to config.main") if not config_module.db_url: try: @@ -244,11 +284,16 @@ async def load_static_config( username=db_config["user"], password=db_pw, ) + logger.debug("Database URL configured from static config") except KeyError as e: logger.exception(e) raise ConfigError("Missing database configuration (db block)") + else: + logger.debug("Database URL unchanged (already set)") if n["email"] is not None: config_module.emailer = Email(n["email"], config_module.smtp_password) + logger.debug("Emailer configured") else: config_module.emailer = None + logger.debug("Email disabled, emailer not configured") From 0bd9bbeef782da668906174c91253422d6d92649 Mon Sep 17 00:00:00 2001 From: Stephen Gallagher Date: Tue, 16 Jun 2026 10:12:40 -0400 Subject: [PATCH 7/7] config: Rename the config git repo file Signed-off-by: Stephen Gallagher --- elnbuildsync/config/dynamic.py | 8 +++++--- tests/test_parse_config.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/elnbuildsync/config/dynamic.py b/elnbuildsync/config/dynamic.py index 82abd54..64382dd 100644 --- a/elnbuildsync/config/dynamic.py +++ b/elnbuildsync/config/dynamic.py @@ -29,6 +29,8 @@ logger = logging.getLogger(__name__) DEFAULT_CONTENT_RESOLVER = "https://tiny.distro.builders" +DYNAMIC_CONFIG_FILENAME = "elnbuildsync_dynamic.yaml" +TEMP_DIR_PREFIX = "elnbuildsync-" def _parse_control(cnf_control, ConfigError): @@ -242,7 +244,7 @@ async def _fetch_dynamic_config_file( "Cloning dynamic config from %s at ref %s", scm["link"], scm["ref"] ) - with tempfile.TemporaryDirectory(prefix="distrobaker-") as cdir: + with tempfile.TemporaryDirectory(prefix=TEMP_DIR_PREFIX) as cdir: for attempt in range(retry): try: repo = await deferToThread(git.Repo.clone_from, scm["link"], cdir) @@ -260,10 +262,10 @@ async def _fetch_dynamic_config_file( else: raise ConfigError("Failed to fetch configuration, giving up.") - config_path = os.path.join(cdir, "distrobaker.yaml") + config_path = os.path.join(cdir, DYNAMIC_CONFIG_FILENAME) if not os.path.isfile(config_path): raise ConfigError( - "Configuration repository does not contain distrobaker.yaml." + f"Configuration repository does not contain {DYNAMIC_CONFIG_FILENAME}." ) try: diff --git a/tests/test_parse_config.py b/tests/test_parse_config.py index 627d1ed..ea68d0f 100644 --- a/tests/test_parse_config.py +++ b/tests/test_parse_config.py @@ -713,7 +713,7 @@ async def test_load_dynamic_config_trigger_tag_rawhide_resolved_via_bodhi(self): async def test_load_config_missing_file_raises(self): with pytest.raises(ConfigError, match="Could not parse"): await load_dynamic_config( - dynamic_config_file="/nonexistent/path/distrobaker.yaml" + dynamic_config_file="/nonexistent/path/elnbuildsync_dynamic.yaml" ) @pytest.mark.asyncio