From d1c42b7c220a2e5d12f05988aab379e8ace72832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nikola=20Forr=C3=B3?= Date: Mon, 30 Mar 2026 10:03:19 +0200 Subject: [PATCH] Support multiple release streams in sync release jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nikola Forró Assisted-by: Claude Opus 4.6 via Claude Code Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- packit_service/events/anitya/abstract.py | 50 ++-- packit_service/events/anitya/update.py | 20 +- packit_service/events/event_data.py | 35 +-- packit_service/worker/checker/distgit.py | 6 +- .../worker/checker/run_condition.py | 16 +- packit_service/worker/handlers/distgit.py | 140 +++++++--- packit_service/worker/mixin.py | 24 +- packit_service/worker/parser.py | 6 +- tests/integration/test_new_hotness_update.py | 263 +++++++++++++++++- tests/unit/events/test_anitya.py | 52 ++-- tests/unit/test_distgit.py | 2 + 11 files changed, 447 insertions(+), 167 deletions(-) diff --git a/packit_service/events/anitya/abstract.py b/packit_service/events/anitya/abstract.py index 367f6eb68..d017353eb 100644 --- a/packit_service/events/anitya/abstract.py +++ b/packit_service/events/anitya/abstract.py @@ -43,7 +43,7 @@ def __init__( @property @abstractmethod - def version(self) -> str: ... + def versions(self) -> list[str]: ... @cached_property def project(self) -> Optional[GitProject]: @@ -63,33 +63,14 @@ def base_project(self): def _add_release_and_event(self): if not self._db_project_object or not self._db_project_event: - if not self.project_url: - ( - self._db_project_object, - self._db_project_event, - ) = ProjectEventModel.add_anitya_version_event( - version=self.version, - project_name=self.anitya_project_name, - project_id=self.anitya_project_id, - package=self.package_name, - ) - return - - if not (self.tag_name and self.repo_name and self.repo_namespace and self.project_url): - logger.info( - "Not going to create the DB project event, not valid arguments.", - ) - return - ( self._db_project_object, self._db_project_event, - ) = ProjectEventModel.add_release_event( - tag_name=self.tag_name, - namespace=self.repo_namespace, - repo_name=self.repo_name, - project_url=self.project_url, - commit_hash=None, + ) = ProjectEventModel.add_anitya_multiple_versions_event( + versions=self.versions, + project_name=self.anitya_project_name, + project_id=self.anitya_project_id, + package=self.package_name, ) @property @@ -139,22 +120,25 @@ def repo_name(self) -> Optional[str]: return self.repo_url.repo if self.repo_url else None @property - def tag_name(self): + def tag_names(self) -> list[str]: if not (self.packages_config and self.packages_config.upstream_tag_template): - return self.version + return list(self.versions) - return self.packages_config.upstream_tag_template.format( - version=self.version, - upstream_package_name=self.packages_config.upstream_package_name, - ) + return [ + self.packages_config.upstream_tag_template.format( + version=version, + upstream_package_name=self.packages_config.upstream_package_name, + ) + for version in self.versions + ] def get_dict(self, default_dict: Optional[dict] = None) -> dict: d = self.__dict__ d["project_url"] = self.project_url - d["tag_name"] = self.tag_name + d["tag_names"] = self.tag_names d["repo_name"] = self.repo_name d["repo_namespace"] = self.repo_namespace - d["version"] = self.version + d["versions"] = self.versions result = super().get_dict(d) result.pop("project") result.pop("repo_url") diff --git a/packit_service/events/anitya/update.py b/packit_service/events/anitya/update.py index c6b4ab90b..b71d57998 100644 --- a/packit_service/events/anitya/update.py +++ b/packit_service/events/anitya/update.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: MIT from logging import getLogger -from typing import Optional from packit.config import JobConfigTriggerType @@ -21,7 +20,7 @@ class NewHotness(AnityaUpdate): def __init__( self, package_name: str, - version: str, + versions: list[str], distgit_project_url: str, bug_id: int, anitya_project_id: int, @@ -33,7 +32,7 @@ def __init__( anitya_project_id=anitya_project_id, anitya_project_name=anitya_project_name, ) - self._version = version + self._versions = versions self.bug_id = bug_id @classmethod @@ -41,14 +40,14 @@ def event_type(cls) -> str: return "anitya.NewHotness" @property - def version(self) -> str: - return self._version + def versions(self) -> list[str]: + return self._versions @classmethod def from_event_dict(cls, event: dict) -> "NewHotness": return cls( package_name=event.get("package_name"), - version=event.get("version"), + versions=event.get("versions"), distgit_project_url=event.get("distgit_project_url"), bug_id=event.get("bug_id"), anitya_project_id=event.get("anitya_project_id"), @@ -82,11 +81,8 @@ def event_type(cls) -> str: return "anitya.VersionUpdate" @property - def version(self) -> Optional[str]: - # we will decide the version just when syncing release - # (for the particular branch etc.), - # until that we work with all the new versions - return None + def versions(self) -> list[str]: + return self._versions def _add_release_and_event(self): if not self._db_project_object or not self._db_project_event: @@ -94,7 +90,7 @@ def _add_release_and_event(self): self._db_project_object, self._db_project_event, ) = ProjectEventModel.add_anitya_multiple_versions_event( - versions=self._versions, + versions=self.versions, project_name=self.anitya_project_name, project_id=self.anitya_project_id, package=self.package_name, diff --git a/packit_service/events/event_data.py b/packit_service/events/event_data.py index c5a4e22b4..7ceca2124 100644 --- a/packit_service/events/event_data.py +++ b/packit_service/events/event_data.py @@ -7,7 +7,6 @@ from typing import Optional from ogr.abstract import GitProject -from ogr.parsing import RepoUrl from packit_service.config import ServiceConfig from packit_service.models import ( @@ -32,6 +31,7 @@ def __init__( event_id: int, project_url: str, tag_name: Optional[str], + tag_names: Optional[list[str]], git_ref: Optional[str], pr_id: Optional[int], commit_sha: Optional[str], @@ -49,6 +49,7 @@ def __init__( self.event_id = event_id self.project_url = project_url self.tag_name = tag_name + self.tag_names = tag_names self.git_ref = git_ref self.pr_id = pr_id self.commit_sha = commit_sha @@ -78,6 +79,7 @@ def from_event_dict(cls, event: dict): event_id = event.get("event_id") project_url = event.get("project_url") tag_name = event.get("tag_name") + tag_names = event.get("tag_names") git_ref = event.get("git_ref") # event has _pr_id as the attribute while pr_id is a getter property pr_id = event.get("_pr_id") or event.get("pr_id") @@ -107,6 +109,7 @@ def from_event_dict(cls, event: dict): event_id=event_id, project_url=project_url, tag_name=tag_name, + tag_names=tag_names, git_ref=git_ref, pr_id=pr_id, commit_sha=commit_sha, @@ -219,34 +222,14 @@ def _add_project_object_and_event(self): elif self.event_type in { "anitya.NewHotness", }: - if not self.project_url: - ( - self._db_project_object, - self._db_project_event, - ) = ProjectEventModel.add_anitya_version_event( - version=self.event_dict.get("version"), - project_name=self.event_dict.get("anitya_project_name"), - project_id=self.event_dict.get("anitya_project_id"), - package=self.event_dict.get("package_name"), - ) - return - - if self.project: - namespace = self.project.namespace - repo_name = self.project.repo - else: - repo_url = RepoUrl.parse(self.project_url) - namespace = repo_url.namespace - repo_name = repo_url.repo ( self._db_project_object, self._db_project_event, - ) = ProjectEventModel.add_release_event( - tag_name=self.tag_name, - namespace=namespace, - repo_name=repo_name, - project_url=self.project_url, - commit_hash=self.commit_sha, + ) = ProjectEventModel.add_anitya_multiple_versions_event( + versions=self.event_dict.get("versions"), + project_name=self.event_dict.get("anitya_project_name"), + project_id=self.event_dict.get("anitya_project_id"), + package=self.event_dict.get("package_name"), ) elif self.event_type in { "github.issue.Comment", diff --git a/packit_service/worker/checker/distgit.py b/packit_service/worker/checker/distgit.py index 32daca925..aca897213 100644 --- a/packit_service/worker/checker/distgit.py +++ b/packit_service/worker/checker/distgit.py @@ -272,7 +272,7 @@ def pre_check(self) -> bool: valid = True msg_to_report = None issue_title = ( - f"Pull from upstream could not be run for update {self.data.event_dict.get('version')}" + f"Pull from upstream could not be run for update {self.data.event_dict.get('versions')}" ) if self.package_config.upstream_project_url and not ( @@ -286,9 +286,9 @@ def pre_check(self) -> bool: valid = False if self.package_config.upstream_project_url and ( - self.data.event_type in (anitya.NewHotness.event_type(),) and not self.data.tag_name + self.data.event_type in (anitya.NewHotness.event_type(),) and not self.data.tag_names ): - msg_to_report = "We were not able to get the upstream tag name." + msg_to_report = "We were not able to get the upstream tag name(s)." valid = False if self.data.event_type in (pagure.pr.Comment.event_type(),): diff --git a/packit_service/worker/checker/run_condition.py b/packit_service/worker/checker/run_condition.py index 7e8c5f444..7b4b39b50 100644 --- a/packit_service/worker/checker/run_condition.py +++ b/packit_service/worker/checker/run_condition.py @@ -5,6 +5,7 @@ import logging import shutil import tempfile +from functools import cmp_to_key from pathlib import Path from typing import Optional @@ -18,6 +19,7 @@ ) from packit.config import JobConfig, PackageConfig from packit.exceptions import PackitCommandFailedError +from packit.utils.versions import compare_versions from specfile import Specfile from packit_service.events import ( @@ -127,8 +129,18 @@ def pre_check(self) -> bool: elif self.data.event_type in (anitya.NewHotness.event_type(),): event = anitya.NewHotness.from_event_dict(self.data.event_dict) project = event.project - git_ref = event.tag_name - version = event.version + # FIXME: for now get the highest version (and tag), in the future the list + # of all versions should be presented to users as an environment variable + version = ( + max(event.versions, key=cmp_to_key(compare_versions)) + if event.versions + else None + ) + git_ref = ( + max(event.tag_names, key=cmp_to_key(compare_versions)) + if event.tag_names + else None + ) elif self.data.event_type in ( github.issue.Comment.event_type(), gitlab.issue.Comment.event_type(), diff --git a/packit_service/worker/handlers/distgit.py b/packit_service/worker/handlers/distgit.py index 0d198d85b..2dc7b6436 100644 --- a/packit_service/worker/handlers/distgit.py +++ b/packit_service/worker/handlers/distgit.py @@ -57,6 +57,7 @@ KojiTagRequestGroupModel, KojiTagRequestTargetModel, PipelineModel, + ProjectEventModel, SyncReleaseJobType, SyncReleaseModel, SyncReleasePullRequestModel, @@ -263,6 +264,8 @@ def sync_branch( self, branch: str, model: SyncReleaseModel, + tag: Optional[str] = None, + version: Optional[str] = None, ) -> Optional[tuple[PullRequest, dict[str, PullRequest]]]: try: branch_suffix = f"update-{self.sync_release_job_type.value}" @@ -287,11 +290,9 @@ def sync_branch( branch, ), } - if not self.packit_api.non_git_upstream: - kwargs["tag"] = self.tag - elif (version := self.data.event_dict.get("version")) or ( - version := self.get_version_from_comment() - ): + if tag: + kwargs["tag"] = tag + elif version: kwargs["versions"] = [version] downstream_pr, additional_prs = self.packit_api.sync_release(**kwargs) except PackitDownloadFailedException as ex: @@ -340,13 +341,13 @@ def sync_branch( return downstream_pr, additional_prs - def _get_or_create_sync_release_run(self) -> SyncReleaseModel: + def _get_or_create_sync_release_run(self, project_event_model=None) -> SyncReleaseModel: if self._sync_release_run_id is not None: return SyncReleaseModel.get_by_id(self._sync_release_run_id) sync_release_model, _ = SyncReleaseModel.create_with_new_run( status=SyncReleaseStatus.running, - project_event_model=self.data.db_project_event, + project_event_model=project_event_model or self.data.db_project_event, job_type=( SyncReleaseJobType.propose_downstream if self.job_config.type == JobType.propose_downstream @@ -368,6 +369,8 @@ def run_for_target( self, sync_release_run_model: SyncReleaseModel, model: SyncReleaseTargetModel, + tag: Optional[str] = None, + version: Optional[str] = None, ) -> Optional[str]: """ Run sync-release for the single target specified by the given model. @@ -415,6 +418,8 @@ def run_for_target( downstream_pr, additional_prs = self.sync_branch( branch=branch, model=sync_release_run_model, + tag=tag, + version=version, ) logger.debug("Downstream PR(s) created successfully.") model.set_downstream_pr_url(downstream_pr_url=downstream_pr.url) @@ -488,18 +493,98 @@ def run_for_target( # no error occurred return None + def _get_releases_to_sync(self) -> list[tuple[Optional[str], Optional[str]]]: + """Get list of (tag, version) pairs to sync. + + For git upstreams, returns (tag, None) pairs. + For non-git upstreams, returns (None, version) pairs. + """ + if not self.packit_api.non_git_upstream: + return [(tag, None) for tag in self.tags] + + versions = self.data.event_dict.get("versions") or [] + if not versions: + version = self.get_version_from_comment() + if version: + versions = [version] + if versions: + return [(None, version) for version in versions] + # non-git upstream with no version info — run without tag/version + return [(None, None)] + def _run(self) -> TaskResults: """ Sync the upstream release to dist-git as a pull request. """ + all_errors = {} + + try: + for tag, version in self._get_releases_to_sync(): + errors = self._run_for_release(tag=tag, version=version) + all_errors.update(errors) + except AbortSyncRelease: + return TaskResults( + success=True, # do not create a Sentry issue + details={"msg": "Not able to download archive. Task will be retried."}, + ) + finally: + # remove temporary dist-git clone after we're done here - context: + # 1. the dist-git repo could be cloned on worker, not sandbox + # 2. in such case it's stored in /tmp, not in the mirrored sandbox PV + # 3. it's not being cleaned up and it wastes pod's filesystem space + shutil.rmtree(self.packit_api.dg.local_project.working_dir) + + if all_errors: + return TaskResults( + success=False, + details={ + "msg": f"{self.sync_release_job_type} failed.", + "errors": all_errors, + }, + ) + + return TaskResults(success=True, details={}) + + def _create_release_event(self, tag: str) -> ProjectEventModel: + """Create a ProjectReleaseModel and ProjectEventModel for a specific tag.""" + _, event = ProjectEventModel.add_release_event( + tag_name=tag, + namespace=self.data.event_dict.get("repo_namespace"), + repo_name=self.data.event_dict.get("repo_name"), + project_url=self.data.project_url, + commit_hash=self.data.commit_sha, + ) + return event + + def _run_for_release( + self, + tag: Optional[str] = None, + version: Optional[str] = None, + ) -> dict[str, str]: + """ + Run the sync-release pipeline for a single tag/version across all branches. + + Returns: + Dict of branch → error message for branches that failed. + """ errors = {} - sync_release_run_model = self._get_or_create_sync_release_run() + release_event = ( + self._create_release_event(tag) + if tag and self.data.event_type == anitya.NewHotness.event_type() + else None + ) + sync_release_run_model = self._get_or_create_sync_release_run(release_event) branches_to_run = [target.branch for target in sync_release_run_model.sync_release_targets] - logger.debug(f"Branches to run {self.job_config.type}: {branches_to_run}") + logger.debug( + f"Branches to run {self.job_config.type} " + f"(tag={tag}, version={version}): {branches_to_run}" + ) try: for model in sync_release_run_model.sync_release_targets: - if error := self.run_for_target(sync_release_run_model, model): + if error := self.run_for_target( + sync_release_run_model, model, tag=tag, version=version + ): errors[model.branch] = error except AbortSyncRelease: logger.debug( @@ -518,16 +603,7 @@ def _run(self) -> TaskResults: url="", ) - return TaskResults( - success=True, # do not create a Sentry issue - details={"msg": "Not able to download archive. Task will be retried."}, - ) - finally: - # remove temporary dist-git clone after we're done here - context: - # 1. the dist-git repo could be cloned on worker, not sandbox - # 2. in such case it's stored in /tmp, not in the mirrored sandbox PV - # 3. it's not being cleaned up and it wastes pod's filesystem space - shutil.rmtree(self.packit_api.dg.local_project.working_dir) + raise models_with_errors = [ target @@ -557,20 +633,14 @@ def _run(self) -> TaskResults: command="pull-from-upstream", ) - self._report_errors_for_each_branch(body_msg) + self._report_errors_for_each_branch(body_msg, release=tag or version) sync_release_run_model.set_status(status=SyncReleaseStatus.error) - return TaskResults( - success=False, - details={ - "msg": f"{self.sync_release_job_type} failed.", - "errors": errors, - }, - ) + else: + sync_release_run_model.set_status(status=SyncReleaseStatus.finished) - sync_release_run_model.set_status(status=SyncReleaseStatus.finished) - return TaskResults(success=True, details={}) + return errors - def _report_errors_for_each_branch(self, message: str): + def _report_errors_for_each_branch(self, message: str, release: Optional[str] = None): raise NotImplementedError("Use subclass.") def get_resolved_bugs(self): @@ -631,7 +701,7 @@ def __init__( def get_checkers() -> tuple[type[Checker], ...]: return (IsUpstreamTagMatchingConfig,) - def _report_errors_for_each_branch(self, message: str) -> None: + def _report_errors_for_each_branch(self, message: str, release: Optional[str] = None) -> None: if not self.job_config.notifications.failure_issue.create: logger.debug("Reporting via issues disabled in config, skipping.") return @@ -651,7 +721,7 @@ def _report_errors_for_each_branch(self, message: str) -> None: create_issue_if_needed( project=self.project, - title=f"{self.job_name_for_reporting.capitalize()} failed for release {self.tag}", + title=f"{self.job_name_for_reporting.capitalize()} failed for release {release}", message=body_msg, comment_to_existing=body_msg, ) @@ -738,7 +808,7 @@ def get_resolved_bugs(self) -> list[str]: ) return bugs.split(",") - def _report_errors_for_each_branch(self, message: str) -> None: + def _report_errors_for_each_branch(self, message: str, release: Optional[str] = None) -> None: body_msg = ( f"{message}\n\n---\n\n*Get in [touch with us]({CONTACTS_URL}) if you need some help.*\n" ) @@ -753,7 +823,7 @@ def _report_errors_for_each_branch(self, message: str) -> None: report_in_issue_repository( issue_repository=self.job_config.issue_repository, service_config=self.service_config, - title=f"Pull from upstream failed for release {self.tag}", + title=f"Pull from upstream failed for release {release}", message=long_message, comment_to_existing=short_message, ) diff --git a/packit_service/worker/mixin.py b/packit_service/worker/mixin.py index f4b2e402b..cbb29cb1a 100644 --- a/packit_service/worker/mixin.py +++ b/packit_service/worker/mixin.py @@ -198,7 +198,7 @@ def clean_api(self) -> None: class GetSyncReleaseTagMixin(PackitAPIWithUpstreamMixin): - _tag: Optional[str] = None + _tags: Optional[list[str]] = None def get_version_from_comment(self) -> Optional[str]: """ @@ -223,14 +223,20 @@ def get_version_from_comment(self) -> Optional[str]: return args[idx + 1] if idx < len(args) - 1 else None @property - def tag(self) -> Optional[str]: - self._tag = self.data.tag_name - if not self._tag and not self.non_git_upstream: - # there is no tag information when retriggering pull-from-upstream - # from dist-git PR, use the version from the comment if provided, - # otherwise fall back to the last tag - self._tag = self.get_version_from_comment() or self.packit_api.up.get_last_tag() - return self._tag + def tags(self) -> list[str]: + self._tags = self.data.tag_names + if not self._tags: + if self.data.tag_name: + self._tags = [self.data.tag_name] + elif not self.non_git_upstream: + # there is no tag information when retriggering pull-from-upstream + # from dist-git PR, use the version from the comment if provided, + # otherwise fall back to the last tag + tag = self.get_version_from_comment() or self.packit_api.up.get_last_tag() + self._tags = [tag] if tag else [] + else: + self._tags = [] + return self._tags class LocalProjectMixin(Config): diff --git a/packit_service/worker/parser.py b/packit_service/worker/parser.py index d897e986c..f548f957b 100644 --- a/packit_service/worker/parser.py +++ b/packit_service/worker/parser.py @@ -1770,20 +1770,20 @@ def parse_new_hotness_update_event(event) -> Optional[anitya.NewHotness]: distgit_project_url = f"{dg_base_url}rpms/{package_name}" - version = nested_get(event, "trigger", "msg", "project", "version") + versions = nested_get(event, "trigger", "msg", "message", "upstream_versions") bug_id = nested_get(event, "bug", "bug_id") anitya_project_id = nested_get(event, "trigger", "msg", "project", "id") anitya_project_name = nested_get(event, "trigger", "msg", "project", "name") logger.info( - f"New hotness update event for package: {package_name}, version: {version}," + f"New hotness update event for package: {package_name}, versions: {versions}," f" bug ID: {bug_id}", ) return anitya.NewHotness( package_name=package_name, - version=version, + versions=versions, distgit_project_url=distgit_project_url, bug_id=bug_id, anitya_project_id=anitya_project_id, diff --git a/tests/integration/test_new_hotness_update.py b/tests/integration/test_new_hotness_update.py index 3d0d1a042..c19e4eeba 100644 --- a/tests/integration/test_new_hotness_update.py +++ b/tests/integration/test_new_hotness_update.py @@ -17,8 +17,8 @@ import packit_service.worker.checker.distgit from packit_service.config import ServiceConfig from packit_service.models import ( + AnityaMultipleVersionsModel, AnityaProjectModel, - AnityaVersionModel, PipelineModel, ProjectEventModel, ProjectEventModelType, @@ -47,31 +47,55 @@ def fedora_branches(): @pytest.fixture def sync_release_model(): - db_project_object = flexmock( + # Anitya event creates AnityaMultipleVersionsModel as the representative event + anitya_db_object = flexmock( id=12, - project_event_model_type=ProjectEventModelType.release, + project_event_model_type=ProjectEventModelType.anitya_multiple_versions, job_config_trigger_type=JobConfigTriggerType.release, + project=flexmock(AnityaProjectModel), ) - project_event = ( - flexmock().should_receive("get_project_event_object").and_return(db_project_object).mock() + anitya_event = ( + flexmock().should_receive("get_project_event_object").and_return(anitya_db_object).mock() ) - run_model = flexmock(PipelineModel) + flexmock(AnityaMultipleVersionsModel).should_receive("get_or_create").with_args( + versions=["7.0.3"], + project_name="redis", + project_id=4181, + package="redis", + ).and_return(anitya_db_object) flexmock(ProjectEventModel).should_receive("get_or_create").with_args( - type=ProjectEventModelType.release, + type=ProjectEventModelType.anitya_multiple_versions, event_id=12, commit_sha=None, - ).and_return(project_event) + ).and_return(anitya_event) + + # Per-tag release event created by the handler + release_db_object = flexmock( + id=13, + project_event_model_type=ProjectEventModelType.release, + job_config_trigger_type=JobConfigTriggerType.release, + ) + release_event = ( + flexmock().should_receive("get_project_event_object").and_return(release_db_object).mock() + ) flexmock(ProjectReleaseModel).should_receive("get_or_create").with_args( tag_name="7.0.3", namespace="packit-service", repo_name="hello-world", project_url="https://github.com/packit-service/hello-world", commit_hash=None, - ).and_return(db_project_object) + ).and_return(release_db_object) + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=ProjectEventModelType.release, + event_id=13, + commit_sha=None, + ).and_return(release_event) + + run_model = flexmock(PipelineModel) sync_release_model = flexmock(id=123, sync_release_targets=[]) flexmock(SyncReleaseModel).should_receive("create_with_new_run").with_args( status=SyncReleaseStatus.running, - project_event_model=project_event, + project_event_model=release_event, job_type=SyncReleaseJobType.pull_from_upstream, package_name="redis", ).and_return(sync_release_model, run_model).once() @@ -86,7 +110,7 @@ class AnityaTestProjectModel(AnityaProjectModel): db_project_object = flexmock( id=12, - project_event_model_type=ProjectEventModelType.release, + project_event_model_type=ProjectEventModelType.anitya_multiple_versions, job_config_trigger_type=JobConfigTriggerType.release, project=AnityaTestProjectModel(), ) @@ -95,12 +119,12 @@ class AnityaTestProjectModel(AnityaProjectModel): ) run_model = flexmock(PipelineModel) flexmock(ProjectEventModel).should_receive("get_or_create").with_args( - type=ProjectEventModelType.release, + type=ProjectEventModelType.anitya_multiple_versions, event_id=12, commit_sha=None, ).and_return(project_event) - flexmock(AnityaVersionModel).should_receive("get_or_create").with_args( - version="7.0.3", + flexmock(AnityaMultipleVersionsModel).should_receive("get_or_create").with_args( + versions=["7.0.3"], project_name="redis", project_id=4181, package="redis", @@ -302,6 +326,28 @@ def test_new_hotness_update_pre_check_fail(new_hotness_update): flexmock(Allowlist, check_and_report=True) + # Anitya event now always creates AnityaMultipleVersionsModel + anitya_db_object = flexmock( + id=12, + project_event_model_type=ProjectEventModelType.anitya_multiple_versions, + job_config_trigger_type=JobConfigTriggerType.release, + project=flexmock(AnityaProjectModel), + ) + anitya_event = ( + flexmock().should_receive("get_project_event_object").and_return(anitya_db_object).mock() + ) + flexmock(AnityaMultipleVersionsModel).should_receive("get_or_create").with_args( + versions=["7.0.3"], + project_name="redis", + project_id=4181, + package="redis", + ).and_return(anitya_db_object) + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=ProjectEventModelType.anitya_multiple_versions, + event_id=12, + commit_sha=None, + ).and_return(anitya_event) + flexmock(Pushgateway).should_receive("push").times(1).and_return() service_config = ServiceConfig().get_service_config() flexmock(service_config).should_receive("get_project").with_args( @@ -317,7 +363,7 @@ def test_new_hotness_update_pre_check_fail(new_hotness_update): ).with_args( issue_repository="https://github.com/packit/issue_repository", service_config=service_config, - title="Pull from upstream could not be run for update 7.0.3", + title="Pull from upstream could not be run for update ['7.0.3']", message=msg, comment_to_existing=msg, ) @@ -325,6 +371,193 @@ def test_new_hotness_update_pre_check_fail(new_hotness_update): SteveJobs().process_message(new_hotness_update) +def test_new_hotness_update_non_git_multiple_versions(new_hotness_update): + """Test that multiple upstream versions each trigger a separate sync_release call.""" + # Modify the event to carry two versions + new_hotness_update["trigger"]["msg"]["message"]["upstream_versions"] = ["7.0.3", "7.0.4"] + + class AnityaTestProjectModel(AnityaProjectModel): + pass + + db_project_object = flexmock( + id=12, + project_event_model_type=ProjectEventModelType.anitya_multiple_versions, + job_config_trigger_type=JobConfigTriggerType.release, + project=AnityaTestProjectModel(), + ) + project_event = ( + flexmock().should_receive("get_project_event_object").and_return(db_project_object).mock() + ) + run_model = flexmock(PipelineModel) + flexmock(ProjectEventModel).should_receive("get_or_create").with_args( + type=ProjectEventModelType.anitya_multiple_versions, + event_id=12, + commit_sha=None, + ).and_return(project_event) + flexmock(AnityaMultipleVersionsModel).should_receive("get_or_create").with_args( + versions=["7.0.3", "7.0.4"], + project_name="redis", + project_id=4181, + package="redis", + ).and_return(db_project_object) + + # Two sync release models – one per version + sync_release_model_1 = flexmock(id=123, sync_release_targets=[]) + sync_release_model_2 = flexmock(id=124, sync_release_targets=[]) + flexmock(SyncReleaseModel).should_receive("create_with_new_run").with_args( + status=SyncReleaseStatus.running, + project_event_model=project_event, + job_type=SyncReleaseJobType.pull_from_upstream, + package_name="redis", + ).and_return(sync_release_model_1, run_model).and_return( + sync_release_model_2, run_model + ).twice() + + # One target model per version + model_1 = flexmock(status="queued", id=1234, branch="main") + model_2 = flexmock(status="queued", id=1235, branch="main") + flexmock(SyncReleaseTargetModel).should_receive("create").with_args( + status=SyncReleaseTargetStatus.queued, + branch="main", + ).and_return(model_1).and_return(model_2).twice() + flexmock(SyncReleasePullRequestModel).should_receive("get_or_create").with_args( + pr_id=21, + namespace="downstream-namespace", + repo_name="downstream-repo", + project_url="https://src.fedoraproject.org/rpms/downstream-repo", + target_branch=str, + url=str, + ).and_return(flexmock(sync_release_targets=[flexmock()])) + + packit_yaml = ( + "{'specfile_path': 'hello-world.spec', " + "jobs: [{trigger: release, job: pull_from_upstream, metadata: {targets:[]}}]}" + ) + flexmock(Github, get_repo=lambda full_name_or_id: None) + distgit_project = flexmock( + get_files=lambda ref, recursive: [".packit.yaml"], + get_file_content=lambda path, ref, headers: packit_yaml, + full_repo_name="rpms/redis", + repo="redis", + namespace="rpms", + is_private=lambda: False, + default_branch="main", + ) + + lp = flexmock(LocalProject, refresh_the_arguments=lambda: None) + flexmock(LocalProjectBuilder, _refresh_the_state=lambda *args: lp) + lp.working_dir = "" + flexmock(DistGit).should_receive("local_project").and_return(lp) + + flexmock(Allowlist, check_and_report=True) + + service_config = ServiceConfig().get_service_config() + flexmock(service_config).should_receive("get_project").with_args( + "https://src.fedoraproject.org/rpms/redis", + required=False, + ).and_return(distgit_project) + flexmock(service_config).should_receive("get_project").with_args( + "https://src.fedoraproject.org/rpms/redis", + ).and_return(distgit_project) + + target_project = ( + flexmock(namespace="downstream-namespace", repo="downstream-repo") + .should_receive("get_web_url") + .and_return("https://src.fedoraproject.org/rpms/downstream-repo") + .mock() + ) + pr = ( + flexmock( + id=21, + url="some_url", + target_project=target_project, + description="some-title", + ) + .should_receive("comment") + .mock() + ) + # sync_release should be called once per version + flexmock(PackitAPI).should_receive("sync_release").with_args( + dist_git_branch="main", + versions=["7.0.3"], + create_pr=True, + local_pr_branch_suffix="update-pull_from_upstream", + use_downstream_specfile=True, + add_pr_instructions=True, + resolved_bugs=["rhbz#2106196"], + release_monitoring_project_id=4181, + sync_acls=True, + pr_description_footer=DistgitAnnouncement.get_announcement(), + add_new_sources=True, + fast_forward_merge_branches=set(), + ).and_return((pr, {})).once() + flexmock(PackitAPI).should_receive("sync_release").with_args( + dist_git_branch="main", + versions=["7.0.4"], + create_pr=True, + local_pr_branch_suffix="update-pull_from_upstream", + use_downstream_specfile=True, + add_pr_instructions=True, + resolved_bugs=["rhbz#2106196"], + release_monitoring_project_id=4181, + sync_acls=True, + pr_description_footer=DistgitAnnouncement.get_announcement(), + add_new_sources=True, + fast_forward_merge_branches=set(), + ).and_return((pr, {})).once() + flexmock(PackitAPI).should_receive("clean") + + for model, sync_release_model in [ + (model_1, sync_release_model_1), + (model_2, sync_release_model_2), + ]: + flexmock(model).should_receive("set_status").with_args( + status=SyncReleaseTargetStatus.running, + ).once() + flexmock(model).should_receive("set_downstream_pr_url").with_args( + downstream_pr_url="some_url", + ) + flexmock(model).should_receive("set_downstream_prs").with_args( + downstream_prs=list, + ).once() + flexmock(model).should_receive("set_status").with_args( + status=SyncReleaseTargetStatus.submitted, + ).once() + flexmock(model).should_receive("set_start_time").once() + flexmock(model).should_receive("set_finished_time").once() + flexmock(model).should_receive("set_logs").once() + flexmock(sync_release_model).should_receive("set_status").with_args( + status=SyncReleaseStatus.finished, + ).once() + sync_release_model.should_receive("get_package_name").and_return(None) + + flexmock(IsRunConditionSatisfied).should_receive("pre_check").and_return(True) + + flexmock(AddReleaseEventToDb).should_receive("db_project_object").and_return( + flexmock( + job_config_trigger_type=JobConfigTriggerType.release, + id=123, + project_event_model_type=ProjectEventModelType.release, + ), + ) + flexmock(group).should_receive("apply_async").once() + flexmock(Pushgateway).should_receive("push").times(2).and_return() + flexmock(shutil).should_receive("rmtree").with_args("") + + processing_results = SteveJobs().process_message(new_hotness_update) + event_dict, _, job_config, package_config = get_parameters_from_results( + processing_results, + ) + assert json.dumps(event_dict) + + results = run_pull_from_upstream_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + assert first_dict_value(results["job"])["success"] + + def test_new_hotness_update_non_git(new_hotness_update, sync_release_model_non_git): model = flexmock(status="queued", id=1234, branch="main") flexmock(SyncReleaseTargetModel).should_receive("create").with_args( diff --git a/tests/unit/events/test_anitya.py b/tests/unit/events/test_anitya.py index a7e4a3c2f..f338d6b11 100644 --- a/tests/unit/events/test_anitya.py +++ b/tests/unit/events/test_anitya.py @@ -10,9 +10,9 @@ from packit_service.events.anitya import NewHotness, VersionUpdate from packit_service.models import ( + AnityaMultipleVersionsModel, ProjectEventModel, ProjectEventModelType, - ProjectReleaseModel, ) from packit_service.package_config_getter import PackageConfigGetter from packit_service.worker.parser import Parser @@ -32,46 +32,40 @@ def anitya_version_update(): @pytest.mark.parametrize( - "upstream_project_url, upstream_tag_template, create_db_trigger, " - "tag_name, repo_namespace, repo_name", + "upstream_project_url, upstream_tag_template, tag_names, repo_namespace, repo_name", [ ( "https://github.com/redis-namespace/redis", None, - True, - "7.0.3", + ["7.0.3"], "redis-namespace", "redis", ), ( "https://github.com/redis-namespace/redis", "no-version-tag", - True, - "no-version-tag", + ["no-version-tag"], "redis-namespace", "redis", ), ( "https://github.com/redis-namespace/redis", "v{version}", - True, - "v7.0.3", + ["v7.0.3"], "redis-namespace", "redis", ), ( "https://github.com/redis-namespace", None, - False, - "7.0.3", + ["7.0.3"], None, "redis-namespace", ), ( "https://github.com/redis-namespace/another-level/redis", None, - True, - "7.0.3", + ["7.0.3"], "redis-namespace/another-level", "redis", ), @@ -81,8 +75,7 @@ def test_parse_new_hotness_update( new_hotness_update, upstream_project_url, upstream_tag_template, - create_db_trigger, - tag_name, + tag_names, repo_namespace, repo_name, ): @@ -105,20 +98,23 @@ def test_parse_new_hotness_update( ), ).once() + # Event class now always creates AnityaMultipleVersionsModel + flexmock(AnityaMultipleVersionsModel).should_receive("get_or_create").with_args( + versions=["7.0.3"], + project_name="redis", + project_id=4181, + package="redis", + ).and_return( + flexmock( + project_event_model_type=ProjectEventModelType.anitya_multiple_versions, + id="123", + ), + ) flexmock(ProjectEventModel).should_receive("get_or_create").with_args( - type=ProjectEventModelType.release, + type=ProjectEventModelType.anitya_multiple_versions, event_id="123", commit_sha=None, ).and_return(flexmock()) - flexmock(ProjectReleaseModel).should_receive("get_or_create").with_args( - tag_name=tag_name, - namespace=repo_namespace, - repo_name=repo_name, - project_url=upstream_project_url, - commit_hash=None, - ).and_return( - flexmock(project_event_model_type=ProjectEventModelType.release, id="123"), - ) assert isinstance(event_object, NewHotness) assert isinstance(event_object.project, PagureProject) @@ -127,11 +123,9 @@ def test_parse_new_hotness_update( assert event_object.repo_namespace == repo_namespace assert event_object.repo_name == repo_name assert event_object.distgit_project_url == "https://src.fedoraproject.org/rpms/redis" - assert event_object.tag_name == tag_name + assert event_object.tag_names == tag_names assert event_object.packages_config - - if create_db_trigger: - assert event_object.db_project_object + assert event_object.db_project_object # [NOTE] doesn't exist in CentOS… I've added CentOS entry to the event payload diff --git a/tests/unit/test_distgit.py b/tests/unit/test_distgit.py index a82d5b3a9..1cee79fb9 100644 --- a/tests/unit/test_distgit.py +++ b/tests/unit/test_distgit.py @@ -64,12 +64,14 @@ def test_create_one_issue_for_pr(): "f34": "Propose downstream failed for release 056", "f35": "Propose downstream failed for release 056", }, + release="056", ) handler._report_errors_for_each_branch( { "f34": "Propose downstream failed for release 056", "f35": "Propose downstream failed for release 056", }, + release="056", )