Skip to content

[PR #1424/482bd0d4 backport][3.8] Allow publication of empty repositories #1455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
patchback wants to merge 1 commit into
base: 3.8
Choose a base branch
from
Open

[PR #1424/482bd0d4 backport][3.8] Allow publication of empty repositories #1455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES/+publish_empty_repos.feature
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Provide minimal metadata files for empty repository versions during structured publish.
4 changes: 4 additions & 0 deletions pulp_deb/app/settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,7 @@
FORBIDDEN_CHECKSUM_WARNINGS = True
FORCE_IGNORE_MISSING_PACKAGE_INDICES = False
PERMISSIVE_SYNC = False

STRUCTURED_EMPTY_REPO_DISTRIBUTION = "default"
STRUCTURED_EMPTY_REPO_COMPONENT = "empty"
STRUCTURED_EMPTY_REPO_ARCHITECTURES = ["all"]
180 changes: 107 additions & 73 deletions pulp_deb/app/tasks/publishing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ def publish(
"distribution", flat=True
)
)
has_structured_distribution = bool(distributions)

if simple and "default" in distributions:
message = (
Expand All @@ -186,55 +187,23 @@ def publish(
distributions.remove("default")

release_helpers = []
for distribution in distributions:
architectures = list(
ReleaseArchitecture.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
distribution=distribution,
)
.distinct("architecture")
.values_list("architecture", flat=True)
)

if not has_structured_distribution and not simple:
distribution = settings.STRUCTURED_EMPTY_REPO_DISTRIBUTION
components = [settings.STRUCTURED_EMPTY_REPO_COMPONENT]
architectures = list(settings.STRUCTURED_EMPTY_REPO_ARCHITECTURES)
if "all" not in architectures:
architectures.append("all")

release = Release.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
codename = distribution.strip("/").split("/")[0]
release = Release(
distribution=distribution,
).first()
publish_upstream = (
publish_upstream_release_fields
if publish_upstream_release_fields is not None
else repository.publish_upstream_release_fields
)
if not release:
codename = distribution.strip("/").split("/")[0]
release = Release(
distribution=distribution,
codename=codename,
suite=codename,
origin="Pulp 3",
)
if repository.description:
release.description = repository.description
elif not publish_upstream:
release = Release(
distribution=release.distribution,
codename=release.codename,
suite=release.suite,
origin="Pulp 3",
)
if repository.description:
release.description = repository.description

release_components_filtered = release_components.filter(
distribution=distribution
)
components = list(
release_components_filtered.distinct("component").values_list(
"component", flat=True
)
codename=codename,
suite=codename,
origin="Pulp 3",
)
if repository.description:
release.description = repository.description

signing_service = repository.release_signing_service(release)

Expand All @@ -246,43 +215,108 @@ def publish(
temp_dir=temp_dir,
signing_service=signing_service,
)
release_helper.save_unsigned_metadata()
release_helpers.append(release_helper)
else:
for distribution in distributions:
architectures = list(
ReleaseArchitecture.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
distribution=distribution,
)
.distinct("architecture")
.values_list("architecture", flat=True)
)
if "all" not in architectures:
architectures.append("all")

package_release_components = PackageReleaseComponent.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
release_component__in=release_components_filtered,
).select_related("release_component", "package")
release = Release.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
distribution=distribution,
).first()
publish_upstream = (
publish_upstream_release_fields
if publish_upstream_release_fields is not None
else repository.publish_upstream_release_fields
)
if not release:
codename = distribution.strip("/").split("/")[0]
release = Release(
distribution=distribution,
codename=codename,
suite=codename,
origin="Pulp 3",
)
if repository.description:
release.description = repository.description
elif not publish_upstream:
release = Release(
distribution=release.distribution,
codename=release.codename,
suite=release.suite,
origin="Pulp 3",
)
if repository.description:
release.description = repository.description

release_components_filtered = release_components.filter(
distribution=distribution
)
components = list(
release_components_filtered.distinct("component").values_list(
"component", flat=True
)
)

signing_service = repository.release_signing_service(release)

source_package_release_components = (
SourcePackageReleaseComponent.objects.filter(
release_helper = _ReleaseHelper(
publication=publication,
components=components,
architectures=architectures,
release=release,
temp_dir=temp_dir,
signing_service=signing_service,
)

package_release_components = PackageReleaseComponent.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
release_component__in=release_components_filtered,
).select_related("release_component", "source_package")
)
).select_related("release_component", "package")

for component in components:
packages = Package.objects.filter(
pk__in=[
prc.package.pk
for prc in package_release_components
if prc.release_component.component == component
]
).prefetch_related("contentartifact_set", "_artifacts")
artifact_dict, remote_artifact_dict = _batch_fetch_artifacts(packages)
release_helper.components[component].add_packages(
packages,
artifact_dict,
remote_artifact_dict,
source_package_release_components = (
SourcePackageReleaseComponent.objects.filter(
pk__in=repo_version.content.order_by("-pulp_created"),
release_component__in=release_components_filtered,
).select_related("release_component", "source_package")
)

source_packages = [
drc.source_package
for drc in source_package_release_components
if drc.release_component.component == component
]
release_helper.components[component].add_source_packages(source_packages)
for component in components:
packages = Package.objects.filter(
pk__in=[
prc.package.pk
for prc in package_release_components
if prc.release_component.component == component
]
).prefetch_related("contentartifact_set", "_artifacts")
artifact_dict, remote_artifact_dict = _batch_fetch_artifacts(packages)
release_helper.components[component].add_packages(
packages,
artifact_dict,
remote_artifact_dict,
)

source_packages = [
drc.source_package
for drc in source_package_release_components
if drc.release_component.component == component
]
release_helper.components[component].add_source_packages(
source_packages
)

release_helper.save_unsigned_metadata()
release_helpers.append(release_helper)
release_helper.save_unsigned_metadata()
release_helpers.append(release_helper)

asyncio.run(_concurrently_sign_metadata(release_helpers))
for release_helper in release_helpers:
Expand Down
83 changes: 83 additions & 0 deletions pulp_deb/tests/functional/api/test_publish.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import pytest
from debian import deb822
from django.conf import settings

from pulpcore.client.pulp_deb.exceptions import ApiException

Expand Down Expand Up @@ -594,6 +595,88 @@ def test_remove_all_content_from_repository(
assert len(prcs) == 0


@pytest.mark.parallel
def test_publish_truly_empty_repository_structured(
apt_distribution_api,
deb_distribution_factory,
deb_publication_factory,
deb_repository_factory,
download_content_unit,
):
"""Test that a truly empty repository can be published in structured mode.

The publication should synthesize a default distribution/component and publish
empty package metadata that apt can consume.
"""
repo = deb_repository_factory()

publication = deb_publication_factory(
repo,
simple=False,
structured=True,
)

distribution = deb_distribution_factory(publication)
distribution = apt_distribution_api.read(distribution.pulp_href)

base_path = distribution.to_dict()["base_path"]

dist_path = "dists/" + settings.STRUCTURED_EMPTY_REPO_DISTRIBUTION + "/"
comp_path = dist_path + settings.STRUCTURED_EMPTY_REPO_COMPONENT + "/"

release_path = dist_path + "Release"
packages_path = comp_path + "binary-all/Packages"
packages_gz_path = comp_path + "binary-all/Packages.gz"

release = download_content_unit(base_path, release_path).decode("utf-8")
packages = download_content_unit(base_path, packages_path)
packages_gz = download_content_unit(base_path, packages_gz_path)

assert "404: Not Found" not in release
assert packages is not None
assert packages_gz is not None

assert "Codename: " + settings.STRUCTURED_EMPTY_REPO_DISTRIBUTION in release
assert "Suite: " + settings.STRUCTURED_EMPTY_REPO_DISTRIBUTION in release
assert "Components: " + settings.STRUCTURED_EMPTY_REPO_COMPONENT in release
assert "Architectures: all" in release
assert settings.STRUCTURED_EMPTY_REPO_COMPONENT + "/binary-all/Packages" in release
assert settings.STRUCTURED_EMPTY_REPO_COMPONENT + "/binary-all/Packages.gz" in release

assert packages.decode("utf-8") == ""


@pytest.mark.parallel
def test_publish_truly_empty_repository_simple_only(
apt_distribution_api,
deb_distribution_factory,
deb_publication_factory,
deb_repository_factory,
download_content_unit,
):
"""Test that simple mode keeps its existing empty-repo behavior."""
repo = deb_repository_factory()

publication = deb_publication_factory(
repo,
simple=True,
structured=False,
)

distribution = deb_distribution_factory(publication)
distribution = apt_distribution_api.read(distribution.pulp_href)

base_path = distribution.to_dict()["base_path"]

release = download_content_unit(base_path, "dists/default/Release").decode("utf-8")
packages = download_content_unit(base_path, "dists/default/all/binary-all/Packages")

assert "Codename: default" in release
assert "Components: all" in release
assert "Architectures: all" in release
assert packages.decode("utf-8") == ""


def assert_equal_package_index(orig, new):
"""In-detail check of two PackageIndex file-strings"""
parsed_orig = parse_package_index(orig)
Expand Down
Loading