diff --git a/CHANGES/2096.bugfix b/CHANGES/2096.bugfix new file mode 100644 index 000000000..18626b9c3 --- /dev/null +++ b/CHANGES/2096.bugfix @@ -0,0 +1 @@ +Fixed syncing of cosign signatures, attestations, and SBOMs (stored as companion tags) being silently skipped when `include_tags` was set on the remote. diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index 12090db09..244b8ba52 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -40,6 +40,8 @@ log = logging.getLogger(__name__) +COSIGN_TAG_SUFFIXES = (".sig", ".att", ".sbom") + class ContainerFirstStage(Stage): """ @@ -60,6 +62,9 @@ def __init__(self, remote, signed_only): self.manifest_list_dcs = [] self.manifest_dcs = [] self.signature_dcs = [] + self._synced_digests = set() + self._full_tag_list = [] + self._cosign_tags = [] async def _download_manifest_data(self, manifest_url): downloader = self.remote.get_downloader(url=manifest_url) @@ -92,11 +97,6 @@ async def run(self): """ ContainerFirstStage. """ - - to_download = [] - BATCH_SIZE = 500 - - # it can be whether a separate sigstore location or registry with extended signatures API signature_source = await self.get_signature_source() async with ProgressReport( @@ -104,12 +104,50 @@ async def run(self): ) as pb: repo_name = self.remote.namespaced_upstream_name tag_list_url = "/v2/{name}/tags/list".format(name=repo_name) - tag_list = await self.get_paginated_tag_list(tag_list_url, repo_name) - tag_list = filter_resources( - tag_list, self.remote.include_tags, self.remote.exclude_tags + self._full_tag_list = await self.get_paginated_tag_list(tag_list_url, repo_name) + self._cosign_tags = filter_resources( + self._full_tag_list, ["sha256-*"], self.remote.exclude_tags ) + if self.remote.include_tags or self.remote.exclude_tags: + # Split sync into two parts, first all non-cosign tags, then cosign tags + exclude_tags_and_cosign = (self.remote.exclude_tags or []) + ["sha256-*"] + tag_list = filter_resources( + self._full_tag_list, self.remote.include_tags, exclude_tags_and_cosign + ) + else: + tag_list = self._full_tag_list await pb.aincrement() + await self._process_tags(tag_list, signature_source) + + if self.remote.include_tags or self.remote.exclude_tags: + # Process cosign companion tags after all non-cosign tags are synced + companion_tags = self._find_cosign_companion_tags() + if companion_tags: + log.info( + "Syncing %d cosign companion tag(s) for filtered images", + len(companion_tags), + ) + await self._process_tags( + companion_tags, signature_source, msg="Processing Cosign Companion Tags" + ) + + def _find_cosign_companion_tags(self): + """Find cosign companion tags for synced digests.""" + companion_tags = [] + for tag in self._cosign_tags: + # Convert sha256-[.sig|.att|.sbom] to sha256: + tag_without_suffix = tag.rsplit(".", 1)[0] + digest = tag_without_suffix.replace("-", ":", 1) + if digest in self._synced_digests: + companion_tags.append(tag) + return companion_tags + + async def _process_tags(self, tag_list, signature_source, msg="Processing Tags"): + """Download and process a batch of tags, creating declarative content objects.""" + BATCH_SIZE = 500 + to_download = [] + for tag_name in tag_list: relative_url = "/v2/{name}/manifests/{tag}".format( name=self.remote.namespaced_upstream_name, tag=tag_name @@ -121,7 +159,7 @@ async def run(self): ) async with ProgressReport( - message="Processing Tags", + message=msg, code="sync.processing.tag", total=len(tag_list), ) as pb_parsed_tags: @@ -135,25 +173,21 @@ async def run(self): digest = calculate_digest(raw_text_data) tag_name = response.url.split("/")[-1] + media_type = determine_media_type(content_data, response) - # Look for cosign signatures - # cosign signature has a tag convention 'sha256-1234.sig' if self.signed_only and not signature_source: - if ( - not (tag_name.endswith(".sig") and tag_name.startswith("sha256-")) - and f"sha256-{digest.removeprefix('sha256:')}.sig" not in tag_list + if not ( + self._is_cosign_companion_tag(tag_name, media_type, content_data) + or await self._has_cosign_signature(digest) ): - # skip this tag, there is no corresponding signature log.info( "The unsigned image {digest} can't be synced " "due to a requirement to sync signed content " "only.".format(digest=digest) ) - # Count the skipped tagks as parsed too. await pb_parsed_tags.aincrement() continue - media_type = determine_media_type(content_data, response) validate_manifest(content_data, media_type, digest) tag_dc = DeclarativeContent(Tag(name=tag_name)) @@ -183,8 +217,6 @@ async def run(self): tag=tag_name, ) ) - # do not pass down the pipeline a manifest list with unsigned - # manifests. break self.signature_dcs.extend(man_sig_dcs) list_dc.extra_data["listed_manifests"].append(listed_manifest) @@ -192,14 +224,14 @@ async def run(self): else: # Manifest indices can be signed too. It is not mandatory. # If signature is available mirror it. + self._synced_digests.add(digest) if signature_source is not None: list_sig_dcs = await self.create_signatures(list_dc, signature_source) if list_sig_dcs: self.signature_dcs.extend(list_sig_dcs) - # only pass the manifest list and tag down the pipeline if there were no - # issues with signatures (no `break` in the `for` loop) tag_dc.extra_data["tagged_manifest_dc"] = list_dc for listed_manifest in list_dc.extra_data["listed_manifests"]: + self._synced_digests.add(listed_manifest["manifest_dc"].content.digest) await self.handle_blobs( listed_manifest["manifest_dc"], listed_manifest["content_data"] ) @@ -215,9 +247,9 @@ async def run(self): if signature_source is not None: man_sig_dcs = await self.create_signatures(man_dc, signature_source) if self.signed_only and not man_sig_dcs: - # do not pass down the pipeline unsigned manifests continue self.signature_dcs.extend(man_sig_dcs) + self._synced_digests.add(digest) tag_dc.extra_data["tagged_manifest_dc"] = man_dc await self.handle_blobs(man_dc, content_data) self.tag_dcs.append(tag_dc) @@ -239,6 +271,36 @@ async def run(self): await self.resolve_flush() + async def _has_cosign_signature(self, digest): + """Check if a digest has a cosign signature.""" + cosign_digest = digest.replace("sha256:", "sha256-") + if f"{cosign_digest}.sig" in self._cosign_tags: + return True + if cosign_digest in self._cosign_tags: + # Potential V3 cosign tag needs to be checked if it is a cosign companion tag + relative_url = f"/v2/{self.remote.namespaced_upstream_name}/manifests/{cosign_digest}" + tag_url = urljoin(self.remote.url, relative_url) + content_data, raw_text_data, response = await self._download_manifest_data(tag_url) + media_type = determine_media_type(content_data, response) + if self._is_cosign_companion_tag(cosign_digest, media_type, content_data): + return True + return False + + def _is_cosign_companion_tag(self, tag_name, media_type, content_data): + """Check if a fetched tag is a cosign companion tag.""" + if tag_name.startswith("sha256-"): + # V3 tags have format: sha256- with no suffixes + if len(tag_name) == 71: + # V3 tags are index lists with each entry having an artifactType + if media_type == MEDIA_TYPE.INDEX_OCI: + if manifests := content_data.get("manifests", []): + if all("artifactType" in entry for entry in manifests): + return True + # V2 tags have format: sha256-. + elif any(tag_name.endswith(s) for s in COSIGN_TAG_SUFFIXES): + return True + return False + async def get_signature_source(self): """ Find out where signatures come from: sigstore, extension API or not available at all. diff --git a/pulp_container/tests/functional/api/test_sync.py b/pulp_container/tests/functional/api/test_sync.py index 33f4ace9a..b350aa70d 100644 --- a/pulp_container/tests/functional/api/test_sync.py +++ b/pulp_container/tests/functional/api/test_sync.py @@ -8,12 +8,22 @@ from pulp_container.constants import MANIFEST_TYPE, MEDIA_TYPE from pulp_container.tests.functional.constants import ( + PULP_COSIGN_COMPANION_TAGS, + PULP_COSIGN_TAGS_MANIFEST_A_DIGEST, + PULP_COSIGN_TAGS_MANIFEST_B_DIGEST, + PULP_COSIGN_TAGS_MANIFEST_C_DIGEST, PULP_FIXTURE_1, PULP_HELLO_WORLD_LINUX_AMD64_DIGEST, PULP_LABELED_FIXTURE, REGISTRY_V2_FEED_URL, ) + +def _cosign_registry_tag_name(image_digest: str) -> str: + """Cosign companion tags use ``sha256-`` instead of ``sha256:``.""" + return image_digest.replace("sha256:", "sha256-", 1) + + # there is a manifest list and a listed manifest BOOTABLE_MANIFESTS_COUNT = 2 @@ -23,12 +33,12 @@ def synced_container_repository_factory( container_repository_factory, container_remote_factory, container_repository_api, container_sync ): def _synced_container_repository_factory( - url=REGISTRY_V2_FEED_URL, include_tags=None, exclude_tags=None + url=REGISTRY_V2_FEED_URL, include_tags=None, exclude_tags=None, upstream_name=PULP_FIXTURE_1 ): """Sync a new repository with the included tags passed as an argument.""" remote = container_remote_factory( url, - upstream_name=PULP_FIXTURE_1, + upstream_name=upstream_name, include_tags=include_tags, exclude_tags=exclude_tags, ) @@ -187,3 +197,81 @@ def test_sync_with_complex_filtering( tags = container_tag_api.list(repository_version=synced_repo.latest_version_href).results assert sorted(include_tags) == sorted(tag.name for tag in tags) + + +@pytest.mark.parallel +def test_sync_cosign_companion_tags( + synced_container_repository_factory, container_tag_api, container_manifest_api +): + """Test syncing a repository with cosign companion tags.""" + synced_repo = synced_container_repository_factory(upstream_name=PULP_COSIGN_COMPANION_TAGS) + + tags = container_tag_api.list(repository_version=synced_repo.latest_version_href) + manifests = container_manifest_api.list(repository_version=synced_repo.latest_version_href) + assert tags.count == 9 + cr = _cosign_registry_tag_name + expected_tag_names = { + "manifest_a", + "manifest_b", + "manifest_c", + "manifest_d", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_A_DIGEST)}.sig", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_A_DIGEST)}.att", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_B_DIGEST)}.sig", + cr(PULP_COSIGN_TAGS_MANIFEST_B_DIGEST), + cr(PULP_COSIGN_TAGS_MANIFEST_C_DIGEST), + } + assert {t.name for t in tags.results} == expected_tag_names + assert manifests.count == 13 + + +@pytest.mark.parallel +def test_sync_cosign_companion_tags_with_filtering( + synced_container_repository_factory, container_tag_api, container_manifest_api +): + """Test syncing a repository with cosign companion tags and filtering.""" + synced_repo = synced_container_repository_factory( + upstream_name=PULP_COSIGN_COMPANION_TAGS, include_tags=["manifest_a"] + ) + + tags = container_tag_api.list(repository_version=synced_repo.latest_version_href) + manifests = container_manifest_api.list(repository_version=synced_repo.latest_version_href) + assert tags.count == 3 + cr = _cosign_registry_tag_name + assert {t.name for t in tags.results} == { + "manifest_a", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_A_DIGEST)}.sig", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_A_DIGEST)}.att", + } + assert manifests.count == 3 + + synced_repo = synced_container_repository_factory( + upstream_name=PULP_COSIGN_COMPANION_TAGS, include_tags=["manifest_b"] + ) + + tags = container_tag_api.list(repository_version=synced_repo.latest_version_href) + manifests = container_manifest_api.list(repository_version=synced_repo.latest_version_href) + assert tags.count == 3 + assert {t.name for t in tags.results} == { + "manifest_b", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_B_DIGEST)}.sig", + cr(PULP_COSIGN_TAGS_MANIFEST_B_DIGEST), + } + assert manifests.count == 5 # The V3 sig is a manifest list with 2 manifests + + synced_repo = synced_container_repository_factory( + upstream_name=PULP_COSIGN_COMPANION_TAGS, exclude_tags=["manifest_a"] + ) + + tags = container_tag_api.list(repository_version=synced_repo.latest_version_href) + manifests = container_manifest_api.list(repository_version=synced_repo.latest_version_href) + assert tags.count == 6 + assert {t.name for t in tags.results} == { + "manifest_b", + "manifest_c", + "manifest_d", + f"{cr(PULP_COSIGN_TAGS_MANIFEST_B_DIGEST)}.sig", + cr(PULP_COSIGN_TAGS_MANIFEST_B_DIGEST), + cr(PULP_COSIGN_TAGS_MANIFEST_C_DIGEST), + } + assert manifests.count == 10 diff --git a/pulp_container/tests/functional/api/test_sync_signatures.py b/pulp_container/tests/functional/api/test_sync_signatures.py index 83267c39a..3cbd1621a 100644 --- a/pulp_container/tests/functional/api/test_sync_signatures.py +++ b/pulp_container/tests/functional/api/test_sync_signatures.py @@ -27,6 +27,7 @@ def synced_repository( upstream_name=DEPRECATED_REPOSITORY_NAME, policy="on_demand", include_tags=[MANIFEST_LIST_TAG, IMAGE_MANIFEST_TAG], + exclude_tags=["sha256-*"], # exclude cosign companion tags ) if request.param["sigstore"]: @@ -137,6 +138,7 @@ def test_sync_image_with_pqc_signatures( upstream_name=UBI10_MICRO_REPOSITORY_NAME, policy="on_demand", include_tags=[UBI10_MICRO_TAG], + exclude_tags=["sha256-*"], sigstore=SIGSTORE_URL, ) remote = container_remote_factory(**data) diff --git a/pulp_container/tests/functional/constants.py b/pulp_container/tests/functional/constants.py index faa39b438..9183f61c7 100644 --- a/pulp_container/tests/functional/constants.py +++ b/pulp_container/tests/functional/constants.py @@ -24,3 +24,26 @@ REGISTRY_V2_REPO_PULP = f"{REGISTRY_V2}/{PULP_FIXTURE_1}" REGISTRY_V2_REPO_HELLO_WORLD = f"{REGISTRY_V2}/{PULP_HELLO_WORLD_REPO}" + +# a repository containing cosign companion tags +PULP_COSIGN_COMPANION_TAGS = "pulp/cosign-tags" +# It contains 4 normal tags: +# manifest_a, manifest_b, manifest_c, manifest_d +# and 5 cosign companion tags: +# 2 for manifest_a: sha256-.sig (v2: 1 signature), sha256-.att (v2: 1 attestation) +# 2 for manifest_b: sha256-.sig (v2: 2 signatures), sha256- (v3: 2 signatures) +# 1 for manifest_c: sha256- (v3: 1 signature, 1 attestation) +# V2 signatures are stored in one manifest with each signature in a separate layer +# V3 signatures are collected in one manifest list with each signature getting its own manifest +# Repo total contains 2 manifest lists and 11 manifests + +PULP_COSIGN_TAGS_MANIFEST_A_DIGEST = PULP_FIXTURE_1_MANIFEST_A_DIGEST +PULP_COSIGN_TAGS_MANIFEST_B_DIGEST = ( + "sha256:f8634bb68dccf0dc2a3113933a67f91dc10c4ac17dee90988cb6bc4ae55cf802" +) +PULP_COSIGN_TAGS_MANIFEST_C_DIGEST = ( + "sha256:6489ee892f64e59755435ee53f7d10cce5588a7788b4b2ae4a510a8bbc92704d" +) +PULP_COSIGN_TAGS_MANIFEST_D_DIGEST = ( + "sha256:badde852ff2ee4daeff0cf1c2b1e9c01a193ca6e93e0fce8acce8a7d6a4ade06" +) diff --git a/pulp_container/tests/unit/test_sync_stages.py b/pulp_container/tests/unit/test_sync_stages.py new file mode 100644 index 000000000..835347cb7 --- /dev/null +++ b/pulp_container/tests/unit/test_sync_stages.py @@ -0,0 +1,156 @@ +"""Unit tests for cosign companion tag helpers on ContainerFirstStage.""" + +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +from pulp_container.app.tasks.sync_stages import COSIGN_TAG_SUFFIXES, ContainerFirstStage +from pulp_container.constants import MEDIA_TYPE + + +def _bare_cosign_digest() -> tuple[str, str]: + """Return (tag_name, docker digest form) for a 71-char V3 cosign tag name.""" + tag = "sha256-" + "a" * 64 + digest = "sha256:" + "a" * 64 + return tag, digest + + +class TestCosignCompanionHelpers(unittest.IsolatedAsyncioTestCase): + """Exercise cosign tagging helpers without the full sync pipeline.""" + + def setUp(self): + remote = MagicMock() + remote.policy = MagicMock() + remote.namespaced_upstream_name = "library/test" + remote.url = "https://registry.example/" + remote.get_downloader = MagicMock() + + self.stage = ContainerFirstStage(remote=remote, signed_only=False) + + def test_is_cosign_companion_tag_v2_suffixes(self): + """V2 companions use sha256-. where suffix is .sig / .att / .sbom.""" + tag, _ = _bare_cosign_digest() + for suffix in COSIGN_TAG_SUFFIXES: + with self.subTest(suffix=suffix): + name = f"{tag}{suffix}" + self.assertTrue( + self.stage._is_cosign_companion_tag(name, MEDIA_TYPE.MANIFEST_LIST, {}) + ) + + def test_is_cosign_companion_tag_v2_not_companion_with_wrong_suffix(self): + tag, _ = _bare_cosign_digest() + self.assertFalse( + self.stage._is_cosign_companion_tag(f"{tag}.other", MEDIA_TYPE.MANIFEST_LIST, {}) + ) + + def test_is_cosign_companion_tag_non_sha256_prefix(self): + self.assertFalse( + self.stage._is_cosign_companion_tag("latest", MEDIA_TYPE.MANIFEST_LIST, {}) + ) + + def test_is_cosign_companion_tag_v3_oci_index_with_artifact_types(self): + tag, _ = _bare_cosign_digest() + content = { + "manifests": [ + {"artifactType": "application/vnd.dev.cosign.simplesigning.v1+json"}, + {"artifactType": "application/vnd.oci.image.config.v1+json"}, + ] + } + self.assertTrue(self.stage._is_cosign_companion_tag(tag, MEDIA_TYPE.INDEX_OCI, content)) + + def test_is_cosign_companion_tag_v3_requires_all_artifact_types(self): + tag, _ = _bare_cosign_digest() + content = { + "manifests": [ + {"artifactType": "application/vnd.dev.cosign.simplesigning.v1+json"}, + {"mediaType": "application/vnd.oci.image.manifest.v1+json"}, + ] + } + self.assertFalse(self.stage._is_cosign_companion_tag(tag, MEDIA_TYPE.INDEX_OCI, content)) + + def test_is_cosign_companion_tag_v3_wrong_media_type(self): + tag, _ = _bare_cosign_digest() + content = { + "manifests": [{"artifactType": "application/vnd.dev.cosign.simplesigning.v1+json"}] + } + self.assertFalse( + self.stage._is_cosign_companion_tag(tag, MEDIA_TYPE.MANIFEST_LIST, content) + ) + + def test_find_cosign_companion_tags_filters_by_synced_digests(self): + tag_sig, digest = _bare_cosign_digest() + tag_sig = f"{tag_sig}.sig" + tag_att = tag_sig.replace(".sig", ".att") + + self.stage._cosign_tags = [tag_sig, tag_att, "sha256-" + "b" * 64 + ".sig"] + self.stage._synced_digests = {digest} + + found = self.stage._find_cosign_companion_tags() + self.assertCountEqual(found, [tag_sig, tag_att]) + + def test_find_cosign_companion_tags_empty_when_nothing_synced(self): + tag_sig, _ = _bare_cosign_digest() + self.stage._cosign_tags = [f"{tag_sig}.sig"] + self.stage._synced_digests = set() + self.assertEqual(self.stage._find_cosign_companion_tags(), []) + + async def test_has_cosign_signature_true_when_sig_tag_present(self): + _, digest = _bare_cosign_digest() + cosign_key = digest.replace("sha256:", "sha256-") + self.stage._cosign_tags = [f"{cosign_key}.sig"] + + self.assertTrue(await self.stage._has_cosign_signature(digest)) + self.stage.remote.get_downloader.assert_not_called() + + async def test_has_cosign_signature_true_after_fetching_v3_index(self): + tag, digest = _bare_cosign_digest() + self.stage._cosign_tags = [tag] + + content_data = { + "manifests": [ + {"artifactType": "application/vnd.dev.cosign.simplesigning.v1+json"}, + ] + } + raw = '{"manifests":[]}' + + mock_response = MagicMock() + mock_response.url = f"https://registry.example/v2/foo/manifests/{tag}" + + self.stage._download_manifest_data = AsyncMock( + return_value=(content_data, raw, mock_response) + ) + + with patch( + "pulp_container.app.tasks.sync_stages.determine_media_type", + return_value=MEDIA_TYPE.INDEX_OCI, + ): + self.assertTrue(await self.stage._has_cosign_signature(digest)) + + self.stage._download_manifest_data.assert_awaited_once() + + async def test_has_cosign_signature_false_when_bare_tag_not_companion(self): + tag, digest = _bare_cosign_digest() + self.stage._cosign_tags = [tag] + + content_data = {"manifests": []} + raw = "{}" + mock_response = MagicMock() + mock_response.url = f"https://registry.example/v2/foo/manifests/{tag}" + + self.stage._download_manifest_data = AsyncMock( + return_value=(content_data, raw, mock_response) + ) + + with patch( + "pulp_container.app.tasks.sync_stages.determine_media_type", + return_value=MEDIA_TYPE.INDEX_OCI, + ): + self.assertFalse(await self.stage._has_cosign_signature(digest)) + + async def test_has_cosign_signature_false_when_no_cosign_tags(self): + _, digest = _bare_cosign_digest() + self.stage._cosign_tags = [] + self.assertFalse(await self.stage._has_cosign_signature(digest)) + + +if __name__ == "__main__": + unittest.main()