From 3859e1b452ceefdf2839174c750dd7b6a6240074 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Mon, 9 Mar 2026 09:57:20 +0100 Subject: [PATCH 1/2] fix: New PyYAML package has issues on older Python 3.8 and Linux ARMv7 --- exclude_list.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/exclude_list.yaml b/exclude_list.yaml index f513fdb..872890b 100644 --- a/exclude_list.yaml +++ b/exclude_list.yaml @@ -140,3 +140,10 @@ # https://pypi.org/project/mcp/ - package_name: 'mcp' python: ['==3.8', '==3.9'] + +# PyYAML 6.0.3 wheel is invalid on Linux ARMv7 + Python 3.8 (BadZipFile: could not read WHEEL) +# https://github.com/espressif/idf-python-wheels/actions/runs/22810198979/job/66174320073 +- package_name: 'pyyaml' + version: '==6.0.3' + platform: ['linux_armv7'] + python: '==3.8' From bb48e9b76325823fd2e23968defc20a82262f098 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Mon, 9 Mar 2026 11:22:29 +0100 Subject: [PATCH 2/2] fix: Added architecture specific resolving for S3 verification --- _helper_functions.py | 103 +++++++++++++++++++++++++++---------------- test_build_wheels.py | 64 +++++++++++++++++++++++++++ verify_s3_wheels.py | 6 +-- yaml_list_adapter.py | 30 +++++++++---- 4 files changed, 155 insertions(+), 48 deletions(-) diff --git a/_helper_functions.py b/_helper_functions.py index c858feb..01b161a 100644 --- a/_helper_functions.py +++ b/_helper_functions.py @@ -237,13 +237,13 @@ def get_wheel_sys_platforms(wheel_name: str) -> list[str] | None: """ Derive sys_platform value(s) from the wheel filename for marker evaluation. - Uses the wheel's platform tag(s) from parse_wheel_filename. For universal - wheels (platform tag "any"), returns all three so platform-specific - exclusions can be checked against every platform the wheel targets. + Returns generic ("win32", "linux", "darwin") and, when detectable from the + tag, the runner-specific platform (e.g. linux_armv7, linux_x86_64) so + exclude_list rules that target a single arch (with preserve_arch_in_markers) + match only the intended wheels. Returns: - List of sys_platform values ("win32", "linux", "darwin"), or None - if the filename cannot be parsed. + List of sys_platform values, or None if the filename cannot be parsed. """ try: _name, _version, _build, tags = parse_wheel_filename(wheel_name) @@ -258,10 +258,63 @@ def get_wheel_sys_platforms(wheel_name: str) -> list[str] | None: for prefixes, sys_plat in _WHEEL_PLATFORM_TO_SYS: if any(pt.startswith(p) for p in prefixes): platforms.add(sys_plat) + # Add runner-style platform for arch-specific marker matching (S3 verification) + if "armv7l" in pt: + platforms.add("linux_armv7") + elif "aarch64" in pt: + platforms.add("linux_arm64") + elif ("x86_64" in pt or "amd64" in pt) and ("manylinux" in pt or pt.startswith("linux_")): + platforms.add("linux_x86_64") + elif "macosx" in pt and "arm64" in pt: + platforms.add("macos_arm64") + elif "macosx" in pt and ("x86_64" in pt or "amd64" in pt): + platforms.add("macos_x86_64") break return list(platforms) if platforms else None +def get_wheel_runner_platform(wheel_name: str) -> str | None: + """ + Derive the runner-style platform from the wheel filename for exclude_list matching. + + Returns the same convention as get_current_platform(): linux_armv7, linux_arm64, + linux_x86_64, linux, windows, darwin, macos_arm64, macos_x86_64, macos. + Used so S3 verification can apply platform-specific exclude rules (e.g. only + linux_armv7) instead of all Linux wheels. + + Returns: + Platform string or None if the filename cannot be parsed. + """ + try: + _name, _version, _build, tags = parse_wheel_filename(wheel_name) + except InvalidWheelFilename: + return None + for tag in tags: + pt = tag.platform + if pt == "any": + return None # Universal wheel, no single platform + if "armv7l" in pt: + return "linux_armv7" + if "aarch64" in pt: + return "linux_arm64" + if "x86_64" in pt or "amd64" in pt: + if "manylinux" in pt or pt.startswith("linux_"): + return "linux_x86_64" + if "macosx" in pt: + return "macos_x86_64" + if "win" in pt: + return "windows" + if "win_amd64" in pt or "win32" in pt: + return "windows" + if "macosx" in pt: + if "arm64" in pt: + return "macos_arm64" + return "macos_x86_64" # or generic "macos" for older tags + if pt.startswith("manylinux") or pt.startswith("linux_"): + return "linux" + return None + + def should_exclude_wheel_s3( wheel_name: str, exclude_requirements: set, @@ -270,29 +323,10 @@ def should_exclude_wheel_s3( """ Check if a wheel should be excluded for S3 verification. - Uses DIRECT exclusion logic (not inverted): - - If marker is True → exclusion applies → EXCLUDE - - If marker is False → exclusion doesn't apply → KEEP - - If version matches specifier → EXCLUDE - - Derives the wheel's target platform from its filename (e.g. win_amd64 - -> win32, manylinux_* -> linux) and evaluates sys_platform markers - against that instead of skipping them, so platform-only exclusions - in exclude_list.yaml are reported as S3 violations when applicable. - - For universal wheels (no cpXY tag, e.g. py3-none-any), python_version - markers are evaluated against supported_python_versions when provided, - so exclusions that apply only to older supported versions are not missed. - - Args: - wheel_name: The wheel filename - exclude_requirements: Set of Requirement objects from YAMLListAdapter (exclude=False) - supported_python_versions: When the wheel has no cpXY tag, evaluate - python_version markers against these versions (e.g. ["3.8", "3.9", "3.10", ...]). - If None, falls back to the runner's Python (may miss version-specific exclusions). - - Returns: - tuple: (should_exclude: bool, reason: str) + Uses merged requirements from YAMLListAdapter. When the adapter is loaded with + preserve_arch_in_markers=True, rules that target a single arch (e.g. linux_armv7) + are preserved and get_wheel_sys_platforms() returns that arch so only matching + wheels are flagged. """ parsed = parse_wheel_name(wheel_name) if not parsed: @@ -303,7 +337,6 @@ def should_exclude_wheel_s3( wheel_python = get_wheel_python_version(wheel_name) wheel_sys_platforms = get_wheel_sys_platforms(wheel_name) - # For universal wheels (no cpXY), evaluate python_version against these if provided python_versions_to_try: list[str | None] = [] if wheel_python is not None: python_versions_to_try.append(wheel_python) @@ -316,11 +349,10 @@ def should_exclude_wheel_s3( if canonicalize_name(req.name) != canonical_name: continue - # Evaluate markers (including sys_platform) using wheel's target platform and Python if req.marker: if "sys_platform" in str(req.marker): if not wheel_sys_platforms: - continue # Cannot derive platform from filename → skip rule + continue marker_matches = False for sys_plat in wheel_sys_platforms: for pv in python_versions_to_try: @@ -333,7 +365,7 @@ def should_exclude_wheel_s3( if marker_matches: break if not marker_matches: - continue # Exclusion condition not met for this wheel's platform(s) + continue else: marker_matches = False for pv in python_versions_to_try: @@ -342,17 +374,14 @@ def should_exclude_wheel_s3( marker_matches = True break if not marker_matches: - continue # Exclusion condition not met → keep + continue - # If we get here, marker is True (or no marker) - # Check version specifier - if version matches, EXCLUDE if req.specifier and wheel_version: try: if Version(wheel_version) not in req.specifier: - continue # Version doesn't match exclusion → keep + continue except Exception: pass return True, f"matches exclude rule: {req}" - return False, "" diff --git a/test_build_wheels.py b/test_build_wheels.py index be1f04d..c2168ee 100644 --- a/test_build_wheels.py +++ b/test_build_wheels.py @@ -17,7 +17,10 @@ from packaging.requirements import Requirement from _helper_functions import get_no_binary_args +from _helper_functions import get_wheel_runner_platform +from _helper_functions import get_wheel_sys_platforms from _helper_functions import merge_requirements +from _helper_functions import should_exclude_wheel_s3 from build_wheels import _add_into_requirements from build_wheels import get_used_idf_branches from yaml_list_adapter import YAMLListAdapter @@ -279,6 +282,67 @@ def test_exclude_with_version_range(self): self.assertFalse(result) +class TestGetWheelSysPlatforms(unittest.TestCase): + """Test get_wheel_sys_platforms includes arch-specific value for marker matching.""" + + def test_linux_armv7_wheel_includes_linux_armv7(self): + platforms = get_wheel_sys_platforms("PyYAML-6.0.3-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.whl") + self.assertIn("linux_armv7", platforms) + self.assertIn("linux", platforms) + + def test_linux_x86_64_wheel_includes_linux_x86_64(self): + platforms = get_wheel_sys_platforms("PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl") + self.assertIn("linux_x86_64", platforms) + self.assertIn("linux", platforms) + + +class TestGetWheelRunnerPlatform(unittest.TestCase): + """Test get_wheel_runner_platform for runner-style platform from wheel name.""" + + def test_linux_armv7(self): + self.assertEqual( + get_wheel_runner_platform("PyYAML-6.0.3-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.whl"), + "linux_armv7", + ) + + def test_linux_x86_64(self): + self.assertEqual( + get_wheel_runner_platform( + "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" + ), + "linux_x86_64", + ) + + def test_linux_aarch64(self): + self.assertEqual( + get_wheel_runner_platform("PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.whl"), + "linux_arm64", + ) + + +class TestShouldExcludeWheelS3(unittest.TestCase): + """Test should_exclude_wheel_s3 with arch-specific marker (preserve_arch_in_markers).""" + + def test_arch_specific_marker_matches_only_that_arch(self): + # Requirement with sys_platform == 'linux_armv7' (from preserve_arch_in_markers) matches only armv7l wheel + req = Requirement('pyyaml==6.0.3; sys_platform == "linux_armv7" and python_version == "3.8"') + exclude_requirements = {req} + + excluded, _ = should_exclude_wheel_s3( + "PyYAML-6.0.3-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", + exclude_requirements, + supported_python_versions=["3.8", "3.9"], + ) + self.assertTrue(excluded) + + not_excluded, _ = should_exclude_wheel_s3( + "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + exclude_requirements, + supported_python_versions=["3.8", "3.9"], + ) + self.assertFalse(not_excluded) + + class TestGetUsedIdfBranches(unittest.TestCase): """Test the get_used_idf_branches function.""" diff --git a/verify_s3_wheels.py b/verify_s3_wheels.py index 839b32f..415a9a6 100644 --- a/verify_s3_wheels.py +++ b/verify_s3_wheels.py @@ -97,8 +97,8 @@ def main(): s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) - # Load exclude requirements (direct logic, no inversion) - exclude_requirements = YAMLListAdapter(EXCLUDE_LIST_PATH, exclude=False).requirements + # preserve_arch_in_markers=True so rules like linux_armv7-only match only that arch + exclude_requirements = YAMLListAdapter(EXCLUDE_LIST_PATH, exclude=False, preserve_arch_in_markers=True).requirements print(f"Loaded {len(exclude_requirements)} exclude rules\n") # Get all wheels from S3 @@ -123,7 +123,7 @@ def main(): old_python_wheels.append((wheel, reason)) continue - # Check against exclude_list (actual violations) + # Check against exclude_list (merged requirements; arch preserved for platform-specific rules) should_exclude, reason = should_exclude_wheel_s3( wheel, exclude_requirements, supported_python_versions=supported_python_versions ) diff --git a/yaml_list_adapter.py b/yaml_list_adapter.py index 5cc3ee8..b19fff3 100644 --- a/yaml_list_adapter.py +++ b/yaml_list_adapter.py @@ -29,8 +29,13 @@ } -def _platform_for_marker(platform): - """Normalize platform to sys_platform value for pip markers.""" +def _platform_for_marker(platform: str, preserve_arch: bool = False) -> str: + """Normalize platform to sys_platform value for pip markers. + When preserve_arch is True (S3 verification), keep arch-specific values so + e.g. linux_armv7 stays linux_armv7 and only that wheel arch matches. + """ + if preserve_arch and platform in SYS_PLATFORM_MAP: + return platform return SYS_PLATFORM_MAP.get(platform, platform) @@ -83,13 +88,20 @@ class YAMLListAdapter: exclude: bool = False requirements: set = set() - def __init__(self, yaml_file: str, exclude: bool = False, current_platform: Optional[str] = None) -> None: + def __init__( + self, + yaml_file: str, + exclude: bool = False, + current_platform: Optional[str] = None, + preserve_arch_in_markers: bool = False, + ) -> None: try: with open(yaml_file, "r") as f: self._yaml_list = yaml.load(f, yaml.Loader) except FileNotFoundError: print_color(f"File not found, please check the file: {yaml_file}", Fore.RED) self.exclude = exclude + self.preserve_arch_in_markers = preserve_arch_in_markers # When building wheels: only exclude entries that apply to this platform if current_platform and self._yaml_list: @@ -178,13 +190,15 @@ def _yaml_to_requirement(self, yaml: list, exclude: bool = False) -> set: # get attributes of the package if defined to reduce unnecessary complexity package_version = package["version"] if "version" in package else "" raw_platform = package["platform"] if "platform" in package else "" - # Map linux_armv7/arm64/x86_64 to "linux" for pip markers, preserving str | list[str] + preserve = getattr(self, "preserve_arch_in_markers", False) + # Map linux_armv7/arm64/x86_64 to "linux" for pip markers unless preserve_arch_in_markers + package_platform: str | list[str] = "" if isinstance(raw_platform, str) and raw_platform: - package_platform = _platform_for_marker(raw_platform) + package_platform = _platform_for_marker(raw_platform, preserve_arch=preserve) elif isinstance(raw_platform, list) and raw_platform: - package_platform = list(dict.fromkeys(_platform_for_marker(p) for p in raw_platform)) - else: - package_platform = "" + package_platform = list( + dict.fromkeys(_platform_for_marker(p, preserve_arch=preserve) for p in raw_platform) + ) package_python = package["python"] if "python" in package else "" requirement_str_list = [f"{package['package_name']}"]