From e9776dc88f59f8421cded32bec73b3cf6acfe7b2 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 20 Mar 2026 11:22:10 +0800 Subject: [PATCH 1/6] verify: run stdlib tests --- .github/workflows/linux.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/windows.yml | 2 +- cpython-unix/build-cpython.sh | 16 +- cpython-unix/build.py | 19 +- cpython-windows/build.py | 16 +- pyproject.toml | 1 + pythonbuild/cpython.py | 198 +++ pythonbuild/testdist.py | 378 ++++- pythonbuild/utils.py | 16 +- stdlib-test-annotations.yml | 2734 +++++++++++++++++++++++++++++++++ uv.lock | 11 + 12 files changed, 3371 insertions(+), 24 deletions(-) create mode 100644 stdlib-test-annotations.yml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2e934c5fd..03de98a0e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -329,7 +329,7 @@ jobs: unset LD_LIBRARY_PATH fi - ./test-distribution.py dist/*.tar.zst + ./test-distribution.py --stdlib dist/*.tar.zst fi env: MATRIX_RUN: ${{ matrix.run }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 3a992a9a4..12885b885 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -181,7 +181,7 @@ jobs: build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst if [ "${MATRIX_RUN}" == "true" ]; then - ./test-distribution.py dist/*.tar.zst + ./test-distribution.py --stdlib dist/*.tar.zst fi env: MATRIX_RUN: ${{ matrix.run }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index cc8b866f0..eb61f1a73 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -186,4 +186,4 @@ jobs: run: | $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative .\pythonbuild.exe validate-distribution $Dists - uv run --no-dev test-distribution.py $Dists + uv run --no-dev test-distribution.py --stdlib $Dists diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index f010750f4..730e6d18e 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -51,10 +51,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz" PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl" SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl" -cat Setup.local -mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local" +# Put critical config files in logs to aid debugging. +for f in Setup.local Makefile.extra stdlib-test-annotations.json; do + echo "BEGIN $f" + cat $f + echo "END $f" +done -cat Makefile.extra +mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local" pushd "Python-${PYTHON_VERSION}" @@ -1386,7 +1390,7 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/" cp -av Python/frozen.c "${ROOT}/out/python/build/Python/" cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/" -# Copy the test hardness runner for convenience. +# Copy the test harness runner for convenience. # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py" @@ -1394,6 +1398,10 @@ else cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/" fi +# Copy standard library test annotations so it is in the artifact and the +# distribution self-describes expected test wonkiness. +cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json" + # Don't hard-code the build-time prefix into the pkg-config files. See # the description of `pcfiledir` in `man pkg-config`. find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \ diff --git a/cpython-unix/build.py b/cpython-unix/build.py index eb0e6ae86..f78c340c4 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -24,6 +24,7 @@ meets_python_maximum_version, meets_python_minimum_version, parse_setup_line, + stdlib_test_annotations, ) from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles from pythonbuild.downloads import DOWNLOADS @@ -50,6 +51,7 @@ SUPPORT = ROOT / "cpython-unix" EXTENSION_MODULES = SUPPORT / "extension-modules.yml" TARGETS_CONFIG = SUPPORT / "targets.yml" +STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml" LINUX_ALLOW_SYSTEM_LIBRARIES = { "c", @@ -732,6 +734,13 @@ def build_cpython( setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) pip_archive = download_entry("pip", DOWNLOADS_PATH) + test_annotations = stdlib_test_annotations( + STDLIB_TEST_ANNOTATIONS, + python_version, + target_triple, + parsed_build_options, + ) + ems = extension_modules_config(EXTENSION_MODULES) setup = derive_setup_local( @@ -804,6 +813,14 @@ def build_cpython( build_env.copy_file(fh.name, dest_name="Makefile.extra") + # Install the derived test annotations. + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh: + os.chmod(fh.name, 0o644) + test_annotations.json_dump(fh) + fh.flush() + + build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json") + env = { "PIP_VERSION": DOWNLOADS["pip"]["version"], "PYTHON_VERSION": python_version, @@ -1070,7 +1087,7 @@ def main(): write_dockerfiles(SUPPORT, BUILD) elif action == "makefiles": targets = get_targets(TARGETS_CONFIG) - write_triples_makefiles(targets, BUILD, SUPPORT) + write_triples_makefiles(targets, ROOT, BUILD, SUPPORT) write_target_settings(targets, BUILD / "targets") write_package_versions(BUILD / "versions") diff --git a/cpython-windows/build.py b/cpython-windows/build.py index f93a59a50..de91aba6c 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -22,6 +22,7 @@ meets_python_maximum_version, meets_python_minimum_version, parse_config_c, + stdlib_test_annotations, ) from pythonbuild.downloads import DOWNLOADS from pythonbuild.utils import ( @@ -39,6 +40,7 @@ BUILD = ROOT / "build" DIST = ROOT / "dist" SUPPORT = ROOT / "cpython-windows" +STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml" LOG_PREFIX = [None] LOG_FH = [None] @@ -128,7 +130,6 @@ "_zstd": ["zstd"], } - # Tests to run during PGO profiling. # # This set was copied from test.libregrtest.pgo in the CPython source @@ -1407,6 +1408,13 @@ def build_cpython( else: raise Exception("unhandled architecture: %s" % arch) + test_annotations = stdlib_test_annotations( + STDLIB_TEST_ANNOTATIONS, + python_version, + target_triple, + parsed_build_options, + ) + tempdir_opts = ( {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {} ) @@ -1763,6 +1771,12 @@ def build_cpython( out_dir / "python" / "build" / "run_tests.py", ) + # Install a JSON file annotating tests. + with (out_dir / "python" / "build" / "stdlib-test-annotations.json").open( + "w", encoding="utf-8" + ) as fh: + test_annotations.json_dump(fh) + licenses_dir = out_dir / "python" / "licenses" licenses_dir.mkdir() for f in sorted(os.listdir(ROOT)): diff --git a/pyproject.toml b/pyproject.toml index 43e5f4eda..de66b871c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "docker>=7.1.0", "jinja2>=3.1.5", "jsonschema>=4.23.0", + "junitparser>=4.0.2", "pyyaml>=6.0.2", "six>=1.17.0", "tomli>=2.2.1", diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 3221760ab..d342f1ad9 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -2,9 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import dataclasses +import json import pathlib import re import tarfile +from typing import Optional import jsonschema import yaml @@ -137,6 +140,65 @@ }, } +STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES = { + "reason": {"type": "string"}, + "targets": { + "type": "array", + "items": {"type": "string"}, + }, + "ignore-targets": { + "type": "array", + "items": {"type": "string"}, + }, + "minimum-python-version": {"type": "string"}, + "maximum-python-version": {"type": "string"}, + "build-option": {"type": "string"}, + "no-build-option": {"type": "string"}, +} + +STDLIB_TEST_ANNOTATIONS_SCHEMA = { + "type": "object", + "properties": { + "harness-skips": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["name", "reason"], + }, + }, + "module-excludes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "module": {"type": "string"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["module", "reason"], + }, + }, + "expected-failures": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "dont-verify": {"type": "boolean"}, + "intermittent": {"type": "boolean"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["name", "reason"], + }, + }, + }, +} # Packages that define tests. STDLIB_TEST_PACKAGES = { @@ -750,3 +812,139 @@ def extension_modules_config(yaml_path: pathlib.Path): jsonschema.validate(data, EXTENSION_MODULES_SCHEMA) return data + + +TEST_ANNOTATION_HARNESS_SKIP = "harness-skip" +TEST_ANNOTATION_MODULE_EXCLUDE = "module-exclude" +TEST_ANNOTATION_TEST_FAILURE = "test-failure" + + +@dataclasses.dataclass +class TestAnnotation: + # Describes the type of annotation. + flavor: str + # Name/pattern of test. + name: str + # Describes why the annotation exists. + reason: str + # Whether the test is expected to fail. + expect_test_failure: bool + # Whether to skip verification of failures. + dont_verify: bool + # Whether test failure is intermittent. Should only be true if + # expect_test_failure also true. + intermittent_test_failure: bool + # Whether to exclude loading the test module when running tests. + exclude_testing: bool + + +@dataclasses.dataclass +class TestAnnotations: + annotations: list[TestAnnotation] + + def json_dump(self, of): + data = [dataclasses.asdict(a) for a in self.annotations] + json.dump(data, of, indent=2) + + +def filter_stdlib_test_entry( + flavor: str, + test, + python_version: str, + target_triple: str, + build_options: set[str], +) -> Optional[TestAnnotation]: + name = test["name"] + + if targets := test.get("targets"): + matches_target = any(re.match(p, target_triple) for p in targets) + else: + matches_target = True + + for m in test.get("ignore-targets", []): + if re.match(m, target_triple): + matches_target = False + + if not matches_target: + log(f"ignoring {flavor} rule (target doesn't match): {name}") + return None + + python_minimum_version = test.get("minimum-python-version", "1.0") + python_maximum_version = test.get("maximum-python-version", "100.0") + + if not meets_python_minimum_version(python_version, python_minimum_version): + log( + f"ignoring {flavor} rule ({python_version} < {python_minimum_version} (min)): {name}" + ) + return None + + if not meets_python_maximum_version(python_version, python_maximum_version): + log( + f"ignoring {flavor} rule ({python_version} > {python_maximum_version} (max)): {name}" + ) + return None + + if option := test.get("build-option"): + if option not in build_options: + log(f"ignoring {flavor} rule (build option {option} not present): {name}") + return None + + if option := test.get("no-build-option"): + if option in build_options: + log(f"ignoring {flavor} rule (build option {option} is present): {name}") + return None + + # Filtering complete. This rule applies to the current build. + + log(f"relevant {flavor} test rule: {name}: {test['reason']}") + + return TestAnnotation( + flavor=flavor, + name=name, + reason=test["reason"], + expect_test_failure=True, + dont_verify=test.get("dont-verify", False), + intermittent_test_failure=test.get("intermittent", False), + exclude_testing=test.get("exclude", False), + ) + + +def stdlib_test_annotations( + yaml_path: pathlib.Path, + python_version: str, + target_triple: str, + build_options: set[str], +) -> TestAnnotations: + """Processes the test-annotations.yml file for a given build configuration.""" + with yaml_path.open("r", encoding="utf-8") as fh: + data = yaml.load(fh, Loader=yaml.SafeLoader) + + jsonschema.validate(data, STDLIB_TEST_ANNOTATIONS_SCHEMA) + + annotations = [] + + log(f"processing {len(data['expected-failures'])} stdlib test annotations") + + raw_entries = [] + + for entry in data["harness-skips"]: + raw_entries.append((TEST_ANNOTATION_HARNESS_SKIP, entry)) + + for entry in data["module-excludes"]: + entry["name"] = entry["module"] + raw_entries.append((TEST_ANNOTATION_MODULE_EXCLUDE, entry)) + + for entry in data["expected-failures"]: + raw_entries.append((TEST_ANNOTATION_TEST_FAILURE, entry)) + + for flavor, entry in raw_entries: + if a := filter_stdlib_test_entry( + flavor, + entry, + python_version, + target_triple, + build_options, + ): + annotations.append(a) + + return TestAnnotations(annotations) diff --git a/pythonbuild/testdist.py b/pythonbuild/testdist.py index 96f65790e..3a07637d5 100644 --- a/pythonbuild/testdist.py +++ b/pythonbuild/testdist.py @@ -3,21 +3,59 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import argparse +import concurrent.futures +import dataclasses import json import os +import shlex import subprocess import tempfile from pathlib import Path from typing import Optional +import junitparser + +from .cpython import ( + TEST_ANNOTATION_HARNESS_SKIP, + TEST_ANNOTATION_MODULE_EXCLUDE, + TEST_ANNOTATION_TEST_FAILURE, + TestAnnotation, + meets_python_minimum_version, +) from .utils import extract_python_archive +# Tests in the standard library that are slow. +# +# Roughly in order from slowest to fastest. +STDLIB_SLOW_TESTS = [ + "test_subprocess", + "test_signal", + "test_socket", + "test.test_multiprocessing_spawn.test_processes", + "test.test_multiprocessing_forkserver.test_processes", + "test_regrtest", + "test.test_concurrent_futures.test_process_pool", + "test_tarfile", + "test_socket", + "test_remote_pdb", + "test_threading", + "test.test_concurrent_futures.test_as_completed", + "test.test_multiprocessing_spawn.test_misc", + "test.test_asyncio.test_tasks", + "test_zipfile", + "test_pickle", +] + +# Maximum wall run time of a single test before timing it out. +TIMEOUT_SECONDS = 300 + def run_dist_python( dist_root: Path, python_info, args: list[str], extra_env: Optional[dict[str, str]] = None, + log_exec=False, **runargs, ) -> subprocess.CompletedProcess[str]: """Runs a `python` process from an extracted PBS distribution. @@ -35,8 +73,13 @@ def run_dist_python( if extra_env: env.update(extra_env) + all_args = [str(dist_root / python_info["python_exe"])] + args + + if log_exec: + print(f"executing: {shlex.join(all_args)}") + return subprocess.run( - [str(dist_root / python_info["python_exe"])] + args, + all_args, cwd=dist_root, env=env, **runargs, @@ -63,44 +106,351 @@ def run_custom_unittests(pbs_source_dir: Path, dist_root: Path, python_info) -> return res.returncode -def run_stdlib_tests(dist_root: Path, python_info, harness_args: list[str]) -> int: +def run_stdlib_tests( + dist_root: Path, + python_info, + skip_main=False, + verbose_expected_failures=False, + raw_harness_args: Optional[list[str]] = None, +) -> int: """Run Python stdlib tests for a PBS distribution. The passed path is the `python` directory from the extracted distribution archive. """ - args = [ - str(dist_root / python_info["run_tests"]), + + dist_name = f"cpython-{python_info['python_version']}-{python_info['build_options']}-{python_info['target_triple']}" + + stdlib_test_annotations_path = dist_root / "build" / "stdlib-test-annotations.json" + + with stdlib_test_annotations_path.open("r", encoding="utf-8") as fh: + annotations = json.load(fh) + + expect_failures = set() + module_excludes = set() + intermittent = set() + dont_verify = set() + + for raw_annotation in annotations: + annotation = TestAnnotation(**raw_annotation) + + name = annotation.name + flavor = annotation.flavor + reason = annotation.reason + + if flavor == TEST_ANNOTATION_HARNESS_SKIP: + print(f"not running the stdlib test harness: {reason}") + + # Add a GitHub Actions annotation to improve observability of this scenario. + + if "CI" in os.environ: + print( + f"::notice title={dist_name} stdlib test harness skipped::{reason}" + ) + + return 0 + + elif flavor == TEST_ANNOTATION_MODULE_EXCLUDE: + print(f"excluding module {name}: {reason}") + module_excludes.add(name) + continue + + elif flavor == TEST_ANNOTATION_TEST_FAILURE: + print(f"expected test failure {name}: {reason}") + expect_failures.add(name) + + if annotation.intermittent_test_failure: + intermittent.add(name) + if annotation.dont_verify: + dont_verify.add(name) + + else: + raise Exception(f"unhandled test annotation flavor: {flavor}") + + base_args = [ + "-u", + "-W", + "default", + "-bb", + "-E", + "-m", + "test", ] - args.extend(harness_args) + args = list(base_args) + + if raw_harness_args: + args.extend(raw_harness_args) + else: + args.extend( + [ + # Display test output on failure. Makes it easier to debug failures. + "-W", + # Re-run failed tests in verbose mode to aid debugging. + "-w", + # Make order non-deterministic to help flush out failures. + "--randomize", + # Run tests in parallel using all available CPUs. + "-j", + "0", + # Force abort tests taking too long to execute. This can prevent + # some runaway tests in CI. + "--timeout", + str(TIMEOUT_SECONDS), + # Print slowest tests to make it easier to audit our slow tests list. + "-o", + ] + ) + + # 3.14 added a test harness feature to prioritize running tests. Prioritize + # running slow tests first to mitigate long poles slowing down harness execution. + if meets_python_minimum_version(python_info["python_version"], "3.14"): + args.extend(["--prioritize", ",".join(STDLIB_SLOW_TESTS)]) + + for test_name in sorted(expect_failures): + args.extend(["--ignore", test_name]) + + if module_excludes: + # --exclude is a boolean argument that changes positional arguments to + # denotes excludes. As of at least 3.15 there doesn't appear to be a + # way to select tests to run via positional arguments while also + # excluding certain modules from being loaded. + args.append("--exclude") + args.extend(sorted(module_excludes)) + + codes = [] + + if not skip_main: + codes.append( + run_dist_python(dist_root, python_info, args, log_exec=True).returncode + ) + if codes[-1] != 0: + print(f"main test harness failed ({codes[-1]})") + else: + print("main test harness passed") + else: + print("main test harness skipped") + + if not raw_harness_args: + codes.extend( + _run_stdlib_expected_failures( + dist_root, + python_info, + base_args, + expect_failures, + intermittent, + dont_verify, + verbose_expected_failures=verbose_expected_failures, + ) + ) + + if any(code != 0 for code in codes): + return 1 + else: + return 0 + + +def _run_stdlib_expected_failures( + dist_root: Path, + python_info, + base_args: list[str], + expect_failures: set[str], + intermittent: set[str], + dont_verify: set[str], + verbose_expected_failures: bool, +) -> list[int]: + # Run ignored / expected failures tests and verify they actually fail. + unexpected_tests: set[str] = set() + codes = [] + + with concurrent.futures.ThreadPoolExecutor() as executor: + fs = [] + + for test_name in sorted(expect_failures): + fs.append( + executor.submit( + _check_stdlib_expected_failure, + dist_root, + python_info, + base_args, + test_name, + test_name in intermittent, + test_name in dont_verify, + ) + ) + + for i, f in enumerate(concurrent.futures.as_completed(fs)): + res: ExpectedFailureResult = f.result() + + if res.unexpected: + unexpected_tests.add(res.test_name) + codes.append(1) + else: + codes.append(0) + + unexpected_suffix = "" + if len(unexpected_tests) > 0: + unexpected_suffix = f"/{len(unexpected_tests)}" + + print( + f"[{i + 1}/{len(expect_failures)}{unexpected_suffix}] verifying {res.test_name} fails... {res.status}" + ) + + print_output = False + + # Show output if we opted into verbose failures. + if verbose_expected_failures and res.code != 0: + print_output = True + + # Show output if test failed and this is a dont-verify test (helps + # with debugging). + if res.code != 0 and res.dont_verify: + print_output = True + + if print_output: + print(res.output.decode("utf-8", errors="ignore")) + + if unexpected_tests: + print("unexpected test results:") + for t in sorted(unexpected_tests): + print(t) + + return codes + + +@dataclasses.dataclass +class ExpectedFailureResult: + test_name: str + code: int + output: bytes + status: str + unexpected: bool + intermittent: bool + dont_verify: bool + junit: Optional[junitparser.JUnitXml] + + +def _check_stdlib_expected_failure( + dist_root: Path, + python_info, + base_args: list[str], + test_name: str, + intermittent: bool, + dont_verify: bool, +) -> ExpectedFailureResult: + args = list(base_args) + + args.extend( + [ + "-v", + "--timeout", + str(TIMEOUT_SECONDS), + ] + ) - return run_dist_python(dist_root, python_info, args).returncode + # Always provide the test_* name as a positional argument to limit + # which tests are loaded. Otherwise we load all test files and incur + # substantial overhead. + parts = test_name.split(".") + if parts[0] == "test": + args.extend(["-m", test_name, parts[1]]) + else: + args.append(test_name) + + with tempfile.TemporaryDirectory() as td: + junit_path = Path(td) / "junit.xml" + args.extend(["--junit-xml", str(junit_path)]) + + res = run_dist_python(dist_root, python_info, args, capture_output=True) + + try: + junit = junitparser.JUnitXml.fromfile(str(junit_path)) + except FileNotFoundError: + junit = None + + unexpected = False + if res.returncode != 0: + status = "yes" + # If we print failing output here, GitHub Actions can sniff the + # output and convert to a failing test annotation, which is semantically + # incorrect. While it can be useful to print failing test details to aid + # debugging, we don't do it here because of the undesirable CI effects. + elif intermittent: + status = "no (intermittent pass allowed)" + elif dont_verify: + status = "no (ignoring pass per dont-verify annotation)" + else: + status = "no (unexpected pass)" + unexpected = True + + return ExpectedFailureResult( + test_name, + res.returncode, + res.stdout, # type: ignore + status, + unexpected, + intermittent, + dont_verify, + junit=junit, + ) def main(pbs_source_dir: Path, raw_args: list[str]) -> int: """test-distribution.py functionality.""" - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description=""" + Runs unit tests against a Python distribution. + + By default, this executes custom PBS unit tests. The unittests from + the distribution's Python standard library can be ran by adding --stdlib. + + By default, the stdlib test harness is run with opinionated behavior + where we automatically exclude running tests annotated as failing followed + by running each of these annotated tests in isolation to validate their + test annotations. e.g. we validate that failures actually fail. + + If arguments are passed after a `--` separator argument, the execution + behavior changes to a proxy to the stdlib test harness and arguments passed + after `--` are passed directly to `python -m test`, allowing you to invoke + the stdlib test harness with arbitrary arguments. + + The --stdlib-no-main (implies --stdlib) can be used to just run annotated + test failures. This can be useful for debugging these tests. + """ + ) parser.add_argument( "--stdlib", action="store_true", help="Run the stdlib test harness", ) + parser.add_argument( + "--stdlib-no-main", + action="store_true", + help="Skip running the main invocation of the stdlib test harness - only failures will be processed", + ) + parser.add_argument( + "--verbose-expected-failures", + action="store_true", + help="Print details of tests that failed expectedly (helps with debugging failures)", + ) parser.add_argument( "dist", nargs=1, help="Path to distribution to test", ) parser.add_argument( - "harness_args", + "stdlib_harness_args", nargs=argparse.REMAINDER, - help="Raw arguments to pass to Python's test harness", + help="Raw arguments to pass to the stdlib test harness", ) args = parser.parse_args(raw_args) + if args.stdlib_no_main: + args.stdlib = True + dist_path_raw = Path(args.dist[0]) td = None @@ -121,7 +471,15 @@ def main(pbs_source_dir: Path, raw_args: list[str]) -> int: codes.append(run_custom_unittests(pbs_source_dir, dist_path, python_info)) if args.stdlib: - codes.append(run_stdlib_tests(dist_path, python_info, args.harness_args)) + codes.append( + run_stdlib_tests( + dist_path, + python_info, + skip_main=args.stdlib_no_main, + verbose_expected_failures=args.verbose_expected_failures, + raw_harness_args=args.stdlib_harness_args, + ) + ) if len(codes) == 0: print("no tests run") diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index f17ab12f2..a5c3b1b66 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -174,7 +174,10 @@ def write_if_different(p: pathlib.Path, data: bytes): def write_triples_makefiles( - targets, dest_dir: pathlib.Path, support_search_dir: pathlib.Path + targets, + root_dir: pathlib.Path, + dest_dir: pathlib.Path, + support_search_dir: pathlib.Path, ): """Write out makefiles containing make variable settings derived from config.""" dest_dir.mkdir(parents=True, exist_ok=True) @@ -211,10 +214,13 @@ def write_triples_makefiles( % (entry, DOWNLOADS[entry]["version"], host_platform) ) - lines.append( - "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" - % (support_search_dir / "extension-modules.yml") - ) + extra_support_files = [ + support_search_dir / "extension-modules.yml", + root_dir / "stdlib-test-annotations.yml", + ] + + for p in extra_support_files: + lines.append(f"PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) {p}\n") write_if_different(makefile_path, "".join(lines).encode("ascii")) diff --git a/stdlib-test-annotations.yml b/stdlib-test-annotations.yml new file mode 100644 index 000000000..ad530be3d --- /dev/null +++ b/stdlib-test-annotations.yml @@ -0,0 +1,2734 @@ +--- +# This file defines metadata about Python standard library tests. +# +# The file consists of keys defining different groups of annotations. +# See each key for documentation. +# +# All entries share a common set of fields to support describing the +# annotation and filtering which annotations apply to which builds. +# +# * reason - human readable context describing the failure +# * targets - list of regular expressions matching target triples to +#. determine if rule is active. Missing or empty matches all targets. +# * ignore-targets - list of regular expressions matching target tripels +#. to determine if rule is ignored. Takes precedence over "targets." +# * minimum-python-version - X.Y Python version we must be equal or newer +#. to for the rule to apply. +# * maximum-python-version - X.Y Python version we must be equal or less +#. than for the rule to apply. +# * build-option - A build option (e.g. `pgo`) that must be present for this +#. rule to be enabled. +# * no-build-option - A build option (e.g. `freethreaded`) that must not be +#. present for this rule to be enabled. + +# harness-skips declares builds to skip running the Python stdlib test harness. +# +# Ideally this feature doesn't need to exist. But sometimes the test harness is +# so broken we need to do this. +harness-skips: + - name: "musl-debug-skip-3.14+" + reason: "`cannot get C stack on this system` error leads to widespread failure" + minimum-python-version: "3.14" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + +# module-excludes annotates modules to exclude from loading. +# +# The stdlib test runner applies this filter based on the name of the +# module file/directory within a directory. So values here need to be +# child most module paths. e.g. to exclude module `foo.bar.baz`, you +# need to use module name `baz` and not `foo.bar.baz`. +module-excludes: + - module: test_call + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "import failure due to missing _testcapi" + maximum-python-version: "3.12" + + - module: test_capi + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "import failure due to missing _testcapi" + + - module: test_code + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "import failure due to missing _testcapi" + + - module: test_ctypes + targets: + - ".+-unknown-linux-musl" + reason: "Module import failures" + + - module: test_datetime + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "import failure due to missing _testcapi" + + - module: test_sqlite3 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "import failure due to missing _testcapi" + + - module: test_stable_abi_ctypes + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "import failure due to missing _testcapi" + + - module: test_regrtest_noop1 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "import failure due to missing _testcapi" + + # Tons of failures. Just exclude entirely instead of running. + - module: test_traceback + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: tons of failures, mostly due to missing _testcapi + +# expected-failures is a list of tests that are expected to fail. +# +# * name - pattern identifying the test or group of tests +# * intermittent - indicates the failure is intermittent. +# * dont-verify - boolean indicating to not verify the failing test actually +#. fails. Useful for intermittent / flaky tests. +expected-failures: + - name: distutils.tests.test_archive_util.ArchiveUtilTestCase.test_tarfile_vs_tar + minimum-python-version: "3.11" + maximum-python-version: "3.11" + reason: "differing element" + + - name: distutils.tests.test_install_lib.InstallLibTestCase.test_byte_compile + maximum-python-version: "3.11" + reason: "TypeError: byte_compile() got an unexpected keyword argument 'dry_run'" + + - name: distutils.tests.test_build_py.BuildPyTestCase.test_byte_compile_optimized + maximum-python-version: "3.11" + reason: "TypeError: byte_compile() got an unexpected keyword argument 'dry_run'" + + - name: distutils.tests.test_sysconfig.SysconfigTestCase.test_srcdir + reason: path resolution error + maximum-python-version: "3.11" + targets: + - ".+-apple-darwin" + + - name: distutils.tests.test_build_ext.BuildExtTestCase.test_deployment_target_higher_ok + reason: wrong deployment target + maximum-python-version: "3.11" + targets: + - ".+-apple-darwin" + + - name: distutils.tests.test_build_ext.ParallelBuildExtTestCase.test_deployment_target_higher_ok + reason: wrong deployment target + maximum-python-version: "3.11" + targets: + - ".+-apple-darwin" + + - name: test.test_array.ByteTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.DoubleTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.FloatTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.IntTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.LongLongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.LongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.ShortTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnicodeTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedByteTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedIntTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedLongLongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedLongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedShortTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_ast.ModuleStateTests.test_subinterpreter + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_ast.test_ast.ModuleStateTests.test_subinterpreter + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_atexit_with_low_memory + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_callback_on_subinterpreter_teardown + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_callbacks_leak + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_callbacks_leak_refcycle + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_audit.AuditTest.test_unraisablehook + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "AssertionError" + + - name: test.test_audit.AuditTest.test_builtin__import__ + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "ModuleNotFoundError: No module named '_testcapi'. Did you mean: 'test_capi'?" + + - name: test.test_audit.AuditTest.test_import_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "ModuleNotFoundError: No module named '_testcapi'. Did you mean: 'test_capi'?" + + - name: test.test_audit.AuditTest.test_import_statement + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "ModuleNotFoundError: No module named '_testcapi'. Did you mean: 'test_capi'?" + + # Looks like a race condition in freethreaded builds. + - name: test.test_buffer.TestBufferProtocol.test_array_alignment + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "AssertionError: Lists differ: [0, 4, 0, 0, 0, 0, 0, 0, 0, 0] != [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" + intermittent: true + + - name: test.test_buffer.TestBufferProtocol.test_pybuffer_size_from_format + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_build_details.CPythonBuildDetailsTests.test_platform + reason: platform key doesn't match when cross compiling - possible PBS quirk/bug. + targets: + - ".+-apple-darwin" + - ".+-unknown-linux-.+" + dont-verify: true + + - name: test.test_bytes.ByteArrayTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_set + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_set + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_zero + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_zero + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_zero + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_zero + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_warn + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_warn + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_call.TestRecursion.test_margin_is_sufficient + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: missing _testcapi + minimum-python-version: "3.14" + + - name: test.test_capi.test_run.CAPITest.test_run_fileexflags + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.12" + reason: "crash" + + - name: test.test_cmath.CMathTests.test_polar_errno + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_cmd_line.CmdLineTest.test_pythonmalloc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_codecs.BasicUnicodeTest.test_basics_capi + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_concurrent_futures.test_interpreter_pool.InterpreterPoolExecutorTest.test_blocking + build-option: "freethreaded" + minimum-python-version: "3.15" + reason: "sometimes times out - possible test deadlock" + intermittent: true + + - name: test.test_concurrent_futures.test_shutdown.ProcessPoolForkProcessPoolShutdownTest.test_interpreter_shutdown + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_concurrent_futures.test_shutdown.ProcessPoolForkserverProcessPoolShutdownTest.test_interpreter_shutdown + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_coroutines.CAPITest.test_tp_await_1 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_coroutines.CAPITest.test_tp_await_2 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_coroutines.CAPITest.test_tp_await_3 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_ctypes.test_callbacks.Callbacks.test_i38748_stackCorruption + targets: + - "i686-pc-windows-msvc" + minimum-python-version: "3.12" + reason: "OSError: exception: access violation writing 0x00000005" + + - name: test.test_ctypes.test_callbacks.StdcallCallbacks.test_i38748_stackCorruption + targets: + - "i686-pc-windows-msvc" + minimum-python-version: "3.12" + reason: "OSError: exception: access violation writing 0x00000005" + + # Looks racy on freethreaded builds. + - name: test.datetimetester.ExtensionModuleTests_Fast.test_concurrent_initialization_subinterpreter + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "AssertionError: b'aaaaaaa' != b'aaaaaaaa'" + intermittent: true + + - name: test.datetimetester.ZoneInfoTest_Fast.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.datetimetester.IranTest_Pure.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.datetimetester.ZoneInfoTest_Pure.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.datetimetester.IranTest_Fast.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.test_dict.CAPITest.test_getitem_knownhash + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: missing _testcapi + + - name: test.test_dict.CAPITest.test_splittable_setattr_after_pop + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + reason: missing _testcapi + + - name: test.test_dict.DictTest.test_splittable_setattr_after_pop + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: missing _testcapi + + - name: test.test_docxmlrpc.DocXMLRPCHTTPGETServer.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_exceptions.ExceptionTests.testSettingException + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_memory_error_in_subinterp + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_MemoryError + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_exception_with_doc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_memory_error_cleanup + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_memory_error_in_PyErr_PrintEx + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_infinite_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_with_no_memory + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.13" + maximum-python-version: "3.13" + reason: "_testcapi missing" + + - name: test.test_faulthandler.FaultHandlerTests.test_fatal_error + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_faulthandler.FaultHandlerTests.test_fatal_error_without_gil + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_fcntl.TestFcntl.test_fcntl_bad_file_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_fcntl.TestFcntl.test_flock_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_fileio.COtherFileTests.testInvalidFd_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.LegacyFinalizationTest.test_legacy + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.LegacyFinalizationTest.test_legacy_resurrect + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.LegacyFinalizationTest.test_legacy_self_cycle + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.SimpleFinalizationTest.test_non_gc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.SimpleFinalizationTest.test_non_gc_resurrect + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_float.IEEEFormatTestCase.test_serialized_float_rounding + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_format.FormatTest.test_precision_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_frame.TestFrameCApi.test_basic + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + reason: _testcapi missing + + # This test has a weird pass/fail matrix. Test might be self-skipping or something. + + # aarch64 passes w/o pgo and on freethreaded+pgo + - name: test.test_frame_pointer_unwind.FramePointerUnwindTests.test_manual_unwind_respects_frame_pointers + minimum-python-version: "3.15" + targets: + - aarch64-unknown-linux-gnu + build-option: "pgo" + no-build-option: "freethreaded" + reason: "AssertionError: 0 not greater than 0 : expected to find Python frames on aarch64 with env {'PYTHON_JIT': '1'}" + + - name: test.test_frame_pointer_unwind.FramePointerUnwindTests.test_manual_unwind_respects_frame_pointers + minimum-python-version: "3.15" + targets: + - x86_64.*-unknown-linux-musl + build-option: "static" + reason: "AssertionError: 0 not greater than 0 : expected to find Python frames on aarch64 with env {'PYTHON_JIT': '1'}" + + - name: test.test_functools.TestPartialC.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_functools.TestPartialCSubclass.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_functools.TestPartialPy.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_functools.TestPartialPySubclass.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_gc.GCCallbackTests.test_collect_garbage + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_gc.GCCallbackTests.test_refcount_errors + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "AssertionError: Regex didn't match: b'gcmodule\\.c:[0-9]+:" + intermittent: true + + - name: test.test_gc.GCTests.test_legacy_finalizer + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_gc.GCTests.test_legacy_finalizer_newclass + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_gc.GCTests.test_garbage_at_shutdown + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "AssertionError: 1 != 0" + + - name: test.test_genericclass.CAPITest.test_c_class + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + # Test appears to be racy on freethreaded builds. + - name: test.test_hashlib.HashlibTestCase.test_threaded_hashing_fast + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "appears to be racy on freethreaded builds" + intermittent: true + + - name: test.test_httpservers.BaseHTTPServerTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_httpservers.CGIHTTPServerTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_httpservers.CGIHTTPServerTestCase.test_large_content_length_truncated + reason: "intermittent failure: http.client.RemoteDisconnected: Remote end closed connection without response" + intermittent: true + + - name: test.test_httpservers.RequestHandlerLoggingTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_httpservers.HTTP09ServerTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test__interpchannels.ChannelTests.test_send_recv_different_interpreters_and_threads + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.CreateTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.CreateTests.test_in_threaded_subinterpreter + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.RunFuncTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.RunStringTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_imp.ImportTests.test_create_builtin_subinterp + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "_testcapi missing" + + - name: test.test_imp.ImportTests.test_issue16421_multiple_modules_in_one_dll + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "_testcapi missing" + + - name: test.test_imp.ImportTests.test_issue24748_load_module_skips_sys_modules_check + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "_testcapi missing" + + - name: test.test_import.CircularImportTests.test_singlephase_circular + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + reason: _testsinglephase shared library expectation + + - name: test.test_import.ImportTests.test_from_import_missing_attr_has_name_and_so_path + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.ModexportTests.test_from_modexport + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_create_nonmodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_create_nonmodule_gil_used + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_empty_slots + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_gil_used + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_null + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_smoke + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.SinglephaseInitTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + reason: shared library expectations + + - name: test.test_import.SubinterpImportTests.test_builtin_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_disallowed_reimport + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testsinglephase shared library expectation" + + - name: test.test_import.SubinterpImportTests.test_frozen_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_isolated_config + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_multi_init_extension_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: shared library expectations + + - name: test.test_import.SubinterpImportTests.test_multi_init_extension_non_isolated_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testmultiphase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_multi_init_extension_per_interpreter_gil_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testmultiphase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_pyinit_function_raises_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testsinglephase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.SubinterpImportTests.test_python_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_singlephase_check_with_setting_and_override + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testsinglephase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_single_init_extension_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testsinglephase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_testmultiphase_exec_multiple + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_is_package + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_load_module_API + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_module_reuse + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_bad_modules + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_load_short_name + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_load_submodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_is_package + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_load_module_API + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_module_reuse + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_bad_modules + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_finder.Frozen_FinderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AssertionError: None is not true" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_unloadable_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_functionality + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_load_twice + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_nonmodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_nonmodule_with_methods + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_null_slots + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_reload + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_try_registration + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_functionality + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_load_short_name + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_load_submodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_load_twice + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_nonmodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_nonmodule_with_methods + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_null_slots + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_reload + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_try_registration + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_unloadable_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "AssertionError: 'fo_6ja' != 'foó'" + + - name: test.test_importlib.extension.test_finder.Source_FinderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AssertionError: None is not true" + + - name: test.test_importlib.test_threaded_import.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_multiprocessing_pool_circular_import + targets: + - ".+-unknown-linux-musl" + reason: "segfault at least on debug builds" + # Test sniffs memory and self-skips. This check is not reliable. + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_concurrent_futures_circular_import + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.11" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_parallel_meta_path + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_parallel_module_init + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_parallel_path_hooks + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_side_effect_import + minimum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_util.IncompatibleExtensionModuleRestrictionsTests.test_incomplete_multi_phase_init_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_inspect.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestSignatureObject.test_signature_on_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestSignatureObject.test_signature_on_builtins_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestSignatureObject.test_signature_on_decorated_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestSignatureObject.test_signature_on_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestSignatureObject.test_signature_on_builtins_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestSignatureObject.test_signature_on_decorated_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_int.IntStrDigitLimitsTests.test_int_max_str_digits_is_per_interpreter + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_int.IntStrDigitLimitsTests.test_denial_of_service_prevented_str_to_int + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + # Seems to be fixed in 3.14. + maximum-python-version: "3.13" + reason: "AssertionError: 0.015625 not less than or equal to 0.0078125" + + - name: test.test_int.IntSubclassStrDigitLimitsTests.test_int_max_str_digits_is_per_interpreter + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_interpreters.CreateTests.test_in_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.TestInterpreterRun.test_in_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.CreateTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.LowLevelTests.test_is_running + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterCall.test_call_in_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.13" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterClose.test_created_with_capi + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.13" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterCall.test_stateless_func_returns_arg + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: missing _testinternalcapi + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterCall.test_stateless_funcs + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: missing _testinternalcapi + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterExec.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterIsRunning.test_created_with_capi + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_itertools.TestBasicOps.test_count_threading + minimum-python-version: "3.13" + build-option: "freethreaded" + reason: "test appears racy on freethreaded builds" + intermittent: true + + - name: test.test_io.CTextIOWrapperTest.test_device_encoding + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_io.PyTextIOWrapperTest.test_device_encoding + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test__locale._LocaleTests.test_alt_digits_nl_langinfo + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.11" + maximum-python-version: "3.13" + reason: "AssertionError: 1 not greater than or equal to 10 : ['0123456789']" + + # This one still fails on static builds on 3.14+. + - name: test.test__locale._LocaleTests.test_alt_digits_nl_langinfo + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: 1 not greater than or equal to 10 : ['0123456789']" + + - name: test.test__locale._LocaleTests.test_lc_numeric_localeconv + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: '.' != ','" + + - name: test.test__locale._LocaleTests.test_lc_numeric_localeconv + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: "AssertionError: '.' != ','" + + - name: test.test__locale._LocaleTests.test_lc_numeric_nl_langinfo + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: '.' != ','" + + # This one still fails on static builds on 3.14+. + - name: test.test__locale._LocaleTests.test_lc_numeric_nl_langinfo + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: '.' != ','" + + - name: test.test_locale.TestEnUSCollation.test_strcoll_with_diacritic + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: 1 not less than 0" + + - name: test.test_locale.TestEnUSCollation.test_strcoll_with_diacritic + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: 1 not less than 0" + + - name: test.test_locale.TestEnUSCollation.test_strxfrm_with_diacritic + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: 1 not less than 0" + + - name: test.test_locale.TestEnUSCollation.test_strxfrm_with_diacritic + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: 1 not less than 0" + + # Passes on x86_64_v3 and x86_64_v4 for some reason. + - name: test.test_math.FMATests.test_fma_zero_result + targets: + - "x86_64(_v2)?-unknown-linux-musl" + minimum-python-version: "3.13" + maximum-python-version: "3.13" + reason: "AssertionError: False is not true : Expected a negative zero, got 0.0" + + - name: test.test_math.FMATests.test_fma_zero_result + targets: + - ".+-unknown-linux-musl" + ignore-targets: + - "x86_64_v[34]-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: False is not true : Expected a negative zero, got 0.0" + + - name: test.test_multibytecodec.Test_IncrementalEncoder.test_subinterp + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_os.TestInvalidFD.test_fpathconf_bad_fd + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.TestInvalidFD.test_fpathconf_bad_fd + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + maximum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.test_os.TestInvalidFD.test_fpathconf_bad_fd + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.test_os.TestInvalidFD.test_pathconf_negative_fd_uses_fd_semantics + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "AssertionError: OSError not raised" + + - name: test.test_os.TestInvalidFD.test_fpathconf + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.test_posix.PosixTester.test_pipe2_c_limits + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "ModuleNotFoundError: No module named '_testcapi'" + + - name: test.test_os.test_posix.PosixTester.test_nodev + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "AssertionError: module 'posix' has no attribute 'NODEV'" + + - name: test.test_poll.PollTests.test_poll_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_posix.PosixTester.test_pipe2_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: missing _testcapi + + - name: test.test_posix.PosixTester.test_makedev + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + reason: "OverflowError: can't convert negative int to unsigned" + + - name: test.test_posix.PosixTester.test_makedev + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "OverflowError: can't convert negative int to unsigned" + + - name: test.test_pty.PtyTest.test_spawn_doesnt_hang + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + reason: AssertionError + intermittent: true + + - name: test.test_pydoc.PydocServerTest.test_server + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_re.ReTests.test_locale_caching + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: None is not true" + + - name: test.test_re.ReTests.test_locale_caching + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: None is not true" + + - name: test.test_re.ReTests.test_locale_compiled + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: None is not true" + + - name: test.test_re.ReTests.test_locale_compiled + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: None is not true" + + - name: test.test_readline.TestHistoryManipulation.test_write_read_append + targets: + - ".+-unknown-linux-musl" + reason: "AssertionError: 4 != 3" + + - name: test.test_readline.TestHistoryManipulation.test_write_read_limited_history + targets: + - ".+-unknown-linux-.+" + minimum-python-version: "3.11" + reason: TESTFN undefined + + - name: test.test_readline.TestReadline.test_write_read_limited_history + targets: + - ".+-unknown-linux-.+" + minimum-python-version: "3.11" + reason: TESTFN undefined + + - name: test.test_repl.TestInteractiveInterpreter.test_no_memory + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "AssertionError: 0 not found in (1, 120)" + + - name: test.test_smtplib.DebuggingServerTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.13" + reason: "segfault" + intermittent: true + + - name: test.test_smtplib.SMTPSimTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "freethreaded" + # Should be thread safe in 3.15. + maximum-python-version: "3.14" + reason: "segfault" + intermittent: true + + - name: test.test_smtplib.SMTPUTF8SimTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "freethreaded" + # Should be thread safe in 3.15. + maximum-python-version: "3.14" + reason: "segfault" + intermittent: true + + - name: test.test_socket.GeneralModuleTests.testNtoHErrors + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.GeneralModuleTests.test_getaddrinfo_int_port_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_socket.GeneralModuleTests.test_listen_backlog_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.NonBlockingTCPTests.testSetBlocking_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.BasicTCPTest.testShutdown_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.BasicTCPTest2.testShutdown_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.GeneralModuleTests.test3542SocketOptions + reason: "AssertionError: False is not true : Missing RFC3542 socket option 'IPV6_RECVDSTOPTS'" + maximum-python-version: "3.10" + targets: + - ".+-apple-darwin" + + - name: test.test_socket.NonBlockingTCPTests.testLargeTimeout + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_ssl.BasicSocketTests.test_read_write_zero + targets: + - ".+-pc-windows-.+" + minimum-python-version: "3.12" + reason: ConnectionAbortedError + intermittent: true + + # These uplink/applink failures should go away once we're patching + # OpenSSL correctly following https://github.com/python/cpython/pull/131839. + # These pass on aarch64 for reasons unknown. + + - name: test.test_ssl.ContextTests.test_load_dh_params + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.SSLErrorTests.test_lib_reason + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.SSLErrorTests.test_dh_params + targets: + - ".+-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.TestSSLDebug.test_keylog_defaults + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.TestSSLDebug.test_keylog_filename + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + # Looks like a race condition in the test. + - name: test.test_str.StrTest.test_raiseMemError + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + minimum-python-version: "3.14" + reason: "AssertionError: 79 != 75" + intermittent: true + + - name: test.test_strptime.StrptimeTests.test_date_locale2 + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + reason: "AssertionError: Tuples differ: (2000, 1, 1) != (1900, 1, 1)" + + - name: test.test_strptime.StrptimeTests.test_date_locale2 + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: Tuples differ: (2000, 1, 1) != (1900, 1, 1)" + + - name: test.test_struct.StructTest.test_endian_table_init_subinterpreters + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "BlockingIOError: [Errno 11] Resource temporarily unavailable: '/tmp/tmpqkxa77l0/python/install/lib/python3.15t/concurrent'" + intermittent: true + + - name: test.test_subprocess.POSIXProcessTestCase.test_vfork_used_when_expected + targets: + - "aarch64-unknown-linux-musl" + minimum-python-version: "3.14" + no-build-option: "debug" + reason: "AssertionError: Regex didn't match: b'(?i)vfork'..." + + - name: test.test_subprocess.ProcessTestCase.test_executable_without_cwd + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: 47 != 1" + + - name: test.test_subprocess.ProcessTestCaseNoPoll.test_executable_without_cwd + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: 47 != 1" + + - name: test.test_support.TestSupport.test_recursion + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.11" + reason: "RecursionError: maximum recursion depth exceeded" + + - name: test.test_sysconfig.TestSysConfig.test_config_vars_depend_on_site_initialization + minimum-python-version: "3.12" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_sysconfig.TestSysConfig.test_config_vars_recalculation_after_site_initialization + minimum-python-version: "3.12" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_sysconfig.TestSysConfig.test_paths_depend_on_site_initialization + minimum-python-version: "3.12" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_sysconfig.TestSysConfig.test_makefile_overwrites_config_vars + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: PBS behavior differences + + # This one fails everywhere. Except on 3.14+ on Windows, which passes. + - name: test.test_sysconfig.TestSysConfig.test_sysconfigdata_json + minimum-python-version: "3.11" + maximum-python-version: "3.13" + reason: PBS behavior differences + + - name: test.test_sysconfig.TestSysConfig.test_sysconfigdata_json + minimum-python-version: "3.14" + ignore-targets: + - ".+-pc-windows-.+" + reason: PBS behavior differences + + - name: test.test_sysconfig.TestSysConfig.test_triplet_in_ext_suffix + targets: + - "x86_64.*-unknown-linux-musl" + maximum-python-version: "3.10" + reason: "AssertionError: False is not true : .cpython-310-x86_64-linux-musl.so" + + - name: test.test_syslog.Test.test_subinterpreter_closelog + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_syslog.Test.test_subinterpreter_openlog + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_syslog.Test.test_subinterpreter_syslog + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + # Looks like a race condition in tests. + - name: test.test_sys.SizeofTest.test_objecttypes + targets: + - ".+-pc-windows-.+" + minimum-python-version: "3.14" + build-option: "freethreaded" + reason: "AssertionError: 137 != 133 : wrong size for : got 137, expected 133" + intermittent: true + + - name: test.test_sys.SysModuleTest.test_getallocatedblocks + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_sys.TestSysJIT.test_jit_is_active + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_custom_unraisablehook + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_custom_unraisablehook_fail + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_original_unraisablehook + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_original_unraisablehook_exception_qualname + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_threading.SubinterpThreadingTests.test_threads_join + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_threads_join_2 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_threads_not_allowed + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_daemon_threads_not_allowed + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_daemon_threads_fatal_error + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.ThreadingExceptionTests.test_print_exception + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "AssertionError" + + - name: test.test_threading.ThreadingExceptionTests.test_print_exception_stderr_is_none_1 + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "AssertionError" + + - name: test.test_threading.ThreadingExceptionTests.test_recursion_limit + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.10" + reason: "AssertionError: -11 != 0" + + - name: test.test_threading.ThreadJoinOnShutdown.test_4_daemon_threads + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused and segfault" + intermittent: true + + - name: test.test_threading.ThreadTests.test_frame_tstate_tracing + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.ThreadTests.test_import_from_another_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_threading.ThreadTests.test_main_thread_after_fork + minimum-python-version: "3.12" + targets: + - "x86_64-apple-darwin" + reason: "AssertionError: b':6: DeprecationWarning: This proc[82 chars]d.\n' != b''" + + - name: test.test_threading.ThreadTests.test_native_id_after_fork + minimum-python-version: "3.12" + targets: + - "x86_64-apple-darwin" + reason: "AssertionError: b':6: DeprecationWarning: This proc[82 chars]d.\n' != b''" + + - name: test.test_threading.ThreadTests.test_PyThreadState_SetAsyncExc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: missing _testcapi + + - name: test.test_threading.ThreadTests.test_finalize_daemon_thread_hang + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.13" + reason: missing _testcapi + + - name: test.test_threading.ThreadTests.test_finalize_running_thread + targets: + - ".+-unknown-linux-mus" + build-option: "static" + reason: "AssertionError: 1 != 42" + + - name: test.test_multiprocessing_forkserver.test_manager.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "unknown; possible segfault" + + - name: test.test_multiprocessing_forkserver.test_misc.TestStartMethod.test_preload_resources + minimum-python-version: "3.12" + targets: + - "x86_64-apple-darwin" + build-option: "debug" + reason: "AssertionError: failed spawning forkserver or grandchild" + + - name: test.test_multiprocessing_fork.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfaults" + + - name: test.test_multiprocessing_fork.test_manager.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_fork.test_processes.WithProcessesTestManagerRestart.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_fork.test_threads.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_forkserver.test_processes.WithProcessesTestManagerRestart.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_forkserver.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_spawn.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_spawn.test_manager.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_spawn.test_processes.WithProcessesTestManagerRestart.test_rapid_restart + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_spawn.test_threads.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.11" + reason: "segfault" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_basic_script + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_basic_script_no_suffix + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_directory + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_directory_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_ipython_workaround + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_module_in_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_module_in_package_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_module_in_subpackage_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_package_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_script_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_zipfile + targets: + - "x86_64-apple-darwin" + no-build-option: "pgo" + minimum-python-version: "3.12" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_zipfile_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_basic_script + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_basic_script_no_suffix + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_directory + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_directory_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_ipython_workaround + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_module_in_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_module_in_package_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_module_in_subpackage_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_package_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_script_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_forkserver.test_processes.WithProcessesTestSpawnedSysPath.test_std_streams_flushed_after_preload + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + build-option: "debug" + reason: "multiprocessing/forkserver.py:278: DeprecationWarning: This process (pid=82723) is multi-threaded, use of fork() may lead to deadlocks in the child." + + - name: test.test_multiprocessing_forkserver.test_threads.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_forkserver.test_misc.MiscTestCase.test_preload_main + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + build-option: "debug" + reason: "AssertionError: b'/private/var/folders/1r/b8l6nzhj4zdbz5jmj[222 chars]()\n' != b'" + + - name: test.test_multiprocessing_forkserver.test_misc.MiscTestCase.test_preload_main_sys_argv + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + build-option: "debug" + reason: "AssertionError: b'/private/var/folders/1r/b8l6nzhj4zdbz5jmj[222 chars]()\n' != b''" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_trampoline_works_after_fork_with_many_code_objects + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.15" + reason: "AssertionError: '/private/var/folders/1r/b8l6nzhj4zdbz5jmj[194 chars]()\n' != ''" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_trampoline_works_with_forks + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.15" + reason: "AssertionError: '/private/var/folders/1r/b8l6nzhj4zdbz5jmj[194 chars]()\n' != ''" + + - name: test.test_regrtest.ArgsTestCase.test_add_python_opts + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: "Something related to _testcapi or similar extension" + + - name: test.test_regrtest.ArgsTestCase.test_findleaks + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "AssertionError: Command ['/tmp/.tmpGZuURa/python/install/bin/python3.10', '-X', 'faulthandler', '-I', '-m', 'test', '--testdir=/tmp/tmp6mrz59ds', '--fail-env-changed', 'test_regrtest_noop1'] failed with exit code 2" + + - name: test.test_regrtest.ArgsTestCase.test_uncollectable + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_tcl.TclTest.test_evalfile_surrogates_in_result + maximum-python-version: "3.11" + ignore-targets: + - ".+-pc-windows-.+" + reason: "_tkinter.TclError: couldn't read file \"@test_169558_tmp\": invalid or incomplete multibyte or wide character" + + - name: test.test_tcl.TclTest.test_getint + maximum-python-version: "3.10" + reason: "AssertionError: 17777777777 != 2147483647" + ignore-targets: + - ".+-pc-windows-.+" + + - name: test.test_unicode.UnicodeTest.test_formatting_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_unicode.UnicodeTest.test_formatting_huge_precision_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_call_args_thread_safe + minimum-python-version: "3.13" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_call_count_thread_safe + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_method_calls_thread_safe + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_mock_calls_thread_safe + minimum-python-version: "3.13" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_wait_until_any_call_with_success + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_wait_until_called_success + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_userstring.UserStringTest.test_formatting_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_venv.BasicTest.test_executable + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_macos_env + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_multiprocessing + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_multiprocessing_recursion + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_prefixes + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_special_chars_bash + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_special_chars_csh + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_sysconfig_symlinks + targets: + - ".+-apple-darwin" + minimum-python-version: "3.11" + maximum-python-version: "3.13" + reason: BytesWarning + + - name: test.test_venv.BasicTest.test_sysconfig + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_sysconfig + minimum-python-version: "3.13" + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + reason: "AssertionError: 3.14t missing suffix - likely PBS quirk" + + - name: test.test_venv.BasicTest.test_venvwlauncher + minimum-python-version: "3.13" + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + reason: "AssertionError: 3.14t missing suffix - likely PBS quirk" + + - name: test.test_venv.BasicTest.test_zippath_from_non_installed_posix + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.EnsurePipTest.test_explicit_no_pip + targets: + - ".+-apple-darwin" + reason: PBS installs pip + + - name: test.test_venv.EnsurePipTest.test_no_pip_by_default + targets: + - ".+-apple-darwin" + reason: PBS installs pip + + - name: test.test_venv.EnsurePipTest.test_with_pip + targets: + - ".+-apple-darwin" + maximum-python-version: "3.10" + reason: logic bug? + + - name: test.test_venv.EnsurePipTest.test_with_pip + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.10" + build-option: "static" + reason: "TBD" + + - name: test.test_weakref.ReferencesTestCase.test_cfunction + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test__xxinterpchannels.ChannelTests.test_send_recv_different_interpreters_and_threads + minimum-python-version: "3.12" + maximum-python-version: "3.12" + targets: + - "aarch64-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__xxsubinterpreters.ChannelTests.test_send_recv_different_interpreters_and_threads + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__xxsubinterpreters.CreateTests.test_in_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test__xxsubinterpreters.CreateTests.test_in_threaded_subinterpreter + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test__xxsubinterpreters.RunStringTests.test_in_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true diff --git a/uv.lock b/uv.lock index 262e79edc..9c7c67700 100644 --- a/uv.lock +++ b/uv.lock @@ -600,6 +600,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "junitparser" +version = "4.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/97/954ee1ef04e50d8494e9f5d82d4051ed71a7618aa2c1514c1b3f24691174/junitparser-4.0.2.tar.gz", hash = "sha256:d5d07cece6d4a600ff3b7b96c8db5ffa45a91eed695cb86c45c3db113c1ca0f8", size = 25646, upload-time = "2025-06-24T04:37:32.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/87/b444f934f62ee2a1be45bb52563cf17a66b0d790eba43af4df9929e7107f/junitparser-4.0.2-py3-none-any.whl", hash = "sha256:94c3570e41fcaedc64cc3c634ca99457fe41a84dd1aa8ff74e9e12e66223a155", size = 14592, upload-time = "2025-06-24T04:37:31.322Z" }, +] + [[package]] name = "lazy-object-proxy" version = "1.12.0" @@ -1191,6 +1200,7 @@ dependencies = [ { name = "docker" }, { name = "jinja2" }, { name = "jsonschema" }, + { name = "junitparser" }, { name = "pyyaml" }, { name = "six" }, { name = "tomli" }, @@ -1217,6 +1227,7 @@ requires-dist = [ { name = "docker", specifier = ">=7.1.0" }, { name = "jinja2", specifier = ">=3.1.5" }, { name = "jsonschema", specifier = ">=4.23.0" }, + { name = "junitparser", specifier = ">=4.0.2" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "six", specifier = ">=1.17.0" }, { name = "tomli", specifier = ">=2.2.1" }, From facf0e3c8c650f903aadb48576684d5848e904f1 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 24 Mar 2026 14:41:23 +0800 Subject: [PATCH 2/6] unix: support building Linux x86-64 from macOS aarch64 Docker on aarch64 macOS will automagically virtualize x86-64 containers if containers are spawned with `platform=linux/amd64`. Performance of spawned containers is a bit slower than native, but not horrible. This functionality means it is viable to develop Linux x86-64 from modern Apple hardware. This commit teaches the build system to support cross-compiling Linux x86-64 from macOS aarch64. Implementing this wasn't too difficult: we need to pass `platform` into Docker's APIs for building and creating containers. We need to teach code to resolve the effective host platform when this scenario is detected. And we need to advertise support for cross-compiling in the `targets.yml` file. In case you are wondering, yes, a similar solution could be employed for Linux too by using emulation. But this requires Docker be configured to support emulation, which isn't common. Rosetta on macOS "just works" and is therefore the lowest hanging fruit to implement. --- cpython-unix/build-main.py | 9 ++++- cpython-unix/build.py | 67 ++++++++++++++++++++++++-------------- cpython-unix/targets.yml | 12 +++++++ pythonbuild/buildenv.py | 5 ++- pythonbuild/docker.py | 35 +++++++++++--------- 5 files changed, 86 insertions(+), 42 deletions(-) diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 0e1f6a7cf..e4d9fe9a2 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -142,7 +142,14 @@ def main(): effective_host_platform = host_platform if building_linux_from_macos: if host_platform == "macos_arm64": - effective_host_platform = "linux_aarch64" + if target_triple.startswith("aarch64"): + effective_host_platform = "linux_aarch64" + elif target_triple.startswith("x86_64"): + effective_host_platform = "linux_x86_64" + else: + raise Exception( + f"unsupported macOS cross-compile: {host_platform} -> {target_triple}" + ) else: raise Exception(f"Unhandled macOS platform: {host_platform}") print( diff --git a/cpython-unix/build.py b/cpython-unix/build.py index f78c340c4..60b3d71fc 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -26,7 +26,12 @@ parse_setup_line, stdlib_test_annotations, ) -from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles +from pythonbuild.docker import ( + build_docker_image, + docker_platform_from_host_platform, + get_image, + write_dockerfiles, +) from pythonbuild.downloads import DOWNLOADS from pythonbuild.logging import log, set_logger from pythonbuild.utils import ( @@ -34,6 +39,7 @@ add_licenses_to_extension_entry, clang_toolchain, create_tar_from_directory, + current_host_platform, download_entry, get_target_settings, get_targets, @@ -100,21 +106,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options) extra_target_ldflags.append("--rtlib=compiler-rt") if build_platform.startswith("linux_"): - machine = platform.machine() + # autoconf is not aware of microarch triples. Normalize those out: + # we force targeting via -march CFLAG. + env["TARGET_TRIPLE"] = ( + target_triple.replace("x86_64_v2-", "x86_64-") + .replace("x86_64_v3-", "x86_64-") + .replace("x86_64_v4-", "x86_64-") + ) - # arm64 allows building for Linux on a macOS host using Docker - if machine == "aarch64" or machine == "arm64": - env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" - env["TARGET_TRIPLE"] = target_triple - elif machine == "x86_64": - env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" - env["TARGET_TRIPLE"] = ( - target_triple.replace("x86_64_v2-", "x86_64-") - .replace("x86_64_v3-", "x86_64-") - .replace("x86_64_v4-", "x86_64-") - ) + # On macOS, we support building Linux in a virtualized container that + # always matches the target platform. Set build/host triple to whatever + # we're building. + # + # Note: we always use the *-gnu triple otherwise autoconf can have + # trouble reasoning about cross-compiling since its detected triple from + # our build environment is always GNU based. + if current_host_platform().startswith("macos_"): + env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu") else: - raise Exception("unhandled Linux machine value: %s" % machine) + # Otherwise assume the container environment matches the machine + # type of the current process. + host_machine = platform.machine() + if host_machine == "aarch64" or host_machine == "arm64": + env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" + elif host_machine == "x86_64": + env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + else: + raise Exception("unhandled Linux machine value: %s" % host_machine) # This will make x86_64_v2, etc count as cross-compiling. This is # semantically correct, since the current machine may not support @@ -977,16 +995,6 @@ def main(): DOWNLOADS_PATH.mkdir(exist_ok=True) (BUILD / "logs").mkdir(exist_ok=True) - if os.environ.get("PYBUILD_NO_DOCKER"): - client = None - else: - try: - client = docker.from_env(timeout=600) - client.ping() - except Exception as e: - print("unable to connect to Docker: %s" % e, file=sys.stderr) - return 1 - # Note these arguments must be synced with `build-main.py` parser = argparse.ArgumentParser() parser.add_argument( @@ -1048,6 +1056,17 @@ def main(): settings = get_target_settings(TARGETS_CONFIG, target_triple) + if os.environ.get("PYBUILD_NO_DOCKER"): + client = None + else: + try: + client = docker.from_env(timeout=600) + client.ping() + client._pbs_platform = docker_platform_from_host_platform(host_platform) + except Exception as e: + print("unable to connect to Docker: %s" % e, file=sys.stderr) + return 1 + if args.action == "dockerfiles": log_name = "dockerfiles" elif args.action == "makefiles": diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 36c59c2c3..432833d69 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -556,6 +556,7 @@ x86_64-apple-darwin: x86_64-unknown-linux-gnu: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -605,6 +606,7 @@ x86_64-unknown-linux-gnu: x86_64_v2-unknown-linux-gnu: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -655,6 +657,7 @@ x86_64_v2-unknown-linux-gnu: x86_64_v3-unknown-linux-gnu: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -705,6 +708,9 @@ x86_64_v3-unknown-linux-gnu: x86_64_v4-unknown-linux-gnu: host_platforms: - linux_x86_64 + # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries + # under Rosetta. But they can build correctly. + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -755,6 +761,9 @@ x86_64_v4-unknown-linux-gnu: x86_64-unknown-linux-musl: host_platforms: - linux_x86_64 + # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries + # under Rosetta. But they can build correctly. + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -802,6 +811,7 @@ x86_64-unknown-linux-musl: x86_64_v2-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -850,6 +860,7 @@ x86_64_v2-unknown-linux-musl: x86_64_v3-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -898,6 +909,7 @@ x86_64_v3-unknown-linux-musl: x86_64_v4-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index a946813ed..37019ca26 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -266,7 +266,10 @@ def find_output_files(self, base_path, pattern): def build_environment(client, image): if client is not None: container = client.containers.run( - image, command=["/bin/sleep", "86400"], detach=True + image, + command=["/bin/sleep", "86400"], + detach=True, + platform=client._pbs_platform, ) td = None context = ContainerContext(container) diff --git a/pythonbuild/docker.py b/pythonbuild/docker.py index 0be78e4f8..dc549658d 100644 --- a/pythonbuild/docker.py +++ b/pythonbuild/docker.py @@ -2,12 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -import contextlib import io import operator import os import pathlib import tarfile +import typing import docker # type: ignore import jinja2 @@ -29,16 +29,31 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path): write_if_different(dest_dir / f, data.encode("utf-8")) +def docker_platform_from_host_platform(host_platform: str) -> typing.Optional[str]: + """Convert a PBS host platform to a Docker platform.""" + if host_platform == "linux_x86_64": + return "linux/amd64" + elif host_platform == "linux_aarch64": + return "linux/arm64" + else: + return None + + def build_docker_image( client, image_data: bytes, image_dir: pathlib.Path, name, host_platform ): image_path = image_dir / f"image-{name}.{host_platform}" - return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path) + return ensure_docker_image( + client, + io.BytesIO(image_data), + image_path=image_path, + platform=docker_platform_from_host_platform(host_platform), + ) -def ensure_docker_image(client, fh, image_path=None): - res = client.api.build(fileobj=fh, decode=True) +def ensure_docker_image(client, fh, image_path=None, platform=None): + res = client.api.build(fileobj=fh, decode=True, platform=platform) image = None @@ -111,18 +126,6 @@ def copy_file_to_container(path, container, container_path, archive_path=None): container.put_archive(container_path, buf.getvalue()) -@contextlib.contextmanager -def run_container(client, image): - container = client.containers.run( - image, command=["/bin/sleep", "86400"], detach=True - ) - try: - yield container - finally: - container.stop(timeout=0) - container.remove() - - def container_exec(container, command, user="build", environment=None): # docker-py's exec_run() won't return the exit code. So we reinvent the # wheel. From 50b7cae5eb083d858fa62dcb0d7c48ffc6782639 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Tue, 24 Mar 2026 11:20:17 +0800 Subject: [PATCH 3/6] unix: disable some tests during PGO training that intermittently fail A couple of tests appear to have race conditions on freethreaded builds on machines with a sufficient number of CPU cores. (These feel like CPython bugs, which I will report later.) We disable these tests during PGO training as a workaround. --- cpython-unix/build-cpython.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 730e6d18e..a9ae7557e 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -625,6 +625,14 @@ if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format" fi +# Some tests aren't race friendly on freethreaded builds. Disable as a mitigation. +if [[ -n "${CPYTHON_FREETHREADED}" ]]; then + PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.FreeThreadingTest.test_free_threading_bytearrayiter" + PROFILE_TASK="${PROFILE_TASK} --ignore test.test_hashlib.HashlibTestCase.test_threaded_hashing_fast" + PROFILE_TASK="${PROFILE_TASK} --ignore test.test_itertools.TestBasicOps.test_count_threading" + PROFILE_TASK="${PROFILE_TASK} --ignore test.test_itertools.TestBasicOps.test_count_with_step_threading" +fi + # ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL, # but on x86-64 that requires v2 and v3 respectively, and on arm64 the performance is bad as noted # in the comments, so just don't even try. (We should check if we can make this conditional) From 0c9e20625e4024cb25e5cff8dce522170a7370cb Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Wed, 25 Mar 2026 15:29:22 +0800 Subject: [PATCH 4/6] HACK reduce compression of Python distribution To test how fast CI can go. --- pythonbuild/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index a5c3b1b66..fd4404774 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -501,10 +501,7 @@ def compress_python_archive( try: with source_path.open("rb") as ifh, temp_path.open("wb") as ofh: - params = zstandard.ZstdCompressionParameters.from_level( - 22, strategy=zstandard.STRATEGY_BTULTRA2 - ) - cctx = zstandard.ZstdCompressor(compression_params=params) + cctx = zstandard.ZstdCompressor() cctx.copy_stream(ifh, ofh, source_path.stat().st_size) temp_path.rename(dest_path) From ca37f1da8a50ccb9c7b96cfa3b5a31f7975b8f5e Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Sat, 21 Mar 2026 15:55:27 +0800 Subject: [PATCH 5/6] ci: use larger Depot Linux runners Default runners have 2 CPUs. We can achieve a nice speed-up (but not quite linear) by leveraging runners with more CPUs. --- ci-runners.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-runners.yaml b/ci-runners.yaml index e81cac74c..ca2182e19 100644 --- a/ci-runners.yaml +++ b/ci-runners.yaml @@ -1,11 +1,11 @@ # Describes the runners that the CI system can use -depot-ubuntu-22.04-4: +depot-ubuntu-22.04-16: arch: x86_64 platform: linux free: false -depot-ubuntu-22.04-arm-4: +depot-ubuntu-22.04-arm-16: arch: aarch64 platform: linux free: false From f87b1f08ca035b9586ab5a78d2548aaf1910fe40 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 20 Mar 2026 11:22:10 +0800 Subject: [PATCH 6/6] unix: use extended PGO training set This will run all tests, ensuring maximal training coverage. As part of this, we had to annotate/ignore every failing test because test failures would otherwise fail the build. --- cpython-unix/build-cpython.sh | 51 +++++--- cpython-unix/build.py | 16 +++ pythonbuild/cpython.py | 43 +++++++ pythonbuild/testdist.py | 11 ++ stdlib-test-annotations.yml | 211 ++++++++++++++++++++++++++++++++++ 5 files changed, 315 insertions(+), 17 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index a9ae7557e..638350e56 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -52,7 +52,7 @@ PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl" SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl" # Put critical config files in logs to aid debugging. -for f in Setup.local Makefile.extra stdlib-test-annotations.json; do +for f in Setup.local Makefile.extra stdlib-test-annotations.json profiling-training-ignores.txt; do echo "BEGIN $f" cat $f echo "END $f" @@ -593,7 +593,27 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]; then fi # Define the base PGO profiling task, which we'll extend below with ignores -export PROFILE_TASK='-m test --pgo' +# +# --pgo-extended implies --pgo. And --pgo mode significantly changes the behavior +# of the test harness. Notably, it disables richer reporting of test failures, +# making it vastly more difficult to debug test failures. This behavior undermines +# our ability to debug failures when running tests for PGO instrumentation. +# +# The only material downside to not using --pgo is some tests that self-skip +# when run under PGO don't do so. When we audited for such tests in 2026-03, +# all such tests had the reason "PGO isn't useful" or some such. There were +# ~10 such tests. Since we run the entire test suite anyway, the inclusion of +# such "worthless" tests isn't impactful. The test failure observability is +# worth their loss. +export PROFILE_TASK='-m test' + +# Display test output on failure. This helps immensely with debugging PGO +# failures. +PROFILE_TASK="${PROFILE_TASK} -W" + +# Force kill tests taking too long. This prevents deadlocks that can cause +# CI jobs to run for potentially hours while doing nothing. +PROFILE_TASK="${PROFILE_TASK} --timeout 300" # Run tests in parallel to reduce wall time. # @@ -608,29 +628,26 @@ export PROFILE_TASK='-m test --pgo' # and there will be no loss in profile quality. PROFILE_TASK="${PROFILE_TASK} -j ${NUM_CPUS}" -# On 3.14+ `test_strftime_y2k` fails when cross-compiling for `x86_64_v2` and `x86_64_v3` targets on -# Linux, so we ignore it. See https://github.com/python/cpython/issues/128104 -if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" && -n "${CROSS_COMPILING}" && "${PYBUILD_PLATFORM}" != macos* ]]; then - PROFILE_TASK="${PROFILE_TASK} --ignore test_strftime_y2k" +IGNORE_TESTS=() + +# Possible race condition. +IGNORE_TESTS+=("test.test_audit.AuditTest.test_time_fail") + +if [[ -n "${CPYTHON_FREETHREADED}" ]]; then + IGNORE_TESTS+=("test.test_bytes.FreeThreadingTest.test_free_threading_bytearrayiter") fi # On 3.14+ `test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding` fails during # PGO due to RecursionError not being raised as expected. See https://github.com/python/cpython/issues/140125 if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then - PROFILE_TASK="${PROFILE_TASK} --ignore test_json" + IGNORE_TESTS+=("test.test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding") fi -# PGO optimized / BOLT instrumented binaries segfault in a test_bytes test. Skip it. -if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* ]]; then - PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format" -fi +PROFILE_TASK="${PROFILE_TASK} --ignorefile ${ROOT}/profiling-training-ignores.txt" -# Some tests aren't race friendly on freethreaded builds. Disable as a mitigation. -if [[ -n "${CPYTHON_FREETHREADED}" ]]; then - PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.FreeThreadingTest.test_free_threading_bytearrayiter" - PROFILE_TASK="${PROFILE_TASK} --ignore test.test_hashlib.HashlibTestCase.test_threaded_hashing_fast" - PROFILE_TASK="${PROFILE_TASK} --ignore test.test_itertools.TestBasicOps.test_count_threading" - PROFILE_TASK="${PROFILE_TASK} --ignore test.test_itertools.TestBasicOps.test_count_with_step_threading" +# Exclude whole modules from profiling based on stdlib test annotations. +if [ -n "${PROFILING_EXCLUDE_MODULES}" ]; then + PROFILE_TASK="${PROFILE_TASK} --exclude ${PROFILING_EXCLUDE_MODULES}" fi # ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL, diff --git a/cpython-unix/build.py b/cpython-unix/build.py index 60b3d71fc..34422d259 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -839,12 +839,28 @@ def build_cpython( build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json") + # Install a file with tests to skip during profile training. + # Also collect module excludes for the profiling run of the test harness. + profiling_exclude_modules = [] + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh: + os.chmod(fh.name, 0o644) + + for ann in test_annotations.annotations: + if ann.profile_training_skip(): + fh.write(f"{ann.name}\n") + if m := ann.profile_training_exclude_module(): + profiling_exclude_modules.append(m) + + fh.flush() + build_env.copy_file(fh.name, dest_name="profiling-training-ignores.txt") + env = { "PIP_VERSION": DOWNLOADS["pip"]["version"], "PYTHON_VERSION": python_version, "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]), "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"], "TOOLCHAIN": "clang-%s" % host_platform, + "PROFILING_EXCLUDE_MODULES": " ".join(profiling_exclude_modules), } # Set environment variables allowing convenient testing for Python diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index d342f1ad9..b5eea371e 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -197,6 +197,30 @@ "required": ["name", "reason"], }, }, + "profiling-excludes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["name", "reason"], + }, + }, + "profiling-skips": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["name", "reason"], + }, + }, }, } @@ -817,6 +841,8 @@ def extension_modules_config(yaml_path: pathlib.Path): TEST_ANNOTATION_HARNESS_SKIP = "harness-skip" TEST_ANNOTATION_MODULE_EXCLUDE = "module-exclude" TEST_ANNOTATION_TEST_FAILURE = "test-failure" +TEST_ANNOTATION_PROFILING_EXCLUDE = "profiling-exclude" +TEST_ANNOTATION_PROFILING_SKIP = "profiling-skip" @dataclasses.dataclass @@ -837,6 +863,17 @@ class TestAnnotation: # Whether to exclude loading the test module when running tests. exclude_testing: bool + def profile_training_skip(self) -> bool: + """Whether to ignore this test during PGO training.""" + return self.flavor in ( + TEST_ANNOTATION_TEST_FAILURE, + TEST_ANNOTATION_PROFILING_SKIP, + ) + + def profile_training_exclude_module(self) -> Optional[str]: + """Name of module to exclude from profiling.""" + return self.name if self.flavor == TEST_ANNOTATION_PROFILING_EXCLUDE else None + @dataclasses.dataclass class TestAnnotations: @@ -937,6 +974,12 @@ def stdlib_test_annotations( for entry in data["expected-failures"]: raw_entries.append((TEST_ANNOTATION_TEST_FAILURE, entry)) + for entry in data["profiling-excludes"]: + raw_entries.append((TEST_ANNOTATION_PROFILING_EXCLUDE, entry)) + + for entry in data["profiling-skips"]: + raw_entries.append((TEST_ANNOTATION_PROFILING_SKIP, entry)) + for flavor, entry in raw_entries: if a := filter_stdlib_test_entry( flavor, diff --git a/pythonbuild/testdist.py b/pythonbuild/testdist.py index 3a07637d5..c25056ec4 100644 --- a/pythonbuild/testdist.py +++ b/pythonbuild/testdist.py @@ -19,6 +19,8 @@ TEST_ANNOTATION_HARNESS_SKIP, TEST_ANNOTATION_MODULE_EXCLUDE, TEST_ANNOTATION_TEST_FAILURE, + TEST_ANNOTATION_PROFILING_EXCLUDE, + TEST_ANNOTATION_PROFILING_SKIP, TestAnnotation, meets_python_minimum_version, ) @@ -164,6 +166,15 @@ def run_stdlib_tests( if annotation.dont_verify: dont_verify.add(name) + elif flavor == "profiling-exclude": + print(f"profiling exclude module ignored {name}") + continue + + elif flavor == "profiling-skip": + print(f"profiling skip test ignored {name}") + + continue + else: raise Exception(f"unhandled test annotation flavor: {flavor}") diff --git a/stdlib-test-annotations.yml b/stdlib-test-annotations.yml index ad530be3d..a0b6e0c36 100644 --- a/stdlib-test-annotations.yml +++ b/stdlib-test-annotations.yml @@ -2732,3 +2732,214 @@ expected-failures: build-option: "debug" reason: "segfault" intermittent: true + +profiling-excludes: + - name: test_capi + reason: "a lot of random segfaults" + + - name: test_ctypes + targets: + - ".+-unknown-linux-.+" + reason: "tons of random segfaults" + + - name: test_frame + targets: + - ".+-unknown-linux-.+" + reason: "segfault during BOLT instrumentation" + +profiling-skips: + - name: test.test_ast.test_ast.AST_Tests.test_ast_recursion_limit + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: "AssertionError: RecursionError not raised : Compiling ('a' + '()' * 500000)" + + - name: test.test_asyncio.test_base_events.BaseEventLoopWithSelectorTests.test_create_connection_service_name + reason: "socket.gaierror: [Errno -8] Servname not supported for ai_socktype" + + - name: test.test_audit.AuditTest.test_ctypes_call_function + minimum-python-version: "3.14" + reason: "AssertionError (possibly intermittent)" + + - name: test.test_bytes.BytesTest.test_from_format + minimum-python-version: "3.13" + targets: + - "x86_64.+" + reason: "PGO optimized / BOLT instrumented binaries segfault" + + - name: test.test_call.TestRecursion.test_super_deep + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: "AssertionError: RecursionError not raised" + + - name: test.test_capi.test_unicode.CAPITest.test_from_format + reason: "segfault in PyUnicode_FromFormat" + + - name: test.test_code.CoExtra.* + targets: + - ".+-unknown-linux-.+" + reason: "SystemError: Objects/codeobject.c:1613: bad argument to internal function" + + - name: test.test_compile.TestSpecifics.test_compiler_recursion_limit + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: "AssertionError: RecursionError not raised : Compiling ('a' + '.b' * 500000)" + + - name: test.test_ctypes.test_find.FindLibraryLinux.test_find_library_with_gcc + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: None == None" + + - name: test.test_ctypes.test_find.FindLibraryLinux.test_find_library_with_ld + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: None == None" + + - name: test.test_dict.DictTest.test_repr_deep + minimum-python-version: "3.14" + reason: "AssertionError: RecursionError not raised by repr" + + - name: test.test_embed.InitConfigTests.test_global_pathconfig + reason: "ValueError: character U+67e58ee0 is not in range [U+0000; U+10ffff] during BOLT" + + - name: test.test_exception_group.DeepRecursionInSplitAndSubgroup.test_deep_split + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: "AssertionError: RecursionError not raised by repr" + + - name: test.test_exception_group.DeepRecursionInSplitAndSubgroup.test_deep_subgroup + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: "AssertionError: RecursionError not raised by repr" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_exception + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: "AssertionError: RecursionError not raised by repr" + + - name: test.test_frame.FrameLocalsProxyMappingTests.test_repr_deep + minimum-python-version: "3.14" + reason: "AssertionError: RecursionError not raised by repr" + + - name: test.test_linecache.LineCacheInvalidationTests.* + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_list.ListTest.test_repr_deep + minimum-python-version: "3.14" + reason: "AssertionError: RecursionError not raised by repr" + + - name: test.test_os.* + reason: "env changed (unclear which test)" + + - name: test.test_free_threading.test_capi.TestImportCAPI.test_pyimport_addmoduleref_thread_safe + minimum-python-version: "3.14" + build-option: "freethreaded" + reason: "segfault" + + - name: test.test_free_threading.test_code.* + build-option: "freethreaded" + reason: "ystemError: Objects/codeobject.c:1676: bad argument to internal function" + + - name: test.test_free_threading.test_dict.TestDict.test_racing_creation_nonmanaged_dict + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_dict.TestDict.test_racing_creation_inline_values_invalid + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_dict.TestDict.test_racing_creation_no_shared_keys + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_dict.TestDict.test_racing_creation_shared_keys + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_functools.TestLRUCache.* + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_heapq.TestHeapq.test_racing_heappush_max + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_heapq.TestHeapq.test_racing_heapreplace + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_free_threading.test_type.TestType.test_attr_cache_consistency_subclass + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_itertools.TestBasicOps.test_count_threading + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_itertools.TestBasicOps.test_count_with_step_threading + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_lazy_import.ThreadSafetyTests.test_concurrent_lazy_modules_set_updates + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_sys_api + maximum-python-version: "3.13" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: 'py::spam:/tmp/test_python_p8t76wgm/tmp6irl7ctw/perftest.py' not found in '7f380e676000 40 py::baz:/tmp/test_python_p8t76wgm/tmp6irl7ctw/perftest.py\n'" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_trampoline_works + maximum-python-version: "3.13" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: unexpectedly None : Could not find py::foo:/tmp/test_python_p8t76wgm/tmp66dkos4l/perftest.py in perf file" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_trampoline_works_with_forks + maximum-python-version: "3.13" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: -11 != 0 (-11 might be segfault)" + + - name: test.test_queue.* + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed in multiple tests" + + - name: test.test_socket.GeneralModuleTests.testGetServBy + targets: + - ".+-unknown-linux-.+" + reason: OSError + + - name: test.test_socket.GeneralModuleTests.testGetaddrinfo + targets: + - ".+-unknown-linux-.+" + reason: "socket.gaierror: [Errno -8] Servname not supported for ai_socktype" + + - name: test.test_syslog.test.test_syslog_threaded + build-option: "freethreaded" + reason: "Fatal Python error: _PySemaphore_Wakeup: parking_lot: sem_post failed" + + - name: test.test_threading.ThreadTests.test_PyThreadState_SetAsyncExc + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: 0 != 1" + + - name: test.test_tools.test_compute_changes.TestProcessChangedFiles.test_ci_fuzz_stdlib + minimum-python-version: "3.15" + reason: "AssertionError: LIBRARY_FUZZER_PATHS contains an invalid entry: PosixPath('.github/workflows/reusable-cifuzz.yml')" + + - name: test.test_venv.* + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-.+" + reason: "env changed (unclear which test)" +