From 887f2c7a872ab21325d385eb7188adaabe645ba1 Mon Sep 17 00:00:00 2001 From: Gregory Szorc Date: Fri, 20 Mar 2026 11:22:10 +0800 Subject: [PATCH] verify: run stdlib tests --- .github/workflows/linux.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/windows.yml | 2 +- cpython-unix/build-cpython.sh | 16 +- cpython-unix/build.py | 19 +- cpython-windows/build.py | 16 +- pyproject.toml | 1 + pythonbuild/cpython.py | 198 +++ pythonbuild/testdist.py | 475 +++++- pythonbuild/utils.py | 16 +- stdlib-test-annotations.yml | 2779 +++++++++++++++++++++++++++++++++ uv.lock | 11 + 12 files changed, 3511 insertions(+), 26 deletions(-) create mode 100644 stdlib-test-annotations.yml diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2e934c5fd..03de98a0e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -329,7 +329,7 @@ jobs: unset LD_LIBRARY_PATH fi - ./test-distribution.py dist/*.tar.zst + ./test-distribution.py --stdlib dist/*.tar.zst fi env: MATRIX_RUN: ${{ matrix.run }} diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 3a992a9a4..12885b885 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -181,7 +181,7 @@ jobs: build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst if [ "${MATRIX_RUN}" == "true" ]; then - ./test-distribution.py dist/*.tar.zst + ./test-distribution.py --stdlib dist/*.tar.zst fi env: MATRIX_RUN: ${{ matrix.run }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index cc8b866f0..eb61f1a73 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -186,4 +186,4 @@ jobs: run: | $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative .\pythonbuild.exe validate-distribution $Dists - uv run --no-dev test-distribution.py $Dists + uv run --no-dev test-distribution.py --stdlib $Dists diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index f010750f4..730e6d18e 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -51,10 +51,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz" PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl" SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl" -cat Setup.local -mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local" +# Put critical config files in logs to aid debugging. +for f in Setup.local Makefile.extra stdlib-test-annotations.json; do + echo "BEGIN $f" + cat $f + echo "END $f" +done -cat Makefile.extra +mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local" pushd "Python-${PYTHON_VERSION}" @@ -1386,7 +1390,7 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/" cp -av Python/frozen.c "${ROOT}/out/python/build/Python/" cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/" -# Copy the test hardness runner for convenience. +# Copy the test harness runner for convenience. # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py" @@ -1394,6 +1398,10 @@ else cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/" fi +# Copy standard library test annotations so it is in the artifact and the +# distribution self-describes expected test wonkiness. +cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json" + # Don't hard-code the build-time prefix into the pkg-config files. See # the description of `pcfiledir` in `man pkg-config`. find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \ diff --git a/cpython-unix/build.py b/cpython-unix/build.py index eb0e6ae86..f78c340c4 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -24,6 +24,7 @@ meets_python_maximum_version, meets_python_minimum_version, parse_setup_line, + stdlib_test_annotations, ) from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles from pythonbuild.downloads import DOWNLOADS @@ -50,6 +51,7 @@ SUPPORT = ROOT / "cpython-unix" EXTENSION_MODULES = SUPPORT / "extension-modules.yml" TARGETS_CONFIG = SUPPORT / "targets.yml" +STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml" LINUX_ALLOW_SYSTEM_LIBRARIES = { "c", @@ -732,6 +734,13 @@ def build_cpython( setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH) pip_archive = download_entry("pip", DOWNLOADS_PATH) + test_annotations = stdlib_test_annotations( + STDLIB_TEST_ANNOTATIONS, + python_version, + target_triple, + parsed_build_options, + ) + ems = extension_modules_config(EXTENSION_MODULES) setup = derive_setup_local( @@ -804,6 +813,14 @@ def build_cpython( build_env.copy_file(fh.name, dest_name="Makefile.extra") + # Install the derived test annotations. + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh: + os.chmod(fh.name, 0o644) + test_annotations.json_dump(fh) + fh.flush() + + build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json") + env = { "PIP_VERSION": DOWNLOADS["pip"]["version"], "PYTHON_VERSION": python_version, @@ -1070,7 +1087,7 @@ def main(): write_dockerfiles(SUPPORT, BUILD) elif action == "makefiles": targets = get_targets(TARGETS_CONFIG) - write_triples_makefiles(targets, BUILD, SUPPORT) + write_triples_makefiles(targets, ROOT, BUILD, SUPPORT) write_target_settings(targets, BUILD / "targets") write_package_versions(BUILD / "versions") diff --git a/cpython-windows/build.py b/cpython-windows/build.py index f93a59a50..de91aba6c 100644 --- a/cpython-windows/build.py +++ b/cpython-windows/build.py @@ -22,6 +22,7 @@ meets_python_maximum_version, meets_python_minimum_version, parse_config_c, + stdlib_test_annotations, ) from pythonbuild.downloads import DOWNLOADS from pythonbuild.utils import ( @@ -39,6 +40,7 @@ BUILD = ROOT / "build" DIST = ROOT / "dist" SUPPORT = ROOT / "cpython-windows" +STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml" LOG_PREFIX = [None] LOG_FH = [None] @@ -128,7 +130,6 @@ "_zstd": ["zstd"], } - # Tests to run during PGO profiling. # # This set was copied from test.libregrtest.pgo in the CPython source @@ -1407,6 +1408,13 @@ def build_cpython( else: raise Exception("unhandled architecture: %s" % arch) + test_annotations = stdlib_test_annotations( + STDLIB_TEST_ANNOTATIONS, + python_version, + target_triple, + parsed_build_options, + ) + tempdir_opts = ( {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {} ) @@ -1763,6 +1771,12 @@ def build_cpython( out_dir / "python" / "build" / "run_tests.py", ) + # Install a JSON file annotating tests. + with (out_dir / "python" / "build" / "stdlib-test-annotations.json").open( + "w", encoding="utf-8" + ) as fh: + test_annotations.json_dump(fh) + licenses_dir = out_dir / "python" / "licenses" licenses_dir.mkdir() for f in sorted(os.listdir(ROOT)): diff --git a/pyproject.toml b/pyproject.toml index 43e5f4eda..de66b871c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "docker>=7.1.0", "jinja2>=3.1.5", "jsonschema>=4.23.0", + "junitparser>=4.0.2", "pyyaml>=6.0.2", "six>=1.17.0", "tomli>=2.2.1", diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 3221760ab..d342f1ad9 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -2,9 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import dataclasses +import json import pathlib import re import tarfile +from typing import Optional import jsonschema import yaml @@ -137,6 +140,65 @@ }, } +STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES = { + "reason": {"type": "string"}, + "targets": { + "type": "array", + "items": {"type": "string"}, + }, + "ignore-targets": { + "type": "array", + "items": {"type": "string"}, + }, + "minimum-python-version": {"type": "string"}, + "maximum-python-version": {"type": "string"}, + "build-option": {"type": "string"}, + "no-build-option": {"type": "string"}, +} + +STDLIB_TEST_ANNOTATIONS_SCHEMA = { + "type": "object", + "properties": { + "harness-skips": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["name", "reason"], + }, + }, + "module-excludes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "module": {"type": "string"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["module", "reason"], + }, + }, + "expected-failures": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "dont-verify": {"type": "boolean"}, + "intermittent": {"type": "boolean"}, + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, + }, + "additionalProperties": False, + "required": ["name", "reason"], + }, + }, + }, +} # Packages that define tests. STDLIB_TEST_PACKAGES = { @@ -750,3 +812,139 @@ def extension_modules_config(yaml_path: pathlib.Path): jsonschema.validate(data, EXTENSION_MODULES_SCHEMA) return data + + +TEST_ANNOTATION_HARNESS_SKIP = "harness-skip" +TEST_ANNOTATION_MODULE_EXCLUDE = "module-exclude" +TEST_ANNOTATION_TEST_FAILURE = "test-failure" + + +@dataclasses.dataclass +class TestAnnotation: + # Describes the type of annotation. + flavor: str + # Name/pattern of test. + name: str + # Describes why the annotation exists. + reason: str + # Whether the test is expected to fail. + expect_test_failure: bool + # Whether to skip verification of failures. + dont_verify: bool + # Whether test failure is intermittent. Should only be true if + # expect_test_failure also true. + intermittent_test_failure: bool + # Whether to exclude loading the test module when running tests. + exclude_testing: bool + + +@dataclasses.dataclass +class TestAnnotations: + annotations: list[TestAnnotation] + + def json_dump(self, of): + data = [dataclasses.asdict(a) for a in self.annotations] + json.dump(data, of, indent=2) + + +def filter_stdlib_test_entry( + flavor: str, + test, + python_version: str, + target_triple: str, + build_options: set[str], +) -> Optional[TestAnnotation]: + name = test["name"] + + if targets := test.get("targets"): + matches_target = any(re.match(p, target_triple) for p in targets) + else: + matches_target = True + + for m in test.get("ignore-targets", []): + if re.match(m, target_triple): + matches_target = False + + if not matches_target: + log(f"ignoring {flavor} rule (target doesn't match): {name}") + return None + + python_minimum_version = test.get("minimum-python-version", "1.0") + python_maximum_version = test.get("maximum-python-version", "100.0") + + if not meets_python_minimum_version(python_version, python_minimum_version): + log( + f"ignoring {flavor} rule ({python_version} < {python_minimum_version} (min)): {name}" + ) + return None + + if not meets_python_maximum_version(python_version, python_maximum_version): + log( + f"ignoring {flavor} rule ({python_version} > {python_maximum_version} (max)): {name}" + ) + return None + + if option := test.get("build-option"): + if option not in build_options: + log(f"ignoring {flavor} rule (build option {option} not present): {name}") + return None + + if option := test.get("no-build-option"): + if option in build_options: + log(f"ignoring {flavor} rule (build option {option} is present): {name}") + return None + + # Filtering complete. This rule applies to the current build. + + log(f"relevant {flavor} test rule: {name}: {test['reason']}") + + return TestAnnotation( + flavor=flavor, + name=name, + reason=test["reason"], + expect_test_failure=True, + dont_verify=test.get("dont-verify", False), + intermittent_test_failure=test.get("intermittent", False), + exclude_testing=test.get("exclude", False), + ) + + +def stdlib_test_annotations( + yaml_path: pathlib.Path, + python_version: str, + target_triple: str, + build_options: set[str], +) -> TestAnnotations: + """Processes the test-annotations.yml file for a given build configuration.""" + with yaml_path.open("r", encoding="utf-8") as fh: + data = yaml.load(fh, Loader=yaml.SafeLoader) + + jsonschema.validate(data, STDLIB_TEST_ANNOTATIONS_SCHEMA) + + annotations = [] + + log(f"processing {len(data['expected-failures'])} stdlib test annotations") + + raw_entries = [] + + for entry in data["harness-skips"]: + raw_entries.append((TEST_ANNOTATION_HARNESS_SKIP, entry)) + + for entry in data["module-excludes"]: + entry["name"] = entry["module"] + raw_entries.append((TEST_ANNOTATION_MODULE_EXCLUDE, entry)) + + for entry in data["expected-failures"]: + raw_entries.append((TEST_ANNOTATION_TEST_FAILURE, entry)) + + for flavor, entry in raw_entries: + if a := filter_stdlib_test_entry( + flavor, + entry, + python_version, + target_triple, + build_options, + ): + annotations.append(a) + + return TestAnnotations(annotations) diff --git a/pythonbuild/testdist.py b/pythonbuild/testdist.py index 96f65790e..534f2b6d4 100644 --- a/pythonbuild/testdist.py +++ b/pythonbuild/testdist.py @@ -3,23 +3,63 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import argparse +import concurrent.futures +import dataclasses +import enum import json import os +import re +import shlex import subprocess import tempfile from pathlib import Path from typing import Optional +import junitparser + +from .cpython import ( + TEST_ANNOTATION_HARNESS_SKIP, + TEST_ANNOTATION_MODULE_EXCLUDE, + TEST_ANNOTATION_TEST_FAILURE, + TestAnnotation, + meets_python_minimum_version, +) from .utils import extract_python_archive +# Tests in the standard library that are slow. +# +# Roughly in order from slowest to fastest. +STDLIB_SLOW_TESTS = [ + "test_subprocess", + "test_signal", + "test_socket", + "test.test_multiprocessing_spawn.test_processes", + "test.test_multiprocessing_forkserver.test_processes", + "test_regrtest", + "test.test_concurrent_futures.test_process_pool", + "test_tarfile", + "test_socket", + "test_remote_pdb", + "test_threading", + "test.test_concurrent_futures.test_as_completed", + "test.test_multiprocessing_spawn.test_misc", + "test.test_asyncio.test_tasks", + "test_zipfile", + "test_pickle", +] + +# Maximum wall run time of a single test before timing it out. +TIMEOUT_SECONDS = 300 + def run_dist_python( dist_root: Path, python_info, args: list[str], extra_env: Optional[dict[str, str]] = None, + log_exec=False, **runargs, -) -> subprocess.CompletedProcess[str]: +) -> subprocess.CompletedProcess[bytes]: """Runs a `python` process from an extracted PBS distribution. This function attempts to isolate the spawned interpreter from any @@ -35,8 +75,13 @@ def run_dist_python( if extra_env: env.update(extra_env) + all_args = [str(dist_root / python_info["python_exe"])] + args + + if log_exec: + print(f"executing: {shlex.join(all_args)}") + return subprocess.run( - [str(dist_root / python_info["python_exe"])] + args, + all_args, cwd=dist_root, env=env, **runargs, @@ -63,44 +108,442 @@ def run_custom_unittests(pbs_source_dir: Path, dist_root: Path, python_info) -> return res.returncode -def run_stdlib_tests(dist_root: Path, python_info, harness_args: list[str]) -> int: +def run_stdlib_tests( + dist_root: Path, + python_info, + skip_main=False, + verbose_expected_failures=False, + raw_harness_args: Optional[list[str]] = None, +) -> int: """Run Python stdlib tests for a PBS distribution. The passed path is the `python` directory from the extracted distribution archive. """ - args = [ - str(dist_root / python_info["run_tests"]), + + dist_name = f"cpython-{python_info['python_version']}-{python_info['build_options']}-{python_info['target_triple']}" + + stdlib_test_annotations_path = dist_root / "build" / "stdlib-test-annotations.json" + + with stdlib_test_annotations_path.open("r", encoding="utf-8") as fh: + annotations = json.load(fh) + + expect_failures = set() + module_excludes = set() + intermittent = set() + dont_verify = set() + + for raw_annotation in annotations: + annotation = TestAnnotation(**raw_annotation) + + name = annotation.name + flavor = annotation.flavor + reason = annotation.reason + + if flavor == TEST_ANNOTATION_HARNESS_SKIP: + print(f"not running the stdlib test harness: {reason}") + + # Add a GitHub Actions annotation to improve observability of this scenario. + + if "CI" in os.environ: + print( + f"::notice title={dist_name} stdlib test harness skipped::{reason}" + ) + + return 0 + + elif flavor == TEST_ANNOTATION_MODULE_EXCLUDE: + print(f"excluding module {name}: {reason}") + module_excludes.add(name) + continue + + elif flavor == TEST_ANNOTATION_TEST_FAILURE: + print(f"expected test failure {name}: {reason}") + expect_failures.add(name) + + if annotation.intermittent_test_failure: + intermittent.add(name) + if annotation.dont_verify: + dont_verify.add(name) + + else: + raise Exception(f"unhandled test annotation flavor: {flavor}") + + base_args = [ + "-u", + "-W", + "default", + "-bb", + "-E", + "-m", + "test", ] - args.extend(harness_args) + args = list(base_args) + + if raw_harness_args: + args.extend(raw_harness_args) + else: + args.extend( + [ + # Display test output on failure. Makes it easier to debug failures. + "-W", + # Re-run failed tests in verbose mode to aid debugging. + "-w", + # Make order non-deterministic to help flush out failures. + "--randomize", + # Run tests in parallel using all available CPUs. + "-j", + "0", + # Force abort tests taking too long to execute. This can prevent + # some runaway tests in CI. + "--timeout", + str(TIMEOUT_SECONDS), + # Print slowest tests to make it easier to audit our slow tests list. + "-o", + ] + ) + + # 3.14 added a test harness feature to prioritize running tests. Prioritize + # running slow tests first to mitigate long poles slowing down harness execution. + if meets_python_minimum_version(python_info["python_version"], "3.14"): + args.extend(["--prioritize", ",".join(STDLIB_SLOW_TESTS)]) + + for test_name in sorted(expect_failures): + args.extend(["--ignore", test_name]) + + if module_excludes: + # --exclude is a boolean argument that changes positional arguments to + # denotes excludes. As of at least 3.15 there doesn't appear to be a + # way to select tests to run via positional arguments while also + # excluding certain modules from being loaded. + args.append("--exclude") + args.extend(sorted(module_excludes)) + + codes = [] + + if not skip_main: + codes.append( + run_dist_python(dist_root, python_info, args, log_exec=True).returncode + ) + if codes[-1] != 0: + print(f"main test harness failed ({codes[-1]})") + else: + print("main test harness passed") + else: + print("main test harness skipped") + + if not raw_harness_args: + codes.extend( + _run_stdlib_expected_failures( + dist_root, + python_info, + base_args, + expect_failures, + intermittent, + dont_verify, + verbose_expected_failures=verbose_expected_failures, + ) + ) + + if any(code != 0 for code in codes): + return 1 + else: + return 0 - return run_dist_python(dist_root, python_info, args).returncode + +def _run_stdlib_expected_failures( + dist_root: Path, + python_info, + base_args: list[str], + expect_failures: set[str], + intermittent: set[str], + dont_verify: set[str], + verbose_expected_failures: bool, +) -> list[int]: + # Run ignored / expected failures tests and verify they actually fail. + unexpected_tests: set[str] = set() + codes = [] + + with concurrent.futures.ThreadPoolExecutor() as executor: + fs = [] + + for test_name in sorted(expect_failures): + fs.append( + executor.submit( + _check_stdlib_expected_failure, + dist_root, + python_info, + base_args, + test_name, + test_name in intermittent, + test_name in dont_verify, + ) + ) + + for i, f in enumerate(concurrent.futures.as_completed(fs)): + res: ExpectedFailureResult = f.result() + + if res.unexpected: + unexpected_tests.add(res.test_name) + codes.append(1) + else: + codes.append(0) + + unexpected_suffix = "" + if len(unexpected_tests) > 0: + unexpected_suffix = f"/{len(unexpected_tests)}" + + fail_result = "yes" if res.fails else "no" + + if res.result_reason != TestResultReason.NONE: + fail_result += f" ({res.result_reason})" + + print( + f"[{i + 1}/{len(expect_failures)}{unexpected_suffix}] verifying {res.test_name} fails... {fail_result}" + ) + + print_output = verbose_expected_failures or res.result_reason in ( + TestResultReason.DONT_VERIFY_FAIL_ALLOWED, + TestResultReason.UNKNOWN_TEST, + ) + + if print_output: + for line in res.output.decode("utf-8", errors="ignore").splitlines(): + print(f"> {line}") + + if unexpected_tests: + print("unexpected test results:") + for t in sorted(unexpected_tests): + print(t) + + return codes + + +class TestResultReason(enum.StrEnum): + NONE = "" + EXPECTED = "expected" + UNKNOWN_TEST = "unknown test" + INTERMITTENT_PASS_ALLOWED = "intermittent pass allowed" + DONT_VERIFY_PASS_ALLOWED = "dont-verify annotation allows pass" + DONT_VERIFY_FAIL_ALLOWED = "dont-verify annotation allows fail" + UNEXPECTED_PASS = "unexpected pass" + + +@dataclasses.dataclass +class ExpectedFailureResult: + test_name: str + code: int + output: bytes + fails: bool + result_reason: TestResultReason + unexpected: bool + intermittent: bool + dont_verify: bool + junit: Optional[junitparser.JUnitXml] + + +RE_TEST_RUN_COUNT = re.compile(rb"^Total tests: run=(\d+)", re.MULTILINE) + +RE_TEST_CRASHED = re.compile(rb"^test test_.+ crashed", re.MULTILINE) + +RE_TEST_UNCAUGHT_EXCEPTION = re.compile( + rb"test_.+ failed \(uncaught exception\)", re.MULTILINE +) + +RE_TEST_SKIPPED = re.compile(rb"^test_.+ skipped", re.MULTILINE) + + +def _check_stdlib_expected_failure( + dist_root: Path, + python_info, + base_args: list[str], + test_name: str, + intermittent: bool, + dont_verify: bool, +) -> ExpectedFailureResult: + args = list(base_args) + + args.extend( + [ + "-v", + "--timeout", + str(TIMEOUT_SECONDS), + ] + ) + + # Always provide the test_* name as a positional argument to limit + # which tests are loaded. Otherwise we load all test files and incur + # substantial overhead. + + parts = test_name.split(".") + + # time.datetimetester is loaded by test_datetime. This pattern was removed + # in 3.11. + if test_name.startswith("test.datetimetester."): + module_name = "test_datetime" + + # distutils.tests is loaded by test_distutils. This pattern was removed + # in 3.12. + elif test_name.startswith("distutils.tests."): + module_name = "test_distutils" + + elif test_name.startswith("test."): + module_name = parts[1] + + else: + raise ValueError(f"unknown test module pattern: {test_name}") + + args.extend(["-m", test_name]) + + with tempfile.TemporaryDirectory() as td: + junit_path = Path(td) / "junit.xml" + args.extend(["--junit-xml", str(junit_path)]) + + args.append(module_name) + + # We sniff stdout for certain string patterns. Non-deterministic color + # sequences undermines those efforts. So prevent the test harness from + # emitting colors. + extra_env = { + "NO_COLOR": "1", + } + + res = run_dist_python( + dist_root, python_info, args, extra_env=extra_env, capture_output=True + ) + + try: + junit = junitparser.JUnitXml.fromfile(str(junit_path)) + except FileNotFoundError: + junit = None + + unexpected = False + fails = False + result_reason = None + + # Since we're running 1 test at a time, the test harness should always + # run at least 1 test. If no tests are executed, it means we have a test + # annotation referencing an unknown test. i.e. the test annotation is wrong + # (likely incorrect minimum/maximum Python versions). Treat this as a hard + # error to force our test annotations track reality. + # + # But this condition can also materialize if there's an error setting up + # tests, including importing them. Said errors take precedence. Ditto for + # if the load of the module itself self-skips. + crashed = RE_TEST_CRASHED.search(res.stdout) is not None + uncaught_exception = RE_TEST_UNCAUGHT_EXCEPTION.search(res.stdout) is not None + load_error = b"\nERROR: setUpClass" in res.stdout + skipped = RE_TEST_SKIPPED.search(res.stdout) is not None + + if not crashed and not uncaught_exception and not load_error and not skipped: + # 3.13+ syntax. + if m := RE_TEST_RUN_COUNT.search(res.stdout): + if m.group(1) == b"0": + fails = False + result_reason = TestResultReason.UNKNOWN_TEST + unexpected = True + + # 3.11-3.12 syntax. + elif b"NO TESTS RAN" in res.stdout: + fails = False + result_reason = TestResultReason.UNKNOWN_TEST + unexpected = True + + # 3.10 syntax. + elif b"NO TEST RUN" in res.stdout: + fails = False + result_reason = TestResultReason.UNKNOWN_TEST + unexpected = True + + if result_reason is None: + if res.returncode != 0: + fails = True + result_reason = ( + TestResultReason.DONT_VERIFY_FAIL_ALLOWED + if dont_verify + else TestResultReason.EXPECTED + ) + elif intermittent: + fails = False + result_reason = TestResultReason.INTERMITTENT_PASS_ALLOWED + elif dont_verify: + fails = False + result_reason = TestResultReason.DONT_VERIFY_PASS_ALLOWED + else: + fails = False + result_reason = TestResultReason.UNEXPECTED_PASS + unexpected = True + + return ExpectedFailureResult( + test_name, + res.returncode, + res.stdout, + fails, + result_reason, + unexpected, + intermittent, + dont_verify, + junit=junit, + ) def main(pbs_source_dir: Path, raw_args: list[str]) -> int: """test-distribution.py functionality.""" - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description=""" + Runs unit tests against a Python distribution. + + By default, this executes custom PBS unit tests. The unittests from + the distribution's Python standard library can be ran by adding --stdlib. + + By default, the stdlib test harness is run with opinionated behavior + where we automatically exclude running tests annotated as failing followed + by running each of these annotated tests in isolation to validate their + test annotations. e.g. we validate that failures actually fail. + + If arguments are passed after a `--` separator argument, the execution + behavior changes to a proxy to the stdlib test harness and arguments passed + after `--` are passed directly to `python -m test`, allowing you to invoke + the stdlib test harness with arbitrary arguments. + + The --stdlib-no-main (implies --stdlib) can be used to just run annotated + test failures. This can be useful for debugging these tests. + """ + ) parser.add_argument( "--stdlib", action="store_true", help="Run the stdlib test harness", ) + parser.add_argument( + "--stdlib-no-main", + action="store_true", + help="Skip running the main invocation of the stdlib test harness - only failures will be processed", + ) + parser.add_argument( + "--verbose-expected-failures", + action="store_true", + help="Print details of tests that failed expectedly (helps with debugging failures)", + ) parser.add_argument( "dist", nargs=1, help="Path to distribution to test", ) parser.add_argument( - "harness_args", + "stdlib_harness_args", nargs=argparse.REMAINDER, - help="Raw arguments to pass to Python's test harness", + help="Raw arguments to pass to the stdlib test harness", ) args = parser.parse_args(raw_args) + if args.stdlib_no_main: + args.stdlib = True + dist_path_raw = Path(args.dist[0]) td = None @@ -120,8 +563,16 @@ def main(pbs_source_dir: Path, raw_args: list[str]) -> int: codes.append(run_custom_unittests(pbs_source_dir, dist_path, python_info)) - if args.stdlib: - codes.append(run_stdlib_tests(dist_path, python_info, args.harness_args)) + if args.stdlib or args.stdlib_harness_args: + codes.append( + run_stdlib_tests( + dist_path, + python_info, + skip_main=args.stdlib_no_main, + verbose_expected_failures=args.verbose_expected_failures, + raw_harness_args=args.stdlib_harness_args, + ) + ) if len(codes) == 0: print("no tests run") diff --git a/pythonbuild/utils.py b/pythonbuild/utils.py index f17ab12f2..a5c3b1b66 100644 --- a/pythonbuild/utils.py +++ b/pythonbuild/utils.py @@ -174,7 +174,10 @@ def write_if_different(p: pathlib.Path, data: bytes): def write_triples_makefiles( - targets, dest_dir: pathlib.Path, support_search_dir: pathlib.Path + targets, + root_dir: pathlib.Path, + dest_dir: pathlib.Path, + support_search_dir: pathlib.Path, ): """Write out makefiles containing make variable settings derived from config.""" dest_dir.mkdir(parents=True, exist_ok=True) @@ -211,10 +214,13 @@ def write_triples_makefiles( % (entry, DOWNLOADS[entry]["version"], host_platform) ) - lines.append( - "PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) %s\n" - % (support_search_dir / "extension-modules.yml") - ) + extra_support_files = [ + support_search_dir / "extension-modules.yml", + root_dir / "stdlib-test-annotations.yml", + ] + + for p in extra_support_files: + lines.append(f"PYTHON_SUPPORT_FILES := $(PYTHON_SUPPORT_FILES) {p}\n") write_if_different(makefile_path, "".join(lines).encode("ascii")) diff --git a/stdlib-test-annotations.yml b/stdlib-test-annotations.yml new file mode 100644 index 000000000..562a1eced --- /dev/null +++ b/stdlib-test-annotations.yml @@ -0,0 +1,2779 @@ +--- +# This file defines metadata about Python standard library tests. +# +# The file consists of keys defining different groups of annotations. +# See each key for documentation. +# +# All entries share a common set of fields to support describing the +# annotation and filtering which annotations apply to which builds. +# +# * reason - human readable context describing the failure +# * targets - list of regular expressions matching target triples to +#. determine if rule is active. Missing or empty matches all targets. +# * ignore-targets - list of regular expressions matching target tripels +#. to determine if rule is ignored. Takes precedence over "targets." +# * minimum-python-version - X.Y Python version we must be equal or newer +#. to for the rule to apply. +# * maximum-python-version - X.Y Python version we must be equal or less +#. than for the rule to apply. +# * build-option - A build option (e.g. `pgo`) that must be present for this +#. rule to be enabled. +# * no-build-option - A build option (e.g. `freethreaded`) that must not be +#. present for this rule to be enabled. + +# harness-skips declares builds to skip running the Python stdlib test harness. +# +# Ideally this feature doesn't need to exist. But sometimes the test harness is +# so broken we need to do this. +harness-skips: + - name: "musl-debug-skip-3.14+" + reason: "`cannot get C stack on this system` error leads to widespread failure" + minimum-python-version: "3.14" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + +# module-excludes annotates modules to exclude from loading. +# +# The stdlib test runner applies this filter based on the name of the +# module file/directory within a directory. So values here need to be +# child most module paths. e.g. to exclude module `foo.bar.baz`, you +# need to use module name `baz` and not `foo.bar.baz`. +module-excludes: + - module: test_call + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "import failure due to missing _testcapi" + maximum-python-version: "3.12" + + - module: test_capi + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "import failure due to missing _testcapi" + + - module: test_code + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "import failure due to missing _testcapi" + + - module: test_ctypes + targets: + - ".+-unknown-linux-musl" + reason: "Module import failures" + + - module: test_datetime + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "import failure due to missing _testcapi" + + - module: test_sqlite3 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "import failure due to missing _testcapi" + + - module: test_stable_abi_ctypes + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "import failure due to missing _testcapi" + + - module: test_regrtest_noop1 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "import failure due to missing _testcapi" + + # Tons of failures. Just exclude entirely instead of running. + - module: test_traceback + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: tons of failures, mostly due to missing _testcapi + + - module: test__xxsubinterpreters + maximum-python-version: "3.12" + build-option: "static" + reason: "skipped due to missing _testcapi" + +# expected-failures is a list of tests that are expected to fail. +# +# * name - pattern identifying the test or group of tests +# * intermittent - indicates the failure is intermittent. +# * dont-verify - boolean indicating to not verify the failing test actually +#. fails. Useful for intermittent / flaky tests. +expected-failures: + - name: distutils.tests.test_archive_util.ArchiveUtilTestCase.test_tarfile_vs_tar + minimum-python-version: "3.11" + maximum-python-version: "3.11" + reason: "differing element" + intermittent: true + + - name: distutils.tests.test_install_lib.InstallLibTestCase.test_byte_compile + maximum-python-version: "3.11" + reason: "TypeError: byte_compile() got an unexpected keyword argument 'dry_run'" + + - name: distutils.tests.test_build_py.BuildPyTestCase.test_byte_compile_optimized + maximum-python-version: "3.11" + reason: "TypeError: byte_compile() got an unexpected keyword argument 'dry_run'" + + - name: distutils.tests.test_build_ext.BuildExtTestCase.test_deployment_target_higher_ok + reason: wrong deployment target + maximum-python-version: "3.11" + targets: + - ".+-apple-darwin" + + - name: distutils.tests.test_build_ext.ParallelBuildExtTestCase.test_deployment_target_higher_ok + reason: wrong deployment target + maximum-python-version: "3.11" + targets: + - ".+-apple-darwin" + + # Looks like it is dependent on the system temp dir, which can vary in CI. + - name: distutils.tests.test_sysconfig.SysconfigTestCase.test_srcdir + maximum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: "AssertionError: '/var/folders/1r/b8l6nzhj4zdbz5jmjdhqbprw00[64 chars]rwin' != '/private/var/folders/1r/b8l6nzhj4zdbz5jmjd[72 chars]rwin'" + intermittent: true + + - name: test.test_array.ByteTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.DoubleTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.FloatTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.IntTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.LongLongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.LongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.ShortTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnicodeTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedByteTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedIntTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedLongLongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedLongTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_array.UnsignedShortTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + # This test moved to a sub-module in 3.12. + - name: test.test_ast.ModuleStateTests.test_subinterpreter + maximum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_ast.test_ast.ModuleStateTests.test_subinterpreter + minimum-python-version: "3.12" + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_atexit_with_low_memory + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_callback_on_subinterpreter_teardown + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_callbacks_leak + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_atexit.SubinterpreterTest.test_callbacks_leak_refcycle + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_audit.AuditTest.test_unraisablehook + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "AssertionError" + + - name: test.test_audit.AuditTest.test_builtin__import__ + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "ModuleNotFoundError: No module named '_testcapi'. Did you mean: 'test_capi'?" + + - name: test.test_audit.AuditTest.test_import_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "ModuleNotFoundError: No module named '_testcapi'. Did you mean: 'test_capi'?" + + - name: test.test_audit.AuditTest.test_import_statement + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "ModuleNotFoundError: No module named '_testcapi'. Did you mean: 'test_capi'?" + + # Looks like a race condition in freethreaded builds. + - name: test.test_buffer.TestBufferProtocol.test_array_alignment + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "AssertionError: Lists differ: [0, 4, 0, 0, 0, 0, 0, 0, 0, 0] != [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" + intermittent: true + + - name: test.test_buffer.TestBufferProtocol.test_pybuffer_size_from_format + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "missing _testcapi" + + - name: test.test_build_details.CPythonBuildDetailsTests.test_platform + minimum-python-version: "3.14" + reason: platform key doesn't match when cross compiling - possible PBS quirk/bug. + targets: + - ".+-apple-darwin" + - ".+-unknown-linux-.+" + dont-verify: true + + - name: test.test_bytes.ByteArrayTest.test_obsolete_write_lock + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_set + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_set + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_zero + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_not_zero + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_zero + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_zero + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_warn + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: AssertionError + + - name: test.test_c_locale_coercion.LocaleCoercionTests.test_PYTHONCOERCECLOCALE_set_to_warn + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: AssertionError + + - name: test.test_call.TestRecursion.test_margin_is_sufficient + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: missing _testcapi + minimum-python-version: "3.14" + + - name: test.test_capi.test_misc.Test_testinternalcapi.test_threadstate_set_stack_protection + minimum-python-version: "3.14" + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + reason: "Assertion failed: ts->c_stack_soft_limit < ts->c_stack_top" + intermittent: true + + - name: test.test_capi.test_run.CAPITest.test_run_fileexflags + minimum-python-version: "3.12" + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + # test_capi is module excluded above in static builds. It fails to + # import / load tests on static builds, leading to a false positive of + # this test not being found. Exclude static builds here to prevent + # our test harness from getting confused. + no-build-option: "static" + reason: "crash" + + - name: test.test_cmath.CMathTests.test_polar_errno + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_cmd_line.CmdLineTest.test_pythonmalloc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_codecs.BasicUnicodeTest.test_basics_capi + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_concurrent_futures.test_interpreter_pool.InterpreterPoolExecutorTest.test_blocking + build-option: "freethreaded" + minimum-python-version: "3.15" + reason: "sometimes times out - possible test deadlock" + intermittent: true + + - name: test.test_concurrent_futures.test_shutdown.ProcessPoolForkProcessPoolShutdownTest.test_interpreter_shutdown + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_concurrent_futures.test_shutdown.ProcessPoolForkserverProcessPoolShutdownTest.test_interpreter_shutdown + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_coroutines.CAPITest.test_tp_await_1 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_coroutines.CAPITest.test_tp_await_2 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_coroutines.CAPITest.test_tp_await_3 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_ctypes.test_callbacks.Callbacks.test_i38748_stackCorruption + targets: + - "i686-pc-windows-msvc" + minimum-python-version: "3.12" + reason: "OSError: exception: access violation writing 0x00000005" + + - name: test.test_ctypes.test_callbacks.StdcallCallbacks.test_i38748_stackCorruption + targets: + - "i686-pc-windows-msvc" + minimum-python-version: "3.12" + reason: "OSError: exception: access violation writing 0x00000005" + + # Looks racy on freethreaded builds. + #- name: test.datetimetester.ExtensionModuleTests_Fast.test_concurrent_initialization_subinterpreter + # minimum-python-version: "3.15" + # build-option: "freethreaded" + # reason: "AssertionError: b'aaaaaaa' != b'aaaaaaaa'" + # intermittent: true + + - name: test.datetimetester.ZoneInfoTest_Fast.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.datetimetester.IranTest_Pure.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.datetimetester.ZoneInfoTest_Pure.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.datetimetester.IranTest_Fast.test_system_transitions + targets: + - "x86_64-apple-darwin" + maximum-python-version: "3.10" + reason: "AttributeError: module 'time' has no attribute 'tzset'" + intermittent: true + + - name: test.test_dict.CAPITest.test_getitem_knownhash + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: missing _testcapi + + #- name: test.test_dict.CAPITest.test_splittable_setattr_after_pop + # targets: + # - ".+-unknown-linux-musl" + # build-option: "static" + # maximum-python-version: "3.10" + # reason: missing _testcapi + + - name: test.test_dict.DictTest.test_splittable_setattr_after_pop + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: missing _testcapi + + - name: test.test_docxmlrpc.DocXMLRPCHTTPGETServer.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_exceptions.ExceptionTests.testSettingException + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_memory_error_in_subinterp + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_MemoryError + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_exception_with_doc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_memory_error_cleanup + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_memory_error_in_PyErr_PrintEx + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_infinite_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_with_no_memory + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_exceptions.ExceptionTests.test_recursion_normalizing_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.13" + maximum-python-version: "3.13" + reason: "_testcapi missing" + + - name: test.test_faulthandler.FaultHandlerTests.test_fatal_error + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_faulthandler.FaultHandlerTests.test_fatal_error_without_gil + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_fcntl.TestFcntl.test_fcntl_bad_file_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_fcntl.TestFcntl.test_flock_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_fileio.COtherFileTests.testInvalidFd_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.LegacyFinalizationTest.test_legacy + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.LegacyFinalizationTest.test_legacy_resurrect + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.LegacyFinalizationTest.test_legacy_self_cycle + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.SimpleFinalizationTest.test_non_gc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_finalization.SimpleFinalizationTest.test_non_gc_resurrect + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_float.IEEEFormatTestCase.test_serialized_float_rounding + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "_testcapi missing" + + - name: test.test_format.FormatTest.test_precision_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_frame.TestFrameCApi.test_basic + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.13" + reason: _testcapi missing + + # This test has a weird pass/fail matrix. Test might be self-skipping or something. + + # aarch64 passes w/o pgo and on freethreaded+pgo + - name: test.test_frame_pointer_unwind.FramePointerUnwindTests.test_manual_unwind_respects_frame_pointers + minimum-python-version: "3.15" + targets: + - aarch64-unknown-linux-gnu + build-option: "pgo" + no-build-option: "freethreaded" + reason: "AssertionError: 0 not greater than 0 : expected to find Python frames on aarch64 with env {'PYTHON_JIT': '1'}" + + - name: test.test_frame_pointer_unwind.FramePointerUnwindTests.test_manual_unwind_respects_frame_pointers + minimum-python-version: "3.15" + targets: + - x86_64.*-unknown-linux-musl + build-option: "static" + reason: "AssertionError: 0 not greater than 0 : expected to find Python frames on aarch64 with env {'PYTHON_JIT': '1'}" + + - name: test.test_functools.TestPartialC.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_functools.TestPartialCSubclass.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_functools.TestPartialPy.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_functools.TestPartialPySubclass.test_recursive_pickle + targets: + - ".+-apple-darwin" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + build-option: "pgo" + reason: crash + intermittent: true + + - name: test.test_gc.GCCallbackTests.test_collect_garbage + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_gc.GCCallbackTests.test_refcount_errors + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "AssertionError: Regex didn't match: b'gcmodule\\.c:[0-9]+:" + intermittent: true + + - name: test.test_gc.GCTests.test_legacy_finalizer + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_gc.GCTests.test_legacy_finalizer_newclass + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_gc.GCTests.test_garbage_at_shutdown + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "AssertionError: 1 != 0" + + - name: test.test_genericclass.CAPITest.test_c_class + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + # Test appears to be racy on freethreaded builds. + #- name: test.test_hashlib.HashlibTestCase.test_threaded_hashing_fast + # minimum-python-version: "3.15" + # build-option: "freethreaded" + # reason: "appears to be racy on freethreaded builds" + # intermittent: true + + - name: test.test_httpservers.BaseHTTPServerTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_httpservers.CGIHTTPServerTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_httpservers.CGIHTTPServerTestCase.test_large_content_length_truncated + maximum-python-version: "3.14" + reason: "intermittent failure: http.client.RemoteDisconnected: Remote end closed connection without response" + intermittent: true + + - name: test.test_httpservers.RequestHandlerLoggingTestCase.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_httpservers.HTTP09ServerTestCase.* + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test__interpchannels.ChannelTests.test_send_recv_different_interpreters_and_threads + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.CreateTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.CreateTests.test_in_threaded_subinterpreter + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.RunFuncTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__interpreters.RunStringTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_imp.ImportTests.test_create_builtin_subinterp + maximum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "_testcapi missing" + + - name: test.test_imp.ImportTests.test_issue16421_multiple_modules_in_one_dll + maximum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "_testcapi missing" + + - name: test.test_imp.ImportTests.test_issue24748_load_module_skips_sys_modules_check + maximum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: "_testcapi missing" + + - name: test.test_import.CircularImportTests.test_singlephase_circular + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.13" + reason: _testsinglephase shared library expectation + + - name: test.test_import.ImportTests.test_from_import_missing_attr_has_name_and_so_path + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.ModexportTests.test_from_modexport + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_create_nonmodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_create_nonmodule_gil_used + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_empty_slots + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_gil_used + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_null + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.ModexportTests.test_from_modexport_smoke + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.SinglephaseInitTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: shared library expectations + + - name: test.test_import.SubinterpImportTests.test_builtin_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_disallowed_reimport + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testsinglephase shared library expectation" + + - name: test.test_import.SubinterpImportTests.test_frozen_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_isolated_config + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_multi_init_extension_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: shared library expectations + + - name: test.test_import.SubinterpImportTests.test_multi_init_extension_non_isolated_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testmultiphase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_multi_init_extension_per_interpreter_gil_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testmultiphase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_pyinit_function_raises_exception + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testsinglephase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_import.SubinterpImportTests.test_python_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_import.SubinterpImportTests.test_singlephase_check_with_setting_and_override + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testsinglephase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_single_init_extension_compat + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: _testsinglephase shared library expectations + + - name: test.test_import.SubinterpImportTests.test_testmultiphase_exec_multiple + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.15" + reason: "AttributeError: module '_testmultiphase' has no attribute '__file__'. Did you mean '.__name__' instead of '.__file__'?" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_is_package + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_load_module_API + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_LoaderTests.test_module_reuse + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_bad_modules + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_load_short_name + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_load_submodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_is_package + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_load_module_API + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_LoaderTests.test_module_reuse + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AttributeError: 'NoneType' object has no attribute 'rfind'" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_bad_modules + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_finder.Frozen_FinderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AssertionError: None is not true" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_unloadable_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_functionality + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_load_twice + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_nonmodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_nonmodule_with_methods + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_null_slots + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_reload + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Frozen_MultiPhaseExtensionModuleTests.test_try_registration + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_functionality + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_load_short_name + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_load_submodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_load_twice + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_nonmodule + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_nonmodule_with_methods + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_null_slots + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_reload + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_try_registration + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "ImportError: Dynamic loading not supported" + + - name: test.test_importlib.extension.test_loader.Source_MultiPhaseExtensionModuleTests.test_unloadable_nonascii + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "AssertionError: 'fo_6ja' != 'foó'" + + - name: test.test_importlib.extension.test_finder.Source_FinderTests.test_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.11" + reason: "AssertionError: None is not true" + + - name: test.test_importlib.test_threaded_import.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_multiprocessing_pool_circular_import + targets: + - ".+-unknown-linux-musl" + reason: "segfault at least on debug builds" + # Test sniffs memory and self-skips. This check is not reliable. + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_concurrent_futures_circular_import + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.11" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_parallel_meta_path + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_parallel_module_init + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_parallel_path_hooks + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_threaded_import.ThreadedImportTests.test_side_effect_import + minimum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_importlib.test_util.IncompatibleExtensionModuleRestrictionsTests.test_incomplete_multi_phase_init_module + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_inspect.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestSignatureObject.test_signature_on_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestSignatureObject.test_signature_on_builtins_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.test_inspect.TestSignatureObject.test_signature_on_decorated_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestClassesAndFunctions.test_getfullargspec_builtin_func_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestSignatureObject.test_signature_on_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestSignatureObject.test_signature_on_builtins_no_signature + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_inspect.TestSignatureObject.test_signature_on_decorated_builtins + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_int.IntStrDigitLimitsTests.test_int_max_str_digits_is_per_interpreter + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_int.IntStrDigitLimitsTests.test_denial_of_service_prevented_str_to_int + targets: + - ".+-pc-windows-.+" + # Seems to be fixed in 3.14. + maximum-python-version: "3.13" + reason: "AssertionError: 0.015625 not less than or equal to 0.0078125" + intermittent: true + + - name: test.test_int.IntSubclassStrDigitLimitsTests.test_int_max_str_digits_is_per_interpreter + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_interpreters.CreateTests.test_in_thread + # Moved to test__interpreters in 3.13. + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.TestInterpreterRun.test_in_thread + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.CreateTests.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.LowLevelTests.test_is_running + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterCall.test_call_in_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.13" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterClose.test_created_with_capi + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.13" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterCall.test_stateless_func_returns_arg + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: missing _testinternalcapi + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterCall.test_stateless_funcs + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: missing _testinternalcapi + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterExec.test_in_thread + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_interpreters.test_api.TestInterpreterIsRunning.test_created_with_capi + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_itertools.TestBasicOps.test_count_threading + minimum-python-version: "3.13" + build-option: "freethreaded" + reason: "test appears racy on freethreaded builds" + intermittent: true + + - name: test.test_io.CTextIOWrapperTest.test_device_encoding + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_io.PyTextIOWrapperTest.test_device_encoding + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test__locale._LocaleTests.test_alt_digits_nl_langinfo + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + reason: "AssertionError: 1 not greater than or equal to 10 : ['0123456789']" + + # This one still fails on static builds on 3.14+. + - name: test.test__locale._LocaleTests.test_alt_digits_nl_langinfo + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: 1 not greater than or equal to 10 : ['0123456789']" + + - name: test.test__locale._LocaleTests.test_lc_numeric_localeconv + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: '.' != ','" + + - name: test.test__locale._LocaleTests.test_lc_numeric_localeconv + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: "AssertionError: '.' != ','" + + - name: test.test__locale._LocaleTests.test_lc_numeric_nl_langinfo + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: '.' != ','" + + # This one still fails on static builds on 3.14+. + - name: test.test__locale._LocaleTests.test_lc_numeric_nl_langinfo + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: '.' != ','" + + - name: test.test_locale.TestEnUSCollation.test_strcoll_with_diacritic + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: 1 not less than 0" + + - name: test.test_locale.TestEnUSCollation.test_strcoll_with_diacritic + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: 1 not less than 0" + + - name: test.test_locale.TestEnUSCollation.test_strxfrm_with_diacritic + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: 1 not less than 0" + + - name: test.test_locale.TestEnUSCollation.test_strxfrm_with_diacritic + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: 1 not less than 0" + + # Passes on x86_64_v3 and x86_64_v4 for some reason. + - name: test.test_math.FMATests.test_fma_zero_result + targets: + - "x86_64(_v2)?-unknown-linux-musl" + minimum-python-version: "3.13" + maximum-python-version: "3.13" + reason: "AssertionError: False is not true : Expected a negative zero, got 0.0" + + - name: test.test_math.FMATests.test_fma_zero_result + targets: + - ".+-unknown-linux-musl" + ignore-targets: + - "x86_64_v[34]-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: False is not true : Expected a negative zero, got 0.0" + + - name: test.test_multibytecodec.Test_IncrementalEncoder.test_subinterp + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_os.TestInvalidFD.test_fpathconf_bad_fd + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + maximum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.test_os.TestInvalidFD.test_fpathconf_bad_fd + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.test_os.TestInvalidFD.test_pathconf_negative_fd_uses_fd_semantics + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "AssertionError: OSError not raised" + + - name: test.test_os.TestInvalidFD.test_fpathconf + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: didn't raise an OSError with a bad file descriptor" + + - name: test.test_os.test_posix.PosixTester.test_pipe2_c_limits + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "ModuleNotFoundError: No module named '_testcapi'" + + - name: test.test_os.test_posix.PosixTester.test_nodev + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.15" + build-option: "static" + reason: "AssertionError: module 'posix' has no attribute 'NODEV'" + + - name: test.test_poll.PollTests.test_poll_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_posix.PosixTester.test_pipe2_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: missing _testcapi + + - name: test.test_posix.PosixTester.test_makedev + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + reason: "OverflowError: can't convert negative int to unsigned" + + - name: test.test_posix.PosixTester.test_makedev + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "OverflowError: can't convert negative int to unsigned" + + - name: test.test_pty.PtyTest.test_spawn_doesnt_hang + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + reason: AssertionError + intermittent: true + + - name: test.test_pydoc.PydocServerTest.test_server + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_re.ReTests.test_locale_caching + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: None is not true" + + - name: test.test_re.ReTests.test_locale_caching + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: None is not true" + + - name: test.test_re.ReTests.test_locale_compiled + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.13" + reason: "AssertionError: None is not true" + + - name: test.test_re.ReTests.test_locale_compiled + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: None is not true" + + - name: test.test_readline.TestHistoryManipulation.test_write_read_append + targets: + - ".+-unknown-linux-musl" + reason: "AssertionError: 4 != 3" + + - name: test.test_readline.TestHistoryManipulation.test_write_read_limited_history + targets: + - ".+-unknown-linux-.+" + minimum-python-version: "3.12" + reason: TESTFN undefined + + - name: test.test_readline.TestReadline.test_write_read_limited_history + targets: + - ".+-unknown-linux-.+" + minimum-python-version: "3.12" + reason: TESTFN undefined + + - name: test.test_repl.TestInteractiveInterpreter.test_no_memory + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "AssertionError: 0 not found in (1, 120)" + + - name: test.test_signal.StressTest.test_stress_delivery_simultaneous + reason: "timeout due to potential deadlock or similar test bug" + intermittent: true + + - name: test.test_smtplib.DebuggingServerTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.13" + reason: "segfault" + intermittent: true + + - name: test.test_smtplib.SMTPSimTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "freethreaded" + # Should be thread safe in 3.15. + maximum-python-version: "3.14" + reason: "segfault" + intermittent: true + + - name: test.test_smtplib.SMTPUTF8SimTests.* + targets: + - ".+-unknown-linux-musl" + build-option: "freethreaded" + # Should be thread safe in 3.15. + maximum-python-version: "3.14" + reason: "segfault" + intermittent: true + + - name: test.test_socket.GeneralModuleTests.testNtoHErrors + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.GeneralModuleTests.test_getaddrinfo_int_port_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_socket.GeneralModuleTests.test_listen_backlog_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.NonBlockingTCPTests.testSetBlocking_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.BasicTCPTest.testShutdown_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.BasicTCPTest2.testShutdown_overflow + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_socket.GeneralModuleTests.test3542SocketOptions + reason: "AssertionError: False is not true : Missing RFC3542 socket option 'IPV6_RECVDSTOPTS'" + maximum-python-version: "3.10" + targets: + - ".+-apple-darwin" + + - name: test.test_socket.NonBlockingTCPTests.testLargeTimeout + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_ssl.BasicSocketTests.test_read_write_zero + targets: + - ".+-pc-windows-.+" + minimum-python-version: "3.12" + reason: ConnectionAbortedError + intermittent: true + + # These uplink/applink failures should go away once we're patching + # OpenSSL correctly following https://github.com/python/cpython/pull/131839. + # These pass on aarch64 for reasons unknown. + + - name: test.test_ssl.ContextTests.test_load_dh_params + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.SSLErrorTests.test_lib_reason + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + #- name: test.test_ssl.SSLErrorTests.test_dh_params + # targets: + # - ".+-pc-windows-.+" + # maximum-python-version: "3.11" + # reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + #- name: test.test_ssl.ThreadedTests.test_dh_params + # targets: + # - ".+-pc-windows-.+" + # minimum-python-version: "3.12" + # reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.TestSSLDebug.test_keylog_defaults + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + - name: test.test_ssl.TestSSLDebug.test_keylog_filename + targets: + - "i686-pc-windows-.+" + - "x86_64-pc-windows-.+" + minimum-python-version: "3.12" + reason: "OPENSSL_Uplink(00007FFD9EB59C60,08): no OPENSSL_Applink" + + # Looks like a race condition in the test. + - name: test.test_str.StrTest.test_raiseMemError + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + minimum-python-version: "3.14" + reason: "AssertionError: 79 != 75" + intermittent: true + + - name: test.test_strptime.StrptimeTests.test_date_locale2 + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.12" + maximum-python-version: "3.13" + reason: "AssertionError: Tuples differ: (2000, 1, 1) != (1900, 1, 1)" + + - name: test.test_strptime.StrptimeTests.test_date_locale2 + targets: + - ".+-unknown-linux-musl" + minimum-python-version: "3.14" + build-option: "static" + reason: "AssertionError: Tuples differ: (2000, 1, 1) != (1900, 1, 1)" + + - name: test.test_struct.StructTest.test_endian_table_init_subinterpreters + minimum-python-version: "3.15" + build-option: "freethreaded" + reason: "BlockingIOError: [Errno 11] Resource temporarily unavailable: '/tmp/tmpqkxa77l0/python/install/lib/python3.15t/concurrent'" + intermittent: true + + - name: test.test_subprocess.POSIXProcessTestCase.test_vfork_used_when_expected + targets: + - "aarch64-unknown-linux-musl" + minimum-python-version: "3.14" + no-build-option: "debug" + reason: "AssertionError: Regex didn't match: b'(?i)vfork'..." + + - name: test.test_subprocess.ProcessTestCase.test_executable_without_cwd + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: 47 != 1" + + - name: test.test_subprocess.ProcessTestCaseNoPoll.test_executable_without_cwd + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-.+" + reason: "AssertionError: 47 != 1" + + - name: test.test_support.TestSupport.test_recursion + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.11" + reason: "RecursionError: maximum recursion depth exceeded" + + - name: test.test_sysconfig.TestSysConfig.test_config_vars_depend_on_site_initialization + minimum-python-version: "3.12" + maximum-python-version: "3.13" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_sysconfig.TestSysConfig.test_config_vars_recalculation_after_site_initialization + minimum-python-version: "3.12" + maximum-python-version: "3.13" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_sysconfig.TestSysConfig.test_paths_depend_on_site_initialization + minimum-python-version: "3.12" + maximum-python-version: "3.13" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_sysconfig.TestSysConfig.test_makefile_overwrites_config_vars + minimum-python-version: "3.14" + targets: + - ".+-apple-darwin" + reason: PBS behavior differences + + - name: test.test_sysconfig.TestSysConfig.test_sysconfigdata_json + minimum-python-version: "3.14" + ignore-targets: + - ".+-pc-windows-.+" + reason: PBS behavior differences + + - name: test.test_sysconfig.TestSysConfig.test_triplet_in_ext_suffix + targets: + - "x86_64.*-unknown-linux-musl" + maximum-python-version: "3.10" + reason: "AssertionError: False is not true : .cpython-310-x86_64-linux-musl.so" + + - name: test.test_syslog.Test.test_subinterpreter_closelog + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_syslog.Test.test_subinterpreter_openlog + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_syslog.Test.test_subinterpreter_syslog + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + # Looks like a race condition in tests. + - name: test.test_sys.SizeofTest.test_objecttypes + targets: + - ".+-pc-windows-.+" + minimum-python-version: "3.14" + build-option: "freethreaded" + reason: "AssertionError: 137 != 133 : wrong size for : got 137, expected 133" + intermittent: true + + - name: test.test_sys.SysModuleTest.test_getallocatedblocks + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "_testcapi missing" + + - name: test.test_sys.TestSysJIT.test_jit_is_active + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_custom_unraisablehook + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_custom_unraisablehook_fail + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_original_unraisablehook + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_sys.UnraisableHookTest.test_original_unraisablehook_exception_qualname + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: missing _testcapi + + - name: test.test_threading.SubinterpThreadingTests.test_threads_join + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_threads_join_2 + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_threads_not_allowed + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_daemon_threads_not_allowed + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.12" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.SubinterpThreadingTests.test_daemon_threads_fatal_error + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.ThreadingExceptionTests.test_print_exception + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "AssertionError" + + - name: test.test_threading.ThreadingExceptionTests.test_print_exception_stderr_is_none_1 + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "AssertionError" + + - name: test.test_threading.ThreadingExceptionTests.test_recursion_limit + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.10" + reason: "AssertionError: -11 != 0" + + - name: test.test_threading.ThreadJoinOnShutdown.test_4_daemon_threads + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused and segfault" + intermittent: true + + - name: test.test_threading.ThreadTests.test_frame_tstate_tracing + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_threading.ThreadTests.test_import_from_another_thread + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_threading.ThreadTests.test_main_thread_after_fork + minimum-python-version: "3.12" + targets: + - "x86_64-apple-darwin" + reason: "AssertionError: b':6: DeprecationWarning: This proc[82 chars]d.\n' != b''" + + - name: test.test_threading.ThreadTests.test_native_id_after_fork + minimum-python-version: "3.13" + targets: + - "x86_64-apple-darwin" + reason: "AssertionError: b':6: DeprecationWarning: This proc[82 chars]d.\n' != b''" + + - name: test.test_threading.ThreadTests.test_PyThreadState_SetAsyncExc + targets: + - ".+-unknown-linux-musl" + build-option: "static" + reason: missing _testcapi + + - name: test.test_threading.ThreadTests.test_finalize_daemon_thread_hang + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.13" + reason: missing _testcapi + + - name: test.test_threading.ThreadTests.test_finalize_running_thread + targets: + - ".+-unknown-linux-mus" + build-option: "static" + reason: "AssertionError: 1 != 42" + + - name: test.test_multiprocessing_forkserver.test_manager.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "unknown; possible segfault" + + - name: test.test_multiprocessing_forkserver.test_misc.TestStartMethod.test_preload_resources + minimum-python-version: "3.12" + targets: + - "x86_64-apple-darwin" + build-option: "debug" + reason: "AssertionError: failed spawning forkserver or grandchild" + + - name: test.test_multiprocessing_fork.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfaults" + + - name: test.test_multiprocessing_fork.test_manager.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_fork.test_processes.WithProcessesTestManagerRestart.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_fork.test_threads.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_forkserver.test_processes.WithProcessesTestManagerRestart.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_forkserver.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_spawn.* + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_spawn.test_manager.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_spawn.test_processes.WithProcessesTestManagerRestart.test_rapid_restart + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "ConnectionRefusedError: [Errno 111] Connection refused" + + - name: test.test_multiprocessing_spawn.test_processes.WithProcessesTestSharedMemory.test_shared_memory_tracking + build-option: "freethreaded" + reason: "timeout possibly due to deadlock" + intermittent: true + + - name: test.test_multiprocessing_spawn.test_threads.* + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + minimum-python-version: "3.11" + reason: "segfault" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_basic_script + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_basic_script_no_suffix + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_directory + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_directory_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_ipython_workaround + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_module_in_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_module_in_package_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_module_in_subpackage_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_package_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_script_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_zipfile + targets: + - "x86_64-apple-darwin" + no-build-option: "pgo" + minimum-python-version: "3.12" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkCmdLineTest.test_zipfile_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_basic_script + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_basic_script_no_suffix + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_directory + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_directory_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_ipython_workaround + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_module_in_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_module_in_package_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_module_in_subpackage_in_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_package + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_package_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_script_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_main_handling.ForkServerCmdLineTest.test_zipfile_compiled + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.12" + build-option: "debug" + reason: "DeprecationWarning: This process (pid=81234) is multi-threaded, use of fork() may lead to deadlocks in the child.\n self.pid = os.fork()\n' is not false" + + - name: test.test_multiprocessing_forkserver.test_processes.WithProcessesTestSpawnedSysPath.test_std_streams_flushed_after_preload + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + build-option: "debug" + reason: "multiprocessing/forkserver.py:278: DeprecationWarning: This process (pid=82723) is multi-threaded, use of fork() may lead to deadlocks in the child." + + - name: test.test_multiprocessing_forkserver.test_threads.* + minimum-python-version: "3.11" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_multiprocessing_forkserver.test_misc.MiscTestCase.test_preload_main + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + build-option: "debug" + reason: "AssertionError: b'/private/var/folders/1r/b8l6nzhj4zdbz5jmj[222 chars]()\n' != b'" + + - name: test.test_multiprocessing_forkserver.test_misc.MiscTestCase.test_preload_main_sys_argv + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.13" + build-option: "debug" + reason: "AssertionError: b'/private/var/folders/1r/b8l6nzhj4zdbz5jmj[222 chars]()\n' != b''" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_trampoline_works_after_fork_with_many_code_objects + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.15" + reason: "AssertionError: '/private/var/folders/1r/b8l6nzhj4zdbz5jmj[194 chars]()\n' != ''" + + - name: test.test_perf_profiler.TestPerfTrampoline.test_trampoline_works_with_forks + targets: + - "x86_64-apple-darwin" + minimum-python-version: "3.15" + reason: "AssertionError: '/private/var/folders/1r/b8l6nzhj4zdbz5jmj[194 chars]()\n' != ''" + + - name: test.test_regrtest.ArgsTestCase.test_add_python_opts + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.14" + reason: "Something related to _testcapi or similar extension" + + - name: test.test_regrtest.ArgsTestCase.test_findleaks + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "AssertionError: Command ['/tmp/.tmpGZuURa/python/install/bin/python3.10', '-X', 'faulthandler', '-I', '-m', 'test', '--testdir=/tmp/tmp6mrz59ds', '--fail-env-changed', 'test_regrtest_noop1'] failed with exit code 2" + + - name: test.test_regrtest.ArgsTestCase.test_uncollectable + targets: + - ".+-unknown-linux-musl" + build-option: "static" + minimum-python-version: "3.11" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test_tcl.TclTest.test_evalfile_surrogates_in_result + maximum-python-version: "3.11" + ignore-targets: + - ".+-pc-windows-.+" + reason: "_tkinter.TclError: couldn't read file \"@test_169558_tmp\": invalid or incomplete multibyte or wide character" + + - name: test.test_tcl.TclTest.test_getint + maximum-python-version: "3.10" + reason: "AssertionError: 17777777777 != 2147483647" + ignore-targets: + - ".+-pc-windows-.+" + + - name: test.test_unicode.UnicodeTest.test_formatting_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_unicode.UnicodeTest.test_formatting_huge_precision_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_call_args_thread_safe + minimum-python-version: "3.13" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_call_count_thread_safe + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_method_calls_thread_safe + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_mock_calls_thread_safe + minimum-python-version: "3.13" + build-option: "debug" + reason: "segfault" + intermittent: true + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_wait_until_any_call_with_success + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_unittest.testmock.testthreadingmock.TestThreadingMock.test_wait_until_called_success + minimum-python-version: "3.13" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test_userstring.UserStringTest.test_formatting_c_limits + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.10" + reason: "missing _testcapi" + + - name: test.test_venv.BasicTest.test_executable + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_macos_env + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_multiprocessing + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_multiprocessing_recursion + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_prefixes + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_special_chars_bash + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_special_chars_csh + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_sysconfig_symlinks + targets: + - ".+-apple-darwin" + minimum-python-version: "3.11" + maximum-python-version: "3.13" + reason: BytesWarning + + - name: test.test_venv.BasicTest.test_sysconfig + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.BasicTest.test_sysconfig + minimum-python-version: "3.13" + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + reason: "AssertionError: 3.14t missing suffix - likely PBS quirk" + + - name: test.test_venv.BasicTest.test_venvwlauncher + minimum-python-version: "3.13" + targets: + - ".+-pc-windows-.+" + build-option: "freethreaded" + reason: "AssertionError: 3.14t missing suffix - likely PBS quirk" + + - name: test.test_venv.BasicTest.test_zippath_from_non_installed_posix + minimum-python-version: "3.11" + targets: + - ".+-apple-darwin" + reason: crashes + + - name: test.test_venv.EnsurePipTest.test_explicit_no_pip + targets: + - ".+-apple-darwin" + reason: PBS installs pip + + - name: test.test_venv.EnsurePipTest.test_no_pip_by_default + targets: + - ".+-apple-darwin" + reason: PBS installs pip + + - name: test.test_venv.EnsurePipTest.test_with_pip + targets: + - ".+-apple-darwin" + maximum-python-version: "3.10" + reason: logic bug? + + - name: test.test_venv.EnsurePipTest.test_with_pip + targets: + - ".+-unknown-linux-musl" + maximum-python-version: "3.10" + build-option: "static" + reason: "TBD" + + - name: test.test_weakref.ReferencesTestCase.test_cfunction + targets: + - ".+-unknown-linux-musl" + build-option: "static" + maximum-python-version: "3.12" + reason: "missing _testcapi" + + - name: test.test__xxinterpchannels.ChannelTests.test_send_recv_different_interpreters_and_threads + minimum-python-version: "3.12" + maximum-python-version: "3.12" + targets: + - "aarch64-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + + - name: test.test__xxsubinterpreters.ChannelTests.test_send_recv_different_interpreters_and_threads + maximum-python-version: "3.10" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + no-build-option: "static" + reason: "segfault" + + - name: test.test__xxsubinterpreters.CreateTests.test_in_thread + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + no-build-option: "static" + intermittent: true + + - name: test.test__xxsubinterpreters.CreateTests.test_in_threaded_subinterpreter + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + no-build-option: "static" + intermittent: true + + - name: test.test__xxsubinterpreters.RunStringTests.test_in_thread + maximum-python-version: "3.12" + targets: + - ".+-unknown-linux-musl" + build-option: "debug" + reason: "segfault" + no-build-option: "static" + intermittent: true diff --git a/uv.lock b/uv.lock index 262e79edc..9c7c67700 100644 --- a/uv.lock +++ b/uv.lock @@ -600,6 +600,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, ] +[[package]] +name = "junitparser" +version = "4.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/97/954ee1ef04e50d8494e9f5d82d4051ed71a7618aa2c1514c1b3f24691174/junitparser-4.0.2.tar.gz", hash = "sha256:d5d07cece6d4a600ff3b7b96c8db5ffa45a91eed695cb86c45c3db113c1ca0f8", size = 25646, upload-time = "2025-06-24T04:37:32.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/87/b444f934f62ee2a1be45bb52563cf17a66b0d790eba43af4df9929e7107f/junitparser-4.0.2-py3-none-any.whl", hash = "sha256:94c3570e41fcaedc64cc3c634ca99457fe41a84dd1aa8ff74e9e12e66223a155", size = 14592, upload-time = "2025-06-24T04:37:31.322Z" }, +] + [[package]] name = "lazy-object-proxy" version = "1.12.0" @@ -1191,6 +1200,7 @@ dependencies = [ { name = "docker" }, { name = "jinja2" }, { name = "jsonschema" }, + { name = "junitparser" }, { name = "pyyaml" }, { name = "six" }, { name = "tomli" }, @@ -1217,6 +1227,7 @@ requires-dist = [ { name = "docker", specifier = ">=7.1.0" }, { name = "jinja2", specifier = ">=3.1.5" }, { name = "jsonschema", specifier = ">=4.23.0" }, + { name = "junitparser", specifier = ">=4.0.2" }, { name = "pyyaml", specifier = ">=6.0.2" }, { name = "six", specifier = ">=1.17.0" }, { name = "tomli", specifier = ">=2.2.1" },