Run stdlib tests

.github/workflows/linux.yml

@@ -329,7 +329,7 @@ jobs:
               unset LD_LIBRARY_PATH
             fi
 
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --stdlib dist/*.tar.zst
           fi
         env:
           MATRIX_RUN: ${{ matrix.run }}

.github/workflows/macos.yml

@@ -181,7 +181,7 @@ jobs:
           build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst
 
           if [ "${MATRIX_RUN}" == "true" ]; then
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --stdlib dist/*.tar.zst
           fi
         env:
           MATRIX_RUN: ${{ matrix.run }}

.github/workflows/windows.yml

@@ -186,4 +186,4 @@ jobs:
         run: |
           $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
           .\pythonbuild.exe validate-distribution $Dists
-          uv run --no-dev test-distribution.py $Dists
+          uv run --no-dev test-distribution.py --stdlib $Dists

cpython-unix/build-cpython.sh

@@ -51,10 +51,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz"
 PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
 SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
 
-cat Setup.local
-mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
+# Put critical config files in logs to aid debugging.
+for f in Setup.local Makefile.extra stdlib-test-annotations.json; do
+  echo "BEGIN $f"
+  cat $f
+  echo "END $f"
+done
 
-cat Makefile.extra
+mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
 
 pushd "Python-${PYTHON_VERSION}"
 
@@ -1386,14 +1390,18 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/"
 cp -av Python/frozen.c "${ROOT}/out/python/build/Python/"
 cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/"
 
-# Copy the test hardness runner for convenience.
+# Copy the test harness runner for convenience.
 # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script
 if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
     cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py"
 else
     cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/"
 fi
 
+# Copy standard library test annotations so it is in the artifact and the
+# distribution self-describes expected test wonkiness.
+cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json"
+
 # Don't hard-code the build-time prefix into the pkg-config files. See
 # the description of `pcfiledir` in `man pkg-config`.
 find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \

cpython-unix/build.py

@@ -24,6 +24,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_setup_line,
+    stdlib_test_annotations,
 )
 from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
 from pythonbuild.downloads import DOWNLOADS
@@ -50,6 +51,7 @@
 SUPPORT = ROOT / "cpython-unix"
 EXTENSION_MODULES = SUPPORT / "extension-modules.yml"
 TARGETS_CONFIG = SUPPORT / "targets.yml"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LINUX_ALLOW_SYSTEM_LIBRARIES = {
     "c",
@@ -732,6 +734,13 @@ def build_cpython(
     setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
     pip_archive = download_entry("pip", DOWNLOADS_PATH)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     ems = extension_modules_config(EXTENSION_MODULES)
 
     setup = derive_setup_local(
@@ -804,6 +813,14 @@ def build_cpython(
 
             build_env.copy_file(fh.name, dest_name="Makefile.extra")
 
+        # Install the derived test annotations.
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+            test_annotations.json_dump(fh)
+            fh.flush()
+
+            build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json")
+
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
             "PYTHON_VERSION": python_version,
@@ -1070,7 +1087,7 @@ def main():
             write_dockerfiles(SUPPORT, BUILD)
         elif action == "makefiles":
             targets = get_targets(TARGETS_CONFIG)
-            write_triples_makefiles(targets, BUILD, SUPPORT)
+            write_triples_makefiles(targets, ROOT, BUILD, SUPPORT)
             write_target_settings(targets, BUILD / "targets")
             write_package_versions(BUILD / "versions")
 

cpython-windows/build.py

@@ -22,6 +22,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_config_c,
+    stdlib_test_annotations,
 )
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.utils import (
@@ -39,6 +40,7 @@
 BUILD = ROOT / "build"
 DIST = ROOT / "dist"
 SUPPORT = ROOT / "cpython-windows"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LOG_PREFIX = [None]
 LOG_FH = [None]
@@ -128,7 +130,6 @@
     "_zstd": ["zstd"],
 }
 
-
 # Tests to run during PGO profiling.
 #
 # This set was copied from test.libregrtest.pgo in the CPython source
@@ -1407,6 +1408,13 @@ def build_cpython(
     else:
         raise Exception("unhandled architecture: %s" % arch)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     tempdir_opts = (
         {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {}
     )
@@ -1763,6 +1771,12 @@ def build_cpython(
                 out_dir / "python" / "build" / "run_tests.py",
             )
 
+        # Install a JSON file annotating tests.
+        with (out_dir / "python" / "build" / "stdlib-test-annotations.json").open(
+            "w", encoding="utf-8"
+        ) as fh:
+            test_annotations.json_dump(fh)
+
         licenses_dir = out_dir / "python" / "licenses"
         licenses_dir.mkdir()
         for f in sorted(os.listdir(ROOT)):

pyproject.toml

@@ -10,6 +10,7 @@ dependencies = [
     "docker>=7.1.0",
     "jinja2>=3.1.5",
     "jsonschema>=4.23.0",
+    "junitparser>=4.0.2",
     "pyyaml>=6.0.2",
     "six>=1.17.0",
     "tomli>=2.2.1",

pythonbuild/cpython.py

@@ -2,9 +2,12 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+import dataclasses
+import json
 import pathlib
 import re
 import tarfile
+from typing import Optional
 
 import jsonschema
 import yaml
@@ -137,6 +140,65 @@
     },
 }
 
+STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES = {
+    "reason": {"type": "string"},
+    "targets": {
+        "type": "array",
+        "items": {"type": "string"},
+    },
+    "ignore-targets": {
+        "type": "array",
+        "items": {"type": "string"},
+    },
+    "minimum-python-version": {"type": "string"},
+    "maximum-python-version": {"type": "string"},
+    "build-option": {"type": "string"},
+    "no-build-option": {"type": "string"},
+}
+
+STDLIB_TEST_ANNOTATIONS_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "harness-skips": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES,
+                },
+                "additionalProperties": False,
+                "required": ["name", "reason"],
+            },
+        },
+        "module-excludes": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "module": {"type": "string"},
+                    **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES,
+                },
+                "additionalProperties": False,
+                "required": ["module", "reason"],
+            },
+        },
+        "expected-failures": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "dont-verify": {"type": "boolean"},
+                    "intermittent": {"type": "boolean"},
+                    **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES,
+                },
+                "additionalProperties": False,
+                "required": ["name", "reason"],
+            },
+        },
+    },
+}
 
 # Packages that define tests.
 STDLIB_TEST_PACKAGES = {
@@ -750,3 +812,139 @@ def extension_modules_config(yaml_path: pathlib.Path):
     jsonschema.validate(data, EXTENSION_MODULES_SCHEMA)
 
     return data
+
+
+TEST_ANNOTATION_HARNESS_SKIP = "harness-skip"
+TEST_ANNOTATION_MODULE_EXCLUDE = "module-exclude"
+TEST_ANNOTATION_TEST_FAILURE = "test-failure"
+
+
+@dataclasses.dataclass
+class TestAnnotation:
+    # Describes the type of annotation.
+    flavor: str
+    # Name/pattern of test.
+    name: str
+    # Describes why the annotation exists.
+    reason: str
+    # Whether the test is expected to fail.
+    expect_test_failure: bool
+    # Whether to skip verification of failures.
+    dont_verify: bool
+    # Whether test failure is intermittent. Should only be true if
+    # expect_test_failure also true.
+    intermittent_test_failure: bool
+    # Whether to exclude loading the test module when running tests.
+    exclude_testing: bool
+
+
+@dataclasses.dataclass
+class TestAnnotations:
+    annotations: list[TestAnnotation]
+
+    def json_dump(self, of):
+        data = [dataclasses.asdict(a) for a in self.annotations]
+        json.dump(data, of, indent=2)
+
+
+def filter_stdlib_test_entry(
+    flavor: str,
+    test,
+    python_version: str,
+    target_triple: str,
+    build_options: set[str],
+) -> Optional[TestAnnotation]:
+    name = test["name"]
+
+    if targets := test.get("targets"):
+        matches_target = any(re.match(p, target_triple) for p in targets)
+    else:
+        matches_target = True
+
+    for m in test.get("ignore-targets", []):
+        if re.match(m, target_triple):
+            matches_target = False
+
+    if not matches_target:
+        log(f"ignoring {flavor} rule (target doesn't match): {name}")
+        return None
+
+    python_minimum_version = test.get("minimum-python-version", "1.0")
+    python_maximum_version = test.get("maximum-python-version", "100.0")
+
+    if not meets_python_minimum_version(python_version, python_minimum_version):
+        log(
+            f"ignoring {flavor} rule ({python_version} < {python_minimum_version} (min)): {name}"
+        )
+        return None
+
+    if not meets_python_maximum_version(python_version, python_maximum_version):
+        log(
+            f"ignoring {flavor} rule ({python_version} > {python_maximum_version} (max)): {name}"
+        )
+        return None
+
+    if option := test.get("build-option"):
+        if option not in build_options:
+            log(f"ignoring {flavor} rule (build option {option} not present): {name}")
+            return None
+
+    if option := test.get("no-build-option"):
+        if option in build_options:
+            log(f"ignoring {flavor} rule (build option {option} is present): {name}")
+            return None
+
+    # Filtering complete. This rule applies to the current build.
+
+    log(f"relevant {flavor} test rule: {name}: {test['reason']}")
+
+    return TestAnnotation(
+        flavor=flavor,
+        name=name,
+        reason=test["reason"],
+        expect_test_failure=True,
+        dont_verify=test.get("dont-verify", False),
+        intermittent_test_failure=test.get("intermittent", False),
+        exclude_testing=test.get("exclude", False),
+    )
+
+
+def stdlib_test_annotations(
+    yaml_path: pathlib.Path,
+    python_version: str,
+    target_triple: str,
+    build_options: set[str],
+) -> TestAnnotations:
+    """Processes the test-annotations.yml file for a given build configuration."""
+    with yaml_path.open("r", encoding="utf-8") as fh:
+        data = yaml.load(fh, Loader=yaml.SafeLoader)
+
+    jsonschema.validate(data, STDLIB_TEST_ANNOTATIONS_SCHEMA)
+
+    annotations = []
+
+    log(f"processing {len(data['expected-failures'])} stdlib test annotations")
+
+    raw_entries = []
+
+    for entry in data["harness-skips"]:
+        raw_entries.append((TEST_ANNOTATION_HARNESS_SKIP, entry))
+
+    for entry in data["module-excludes"]:
+        entry["name"] = entry["module"]
+        raw_entries.append((TEST_ANNOTATION_MODULE_EXCLUDE, entry))
+
+    for entry in data["expected-failures"]:
+        raw_entries.append((TEST_ANNOTATION_TEST_FAILURE, entry))
+
+    for flavor, entry in raw_entries:
+        if a := filter_stdlib_test_entry(
+            flavor,
+            entry,
+            python_version,
+            target_triple,
+            build_options,
+        ):
+            annotations.append(a)
+
+    return TestAnnotations(annotations)