unix: use extended PGO training set

.github/workflows/linux.yml

@@ -329,7 +329,7 @@ jobs:
               unset LD_LIBRARY_PATH
             fi
 
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --stdlib dist/*.tar.zst
           fi
         env:
           MATRIX_RUN: ${{ matrix.run }}

.github/workflows/macos.yml

@@ -181,7 +181,7 @@ jobs:
           build/pythonbuild validate-distribution --macos-sdks-path macosx-sdks dist/*.tar.zst
 
           if [ "${MATRIX_RUN}" == "true" ]; then
-            ./test-distribution.py dist/*.tar.zst
+            ./test-distribution.py --stdlib dist/*.tar.zst
           fi
         env:
           MATRIX_RUN: ${{ matrix.run }}

.github/workflows/windows.yml

@@ -186,4 +186,4 @@ jobs:
         run: |
           $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
           .\pythonbuild.exe validate-distribution $Dists
-          uv run --no-dev test-distribution.py $Dists
+          uv run --no-dev test-distribution.py --stdlib $Dists

ci-runners.yaml

@@ -1,11 +1,11 @@
 # Describes the runners that the CI system can use
 
-depot-ubuntu-22.04-4:
+depot-ubuntu-22.04-16:
   arch: x86_64
   platform: linux
   free: false
 
-depot-ubuntu-22.04-arm-4:
+depot-ubuntu-22.04-arm-16:
   arch: aarch64
   platform: linux
   free: false

cpython-unix/build-cpython.sh

@@ -51,10 +51,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz"
 PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
 SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
 
-cat Setup.local
-mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
+# Put critical config files in logs to aid debugging.
+for f in Setup.local Makefile.extra stdlib-test-annotations.json profiling-training-ignores.txt; do
+  echo "BEGIN $f"
+  cat $f
+  echo "END $f"
+done
 
-cat Makefile.extra
+mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
 
 pushd "Python-${PYTHON_VERSION}"
 
@@ -589,7 +593,27 @@ if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}"  ]; then
 fi
 
 # Define the base PGO profiling task, which we'll extend below with ignores
-export PROFILE_TASK='-m test --pgo'
+#
+# --pgo-extended implies --pgo. And --pgo mode significantly changes the behavior
+# of the test harness. Notably, it disables richer reporting of test failures,
+# making it vastly more difficult to debug test failures. This behavior undermines
+# our ability to debug failures when running tests for PGO instrumentation.
+#
+# The only material downside to not using --pgo is some tests that self-skip
+# when run under PGO don't do so. When we audited for such tests in 2026-03,
+# all such tests had the reason "PGO isn't useful" or some such. There were
+# ~10 such tests. Since we run the entire test suite anyway, the inclusion of
+# such "worthless" tests isn't impactful. The test failure observability is
+# worth their loss.
+export PROFILE_TASK='-m test'
+
+# Display test output on failure. This helps immensely with debugging PGO
+# failures.
+PROFILE_TASK="${PROFILE_TASK} -W"
+
+# Force kill tests taking too long. This prevents deadlocks that can cause
+# CI jobs to run for potentially hours while doing nothing.
+PROFILE_TASK="${PROFILE_TASK} --timeout 300"
 
 # Run tests in parallel to reduce wall time.
 #
@@ -604,21 +628,26 @@ export PROFILE_TASK='-m test --pgo'
 # and there will be no loss in profile quality.
 PROFILE_TASK="${PROFILE_TASK} -j ${NUM_CPUS}"
 
-# On 3.14+ `test_strftime_y2k` fails when cross-compiling for `x86_64_v2` and `x86_64_v3` targets on
-# Linux, so we ignore it. See https://github.com/python/cpython/issues/128104
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" && -n "${CROSS_COMPILING}" && "${PYBUILD_PLATFORM}" != macos* ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test_strftime_y2k"
+IGNORE_TESTS=()
+
+# Possible race condition.
+IGNORE_TESTS+=("test.test_audit.AuditTest.test_time_fail")
+
+if [[ -n "${CPYTHON_FREETHREADED}" ]]; then
+    IGNORE_TESTS+=("test.test_bytes.FreeThreadingTest.test_free_threading_bytearrayiter")
 fi
 
 # On 3.14+ `test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding` fails during
 # PGO due to RecursionError not being raised as expected. See https://github.com/python/cpython/issues/140125
 if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test_json"
+    IGNORE_TESTS+=("test.test_json.test_recursion.TestCRecursion.test_highly_nested_objects_decoding")
 fi
 
-# PGO optimized / BOLT instrumented binaries segfault in a test_bytes test. Skip it.
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64* ]]; then
-    PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format"
+PROFILE_TASK="${PROFILE_TASK} --ignorefile ${ROOT}/profiling-training-ignores.txt"
+
+# Exclude whole modules from profiling based on stdlib test annotations.
+if [ -n "${PROFILING_EXCLUDE_MODULES}" ]; then
+    PROFILE_TASK="${PROFILE_TASK} --exclude ${PROFILING_EXCLUDE_MODULES}"
 fi
 
 # ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL,
@@ -1386,14 +1415,18 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/"
 cp -av Python/frozen.c "${ROOT}/out/python/build/Python/"
 cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/"
 
-# Copy the test hardness runner for convenience.
+# Copy the test harness runner for convenience.
 # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script
 if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
     cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py"
 else
     cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/"
 fi
 
+# Copy standard library test annotations so it is in the artifact and the
+# distribution self-describes expected test wonkiness.
+cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json"
+
 # Don't hard-code the build-time prefix into the pkg-config files. See
 # the description of `pcfiledir` in `man pkg-config`.
 find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \

cpython-unix/build-main.py

@@ -142,7 +142,14 @@ def main():
     effective_host_platform = host_platform
     if building_linux_from_macos:
         if host_platform == "macos_arm64":
-            effective_host_platform = "linux_aarch64"
+            if target_triple.startswith("aarch64"):
+                effective_host_platform = "linux_aarch64"
+            elif target_triple.startswith("x86_64"):
+                effective_host_platform = "linux_x86_64"
+            else:
+                raise Exception(
+                    f"unsupported macOS cross-compile: {host_platform} -> {target_triple}"
+                )
         else:
             raise Exception(f"Unhandled macOS platform: {host_platform}")
         print(

cpython-unix/build.py

@@ -24,15 +24,22 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_setup_line,
+    stdlib_test_annotations,
+)
+from pythonbuild.docker import (
+    build_docker_image,
+    docker_platform_from_host_platform,
+    get_image,
+    write_dockerfiles,
 )
-from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.logging import log, set_logger
 from pythonbuild.utils import (
     add_env_common,
     add_licenses_to_extension_entry,
     clang_toolchain,
     create_tar_from_directory,
+    current_host_platform,
     download_entry,
     get_target_settings,
     get_targets,
@@ -50,6 +57,7 @@
 SUPPORT = ROOT / "cpython-unix"
 EXTENSION_MODULES = SUPPORT / "extension-modules.yml"
 TARGETS_CONFIG = SUPPORT / "targets.yml"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LINUX_ALLOW_SYSTEM_LIBRARIES = {
     "c",
@@ -98,21 +106,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options)
         extra_target_ldflags.append("--rtlib=compiler-rt")
 
     if build_platform.startswith("linux_"):
-        machine = platform.machine()
+        # autoconf is not aware of microarch triples. Normalize those out:
+        # we force targeting via -march CFLAG.
+        env["TARGET_TRIPLE"] = (
+            target_triple.replace("x86_64_v2-", "x86_64-")
+            .replace("x86_64_v3-", "x86_64-")
+            .replace("x86_64_v4-", "x86_64-")
+        )
 
-        # arm64 allows building for Linux on a macOS host using Docker
-        if machine == "aarch64" or machine == "arm64":
-            env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = target_triple
-        elif machine == "x86_64":
-            env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = (
-                target_triple.replace("x86_64_v2-", "x86_64-")
-                .replace("x86_64_v3-", "x86_64-")
-                .replace("x86_64_v4-", "x86_64-")
-            )
+        # On macOS, we support building Linux in a virtualized container that
+        # always matches the target platform. Set build/host triple to whatever
+        # we're building.
+        #
+        # Note: we always use the *-gnu triple otherwise autoconf can have
+        # trouble reasoning about cross-compiling since its detected triple from
+        # our build environment is always GNU based.
+        if current_host_platform().startswith("macos_"):
+            env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu")
         else:
-            raise Exception("unhandled Linux machine value: %s" % machine)
+            # Otherwise assume the container environment matches the machine
+            # type of the current process.
+            host_machine = platform.machine()
+            if host_machine == "aarch64" or host_machine == "arm64":
+                env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
+            elif host_machine == "x86_64":
+                env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
+            else:
+                raise Exception("unhandled Linux machine value: %s" % host_machine)
 
         # This will make x86_64_v2, etc count as cross-compiling. This is
         # semantically correct, since the current machine may not support
@@ -732,6 +752,13 @@ def build_cpython(
     setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
     pip_archive = download_entry("pip", DOWNLOADS_PATH)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     ems = extension_modules_config(EXTENSION_MODULES)
 
     setup = derive_setup_local(
@@ -804,12 +831,36 @@ def build_cpython(
 
             build_env.copy_file(fh.name, dest_name="Makefile.extra")
 
+        # Install the derived test annotations.
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+            test_annotations.json_dump(fh)
+            fh.flush()
+
+            build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json")
+
+        # Install a file with tests to skip during profile training.
+        # Also collect module excludes for the profiling run of the test harness.
+        profiling_exclude_modules = []
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+
+            for ann in test_annotations.annotations:
+                if ann.profile_training_skip():
+                    fh.write(f"{ann.name}\n")
+                if m := ann.profile_training_exclude_module():
+                    profiling_exclude_modules.append(m)
+
+            fh.flush()
+            build_env.copy_file(fh.name, dest_name="profiling-training-ignores.txt")
+
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
             "PYTHON_VERSION": python_version,
             "PYTHON_MAJMIN_VERSION": ".".join(python_version.split(".")[0:2]),
             "SETUPTOOLS_VERSION": DOWNLOADS["setuptools"]["version"],
             "TOOLCHAIN": "clang-%s" % host_platform,
+            "PROFILING_EXCLUDE_MODULES": " ".join(profiling_exclude_modules),
         }
 
         # Set environment variables allowing convenient testing for Python
@@ -960,16 +1011,6 @@ def main():
     DOWNLOADS_PATH.mkdir(exist_ok=True)
     (BUILD / "logs").mkdir(exist_ok=True)
 
-    if os.environ.get("PYBUILD_NO_DOCKER"):
-        client = None
-    else:
-        try:
-            client = docker.from_env(timeout=600)
-            client.ping()
-        except Exception as e:
-            print("unable to connect to Docker: %s" % e, file=sys.stderr)
-            return 1
-
     # Note these arguments must be synced with `build-main.py`
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -1031,6 +1072,17 @@ def main():
 
     settings = get_target_settings(TARGETS_CONFIG, target_triple)
 
+    if os.environ.get("PYBUILD_NO_DOCKER"):
+        client = None
+    else:
+        try:
+            client = docker.from_env(timeout=600)
+            client.ping()
+            client._pbs_platform = docker_platform_from_host_platform(host_platform)
+        except Exception as e:
+            print("unable to connect to Docker: %s" % e, file=sys.stderr)
+            return 1
+
     if args.action == "dockerfiles":
         log_name = "dockerfiles"
     elif args.action == "makefiles":
@@ -1070,7 +1122,7 @@ def main():
             write_dockerfiles(SUPPORT, BUILD)
         elif action == "makefiles":
             targets = get_targets(TARGETS_CONFIG)
-            write_triples_makefiles(targets, BUILD, SUPPORT)
+            write_triples_makefiles(targets, ROOT, BUILD, SUPPORT)
             write_target_settings(targets, BUILD / "targets")
             write_package_versions(BUILD / "versions")
 

cpython-unix/targets.yml

@@ -556,6 +556,7 @@ x86_64-apple-darwin:
 x86_64-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -605,6 +606,7 @@ x86_64-unknown-linux-gnu:
 x86_64_v2-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -655,6 +657,7 @@ x86_64_v2-unknown-linux-gnu:
 x86_64_v3-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -705,6 +708,9 @@ x86_64_v3-unknown-linux-gnu:
 x86_64_v4-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -755,6 +761,9 @@ x86_64_v4-unknown-linux-gnu:
 x86_64-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -802,6 +811,7 @@ x86_64-unknown-linux-musl:
 x86_64_v2-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -850,6 +860,7 @@ x86_64_v2-unknown-linux-musl:
 x86_64_v3-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -898,6 +909,7 @@ x86_64_v3-unknown-linux-musl:
 x86_64_v4-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'