unix: support building Linux x86-64 from macOS aarch64

cpython-unix/build-main.py

@@ -142,7 +142,14 @@ def main():
     effective_host_platform = host_platform
     if building_linux_from_macos:
         if host_platform == "macos_arm64":
-            effective_host_platform = "linux_aarch64"
+            if target_triple.startswith("aarch64"):
+                effective_host_platform = "linux_aarch64"
+            elif target_triple.startswith("x86_64"):
+                effective_host_platform = "linux_x86_64"
+            else:
+                raise Exception(
+                    f"unsupported macOS cross-compile: {host_platform} -> {target_triple}"
+                )
         else:
             raise Exception(f"Unhandled macOS platform: {host_platform}")
         print(

cpython-unix/build.py

@@ -25,14 +25,20 @@
     meets_python_minimum_version,
     parse_setup_line,
 )
-from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles
+from pythonbuild.docker import (
+    build_docker_image,
+    docker_platform_from_host_platform,
+    get_image,
+    write_dockerfiles,
+)
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.logging import log, set_logger
 from pythonbuild.utils import (
     add_env_common,
     add_licenses_to_extension_entry,
     clang_toolchain,
     create_tar_from_directory,
+    current_host_platform,
     download_entry,
     get_target_settings,
     get_targets,
@@ -98,21 +104,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options)
         extra_target_ldflags.append("--rtlib=compiler-rt")
 
     if build_platform.startswith("linux_"):
-        machine = platform.machine()
+        # autoconf is not aware of microarch triples. Normalize those out:
+        # we force targeting via -march CFLAG.
+        env["TARGET_TRIPLE"] = (
+            target_triple.replace("x86_64_v2-", "x86_64-")
+            .replace("x86_64_v3-", "x86_64-")
+            .replace("x86_64_v4-", "x86_64-")
+        )
 
-        # arm64 allows building for Linux on a macOS host using Docker
-        if machine == "aarch64" or machine == "arm64":
-            env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = target_triple
-        elif machine == "x86_64":
-            env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
-            env["TARGET_TRIPLE"] = (
-                target_triple.replace("x86_64_v2-", "x86_64-")
-                .replace("x86_64_v3-", "x86_64-")
-                .replace("x86_64_v4-", "x86_64-")
-            )
+        # On macOS, we support building Linux in a virtualized container that
+        # always matches the target platform. Set build/host triple to whatever
+        # we're building.
+        #
+        # Note: we always use the *-gnu triple otherwise autoconf can have
+        # trouble reasoning about cross-compiling since its detected triple from
+        # our build environment is always GNU based.
+        if current_host_platform().startswith("macos_"):
+            env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu")
         else:
-            raise Exception("unhandled Linux machine value: %s" % machine)
+            # Otherwise assume the container environment matches the machine
+            # type of the current process.
+            host_machine = platform.machine()
+            if host_machine == "aarch64" or host_machine == "arm64":
+                env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu"
+            elif host_machine == "x86_64":
+                env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
+            else:
+                raise Exception("unhandled Linux machine value: %s" % host_machine)
 
         # This will make x86_64_v2, etc count as cross-compiling. This is
         # semantically correct, since the current machine may not support
@@ -960,16 +978,6 @@ def main():
     DOWNLOADS_PATH.mkdir(exist_ok=True)
     (BUILD / "logs").mkdir(exist_ok=True)
 
-    if os.environ.get("PYBUILD_NO_DOCKER"):
-        client = None
-    else:
-        try:
-            client = docker.from_env(timeout=600)
-            client.ping()
-        except Exception as e:
-            print("unable to connect to Docker: %s" % e, file=sys.stderr)
-            return 1
-
     # Note these arguments must be synced with `build-main.py`
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -1031,6 +1039,17 @@ def main():
 
     settings = get_target_settings(TARGETS_CONFIG, target_triple)
 
+    if os.environ.get("PYBUILD_NO_DOCKER"):
+        client = None
+    else:
+        try:
+            client = docker.from_env(timeout=600)
+            client.ping()
+            client._pbs_platform = docker_platform_from_host_platform(host_platform)
+        except Exception as e:
+            print("unable to connect to Docker: %s" % e, file=sys.stderr)
+            return 1
+
     if args.action == "dockerfiles":
         log_name = "dockerfiles"
     elif args.action == "makefiles":

cpython-unix/targets.yml

@@ -556,6 +556,7 @@ x86_64-apple-darwin:
 x86_64-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -605,6 +606,7 @@ x86_64-unknown-linux-gnu:
 x86_64_v2-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -655,6 +657,7 @@ x86_64_v2-unknown-linux-gnu:
 x86_64_v3-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -705,6 +708,9 @@ x86_64_v3-unknown-linux-gnu:
 x86_64_v4-unknown-linux-gnu:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -755,6 +761,9 @@ x86_64_v4-unknown-linux-gnu:
 x86_64-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries
+    # under Rosetta. But they can build correctly.
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -802,6 +811,7 @@ x86_64-unknown-linux-musl:
 x86_64_v2-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -850,6 +860,7 @@ x86_64_v2-unknown-linux-musl:
 x86_64_v3-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'
@@ -898,6 +909,7 @@ x86_64_v3-unknown-linux-musl:
 x86_64_v4-unknown-linux-musl:
   host_platforms:
     - linux_x86_64
+    - macos_arm64
   pythons_supported:
     - '3.10'
     - '3.11'

pythonbuild/buildenv.py

@@ -266,7 +266,10 @@ def find_output_files(self, base_path, pattern):
 def build_environment(client, image):
     if client is not None:
         container = client.containers.run(
-            image, command=["/bin/sleep", "86400"], detach=True
+            image,
+            command=["/bin/sleep", "86400"],
+            detach=True,
+            platform=client._pbs_platform,
         )
         td = None
         context = ContainerContext(container)

pythonbuild/docker.py

@@ -2,12 +2,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-import contextlib
 import io
 import operator
 import os
 import pathlib
+import sys
 import tarfile
+import typing
 
 import docker  # type: ignore
 import jinja2
@@ -29,16 +30,44 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path):
         write_if_different(dest_dir / f, data.encode("utf-8"))
 
 
+def docker_platform_from_host_platform(host_platform: str) -> typing.Optional[str]:
+    """Convert a PBS host platform to a Docker platform.
+
+    This function assumes we're using a Docker daemon on the local machine.
+    """
+    # On macOS, we only use Docker to build Linux distros. We also support
+    # using Rosetta for x86-64 emulation when building from aarch64. So it
+    # is allowed to specify a non-native platform here.
+    if sys.platform == "darwin":
+        if host_platform == "linux_x86_64":
+            return "linux/amd64"
+        elif host_platform == "linux_aarch64":
+            return "linux/arm64"
+        else:
+            return None
+    else:
+        # On Linux, we currently only support running native architecture
+        # containers. Cross-compiling is achieved by running a native arch
+        # container and cross-compiling within it. We don't specify a platform
+        # here and let the Docker runtime use the defaults.
+        return None
+
+
 def build_docker_image(
     client, image_data: bytes, image_dir: pathlib.Path, name, host_platform
 ):
     image_path = image_dir / f"image-{name}.{host_platform}"
 
-    return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path)
+    return ensure_docker_image(
+        client,
+        io.BytesIO(image_data),
+        image_path=image_path,
+        platform=docker_platform_from_host_platform(host_platform),
+    )
 
 
-def ensure_docker_image(client, fh, image_path=None):
-    res = client.api.build(fileobj=fh, decode=True)
+def ensure_docker_image(client, fh, image_path=None, platform=None):
+    res = client.api.build(fileobj=fh, decode=True, platform=platform)
 
     image = None
 
@@ -111,18 +140,6 @@ def copy_file_to_container(path, container, container_path, archive_path=None):
     container.put_archive(container_path, buf.getvalue())
 
 
-@contextlib.contextmanager
-def run_container(client, image):
-    container = client.containers.run(
-        image, command=["/bin/sleep", "86400"], detach=True
-    )
-    try:
-        yield container
-    finally:
-        container.stop(timeout=0)
-        container.remove()
-
-
 def container_exec(container, command, user="build", environment=None):
     # docker-py's exec_run() won't return the exit code. So we reinvent the
     # wheel.