diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 0e1f6a7cf..e4d9fe9a2 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -142,7 +142,14 @@ def main(): effective_host_platform = host_platform if building_linux_from_macos: if host_platform == "macos_arm64": - effective_host_platform = "linux_aarch64" + if target_triple.startswith("aarch64"): + effective_host_platform = "linux_aarch64" + elif target_triple.startswith("x86_64"): + effective_host_platform = "linux_x86_64" + else: + raise Exception( + f"unsupported macOS cross-compile: {host_platform} -> {target_triple}" + ) else: raise Exception(f"Unhandled macOS platform: {host_platform}") print( diff --git a/cpython-unix/build.py b/cpython-unix/build.py index eb0e6ae86..c028ae0bd 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -25,7 +25,12 @@ meets_python_minimum_version, parse_setup_line, ) -from pythonbuild.docker import build_docker_image, get_image, write_dockerfiles +from pythonbuild.docker import ( + build_docker_image, + docker_platform_from_host_platform, + get_image, + write_dockerfiles, +) from pythonbuild.downloads import DOWNLOADS from pythonbuild.logging import log, set_logger from pythonbuild.utils import ( @@ -33,6 +38,7 @@ add_licenses_to_extension_entry, clang_toolchain, create_tar_from_directory, + current_host_platform, download_entry, get_target_settings, get_targets, @@ -98,21 +104,33 @@ def add_target_env(env, build_platform, target_triple, build_env, build_options) extra_target_ldflags.append("--rtlib=compiler-rt") if build_platform.startswith("linux_"): - machine = platform.machine() + # autoconf is not aware of microarch triples. Normalize those out: + # we force targeting via -march CFLAG. + env["TARGET_TRIPLE"] = ( + target_triple.replace("x86_64_v2-", "x86_64-") + .replace("x86_64_v3-", "x86_64-") + .replace("x86_64_v4-", "x86_64-") + ) - # arm64 allows building for Linux on a macOS host using Docker - if machine == "aarch64" or machine == "arm64": - env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" - env["TARGET_TRIPLE"] = target_triple - elif machine == "x86_64": - env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" - env["TARGET_TRIPLE"] = ( - target_triple.replace("x86_64_v2-", "x86_64-") - .replace("x86_64_v3-", "x86_64-") - .replace("x86_64_v4-", "x86_64-") - ) + # On macOS, we support building Linux in a virtualized container that + # always matches the target platform. Set build/host triple to whatever + # we're building. + # + # Note: we always use the *-gnu triple otherwise autoconf can have + # trouble reasoning about cross-compiling since its detected triple from + # our build environment is always GNU based. + if current_host_platform().startswith("macos_"): + env["BUILD_TRIPLE"] = env["TARGET_TRIPLE"].replace("-musl", "-gnu") else: - raise Exception("unhandled Linux machine value: %s" % machine) + # Otherwise assume the container environment matches the machine + # type of the current process. + host_machine = platform.machine() + if host_machine == "aarch64" or host_machine == "arm64": + env["BUILD_TRIPLE"] = "aarch64-unknown-linux-gnu" + elif host_machine == "x86_64": + env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu" + else: + raise Exception("unhandled Linux machine value: %s" % host_machine) # This will make x86_64_v2, etc count as cross-compiling. This is # semantically correct, since the current machine may not support @@ -960,16 +978,6 @@ def main(): DOWNLOADS_PATH.mkdir(exist_ok=True) (BUILD / "logs").mkdir(exist_ok=True) - if os.environ.get("PYBUILD_NO_DOCKER"): - client = None - else: - try: - client = docker.from_env(timeout=600) - client.ping() - except Exception as e: - print("unable to connect to Docker: %s" % e, file=sys.stderr) - return 1 - # Note these arguments must be synced with `build-main.py` parser = argparse.ArgumentParser() parser.add_argument( @@ -1031,6 +1039,17 @@ def main(): settings = get_target_settings(TARGETS_CONFIG, target_triple) + if os.environ.get("PYBUILD_NO_DOCKER"): + client = None + else: + try: + client = docker.from_env(timeout=600) + client.ping() + client._pbs_platform = docker_platform_from_host_platform(host_platform) + except Exception as e: + print("unable to connect to Docker: %s" % e, file=sys.stderr) + return 1 + if args.action == "dockerfiles": log_name = "dockerfiles" elif args.action == "makefiles": diff --git a/cpython-unix/targets.yml b/cpython-unix/targets.yml index 36c59c2c3..432833d69 100644 --- a/cpython-unix/targets.yml +++ b/cpython-unix/targets.yml @@ -556,6 +556,7 @@ x86_64-apple-darwin: x86_64-unknown-linux-gnu: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -605,6 +606,7 @@ x86_64-unknown-linux-gnu: x86_64_v2-unknown-linux-gnu: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -655,6 +657,7 @@ x86_64_v2-unknown-linux-gnu: x86_64_v3-unknown-linux-gnu: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -705,6 +708,9 @@ x86_64_v3-unknown-linux-gnu: x86_64_v4-unknown-linux-gnu: host_platforms: - linux_x86_64 + # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries + # under Rosetta. But they can build correctly. + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -755,6 +761,9 @@ x86_64_v4-unknown-linux-gnu: x86_64-unknown-linux-musl: host_platforms: - linux_x86_64 + # Rosetta doesn't support AVX-512. So we cannot run x86-64-v4 binaries + # under Rosetta. But they can build correctly. + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -802,6 +811,7 @@ x86_64-unknown-linux-musl: x86_64_v2-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -850,6 +860,7 @@ x86_64_v2-unknown-linux-musl: x86_64_v3-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' @@ -898,6 +909,7 @@ x86_64_v3-unknown-linux-musl: x86_64_v4-unknown-linux-musl: host_platforms: - linux_x86_64 + - macos_arm64 pythons_supported: - '3.10' - '3.11' diff --git a/pythonbuild/buildenv.py b/pythonbuild/buildenv.py index a946813ed..37019ca26 100644 --- a/pythonbuild/buildenv.py +++ b/pythonbuild/buildenv.py @@ -266,7 +266,10 @@ def find_output_files(self, base_path, pattern): def build_environment(client, image): if client is not None: container = client.containers.run( - image, command=["/bin/sleep", "86400"], detach=True + image, + command=["/bin/sleep", "86400"], + detach=True, + platform=client._pbs_platform, ) td = None context = ContainerContext(container) diff --git a/pythonbuild/docker.py b/pythonbuild/docker.py index 0be78e4f8..1cbae354c 100644 --- a/pythonbuild/docker.py +++ b/pythonbuild/docker.py @@ -2,12 +2,13 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -import contextlib import io import operator import os import pathlib +import sys import tarfile +import typing import docker # type: ignore import jinja2 @@ -29,16 +30,44 @@ def write_dockerfiles(source_dir: pathlib.Path, dest_dir: pathlib.Path): write_if_different(dest_dir / f, data.encode("utf-8")) +def docker_platform_from_host_platform(host_platform: str) -> typing.Optional[str]: + """Convert a PBS host platform to a Docker platform. + + This function assumes we're using a Docker daemon on the local machine. + """ + # On macOS, we only use Docker to build Linux distros. We also support + # using Rosetta for x86-64 emulation when building from aarch64. So it + # is allowed to specify a non-native platform here. + if sys.platform == "darwin": + if host_platform == "linux_x86_64": + return "linux/amd64" + elif host_platform == "linux_aarch64": + return "linux/arm64" + else: + return None + else: + # On Linux, we currently only support running native architecture + # containers. Cross-compiling is achieved by running a native arch + # container and cross-compiling within it. We don't specify a platform + # here and let the Docker runtime use the defaults. + return None + + def build_docker_image( client, image_data: bytes, image_dir: pathlib.Path, name, host_platform ): image_path = image_dir / f"image-{name}.{host_platform}" - return ensure_docker_image(client, io.BytesIO(image_data), image_path=image_path) + return ensure_docker_image( + client, + io.BytesIO(image_data), + image_path=image_path, + platform=docker_platform_from_host_platform(host_platform), + ) -def ensure_docker_image(client, fh, image_path=None): - res = client.api.build(fileobj=fh, decode=True) +def ensure_docker_image(client, fh, image_path=None, platform=None): + res = client.api.build(fileobj=fh, decode=True, platform=platform) image = None @@ -111,18 +140,6 @@ def copy_file_to_container(path, container, container_path, archive_path=None): container.put_archive(container_path, buf.getvalue()) -@contextlib.contextmanager -def run_container(client, image): - container = client.containers.run( - image, command=["/bin/sleep", "86400"], detach=True - ) - try: - yield container - finally: - container.stop(timeout=0) - container.remove() - - def container_exec(container, command, user="build", environment=None): # docker-py's exec_run() won't return the exit code. So we reinvent the # wheel.