Build python with bazel on unix as well

bazel/rules/replace_prefix.sh

@@ -28,29 +28,13 @@ for f in "$@"; do
         echo "$f: file not found"
         exit 2
     fi
-    case $f in
-        *.so)
-            ${PATCHELF} --set-rpath "$PREFIX"/lib "$f"
-            ;;
-        *.dylib)
-            install_name_tool -add_rpath "$PREFIX/lib" "$f" 2>/dev/null || true
-            # Get the old install name/ID
-            dylib_name=$(basename "$f")
-            new_id="$PREFIX/lib/$dylib_name"
-
-            # Change the dylib's own ID
-            install_name_tool -id "$new_id" "$f"
+    # We don't want to process symlinks but rather the actual file it's pointing to
+    # Otherwise `file $f` would return that it's a symlink, not an elf/mach-o file
+    if [ -L "$f" ]; then
+        f=$(realpath "$f")
+    fi
 
-            # Update all dependency paths that point to sandbox locations
-            otool -L "$f" | tail -n +2 | awk '{print $1}' | while read -r dep; do
-                if [[ "$dep" == *"sandbox"* ]] || [[ "$dep" == *"bazel-out"* ]]; then
-                    dep_name=$(basename "$dep")
-                    new_dep="$PREFIX/lib/$dep_name"
-                    install_name_tool -change "$dep" "$new_dep" "$f" 2>/dev/null || true
-                    install_name_tool -add_rpath "$PREFIX/lib" "$dep" 2>/dev/null || true
-                fi
-            done
-            ;;
+    case $f in
         *.pc)
             sed -ibak -e "s|^prefix=.*|prefix=$PREFIX|" -e "s|##PREFIX##|$PREFIX|" -e "s|\${EXT_BUILD_DEPS}|$PREFIX|" "$f" && rm -f "${f}bak"
             ;;
@@ -60,12 +44,20 @@ for f in "$@"; do
             elif file "$f" | grep -q "Mach-O"; then
                 # Handle macOS binaries (executables and other Mach-O files)
                 install_name_tool -add_rpath "$PREFIX/lib" "$f" 2>/dev/null || true
+                # Get the old install name/ID
+                dylib_name=$(basename "$f")
+                new_id="$PREFIX/lib/$dylib_name"
+
+                # Change the dylib's own ID
+                install_name_tool -id "$new_id" "$f"
+
                 # Update all dependency paths that point to sandbox locations
                 otool -L "$f" | tail -n +2 | awk '{print $1}' | while read -r dep; do
                     if [[ "$dep" == *"sandbox"* ]] || [[ "$dep" == *"bazel-out"* ]]; then
                         dep_name=$(basename "$dep")
                         new_dep="$PREFIX/lib/$dep_name"
                         install_name_tool -change "$dep" "$new_dep" "$f" 2>/dev/null || true
+                        install_name_tool -add_rpath "$PREFIX/lib" "$dep" 2>/dev/null || true
                     fi
                 done
             else

deps/cpython.BUILD.bazel

@@ -1,7 +1,10 @@
 load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory")
 load("@bazel_lib//lib:run_binary.bzl", "run_binary")
 load("@rules_pkg//pkg:install.bzl", "pkg_install")
-load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files")
+load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files", "pkg_mklink")
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make")
+load("@rules_pkg//pkg:mappings.bzl", "strip_prefix")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes")
 
 # Keep in sync with the ones on repos.MODULE.bazel
 python_externals = {
@@ -14,6 +17,8 @@ python_externals = {
     "tcltk": "8.6.15.0",
 }
 
+VERSION_STR="3.13"
+
 # These rules will make it easier to get a reference to their folder via $(location)
 # and they add the version in the folder name (as some of the vcxproj stuff relies on that)
 [
@@ -86,22 +91,238 @@ run_binary(
     visibility = ["//visibility:public"],
 )
 
-pkg_files(
-    name = "install_files",
-    srcs = select({
-        "@platforms//os:windows": [":python_win"],
+filegroup(
+    name = "all",
+    srcs = glob(["**"], exclude = ["BUILD.bazel"]),
+)
+
+UNIX_BINS = [
+    "pip3",
+    "python{}".format(VERSION_STR),
+]
+
+python_deps = {
+    'libffi': '-lffi',
+    'libsqlite3': '-lsqlite3',
+    'zlib': '-lz',
+    'bzip2': '-lbz2',
+    'liblzma': '-llzma'
+}
+
+# The list of build tools we want to override in sysconfigdata.py
+# as they will not be available when building custom integrations without bazel
+to_override_build_tools = [
+    'ar',
+    'gcc',
+    'g++',
+    'ld',
+]
+
+to_override_flags = [
+    'CFLAGS',
+    'CXXFLAGS',
+    'LDFLAGS',
+]
+
+configure_make(
+    name = "python_unix",
+    configure_options = [
+        "--enable-ipv6",
+        "--with-ensurepip=yes",
+        "--enable-shared",
+        "--without-static-libpython",
+        "--with-dbmliborder=",
+        # Fixes an issue with __DATE__ being set to undefined `redacted`
+        # https://github.com/bazelbuild/rules_foreign_cc/issues/239#issuecomment-478167267
+        "CPPFLAGS='-Dredacted=\\\"redacted\\\"'",
+        "--with-openssl=$$EXT_BUILD_DEPS/openssl",
+        "--with-openssl-rpath=yes",
+    ] + select({
+        "@@//:macos_arm64": ["--with-universal-archs=universal2"],
+        "@@//:macos_x86_64": ["--with-universal-archs=intel"],
+        "//conditions:default": [],
     }),
+    copts = [
+        "-O2",
+    ],
+    env = {
+        "OPT": "-DNDEBUG -fwrapv",
+        # Ensure we don't use the system provided .pc
+        "PKG_CONFIG_LIBDIR": "/does/not/exist",
+    } | {
+        dep.upper() + '_CFLAGS': "-I$$EXT_BUILD_DEPS/include" for dep in python_deps.keys()
+    } | {
+        dep.upper() + '_LIBS': lib for dep, lib in python_deps.items()
+    } | select({
+            "@platforms//os:macos": {
+                # https://github.com/bazelbuild/bazel/issues/5127
+                "AR": "/usr/bin/ar",
+            },
+            "//conditions:default": {},
+    }),
+    lib_source = ":all",
+    # The single dollar sign here isn't a typo, using 2 seems to confuse rules_foreign_cc's substitution
+    # This is meant to allow python to find its dependency during its modules import test.
+    # We will use the install_dir rpath later on
+    linkopts = ["-Wl,-rpath", "$EXT_BUILD_DEPS/lib"],
+    out_binaries = UNIX_BINS,
+    out_data_dirs = [
+        "lib",
+    ],
+    out_include_dir = "include",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@bzip2//:libbz2",
+        "@libffi//:libffi",
+        "@openssl//:openssl",
+        "@sqlite3//:libsqlite3",
+        "@xz//:liblzma",
+        "@zlib//:zlib",
+    ],
+    dynamic_deps = [
+        "@bzip2//:bz2",
+        "@libffi//:ffi",
+        "@sqlite3//:sqlite3",
+        "@xz//:lzma",
+        "@zlib//:z",
+    ],
+    targets = [
+        # Build in parallel but install without parallel execution
+        # (see https://github.com/python/cpython/issues/109796)
+        "-j 16",
+        "install"
+    ],
+    target_compatible_with = select({
+        "@platforms//os:macos": [],
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    # python's build system will output the entire build config to _sysconfigdata_xxx.py
+    # This is later used to build extensions with the same tools/compiler/flags/config as the interpreter
+    # However, in our case, that means using tools that are stored in the build sandbox, so they aren't
+    # usable when building an extension, causing all builds to fail.
+    # Ideally we would want to explicitly replace bazel paths with known alternatives, but we don't
+    # have an environment variable holding the value we want to replace so we have to resort to
+    # a regular expression replacing paths to tools.
+    # We also unset the flags listed in to_override_flags.
+    # If we start using some specific build flags that need to be propagated, we will need to include
+    # them here instead of replacing them by an empty string.
+    postfix_script = " && ".join([
+        "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format(
+            tool=tool,
+            version=VERSION_STR
+        ) for tool in to_override_build_tools
+    ]) + " && " +
+    " && ".join(["perl -i -pe \"s/\'{flag}\': \'.*\',$$/\'{flag}\': \'\',/g\" $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format(
+            flag=flag,
+            version=VERSION_STR
+        ) for flag in to_override_flags
+    ])
+)
+
+pkg_files(
+    name = "install_files_win",
+    srcs = [":python_win"],
     renames = {
         "python_win": REMOVE_BASE_DIRECTORY,
     },
 )
 
+filegroup(
+    name = "libs_unix",
+    srcs = [":python_unix"],
+    output_group = "lib",
+)
+
+# Fix symlinks for libpython3.x shared libraries
+# rules_foreign_cc dereferences symlinks during installation since we're using
+# out_data_dir instead of out_shared_libs, so we need to recreate them
+# For context, we use out_data_dir to copy the entire list of python modules which
+# too long to explicitly list in out_shared_libs, and we can't only copy the
+# lib/python3.X folder as it conflicts with the python3.X executables (rules_foreign_cc
+# output groups are named based on the basename)
+
+# Filter out the dereferenced symlinks - we'll recreate them as proper symlinks
+copy_to_directory(
+    name = "libs_unix_no_symlinks",
+    srcs = [":libs_unix"],
+    # We want to include libpython3.so & libpython3.13.so.1.0, but
+    # exclude libpython3.13.so
+    exclude_srcs_patterns = [
+        "**/libpython3.*.so",
+        "**/python{}/test/**/*".format(VERSION_STR),
+        "**/*.exe",
+        "**/Makefile",
+    ],
+    include_external_repositories = ["*"],
+    root_paths = ["python_unix"],
+)
+
+# Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install)
+pkg_mklink(
+    name = "libpython_symlink",
+    link_name = "lib/libpython{}.so".format(VERSION_STR),
+    target = "libpython{}.so.1.0".format(VERSION_STR),
+    attributes = pkg_attributes("0644")
+)
+
+pkg_mklink(
+    name = "python_bin_symlink",
+    link_name = "bin/python3",
+    target = "python{}".format(VERSION_STR)
+)
+
+filegroup(
+    name = "headers_unix",
+    srcs = [":python_unix"],
+    output_group = "include",
+)
+
+[
+    filegroup(
+        name = "bins_unix_" + bin,
+        srcs = [":python_unix"],
+        output_group = bin,
+    )
+    for bin in UNIX_BINS
+]
+
+pkg_files(
+    name = "install_libs_unix",
+    srcs = [":libs_unix_no_symlinks"],
+    renames = {
+        "libs_unix_no_symlinks": REMOVE_BASE_DIRECTORY,
+    },
+    attributes = pkg_attributes("0644")
+)
+
+pkg_files(
+    name = "install_headers_unix",
+    srcs = [":headers_unix"],
+)
+
+pkg_files(
+    name = "install_bins_unix",
+    srcs = [":bins_unix_" + bin for bin in UNIX_BINS] + [":python_bin_symlink"],
+    prefix = "bin",
+    attributes = pkg_attributes("0755")
+)
+
 pkg_install(
     name = "install",
-    srcs = [":install_files"] + select({
+    srcs = select({
         "@platforms//os:windows": [
             "@openssl//:openssl_exe_file",
+            ":install_files_win"
+        ],
+        "//conditions:default": [
+            ":install_libs_unix",
+            ":install_headers_unix",
+            ":install_bins_unix",
+            ":python_bin_symlink",
         ],
+    }) + select({
+        "@platforms//os:linux": [":libpython_symlink"],
         "//conditions:default": [],
     }),
 )

deps/openssl.BUILD.bazel

@@ -169,7 +169,9 @@ configure_make(
                 done
             done
             LIBS="$$INSTALLDIR/lib/libcrypto.dylib $$INSTALLDIR/lib/libssl.dylib"
-        """ + FIX_OPENSSL_PATHS,
+        """ + FIX_OPENSSL_PATHS + """
+        codesign -s - -f $$INSTALLDIR/lib/libcrypto.dylib $$INSTALLDIR/lib/libssl.dylib
+        """,
         "//conditions:default": """
             LIBS="$$INSTALLDIR/lib/libcrypto.so $$INSTALLDIR/lib/libssl.so"
         """ + FIX_OPENSSL_PATHS,

omnibus/config/software/python3.rb

@@ -8,7 +8,6 @@
   dependency "bzip2"
   dependency "libsqlite3"
   dependency "liblzma"
-  dependency "libyaml"
 end
 dependency "openssl3"
 
@@ -23,49 +22,13 @@
 
   if !windows_target?
     env = with_standard_compiler_flags(with_embedded_path)
-    python_configure_options = [
-      "--without-readline",  # Disables readline support
-      "--with-ensurepip=yes", # We upgrade pip later, in the pip3 software definition
-      "--without-static-libpython" # We only care about the shared library
-    ]
-
-    if mac_os_x?
-      python_configure_options.push("--enable-ipv6",
-                            "--with-universal-archs=#{arm_target? ? "universal2" : "intel"}",
-                            "--enable-shared")
-    elsif linux_target?
-      python_configure_options.push("--enable-shared",
-                            "--enable-ipv6")
-    elsif aix?
-      # something here...
-    end
-
-    python_configure_options.push("--with-dbmliborder=")
-
-    # Force different defaults for the "optimization settings"
-    # This removes the debug symbol generation and doesn't enable all warnings
-    env["OPT"] = "-DNDEBUG -fwrapv"
-    configure(*python_configure_options, :env => env)
-    command "make -j #{workers}", :env => env
-    command "make install", :env => env
-
-    # There exists no configure flag to tell Python to not compile readline support :(
-    major, minor, bugfix = version.split(".")
-
-    # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default
-    # compiler we use would end up being available in the system/docker image used by customers
-    if linux_target? && env["CC"]
-      command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-#{major}.#{minor}-*-linux-gnu/Makefile", :env => env
-      command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env
-    end
-    if linux_target? && env["CXX"]
-      command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-#{major}.#{minor}-*-linux-gnu/Makefile", :env => env
-      command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env
-    end
-    delete "#{install_dir}/embedded/lib/python#{major}.#{minor}/test"
-    block do
-      FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python#{major}.#{minor}/distutils/command/wininst-*.exe"))
-    end
+    command_on_repo_root "bazelisk run -- @cpython//:install --destdir='#{install_dir}/embedded'"
+    sh_lib = if linux_target? then "libpython3.so" else "libpython3.13.dylib" end
+    command_on_repo_root "bazelisk run -- //bazel/rules:replace_prefix --prefix '#{install_dir}/embedded'" \
+      " #{install_dir}/embedded/lib/pkgconfig/python*.pc" \
+      " #{install_dir}/embedded/lib/#{sh_lib}" \
+      " #{install_dir}/embedded/lib/python3.13/lib-dynload/*.so" \
+      " #{install_dir}/embedded/bin/python3*"
   elsif fips_mode?
     ###############################
     # Setup openssl dependency... #