From 7c4806a44b3665828449240677a6ce1f1294b700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 27 Nov 2025 16:41:11 +0100 Subject: [PATCH 01/29] deps: build python with bazel (through rules_foreign_cc) --- deps/cpython.BUILD.bazel | 143 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 138 insertions(+), 5 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index c68286487e1c..5f78fcc39f7c 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -2,6 +2,9 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_lib//lib:run_binary.bzl", "run_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files") +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@rules_pkg//pkg:mappings.bzl", "strip_prefix") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes") # Keep in sync with the ones on repos.MODULE.bazel python_externals = { @@ -86,22 +89,152 @@ run_binary( visibility = ["//visibility:public"], ) -pkg_files( - name = "install_files", - srcs = select({ - "@platforms//os:windows": [":python_win"], +filegroup( + name = "all", + srcs = glob(["**"], exclude = ["BUILD.bazel"]), +) + +UNIX_BINS = [ + "python3", + "pip3", + "python3.13" +] + +configure_make( + name = "python_unix", + configure_options = [ + "--enable-ipv6", + "--with-ensurepip=yes", + "--enable-shared", + "--without-static-libpython", + "--with-dbmliborder=", + # Fixes an issue with __DATE__ being set to undefined `redacted` + # https://github.com/bazelbuild/rules_foreign_cc/issues/239#issuecomment-478167267 + "CPPFLAGS='-Dredacted=\\\"redacted\\\"'", + "--with-openssl=$$EXT_BUILD_DEPS/openssl", + "--with-openssl-rpath=yes", + "--enable-optimizations", + ] + select({ + "@@//:macos_arm64": ["--with-universal-archs=universal2"], + "@@//:macos_x86_64": ["--with-universal-archs=intel"], + "//conditions:default": [], + }), + copts = [ + "-O3", + ], + env = { + "OPT": "-DNDEBUG -fwrapv", + # manually provide some dependencies cflags/ldflags instead of + # relying on pkg-config + "ZLIB_CFLAGS": "-I$$EXT_BUILD_DEPS/include", + "ZLIB_LIBS": "-lz", + "BZIP2_CFLAGS": "-I$$EXT_BUILD_DEPS/include", + "BZIP2_LIBS": "-lbz2", + # Ensure we don't use the system provided .pc + "PKG_CONFIG_LIBDIR": "/does/not/exist", + } | select({ + "@platforms//os:macos": { + # https://github.com/bazelbuild/bazel/issues/5127 + "AR": "ar", + }, + "//conditions:default": {}, }), + lib_source = ":all", + # The single dollar sign here isn't a typo, using 2 seems to confuse rules_foreign_cc's substitution + # This is meant to allow python to find its dependency during its modules import test. + # We will use the install_dir rpath later on + linkopts = ["-Wl,-rpath", "$EXT_BUILD_DEPS/lib"], + out_binaries = UNIX_BINS, + out_data_dirs = [ + "lib", + ], + out_include_dir = "include", + visibility = ["//visibility:public"], + deps = [ + "@libffi//:ffi", + "@openssl//:openssl", + # "@libyaml", + # "@sqlite3", + "@zlib//:zlib", + "@bzip2//:libbz2", + "@xz//:liblzma", + ], + dynamic_deps = [ + "@zlib//:z", + "@bzip2//:bz2", + ], + targets = [ + # Build in parallel but install without parallel execution + # (see https://github.com/python/cpython/issues/109796) + "-j 16", + "install" + ], +) + +pkg_files( + name = "install_files_win", + srcs = [":python_win"], renames = { "python_win": REMOVE_BASE_DIRECTORY, }, ) +filegroup( + name = "libs_unix", + srcs = [":python_unix"], + output_group = "lib", +) + +filegroup( + name = "headers_unix", + srcs = [":python_unix"], + output_group = "include", +) + +[ + filegroup( + name = "bins_unix_" + bin, + srcs = [":python_unix"], + output_group = bin, + ) + for bin in UNIX_BINS +] + +pkg_files( + name = "install_libs_unix", + srcs = [":libs_unix"], +) + +pkg_files( + name = "install_headers_unix", + srcs = [":headers_unix"], +) + +pkg_files( + name = "install_bins_unix", + srcs = [":bins_unix_" + bin for bin in UNIX_BINS], + prefix = "bin", + attributes = pkg_attributes("0755") +) + +pkg_files( + name = "install_files", + srcs = select({ + "@platforms//os:windows": [":python_win"], + "//conditions:default": [] + }) +) + pkg_install( name = "install", srcs = [":install_files"] + select({ "@platforms//os:windows": [ "@openssl//:openssl_exe_file", ], - "//conditions:default": [], + "//conditions:default": [ + ":install_libs_unix", + ":install_headers_unix", + ":install_bins_unix", + ], }), ) From b03ab75b4995ebb21f994e2f65f2b804f6f540ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 28 Nov 2025 09:10:03 +0100 Subject: [PATCH 02/29] omnibus: build python with bazel --- omnibus/config/software/python3.rb | 32 +++++++----------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 322fde0d0592..6eebe06679ca 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -23,31 +23,13 @@ if !windows_target? env = with_standard_compiler_flags(with_embedded_path) - python_configure_options = [ - "--without-readline", # Disables readline support - "--with-ensurepip=yes", # We upgrade pip later, in the pip3 software definition - "--without-static-libpython" # We only care about the shared library - ] - - if mac_os_x? - python_configure_options.push("--enable-ipv6", - "--with-universal-archs=#{arm_target? ? "universal2" : "intel"}", - "--enable-shared") - elsif linux_target? - python_configure_options.push("--enable-shared", - "--enable-ipv6") - elsif aix? - # something here... - end - - python_configure_options.push("--with-dbmliborder=") - - # Force different defaults for the "optimization settings" - # This removes the debug symbol generation and doesn't enable all warnings - env["OPT"] = "-DNDEBUG -fwrapv" - configure(*python_configure_options, :env => env) - command "make -j #{workers}", :env => env - command "make install", :env => env + command_on_repo_root "bazelisk run -- @cpython//:install --destdir='#{install_dir}/embedded'" + sh_lib = if linux_target? then "libpython3.so" else "libpython3.13.dylib" end + command_on_repo_root "bazelisk run -- //bazel/rules:replace_prefix --prefix '#{install_dir}/embedded'" \ + " #{install_dir}/embedded/lib/pkgconfig/python*.pc" \ + " #{install_dir}/embedded/lib/#{sh_lib}" \ + " #{install_dir}/embedded/lib/python3.13/lib-dynload/*.so" \ + " #{install_dir}/embedded/bin/python3*" # There exists no configure flag to tell Python to not compile readline support :( major, minor, bugfix = version.split(".") From 7844d4ab4768c005c4f8591413ba3338a5ce7382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 8 Dec 2025 12:21:48 +0100 Subject: [PATCH 03/29] provide libyaml dependency --- deps/cpython.BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 5f78fcc39f7c..1e16258b07ad 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -153,7 +153,7 @@ configure_make( deps = [ "@libffi//:ffi", "@openssl//:openssl", - # "@libyaml", + "@libyaml//:libyaml", # "@sqlite3", "@zlib//:zlib", "@bzip2//:libbz2", @@ -162,6 +162,7 @@ configure_make( dynamic_deps = [ "@zlib//:z", "@bzip2//:bz2", + "@libyaml//:yaml", ], targets = [ # Build in parallel but install without parallel execution From 12dcf2e524d4f5da1b19394f7f9029afbdb9b0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 9 Dec 2025 08:33:57 +0100 Subject: [PATCH 04/29] add sqlite dep --- deps/cpython.BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 1e16258b07ad..268fe9cebb64 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -154,7 +154,7 @@ configure_make( "@libffi//:ffi", "@openssl//:openssl", "@libyaml//:libyaml", - # "@sqlite3", + "@sqlite3//:libsqlite3", "@zlib//:zlib", "@bzip2//:libbz2", "@xz//:liblzma", @@ -163,6 +163,7 @@ configure_make( "@zlib//:z", "@bzip2//:bz2", "@libyaml//:yaml", + "@sqlite3//:sqlite3", ], targets = [ # Build in parallel but install without parallel execution From 5a2a11b4b9ecb405992f6a3192735d3f7a277721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 9 Jan 2026 08:50:07 +0100 Subject: [PATCH 05/29] fix ffi dependency --- deps/cpython.BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 268fe9cebb64..df4e01e6e3aa 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -151,7 +151,7 @@ configure_make( out_include_dir = "include", visibility = ["//visibility:public"], deps = [ - "@libffi//:ffi", + "@libffi//:libffi", "@openssl//:openssl", "@libyaml//:libyaml", "@sqlite3//:libsqlite3", @@ -164,6 +164,7 @@ configure_make( "@bzip2//:bz2", "@libyaml//:yaml", "@sqlite3//:sqlite3", + "@libffi//:ffi", ], targets = [ # Build in parallel but install without parallel execution From 52ecf4cf6424307334f31df50607029c2b9cdcf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 9 Jan 2026 08:51:02 +0100 Subject: [PATCH 06/29] don't statically link with liblzma --- deps/cpython.BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index df4e01e6e3aa..543fd50c56e4 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -165,6 +165,7 @@ configure_make( "@libyaml//:yaml", "@sqlite3//:sqlite3", "@libffi//:ffi", + "@xz//:lzma", ], targets = [ # Build in parallel but install without parallel execution From 2c6087341a76fec1dae576fd3b783824df35b46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 9 Jan 2026 09:55:06 +0100 Subject: [PATCH 07/29] replace_prefix: don't trust shared libraries extensions --- bazel/rules/replace_prefix.sh | 36 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/bazel/rules/replace_prefix.sh b/bazel/rules/replace_prefix.sh index e6625ee330c2..5c456c227973 100755 --- a/bazel/rules/replace_prefix.sh +++ b/bazel/rules/replace_prefix.sh @@ -23,34 +23,18 @@ if [ -z "$PREFIX" ]; then exit 1 fi +# We don't want to process symlinks but rather the actual file it's pointing to +# Otherwise `file $f` would return that it's a symlink, not an elf/mach-o file +if [ -L "$f" ]; then + f=$(realpath "$f") +fi + for f in "$@"; do if [ ! -f "$f" ]; then echo "$f: file not found" exit 2 fi case $f in - *.so) - ${PATCHELF} --set-rpath "$PREFIX"/lib "$f" - ;; - *.dylib) - install_name_tool -add_rpath "$PREFIX/lib" "$f" 2>/dev/null || true - # Get the old install name/ID - dylib_name=$(basename "$f") - new_id="$PREFIX/lib/$dylib_name" - - # Change the dylib's own ID - install_name_tool -id "$new_id" "$f" - - # Update all dependency paths that point to sandbox locations - otool -L "$f" | tail -n +2 | awk '{print $1}' | while read -r dep; do - if [[ "$dep" == *"sandbox"* ]] || [[ "$dep" == *"bazel-out"* ]]; then - dep_name=$(basename "$dep") - new_dep="$PREFIX/lib/$dep_name" - install_name_tool -change "$dep" "$new_dep" "$f" 2>/dev/null || true - install_name_tool -add_rpath "$PREFIX/lib" "$dep" 2>/dev/null || true - fi - done - ;; *.pc) sed -ibak -e "s|^prefix=.*|prefix=$PREFIX|" -e "s|##PREFIX##|$PREFIX|" -e "s|\${EXT_BUILD_DEPS}|$PREFIX|" "$f" && rm -f "${f}bak" ;; @@ -60,12 +44,20 @@ for f in "$@"; do elif file "$f" | grep -q "Mach-O"; then # Handle macOS binaries (executables and other Mach-O files) install_name_tool -add_rpath "$PREFIX/lib" "$f" 2>/dev/null || true + # Get the old install name/ID + dylib_name=$(basename "$f") + new_id="$PREFIX/lib/$dylib_name" + + # Change the dylib's own ID + install_name_tool -id "$new_id" "$f" + # Update all dependency paths that point to sandbox locations otool -L "$f" | tail -n +2 | awk '{print $1}' | while read -r dep; do if [[ "$dep" == *"sandbox"* ]] || [[ "$dep" == *"bazel-out"* ]]; then dep_name=$(basename "$dep") new_dep="$PREFIX/lib/$dep_name" install_name_tool -change "$dep" "$new_dep" "$f" 2>/dev/null || true + install_name_tool -add_rpath "$PREFIX/lib" "$dep" 2>/dev/null || true fi done else From 23071cd5e8d119857d29a3dea4ca33c95c17ab64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 9 Jan 2026 10:42:39 +0100 Subject: [PATCH 08/29] attempt to fix windows install --- deps/cpython.BUILD.bazel | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 543fd50c56e4..fbfa6a305c4b 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -221,19 +221,12 @@ pkg_files( attributes = pkg_attributes("0755") ) -pkg_files( - name = "install_files", - srcs = select({ - "@platforms//os:windows": [":python_win"], - "//conditions:default": [] - }) -) - pkg_install( name = "install", - srcs = [":install_files"] + select({ + srcs = select({ "@platforms//os:windows": [ "@openssl//:openssl_exe_file", + ":install_files_win" ], "//conditions:default": [ ":install_libs_unix", From b3924ab5cc8caa51d2a90961429cd9b4a4ca88dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 9 Jan 2026 11:14:57 +0100 Subject: [PATCH 09/29] don't build python_unix on windows --- deps/cpython.BUILD.bazel | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index fbfa6a305c4b..3446ed6d0347 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -173,6 +173,11 @@ configure_make( "-j 16", "install" ], + target_compatible_with = select({ + "@platforms//os:osx": [], + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), ) pkg_files( From 758dfaae5d8696ab757757fbce44d71b587a32d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 9 Jan 2026 11:15:13 +0100 Subject: [PATCH 10/29] replace_prefix fixup --- bazel/rules/replace_prefix.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bazel/rules/replace_prefix.sh b/bazel/rules/replace_prefix.sh index 5c456c227973..2f82daf47c2e 100755 --- a/bazel/rules/replace_prefix.sh +++ b/bazel/rules/replace_prefix.sh @@ -23,17 +23,17 @@ if [ -z "$PREFIX" ]; then exit 1 fi -# We don't want to process symlinks but rather the actual file it's pointing to -# Otherwise `file $f` would return that it's a symlink, not an elf/mach-o file -if [ -L "$f" ]; then - f=$(realpath "$f") -fi - for f in "$@"; do if [ ! -f "$f" ]; then echo "$f: file not found" exit 2 fi + # We don't want to process symlinks but rather the actual file it's pointing to + # Otherwise `file $f` would return that it's a symlink, not an elf/mach-o file + if [ -L "$f" ]; then + f=$(realpath "$f") + fi + case $f in *.pc) sed -ibak -e "s|^prefix=.*|prefix=$PREFIX|" -e "s|##PREFIX##|$PREFIX|" -e "s|\${EXT_BUILD_DEPS}|$PREFIX|" "$f" && rm -f "${f}bak" From 698a18b4da1b78339a1e5ebb1d485cccfdee1981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 12 Jan 2026 09:34:17 +0100 Subject: [PATCH 11/29] ensure we amend the openssl signatures This was done in the omnibus recipe, but we don't use the installed file when building with bazel. This resulted in the python module linking to an unsigned shared library, causing the loading process to be instantly killed by macOS on apple silicon --- deps/openssl.BUILD.bazel | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deps/openssl.BUILD.bazel b/deps/openssl.BUILD.bazel index aab74a537a35..8b19745bd86f 100644 --- a/deps/openssl.BUILD.bazel +++ b/deps/openssl.BUILD.bazel @@ -169,7 +169,9 @@ configure_make( done done LIBS="$$INSTALLDIR/lib/libcrypto.dylib $$INSTALLDIR/lib/libssl.dylib" - """ + FIX_OPENSSL_PATHS, + """ + FIX_OPENSSL_PATHS + """ + codesign -s - -f $$INSTALLDIR/lib/libcrypto.dylib $$INSTALLDIR/lib/libssl.dylib + """, "//conditions:default": """ LIBS="$$INSTALLDIR/lib/libcrypto.so $$INSTALLDIR/lib/libssl.so" """ + FIX_OPENSSL_PATHS, From f1385cad7f0fba2ed73b2b36f583daf8a58fa4f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Desgroppes?= Date: Tue, 13 Jan 2026 13:59:03 +0100 Subject: [PATCH 12/29] fix symlink to libpython3.XX.so --- deps/cpython.BUILD.bazel | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 3446ed6d0347..ee54fd821320 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -1,7 +1,7 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_lib//lib:run_binary.bzl", "run_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") -load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files") +load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files", "pkg_mklink") load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") load("@rules_pkg//pkg:mappings.bzl", "strip_prefix") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes") @@ -194,6 +194,34 @@ filegroup( output_group = "lib", ) +# Fix symlinks for libpython3.x shared libraries +# rules_foreign_cc dereferences symlinks during installation since we're using +# out_data_dir instead of out_shared_libs, so we need to recreate them +# For context, we use out_data_dir to copy the entire list of python modules which +# too long to explicitly list in out_shared_libs, and we can't only copy the +# lib/python3.X folder as it conflicts with the python3.X executables (rules_foreign_cc +# output groups are named based on the basename) + +# Filter out the dereferenced symlinks - we'll recreate them as proper symlinks +copy_to_directory( + name = "libs_unix_no_symlinks", + srcs = [":libs_unix"], + exclude_srcs_patterns = [ + "**/libpython3.*.so", + "**/*.pyc", + ], + include_external_repositories = ["*"], + root_paths = ["python_unix"], +) + +# Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install) +pkg_mklink( + name = "libpython_symlink", + link_name = "lib/libpython3.13.so", + target = "libpython3.13.so.1.0", + attributes = pkg_attributes("0755") +) + filegroup( name = "headers_unix", srcs = [":python_unix"], @@ -211,7 +239,11 @@ filegroup( pkg_files( name = "install_libs_unix", - srcs = [":libs_unix"], + srcs = [":libs_unix_no_symlinks"], + renames = { + "libs_unix_no_symlinks": REMOVE_BASE_DIRECTORY, + }, + attributes = pkg_attributes("0755") ) pkg_files( @@ -237,6 +269,7 @@ pkg_install( ":install_libs_unix", ":install_headers_unix", ":install_bins_unix", + ":libpython_symlink", ], }), ) From 2e94975614b76bdcc7f723d0c8c8fd6307269632 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Wed, 14 Jan 2026 10:56:49 +0100 Subject: [PATCH 13/29] fix workaround on macOS --- deps/cpython.BUILD.bazel | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index ee54fd821320..e1c5bbef70e4 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -206,8 +206,12 @@ filegroup( copy_to_directory( name = "libs_unix_no_symlinks", srcs = [":libs_unix"], - exclude_srcs_patterns = [ - "**/libpython3.*.so", + exclude_srcs_patterns = select({ + # We want to include libpython3.so & libpython3.13.so.1.0, but + # exclude libpython3.13.so + "@platforms//os:linux": ["**/libpython3.*.so"], + "//conditions:default": [], + }) + [ "**/*.pyc", ], include_external_repositories = ["*"], @@ -269,7 +273,9 @@ pkg_install( ":install_libs_unix", ":install_headers_unix", ":install_bins_unix", - ":libpython_symlink", ], + }) + select({ + "@platforms//os:linux": [":libpython_symlink"], + "//conditions:default": [], }), ) From 772870d4916d7673b83124dbbdc459a80f2e5349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 15 Jan 2026 09:35:20 +0100 Subject: [PATCH 14/29] fix python dependency providing --- deps/cpython.BUILD.bazel | 20 ++++++++++++-------- omnibus/config/software/python3.rb | 1 - 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index e1c5bbef70e4..297de3775919 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -100,6 +100,14 @@ UNIX_BINS = [ "python3.13" ] +python_deps = { + 'libffi': '-lffi', + 'libsqlite3': '-lsqlite3', + 'zlib': '-lz', + 'bzip2': '-lbz2', + 'liblzma': '-llzma' +} + configure_make( name = "python_unix", configure_options = [ @@ -124,14 +132,12 @@ configure_make( ], env = { "OPT": "-DNDEBUG -fwrapv", - # manually provide some dependencies cflags/ldflags instead of - # relying on pkg-config - "ZLIB_CFLAGS": "-I$$EXT_BUILD_DEPS/include", - "ZLIB_LIBS": "-lz", - "BZIP2_CFLAGS": "-I$$EXT_BUILD_DEPS/include", - "BZIP2_LIBS": "-lbz2", # Ensure we don't use the system provided .pc "PKG_CONFIG_LIBDIR": "/does/not/exist", + } | { + dep.upper() + '_CFLAGS': "-I$$EXT_BUILD_DEPS/include" for dep in python_deps.keys() + } | { + dep.upper() + '_LIBS': lib for dep, lib in python_deps.items() } | select({ "@platforms//os:macos": { # https://github.com/bazelbuild/bazel/issues/5127 @@ -153,7 +159,6 @@ configure_make( deps = [ "@libffi//:libffi", "@openssl//:openssl", - "@libyaml//:libyaml", "@sqlite3//:libsqlite3", "@zlib//:zlib", "@bzip2//:libbz2", @@ -162,7 +167,6 @@ configure_make( dynamic_deps = [ "@zlib//:z", "@bzip2//:bz2", - "@libyaml//:yaml", "@sqlite3//:sqlite3", "@libffi//:ffi", "@xz//:lzma", diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 6eebe06679ca..4a059b9ef885 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -8,7 +8,6 @@ dependency "bzip2" dependency "libsqlite3" dependency "liblzma" - dependency "libyaml" end dependency "openssl3" From ead8645948895d451fda6ffbc6925eb778f51c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 15 Jan 2026 15:56:04 +0100 Subject: [PATCH 15/29] patch sysconfigdata to allow python module to be built down the line --- deps/cpython.BUILD.bazel | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 297de3775919..b047ccd7a93e 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -108,6 +108,15 @@ python_deps = { 'liblzma': '-llzma' } +# The list of build tools we want to override in sysconfigdata.py +# as they will not be available when building custom integrations without bazel +build_tools = [ + 'ar', + 'gcc', + 'g++', + 'ld', +] + configure_make( name = "python_unix", configure_options = [ @@ -182,6 +191,11 @@ configure_make( "@platforms//os:linux": [], "//conditions:default": ["@platforms//:incompatible"], }), + postfix_script = " && ".join([ + "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python3.13/_sysconfigdata__*.py".format( + tool=tool + ) for tool in build_tools + ]) ) pkg_files( From fa114c8b6b0ad74b16b8b42759ddfb23d711c370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Wed, 21 Jan 2026 11:45:31 +0100 Subject: [PATCH 16/29] also override build flags -isystem causes gcc not to be able to find any header --- deps/cpython.BUILD.bazel | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index b047ccd7a93e..ae5be336e592 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -110,13 +110,19 @@ python_deps = { # The list of build tools we want to override in sysconfigdata.py # as they will not be available when building custom integrations without bazel -build_tools = [ +to_override_build_tools = [ 'ar', 'gcc', 'g++', 'ld', ] +to_override_flags = [ + 'CFLAGS', + 'CXXFLAGS', + 'LDFLAGS', +] + configure_make( name = "python_unix", configure_options = [ @@ -194,7 +200,11 @@ configure_make( postfix_script = " && ".join([ "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python3.13/_sysconfigdata__*.py".format( tool=tool - ) for tool in build_tools + ) for tool in to_override_build_tools + ]) + " && " + + " && ".join(["perl -i -pe \"s/\'{flag}\': \'.*\',$$/\'{flag}\': \'\',/g\" $$INSTALLDIR/lib/python3.13/_sysconfigdata__*.py".format( + flag=flag + ) for flag in to_override_flags ]) ) From 5fa396d1271ab0d672a70a6ba9e0839e68d18c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 09:31:11 +0100 Subject: [PATCH 17/29] don't copy exe & test files --- deps/cpython.BUILD.bazel | 2 ++ omnibus/config/software/python3.rb | 18 ------------------ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index ae5be336e592..70fbbe5e16bb 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -241,6 +241,8 @@ copy_to_directory( "//conditions:default": [], }) + [ "**/*.pyc", + "**/python3.13/test/**/*", + "**/*.exe", ], include_external_repositories = ["*"], root_paths = ["python_unix"], diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 4a059b9ef885..f5bcfa05418b 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -29,24 +29,6 @@ " #{install_dir}/embedded/lib/#{sh_lib}" \ " #{install_dir}/embedded/lib/python3.13/lib-dynload/*.so" \ " #{install_dir}/embedded/bin/python3*" - - # There exists no configure flag to tell Python to not compile readline support :( - major, minor, bugfix = version.split(".") - - # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default - # compiler we use would end up being available in the system/docker image used by customers - if linux_target? && env["CC"] - command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-#{major}.#{minor}-*-linux-gnu/Makefile", :env => env - command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env - end - if linux_target? && env["CXX"] - command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-#{major}.#{minor}-*-linux-gnu/Makefile", :env => env - command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env - end - delete "#{install_dir}/embedded/lib/python#{major}.#{minor}/test" - block do - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python#{major}.#{minor}/distutils/command/wininst-*.exe")) - end elsif fips_mode? ############################### # Setup openssl dependency... # From 6ee786e304b2d340f156da5d16f4a0103ae999a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 10:21:42 +0100 Subject: [PATCH 18/29] generate py_compiled_files from bazel --- deps/cpython.BUILD.bazel | 21 +++++++++++++++++++ .../config/software/datadog-agent-finalize.rb | 4 ---- omnibus/config/software/python3.rb | 3 ++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 70fbbe5e16bb..8a2ec00596d6 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -241,6 +241,7 @@ copy_to_directory( "//conditions:default": [], }) + [ "**/*.pyc", + "**/*.pyo", "**/python3.13/test/**/*", "**/*.exe", ], @@ -248,6 +249,17 @@ copy_to_directory( root_paths = ["python_unix"], ) +# The prerm script of the package should use this list to remove the pyc/pyo files +genrule( + name = "py_compiled_files", + srcs = [":libs_unix"], + cmd = """ + echo '# DO NOT REMOVE/MODIFY - used by package removal tasks' > $@ + find $(locations :libs_unix) '(' -name '*.pyc' -o -name '*.pyo' ')' -print | sed 's|$(locations :libs_unix)|##PREFIX##/lib|g' >> $@ + """, + outs = [".py_compiled_files.txt"] +) + # Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install) pkg_mklink( name = "libpython_symlink", @@ -292,6 +304,14 @@ pkg_files( attributes = pkg_attributes("0755") ) +pkg_files( + name = "install_misc_files", + srcs = [ + ":py_compiled_files", + ], + attributes = pkg_attributes("0755") +) + pkg_install( name = "install", srcs = select({ @@ -303,6 +323,7 @@ pkg_install( ":install_libs_unix", ":install_headers_unix", ":install_bins_unix", + ":install_misc_files", ], }) + select({ "@platforms//os:linux": [":libpython_symlink"], diff --git a/omnibus/config/software/datadog-agent-finalize.rb b/omnibus/config/software/datadog-agent-finalize.rb index 7f9d2d597091..6f64e5b4e3fc 100644 --- a/omnibus/config/software/datadog-agent-finalize.rb +++ b/omnibus/config/software/datadog-agent-finalize.rb @@ -116,10 +116,6 @@ # cleanup clutter delete "#{install_dir}/etc" - # The prerm script of the package should use this list to remove the pyc/pyo files - command "echo '# DO NOT REMOVE/MODIFY - used by package removal tasks' > #{install_dir}/embedded/.py_compiled_files.txt" - command "find #{install_dir}/embedded '(' -name '*.pyc' -o -name '*.pyo' ')' -type f -delete -print >> #{install_dir}/embedded/.py_compiled_files.txt" - # The prerm and preinst scripts of the package will use this list to detect which files # have been setup by the installer, this way, on removal, we'll be able to delete only files # which have not been created by the package. diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index f5bcfa05418b..4bb1e8688f16 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -28,7 +28,8 @@ " #{install_dir}/embedded/lib/pkgconfig/python*.pc" \ " #{install_dir}/embedded/lib/#{sh_lib}" \ " #{install_dir}/embedded/lib/python3.13/lib-dynload/*.so" \ - " #{install_dir}/embedded/bin/python3*" + " #{install_dir}/embedded/bin/python3*" \ + " #{install_dir}/embedded/.py_compiled_files.txt" elsif fips_mode? ############################### # Setup openssl dependency... # From 1652c05097decac32287a57e0daffb0d26636355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 12:25:25 +0100 Subject: [PATCH 19/29] don't filter pyc files early we need to have them all, including those added by our integrations, and filter them after listing them all to .py_compiled_files.txt --- deps/cpython.BUILD.bazel | 22 ------------------- .../config/software/datadog-agent-finalize.rb | 4 ++++ 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 8a2ec00596d6..e2af1a348ca4 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -240,8 +240,6 @@ copy_to_directory( "@platforms//os:linux": ["**/libpython3.*.so"], "//conditions:default": [], }) + [ - "**/*.pyc", - "**/*.pyo", "**/python3.13/test/**/*", "**/*.exe", ], @@ -249,17 +247,6 @@ copy_to_directory( root_paths = ["python_unix"], ) -# The prerm script of the package should use this list to remove the pyc/pyo files -genrule( - name = "py_compiled_files", - srcs = [":libs_unix"], - cmd = """ - echo '# DO NOT REMOVE/MODIFY - used by package removal tasks' > $@ - find $(locations :libs_unix) '(' -name '*.pyc' -o -name '*.pyo' ')' -print | sed 's|$(locations :libs_unix)|##PREFIX##/lib|g' >> $@ - """, - outs = [".py_compiled_files.txt"] -) - # Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install) pkg_mklink( name = "libpython_symlink", @@ -304,14 +291,6 @@ pkg_files( attributes = pkg_attributes("0755") ) -pkg_files( - name = "install_misc_files", - srcs = [ - ":py_compiled_files", - ], - attributes = pkg_attributes("0755") -) - pkg_install( name = "install", srcs = select({ @@ -323,7 +302,6 @@ pkg_install( ":install_libs_unix", ":install_headers_unix", ":install_bins_unix", - ":install_misc_files", ], }) + select({ "@platforms//os:linux": [":libpython_symlink"], diff --git a/omnibus/config/software/datadog-agent-finalize.rb b/omnibus/config/software/datadog-agent-finalize.rb index 6f64e5b4e3fc..7f9d2d597091 100644 --- a/omnibus/config/software/datadog-agent-finalize.rb +++ b/omnibus/config/software/datadog-agent-finalize.rb @@ -116,6 +116,10 @@ # cleanup clutter delete "#{install_dir}/etc" + # The prerm script of the package should use this list to remove the pyc/pyo files + command "echo '# DO NOT REMOVE/MODIFY - used by package removal tasks' > #{install_dir}/embedded/.py_compiled_files.txt" + command "find #{install_dir}/embedded '(' -name '*.pyc' -o -name '*.pyo' ')' -type f -delete -print >> #{install_dir}/embedded/.py_compiled_files.txt" + # The prerm and preinst scripts of the package will use this list to detect which files # have been setup by the installer, this way, on removal, we'll be able to delete only files # which have not been created by the package. From 62cb026582e2242c594d6da0c113d28aeaebb3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 12:27:36 +0100 Subject: [PATCH 20/29] use the same permissions as before for libs --- deps/cpython.BUILD.bazel | 2 +- omnibus/config/software/python3.rb | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index e2af1a348ca4..b09baa30f5cd 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -276,7 +276,7 @@ pkg_files( renames = { "libs_unix_no_symlinks": REMOVE_BASE_DIRECTORY, }, - attributes = pkg_attributes("0755") + attributes = pkg_attributes("0644") ) pkg_files( diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 4bb1e8688f16..f5bcfa05418b 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -28,8 +28,7 @@ " #{install_dir}/embedded/lib/pkgconfig/python*.pc" \ " #{install_dir}/embedded/lib/#{sh_lib}" \ " #{install_dir}/embedded/lib/python3.13/lib-dynload/*.so" \ - " #{install_dir}/embedded/bin/python3*" \ - " #{install_dir}/embedded/.py_compiled_files.txt" + " #{install_dir}/embedded/bin/python3*" elsif fips_mode? ############################### # Setup openssl dependency... # From 195b7ab89c97c5e1b4a82b28b0bb0e82e18eb5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 14:17:29 +0100 Subject: [PATCH 21/29] parameterize python version in path --- deps/cpython.BUILD.bazel | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index b09baa30f5cd..ab085dfce9f4 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -17,6 +17,8 @@ python_externals = { "tcltk": "8.6.15.0", } +VERSION_STR="3.13" + # These rules will make it easier to get a reference to their folder via $(location) # and they add the version in the folder name (as some of the vcxproj stuff relies on that) [ @@ -97,7 +99,7 @@ filegroup( UNIX_BINS = [ "python3", "pip3", - "python3.13" + "python{}".format(VERSION_STR), ] python_deps = { @@ -198,12 +200,14 @@ configure_make( "//conditions:default": ["@platforms//:incompatible"], }), postfix_script = " && ".join([ - "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python3.13/_sysconfigdata__*.py".format( - tool=tool + "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format( + tool=tool, + version=VERSION_STR ) for tool in to_override_build_tools ]) + " && " + - " && ".join(["perl -i -pe \"s/\'{flag}\': \'.*\',$$/\'{flag}\': \'\',/g\" $$INSTALLDIR/lib/python3.13/_sysconfigdata__*.py".format( - flag=flag + " && ".join(["perl -i -pe \"s/\'{flag}\': \'.*\',$$/\'{flag}\': \'\',/g\" $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format( + flag=flag, + version=VERSION_STR ) for flag in to_override_flags ]) ) @@ -240,7 +244,7 @@ copy_to_directory( "@platforms//os:linux": ["**/libpython3.*.so"], "//conditions:default": [], }) + [ - "**/python3.13/test/**/*", + "**/python{}/test/**/*".format(VERSION_STR), "**/*.exe", ], include_external_repositories = ["*"], @@ -250,8 +254,8 @@ copy_to_directory( # Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install) pkg_mklink( name = "libpython_symlink", - link_name = "lib/libpython3.13.so", - target = "libpython3.13.so.1.0", + link_name = "lib/libpython{}.so".format(VERSION_STR), + target = "libpython{}.so.1.0".format(VERSION_STR), attributes = pkg_attributes("0755") ) From 7c7da49c020ff9251471c94011de99c1698eef3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 14:17:52 +0100 Subject: [PATCH 22/29] don't copy the python3 executable --- deps/cpython.BUILD.bazel | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index ab085dfce9f4..117d5d7de09d 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -97,7 +97,6 @@ filegroup( ) UNIX_BINS = [ - "python3", "pip3", "python{}".format(VERSION_STR), ] @@ -259,6 +258,12 @@ pkg_mklink( attributes = pkg_attributes("0755") ) +pkg_mklink( + name = "python_bin_symlink", + link_name = "bin/python3", + target = "python{}".format(VERSION_STR) +) + filegroup( name = "headers_unix", srcs = [":python_unix"], @@ -290,7 +295,7 @@ pkg_files( pkg_files( name = "install_bins_unix", - srcs = [":bins_unix_" + bin for bin in UNIX_BINS], + srcs = [":bins_unix_" + bin for bin in UNIX_BINS] + [":python_bin_symlink"], prefix = "bin", attributes = pkg_attributes("0755") ) @@ -306,6 +311,7 @@ pkg_install( ":install_libs_unix", ":install_headers_unix", ":install_bins_unix", + ":python_bin_symlink", ], }) + select({ "@platforms//os:linux": [":libpython_symlink"], From 510b4a66145c52efe85eb098faad4832f8a86cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 15:59:24 +0100 Subject: [PATCH 23/29] fix more permissions --- deps/cpython.BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 117d5d7de09d..0ef3fde23100 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -285,7 +285,7 @@ pkg_files( renames = { "libs_unix_no_symlinks": REMOVE_BASE_DIRECTORY, }, - attributes = pkg_attributes("0644") + attributes = pkg_attributes("0755") ) pkg_files( From 3fbbc7cce96c80ecf9c749c4ac8adf1b479fa277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 16:01:03 +0100 Subject: [PATCH 24/29] don't enable PGO for python just yet --- deps/cpython.BUILD.bazel | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 0ef3fde23100..f71b49a0cbdd 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -137,7 +137,6 @@ configure_make( "CPPFLAGS='-Dredacted=\\\"redacted\\\"'", "--with-openssl=$$EXT_BUILD_DEPS/openssl", "--with-openssl-rpath=yes", - "--enable-optimizations", ] + select({ "@@//:macos_arm64": ["--with-universal-archs=universal2"], "@@//:macos_x86_64": ["--with-universal-archs=intel"], From 15a2dc4da556ba8825356d305332e4af3a87216c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Thu, 22 Jan 2026 16:41:52 +0100 Subject: [PATCH 25/29] fixup permissions --- deps/cpython.BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index f71b49a0cbdd..4fb99dc680b2 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -254,7 +254,7 @@ pkg_mklink( name = "libpython_symlink", link_name = "lib/libpython{}.so".format(VERSION_STR), target = "libpython{}.so.1.0".format(VERSION_STR), - attributes = pkg_attributes("0755") + attributes = pkg_attributes("0644") ) pkg_mklink( @@ -284,7 +284,7 @@ pkg_files( renames = { "libs_unix_no_symlinks": REMOVE_BASE_DIRECTORY, }, - attributes = pkg_attributes("0755") + attributes = pkg_attributes("0644") ) pkg_files( From 41fb3943c6707c8ad4f4bd0fb319b1c9725c680f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Fri, 23 Jan 2026 15:03:35 +0100 Subject: [PATCH 26/29] use the same optimization level as omnibus --- deps/cpython.BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 4fb99dc680b2..c91ae16982af 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -143,7 +143,7 @@ configure_make( "//conditions:default": [], }), copts = [ - "-O3", + "-O2", ], env = { "OPT": "-DNDEBUG -fwrapv", From f3823d676a9a6d05fc8a64fb80c55f237d7d86d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 26 Jan 2026 13:56:49 +0100 Subject: [PATCH 27/29] simplify file exclusion --- deps/cpython.BUILD.bazel | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index c91ae16982af..0731172a6176 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -236,12 +236,10 @@ filegroup( copy_to_directory( name = "libs_unix_no_symlinks", srcs = [":libs_unix"], - exclude_srcs_patterns = select({ - # We want to include libpython3.so & libpython3.13.so.1.0, but - # exclude libpython3.13.so - "@platforms//os:linux": ["**/libpython3.*.so"], - "//conditions:default": [], - }) + [ + # We want to include libpython3.so & libpython3.13.so.1.0, but + # exclude libpython3.13.so + exclude_srcs_patterns = [ + "**/libpython3.*.so", "**/python{}/test/**/*".format(VERSION_STR), "**/*.exe", ], From c888e16d1eca56b2f4d887c74d71c57e6e34f324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Mon, 26 Jan 2026 13:57:20 +0100 Subject: [PATCH 28/29] don't install Makefile's --- deps/cpython.BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 0731172a6176..1c0086474e47 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -242,6 +242,7 @@ copy_to_directory( "**/libpython3.*.so", "**/python{}/test/**/*".format(VERSION_STR), "**/*.exe", + "**/Makefile", ], include_external_repositories = ["*"], root_paths = ["python_unix"], From 4677c0a32268745883c52cefbb85c17df356ac83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20Beauz=C3=A9e-Luyssen?= Date: Tue, 27 Jan 2026 08:24:42 +0100 Subject: [PATCH 29/29] apply review comments --- deps/cpython.BUILD.bazel | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index 1c0086474e47..30fe59145b17 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -156,7 +156,7 @@ configure_make( } | select({ "@platforms//os:macos": { # https://github.com/bazelbuild/bazel/issues/5127 - "AR": "ar", + "AR": "/usr/bin/ar", }, "//conditions:default": {}, }), @@ -172,19 +172,19 @@ configure_make( out_include_dir = "include", visibility = ["//visibility:public"], deps = [ + "@bzip2//:libbz2", "@libffi//:libffi", "@openssl//:openssl", "@sqlite3//:libsqlite3", - "@zlib//:zlib", - "@bzip2//:libbz2", "@xz//:liblzma", + "@zlib//:zlib", ], dynamic_deps = [ - "@zlib//:z", "@bzip2//:bz2", - "@sqlite3//:sqlite3", "@libffi//:ffi", + "@sqlite3//:sqlite3", "@xz//:lzma", + "@zlib//:z", ], targets = [ # Build in parallel but install without parallel execution @@ -193,10 +193,20 @@ configure_make( "install" ], target_compatible_with = select({ - "@platforms//os:osx": [], + "@platforms//os:macos": [], "@platforms//os:linux": [], "//conditions:default": ["@platforms//:incompatible"], }), + # python's build system will output the entire build config to _sysconfigdata_xxx.py + # This is later used to build extensions with the same tools/compiler/flags/config as the interpreter + # However, in our case, that means using tools that are stored in the build sandbox, so they aren't + # usable when building an extension, causing all builds to fail. + # Ideally we would want to explicitly replace bazel paths with known alternatives, but we don't + # have an environment variable holding the value we want to replace so we have to resort to + # a regular expression replacing paths to tools. + # We also unset the flags listed in to_override_flags. + # If we start using some specific build flags that need to be propagated, we will need to include + # them here instead of replacing them by an empty string. postfix_script = " && ".join([ "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format( tool=tool,