diff --git a/bazel/rules/replace_prefix.sh b/bazel/rules/replace_prefix.sh index e6625ee330c2..2f82daf47c2e 100755 --- a/bazel/rules/replace_prefix.sh +++ b/bazel/rules/replace_prefix.sh @@ -28,29 +28,13 @@ for f in "$@"; do echo "$f: file not found" exit 2 fi - case $f in - *.so) - ${PATCHELF} --set-rpath "$PREFIX"/lib "$f" - ;; - *.dylib) - install_name_tool -add_rpath "$PREFIX/lib" "$f" 2>/dev/null || true - # Get the old install name/ID - dylib_name=$(basename "$f") - new_id="$PREFIX/lib/$dylib_name" - - # Change the dylib's own ID - install_name_tool -id "$new_id" "$f" + # We don't want to process symlinks but rather the actual file it's pointing to + # Otherwise `file $f` would return that it's a symlink, not an elf/mach-o file + if [ -L "$f" ]; then + f=$(realpath "$f") + fi - # Update all dependency paths that point to sandbox locations - otool -L "$f" | tail -n +2 | awk '{print $1}' | while read -r dep; do - if [[ "$dep" == *"sandbox"* ]] || [[ "$dep" == *"bazel-out"* ]]; then - dep_name=$(basename "$dep") - new_dep="$PREFIX/lib/$dep_name" - install_name_tool -change "$dep" "$new_dep" "$f" 2>/dev/null || true - install_name_tool -add_rpath "$PREFIX/lib" "$dep" 2>/dev/null || true - fi - done - ;; + case $f in *.pc) sed -ibak -e "s|^prefix=.*|prefix=$PREFIX|" -e "s|##PREFIX##|$PREFIX|" -e "s|\${EXT_BUILD_DEPS}|$PREFIX|" "$f" && rm -f "${f}bak" ;; @@ -60,12 +44,20 @@ for f in "$@"; do elif file "$f" | grep -q "Mach-O"; then # Handle macOS binaries (executables and other Mach-O files) install_name_tool -add_rpath "$PREFIX/lib" "$f" 2>/dev/null || true + # Get the old install name/ID + dylib_name=$(basename "$f") + new_id="$PREFIX/lib/$dylib_name" + + # Change the dylib's own ID + install_name_tool -id "$new_id" "$f" + # Update all dependency paths that point to sandbox locations otool -L "$f" | tail -n +2 | awk '{print $1}' | while read -r dep; do if [[ "$dep" == *"sandbox"* ]] || [[ "$dep" == *"bazel-out"* ]]; then dep_name=$(basename "$dep") new_dep="$PREFIX/lib/$dep_name" install_name_tool -change "$dep" "$new_dep" "$f" 2>/dev/null || true + install_name_tool -add_rpath "$PREFIX/lib" "$dep" 2>/dev/null || true fi done else diff --git a/deps/cpython.BUILD.bazel b/deps/cpython.BUILD.bazel index c68286487e1c..30fe59145b17 100644 --- a/deps/cpython.BUILD.bazel +++ b/deps/cpython.BUILD.bazel @@ -1,7 +1,10 @@ load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_lib//lib:run_binary.bzl", "run_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") -load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files") +load("@rules_pkg//pkg:mappings.bzl", "REMOVE_BASE_DIRECTORY", "pkg_files", "pkg_mklink") +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") +load("@rules_pkg//pkg:mappings.bzl", "strip_prefix") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes") # Keep in sync with the ones on repos.MODULE.bazel python_externals = { @@ -14,6 +17,8 @@ python_externals = { "tcltk": "8.6.15.0", } +VERSION_STR="3.13" + # These rules will make it easier to get a reference to their folder via $(location) # and they add the version in the folder name (as some of the vcxproj stuff relies on that) [ @@ -86,22 +91,238 @@ run_binary( visibility = ["//visibility:public"], ) -pkg_files( - name = "install_files", - srcs = select({ - "@platforms//os:windows": [":python_win"], +filegroup( + name = "all", + srcs = glob(["**"], exclude = ["BUILD.bazel"]), +) + +UNIX_BINS = [ + "pip3", + "python{}".format(VERSION_STR), +] + +python_deps = { + 'libffi': '-lffi', + 'libsqlite3': '-lsqlite3', + 'zlib': '-lz', + 'bzip2': '-lbz2', + 'liblzma': '-llzma' +} + +# The list of build tools we want to override in sysconfigdata.py +# as they will not be available when building custom integrations without bazel +to_override_build_tools = [ + 'ar', + 'gcc', + 'g++', + 'ld', +] + +to_override_flags = [ + 'CFLAGS', + 'CXXFLAGS', + 'LDFLAGS', +] + +configure_make( + name = "python_unix", + configure_options = [ + "--enable-ipv6", + "--with-ensurepip=yes", + "--enable-shared", + "--without-static-libpython", + "--with-dbmliborder=", + # Fixes an issue with __DATE__ being set to undefined `redacted` + # https://github.com/bazelbuild/rules_foreign_cc/issues/239#issuecomment-478167267 + "CPPFLAGS='-Dredacted=\\\"redacted\\\"'", + "--with-openssl=$$EXT_BUILD_DEPS/openssl", + "--with-openssl-rpath=yes", + ] + select({ + "@@//:macos_arm64": ["--with-universal-archs=universal2"], + "@@//:macos_x86_64": ["--with-universal-archs=intel"], + "//conditions:default": [], }), + copts = [ + "-O2", + ], + env = { + "OPT": "-DNDEBUG -fwrapv", + # Ensure we don't use the system provided .pc + "PKG_CONFIG_LIBDIR": "/does/not/exist", + } | { + dep.upper() + '_CFLAGS': "-I$$EXT_BUILD_DEPS/include" for dep in python_deps.keys() + } | { + dep.upper() + '_LIBS': lib for dep, lib in python_deps.items() + } | select({ + "@platforms//os:macos": { + # https://github.com/bazelbuild/bazel/issues/5127 + "AR": "/usr/bin/ar", + }, + "//conditions:default": {}, + }), + lib_source = ":all", + # The single dollar sign here isn't a typo, using 2 seems to confuse rules_foreign_cc's substitution + # This is meant to allow python to find its dependency during its modules import test. + # We will use the install_dir rpath later on + linkopts = ["-Wl,-rpath", "$EXT_BUILD_DEPS/lib"], + out_binaries = UNIX_BINS, + out_data_dirs = [ + "lib", + ], + out_include_dir = "include", + visibility = ["//visibility:public"], + deps = [ + "@bzip2//:libbz2", + "@libffi//:libffi", + "@openssl//:openssl", + "@sqlite3//:libsqlite3", + "@xz//:liblzma", + "@zlib//:zlib", + ], + dynamic_deps = [ + "@bzip2//:bz2", + "@libffi//:ffi", + "@sqlite3//:sqlite3", + "@xz//:lzma", + "@zlib//:z", + ], + targets = [ + # Build in parallel but install without parallel execution + # (see https://github.com/python/cpython/issues/109796) + "-j 16", + "install" + ], + target_compatible_with = select({ + "@platforms//os:macos": [], + "@platforms//os:linux": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + # python's build system will output the entire build config to _sysconfigdata_xxx.py + # This is later used to build extensions with the same tools/compiler/flags/config as the interpreter + # However, in our case, that means using tools that are stored in the build sandbox, so they aren't + # usable when building an extension, causing all builds to fail. + # Ideally we would want to explicitly replace bazel paths with known alternatives, but we don't + # have an environment variable holding the value we want to replace so we have to resort to + # a regular expression replacing paths to tools. + # We also unset the flags listed in to_override_flags. + # If we start using some specific build flags that need to be propagated, we will need to include + # them here instead of replacing them by an empty string. + postfix_script = " && ".join([ + "perl -i -pe 's/(:?[a-zA-Z0-9_+.\\/-]+)\\/{tool}\\b/{tool}/g' $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format( + tool=tool, + version=VERSION_STR + ) for tool in to_override_build_tools + ]) + " && " + + " && ".join(["perl -i -pe \"s/\'{flag}\': \'.*\',$$/\'{flag}\': \'\',/g\" $$INSTALLDIR/lib/python{version}/_sysconfigdata__*.py".format( + flag=flag, + version=VERSION_STR + ) for flag in to_override_flags + ]) +) + +pkg_files( + name = "install_files_win", + srcs = [":python_win"], renames = { "python_win": REMOVE_BASE_DIRECTORY, }, ) +filegroup( + name = "libs_unix", + srcs = [":python_unix"], + output_group = "lib", +) + +# Fix symlinks for libpython3.x shared libraries +# rules_foreign_cc dereferences symlinks during installation since we're using +# out_data_dir instead of out_shared_libs, so we need to recreate them +# For context, we use out_data_dir to copy the entire list of python modules which +# too long to explicitly list in out_shared_libs, and we can't only copy the +# lib/python3.X folder as it conflicts with the python3.X executables (rules_foreign_cc +# output groups are named based on the basename) + +# Filter out the dereferenced symlinks - we'll recreate them as proper symlinks +copy_to_directory( + name = "libs_unix_no_symlinks", + srcs = [":libs_unix"], + # We want to include libpython3.so & libpython3.13.so.1.0, but + # exclude libpython3.13.so + exclude_srcs_patterns = [ + "**/libpython3.*.so", + "**/python{}/test/**/*".format(VERSION_STR), + "**/*.exe", + "**/Makefile", + ], + include_external_repositories = ["*"], + root_paths = ["python_unix"], +) + +# Create symlinks for libpython (rules_pkg 1.2+ supports symlinks in pkg_install) +pkg_mklink( + name = "libpython_symlink", + link_name = "lib/libpython{}.so".format(VERSION_STR), + target = "libpython{}.so.1.0".format(VERSION_STR), + attributes = pkg_attributes("0644") +) + +pkg_mklink( + name = "python_bin_symlink", + link_name = "bin/python3", + target = "python{}".format(VERSION_STR) +) + +filegroup( + name = "headers_unix", + srcs = [":python_unix"], + output_group = "include", +) + +[ + filegroup( + name = "bins_unix_" + bin, + srcs = [":python_unix"], + output_group = bin, + ) + for bin in UNIX_BINS +] + +pkg_files( + name = "install_libs_unix", + srcs = [":libs_unix_no_symlinks"], + renames = { + "libs_unix_no_symlinks": REMOVE_BASE_DIRECTORY, + }, + attributes = pkg_attributes("0644") +) + +pkg_files( + name = "install_headers_unix", + srcs = [":headers_unix"], +) + +pkg_files( + name = "install_bins_unix", + srcs = [":bins_unix_" + bin for bin in UNIX_BINS] + [":python_bin_symlink"], + prefix = "bin", + attributes = pkg_attributes("0755") +) + pkg_install( name = "install", - srcs = [":install_files"] + select({ + srcs = select({ "@platforms//os:windows": [ "@openssl//:openssl_exe_file", + ":install_files_win" + ], + "//conditions:default": [ + ":install_libs_unix", + ":install_headers_unix", + ":install_bins_unix", + ":python_bin_symlink", ], + }) + select({ + "@platforms//os:linux": [":libpython_symlink"], "//conditions:default": [], }), ) diff --git a/deps/openssl.BUILD.bazel b/deps/openssl.BUILD.bazel index aab74a537a35..8b19745bd86f 100644 --- a/deps/openssl.BUILD.bazel +++ b/deps/openssl.BUILD.bazel @@ -169,7 +169,9 @@ configure_make( done done LIBS="$$INSTALLDIR/lib/libcrypto.dylib $$INSTALLDIR/lib/libssl.dylib" - """ + FIX_OPENSSL_PATHS, + """ + FIX_OPENSSL_PATHS + """ + codesign -s - -f $$INSTALLDIR/lib/libcrypto.dylib $$INSTALLDIR/lib/libssl.dylib + """, "//conditions:default": """ LIBS="$$INSTALLDIR/lib/libcrypto.so $$INSTALLDIR/lib/libssl.so" """ + FIX_OPENSSL_PATHS, diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 322fde0d0592..f5bcfa05418b 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -8,7 +8,6 @@ dependency "bzip2" dependency "libsqlite3" dependency "liblzma" - dependency "libyaml" end dependency "openssl3" @@ -23,49 +22,13 @@ if !windows_target? env = with_standard_compiler_flags(with_embedded_path) - python_configure_options = [ - "--without-readline", # Disables readline support - "--with-ensurepip=yes", # We upgrade pip later, in the pip3 software definition - "--without-static-libpython" # We only care about the shared library - ] - - if mac_os_x? - python_configure_options.push("--enable-ipv6", - "--with-universal-archs=#{arm_target? ? "universal2" : "intel"}", - "--enable-shared") - elsif linux_target? - python_configure_options.push("--enable-shared", - "--enable-ipv6") - elsif aix? - # something here... - end - - python_configure_options.push("--with-dbmliborder=") - - # Force different defaults for the "optimization settings" - # This removes the debug symbol generation and doesn't enable all warnings - env["OPT"] = "-DNDEBUG -fwrapv" - configure(*python_configure_options, :env => env) - command "make -j #{workers}", :env => env - command "make install", :env => env - - # There exists no configure flag to tell Python to not compile readline support :( - major, minor, bugfix = version.split(".") - - # Don't forward CC and CXX to python extensions Makefile, it's quite unlikely that any non default - # compiler we use would end up being available in the system/docker image used by customers - if linux_target? && env["CC"] - command "sed -i \"s/^CC=[[:space:]]*${CC}/CC=gcc/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-#{major}.#{minor}-*-linux-gnu/Makefile", :env => env - command "sed -i \"s/${CC}/gcc/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env - end - if linux_target? && env["CXX"] - command "sed -i \"s/^CXX=[[:space:]]*${CXX}/CC=g++/\" #{install_dir}/embedded/lib/python#{major}.#{minor}/config-#{major}.#{minor}-*-linux-gnu/Makefile", :env => env - command "sed -i \"s/${CXX}/g++/g\" #{install_dir}/embedded/lib/python#{major}.#{minor}/_sysconfigdata__linux_*-linux-gnu.py", :env => env - end - delete "#{install_dir}/embedded/lib/python#{major}.#{minor}/test" - block do - FileUtils.rm_f(Dir.glob("#{install_dir}/embedded/lib/python#{major}.#{minor}/distutils/command/wininst-*.exe")) - end + command_on_repo_root "bazelisk run -- @cpython//:install --destdir='#{install_dir}/embedded'" + sh_lib = if linux_target? then "libpython3.so" else "libpython3.13.dylib" end + command_on_repo_root "bazelisk run -- //bazel/rules:replace_prefix --prefix '#{install_dir}/embedded'" \ + " #{install_dir}/embedded/lib/pkgconfig/python*.pc" \ + " #{install_dir}/embedded/lib/#{sh_lib}" \ + " #{install_dir}/embedded/lib/python3.13/lib-dynload/*.so" \ + " #{install_dir}/embedded/bin/python3*" elsif fips_mode? ############################### # Setup openssl dependency... #