diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 97e6e17b878..5098525e7a1 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -134,7 +134,7 @@ either due to lack of resources or GPU quota exhaustion. The custom VM image has NVIDIA drivers, `nvidia-container-toolkit` and `docker` installed. It contains today's date in the name and the image family is set to -`ubuntu-os-docker-gpu-2004-lts`. The latest image from this family is +`ubuntu-os-docker-gpu-2204-lts`. The latest image from this family is used for running CI. #### Step 3: GitHub diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml old mode 100644 new mode 100755 index 48bde94a366..f8db04df617 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -26,7 +26,7 @@ env: BUILD_CUDA_MODULE: OFF jobs: - MacOS: + build-lib: permissions: contents: write # upload id-token: write @@ -66,9 +66,7 @@ jobs: - name: Install dependencies run: | - brew install ccache glslang spirv-cross - # Fix gfortran not found issue. - brew install gcc + brew install ccache glslang spirv-cross gcc ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran ccache -M 2G # See .github/workflows/readme.md for ccache strategy. @@ -76,9 +74,22 @@ jobs: run: | PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH ccache -s + export BUILD_PYTHON_MODULE=OFF ./util/run_ci.sh DEVEL_PKG_NAME="$(basename build/package/open3d-devel-*.tar.xz)" echo "DEVEL_PKG_NAME=$DEVEL_PKG_NAME" >> $GITHUB_ENV + + # Compress build directory for reuse (exclude some large potentially unnecessary files if needed) + # We need 'build' directory. + tar -caf build.tar.xz build + + - name: Upload build artifact + if: ${{ matrix.CONFIG == 'OFF' }} + uses: actions/upload-artifact@v4 + with: + name: open3d-build-${{ matrix.os }} + path: build.tar.xz + retention-days: 1 - name: Build Open3D viewer app if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }} run: | @@ -130,6 +141,7 @@ jobs: fi build-wheel: + needs: build-lib name: Build wheel permissions: contents: write # upload @@ -172,6 +184,14 @@ jobs: ref: main path: ${{ env.OPEN3D_ML_ROOT }} + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: open3d-build-${{ matrix.os }} + + - name: Unpack build artifact + run: tar -xf build.tar.xz + - name: Setup cache uses: actions/cache@v4 with: @@ -183,8 +203,7 @@ jobs: # Restore any ccache cache entry, if none for # ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }} exists. # Common prefix will be used so that ccache can be used across commits. - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache + restore-keys: ${{ runner.os }}-${{ runner.arch }}-ccache - name: Set up Python uses: actions/setup-python@v5 @@ -202,13 +221,10 @@ jobs: source util/ci_utils.sh install_python_dependencies - # Fix macos-14 arm64 runner image issues, see comments in MacOS job. - # brew install gcc is required so that `brew --prefix gcc` resolves; - brew install gcc - ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran - - brew install ccache glslang spirv-cross + brew install ccache glslang spirv-cross gcc ccache -M 2G # See .github/workflows/readme.md for ccache strategy. + # Fix gfortran not found issue + ln -s $(brew --prefix gcc)/bin/gfortran-* /usr/local/bin/gfortran - name: Config and build wheel run: | diff --git a/.github/workflows/ubuntu-cuda.yml b/.github/workflows/ubuntu-cuda.yml index b69b747155f..9b6579922dc 100644 --- a/.github/workflows/ubuntu-cuda.yml +++ b/.github/workflows/ubuntu-cuda.yml @@ -57,14 +57,13 @@ jobs: fail-fast: false matrix: include: - - CI_CONFIG: 2-jammy - - CI_CONFIG: 3-ml-shared-jammy - - CI_CONFIG: 5-ml-noble + - CI_CONFIG: 2-noble + - CI_CONFIG: 3-ml-shared-noble env: # Export everything from matrix to be easily used. # Docker tag and ccache names must be consistent with docker_build.sh CI_CONFIG : ${{ matrix.CI_CONFIG }} - BUILD_PACKAGE : ${{ contains(fromJson('["3-ml-shared-jammy"]'), matrix.CI_CONFIG) }} + BUILD_PACKAGE : ${{ contains(fromJson('["3-ml-shared-noble"]'), matrix.CI_CONFIG) }} GCE_INSTANCE_PREFIX: open3d-ci-${{ matrix.CI_CONFIG }} DOCKER_TAG : open3d-ci:${{ matrix.CI_CONFIG }} CCACHE_TAR_NAME : open3d-ci-${{ matrix.CI_CONFIG }} diff --git a/.github/workflows/ubuntu-openblas.yml b/.github/workflows/ubuntu-openblas.yml index 46ad4b5a1ff..239e8b5095e 100644 --- a/.github/workflows/ubuntu-openblas.yml +++ b/.github/workflows/ubuntu-openblas.yml @@ -34,13 +34,48 @@ jobs: - name: Docker test run: docker/docker_test.sh openblas-amd64-py312-dev - openblas-arm64: + build-lib-arm64: + name: Build Lib ARM64 permissions: contents: write # Release upload id-token: write attestations: write artifact-metadata: write runs-on: ubuntu-24.04-arm # latest + env: + DEVELOPER_BUILD: 'ON' + OPEN3D_CPU_RENDERING: true + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Maximize build space + run: | + source util/ci_utils.sh + maximize_ubuntu_github_actions_build_space + + - name: Docker build lib + run: | + # Use py310 as dummy + docker/docker_build.sh openblas-arm64-py310-dev build-lib + # Image name open3d-ci:openblas-arm64-py310-dev (check valid tag in script) + docker tag open3d-ci:openblas-arm64-py310-dev open3d-lib-arm64:latest + docker save open3d-lib-arm64:latest | gzip > open3d-lib-arm64.tar.gz + + - name: Upload Lib Image + uses: actions/upload-artifact@v4 + with: + name: open3d-lib-arm64 + path: open3d-lib-arm64.tar.gz + retention-days: 1 + + build-wheel-arm64: + name: Build Wheel ARM64 + permissions: + contents: write # Release upload + id-token: write + attestations: write + runs-on: ubuntu-24.04-arm + needs: build-lib-arm64 strategy: fail-fast: false matrix: @@ -68,6 +103,15 @@ jobs: source util/ci_utils.sh maximize_ubuntu_github_actions_build_space + - name: Download Lib Image + uses: actions/download-artifact@v4 + with: + name: open3d-lib-arm64 + path: . + + - name: Load Lib Image + run: docker load -i open3d-lib-arm64.tar.gz # Restore open3d-lib-arm64:latest + - name: Compute Docker tag for this matrix entry run: | # Strip the dot: 3.12 ➜ 312, 3.14 ➜ 314 … @@ -78,6 +122,7 @@ jobs: - name: Docker build run: | + export BASE_IMAGE=open3d-lib-arm64:latest docker/docker_build.sh "${DOCKER_TAG}" shopt -s failglob PIP_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d-[0-9]*.whl)" diff --git a/.github/workflows/ubuntu-sycl.yml b/.github/workflows/ubuntu-sycl.yml index e4903de1890..92ac6f8db38 100644 --- a/.github/workflows/ubuntu-sycl.yml +++ b/.github/workflows/ubuntu-sycl.yml @@ -23,7 +23,8 @@ env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} jobs: - ubuntu-sycl: + build-lib: + name: Build Lib permissions: contents: write # Release upload id-token: write @@ -41,39 +42,37 @@ jobs: run: | source util/ci_utils.sh maximize_ubuntu_github_actions_build_space - - name: Docker build + - name: Docker build lib run: | + # Use py310 as dummy if [ "${{ matrix.BUILD_SHARED_LIBS }}" = "ON" ]; then - docker/docker_build.sh sycl-shared + docker/docker_build.sh sycl-shared build-lib + docker tag open3d-ci:sycl-shared open3d-lib-sycl-shared:latest + docker save open3d-lib-sycl-shared:latest -o open3d-lib-sycl-shared.tar + xz open3d-lib-sycl-shared.tar + ls -lhs open3d-lib-sycl-shared.tar.xz else - docker/docker_build.sh sycl-static - fi - - name: Docker test - run: | - du -hs $PWD - if [ "${{ matrix.BUILD_SHARED_LIBS }}" = "ON" ]; then - docker/docker_test.sh sycl-shared - else - docker/docker_test.sh sycl-static + docker/docker_build.sh sycl-static build-lib + # Static build usually doesn't produce wheels or shared libs for python + # But we might need artifacts. For now keep simple. fi - - name: Generate artifact attestation + - name: Upload Lib Image if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} - uses: actions/attest@v4 + uses: actions/upload-artifact@v4 with: - subject-path: | - ${{ github.workspace }}/open3d*.whl - ${{ github.workspace }}/open3d-devel-*.tar.xz + name: open3d-lib-sycl-shared + path: open3d-lib-sycl-shared.tar.xz + retention-days: 1 - - name: Upload Python wheel and C++ binary package to GitHub artifacts + - name: Upload C++ binary package to GitHub artifacts if: ${{ matrix.BUILD_SHARED_LIBS == 'ON' }} uses: actions/upload-artifact@v4 with: - name: open3d-sycl-linux-wheel-and-binary - path: | - open3d*.whl - open3d-devel-*.tar.xz + name: open3d-devel-sycl + path: open3d-devel-*.tar.xz if-no-files-found: error + - name: Update devel release if: ${{ github.ref == 'refs/heads/main' && matrix.BUILD_SHARED_LIBS == 'ON' }} env: @@ -97,3 +96,73 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} run: | gsutil cp ${GITHUB_WORKSPACE}/open3d-ci-sycl.tar.gz gs://open3d-ci-cache/ || true + + build-wheel: + name: Build Wheel + permissions: + contents: write # Release upload + id-token: write + attestations: write + artifact-metadata: write + runs-on: ubuntu-latest + needs: build-lib + strategy: + fail-fast: false + matrix: + python_version: ['3.10', '3.11', '3.12', '3.13'] + is_main: + - ${{ github.ref == 'refs/heads/main' }} + exclude: + - is_main: false + python_version: '3.10' + - is_main: false + python_version: '3.11' + - is_main: false + python_version: '3.12' + env: + PYTHON_VERSION: ${{ matrix.python_version }} + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Maximize build space + run: | + source util/ci_utils.sh + maximize_ubuntu_github_actions_build_space + + - name: Download Lib Image + uses: actions/download-artifact@v4 + with: + name: open3d-lib-sycl-shared + path: . + + - name: Load Lib Image + run: docker load -i open3d-lib-sycl-shared.tar.xz + + - name: Docker build + run: | + export BASE_IMAGE=open3d-lib-sycl-shared:latest + + # We execute docker buildscript with python version argument. + docker/docker_build.sh sycl-shared py${{ matrix.python_version }} + + - name: Generate artifact attestation + uses: actions/attest@v4 + with: + subject-path: | + ${{ github.workspace }}/open3d*.whl + + - name: Upload wheel to GitHub artifacts + uses: actions/upload-artifact@v4 + with: + name: open3d-sycl-wheel-py${{ matrix.python_version }} + path: open3d*.whl + if-no-files-found: error + + - name: Update devel release + if: ${{ github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release upload main-devel open3d-*.whl --clobber + gh release view main-devel + diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 1b16e1ec168..f1f51bf96e8 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -23,7 +23,66 @@ env: BUILD_CUDA_MODULE: 'ON' jobs: + # Build CPU + CUDA open3d-devel packages, viewer .deb, and C++ tests bundle. + # ML ops are OFF here (Python/Torch/TF ABI is wheel-stage only). + build-lib: + name: Build Lib + runs-on: ubuntu-latest + env: + DEVELOPER_BUILD: 'ON' + CCACHE_TAR_NAME: open3d-ubuntu-2204-cuda-ci-ccache + OPEN3D_CPU_RENDERING: true + BUILD_PYTORCH_OPS: 'OFF' + BUILD_TENSORFLOW_OPS: 'OFF' + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Maximize build space + run: | + source util/ci_utils.sh + maximize_ubuntu_github_actions_build_space + - name: Docker build lib packages + run: | + # py310 is only used to satisfy docker_build.sh's Python arg; lib mode + # does not build Python modules. + docker/docker_build.sh cuda_wheel_py310_dev build-lib + ls -lh open3d-devel-*.tar.xz open3d-viewer-*.deb open3d-cpp-tests.tar.xz + - name: Upload devel packages + uses: actions/upload-artifact@v4 + with: + name: open3d-devel-packages + path: | + open3d-devel-*.tar.xz + open3d-viewer-*.deb + if-no-files-found: error + - name: Upload C++ tests bundle + uses: actions/upload-artifact@v4 + with: + name: open3d-cpp-tests + path: open3d-cpp-tests.tar.xz + if-no-files-found: error + retention-days: 1 + + # Runs in parallel with build-wheel (both need only build-lib). + test-cpp: + name: Test C++ + runs-on: ubuntu-22.04 + needs: [build-lib] + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Download C++ tests bundle + uses: actions/download-artifact@v4 + with: + name: open3d-cpp-tests + path: . + - name: Run C++ unit tests + run: | + source util/ci_utils.sh + run_cpp_tests_from_bundle open3d-cpp-tests.tar.xz + build-wheel: + needs: build-lib permissions: contents: write # Release upload id-token: write @@ -61,10 +120,18 @@ jobs: run: | source util/ci_utils.sh maximize_ubuntu_github_actions_build_space - # Be verbose and explicit here such that a developer can directly copy the - # `docker/docker_build.sh xxx` command to execute locally. + - name: Download devel packages + uses: actions/download-artifact@v4 + with: + name: open3d-devel-packages + path: open3d-artifacts + - name: List devel artifacts + run: ls -lh open3d-artifacts/ + # Stock nvidia/cuda base (no fat open3d-lib image). Wheels build against + # extracted devel prefixes via OPEN3D_USE_INSTALLED_LIBRARY. - name: Docker build run: | + export OPEN3D_ARTIFACTS_DIR="${GITHUB_WORKSPACE}/open3d-artifacts" if [ "${{ env.PYTHON_VERSION }}" = "3.10" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py310_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then diff --git a/.github/workflows/vtk_packages.yml b/.github/workflows/vtk_packages.yml index f447193a5bd..9ff1a736771 100644 --- a/.github/workflows/vtk_packages.yml +++ b/.github/workflows/vtk_packages.yml @@ -11,7 +11,7 @@ jobs: permissions: contents: write # TODO: Convert to docker - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout source code uses: actions/checkout@v4 diff --git a/.github/workflows/webrtc.yml b/.github/workflows/webrtc.yml index 060831645fb..f09e4e780f2 100644 --- a/.github/workflows/webrtc.yml +++ b/.github/workflows/webrtc.yml @@ -45,7 +45,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies (Ubuntu) if: matrix.os == 'ubuntu-22.04' @@ -106,7 +106,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' - name: Prepare work directory shell: pwsh diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml old mode 100644 new mode 100755 index 0b0882b8b6a..22c9572dfea --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -36,7 +36,7 @@ env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} jobs: - windows: + build-lib: permissions: contents: write # upload id-token: write @@ -136,10 +136,15 @@ jobs: $Env:DEVELOPER_BUILD = "ON" } cmake --version + # Keep 3rdparty downloads inside BUILD_DIR so the build-wheel artifact + # zip includes headers (e.g. vk_mem_alloc.hpp) that ExternalProject + # stamps would otherwise skip re-downloading on a fresh checkout. cmake -G "Visual Studio 17 2022" -A x64 ` -DDEVELOPER_BUILD=$Env:DEVELOPER_BUILD ` -DBUILD_EXAMPLES=OFF ` + -DBUILD_PYTHON_MODULE=OFF ` -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` + -DOPEN3D_THIRD_PARTY_DOWNLOAD_DIR="$env:BUILD_DIR/3rdparty_downloads" ` -DBUILD_SHARED_LIBS=${{ matrix.BUILD_SHARED_LIBS }} ` -DSTATIC_WINDOWS_RUNTIME=${{ matrix.STATIC_RUNTIME }} ` -DBUILD_COMMON_ISPC_ISAS=ON ` @@ -255,23 +260,19 @@ jobs: .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test } Remove-Item -LiteralPath $env:INSTALL_DIR -Recurse -Force -ErrorAction SilentlyContinue - - name: Install Open3D python build requirements - working-directory: ${{ env.SRC_DIR }} - run: | - $ErrorActionPreference = 'Stop' - python -m pip install -U pip==${{ env.PIP_VER }} - python -m pip install -U -r python/requirements_build.txt - - name: Install Python package - working-directory: ${{ env.BUILD_DIR }} - run: | - $ErrorActionPreference = 'Stop' - cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - - name: Import python package - # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. - if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.BUILD_CUDA_MODULE == 'OFF' }} # FIXME + - name: Compress build directory + working-directory: "C:/" + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }} run: | - python -c "import open3d; print('Imported:', open3d)" - python -c "import open3d; print('CUDA enabled: ', open3d.core.cuda.is_available())" + Compress-Archive -Path "${{ env.BUILD_DIR }}" -DestinationPath "${{ env.SRC_DIR }}/open3d_build.zip" + + - name: Upload build artifact + if: ${{ matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' && matrix.BUILD_CUDA_MODULE == 'OFF' && matrix.CONFIG == 'Release' }} + uses: actions/upload-artifact@v4 + with: + name: open3d-build-windows + path: ${{ env.SRC_DIR }}/open3d_build.zip + retention-days: 1 - name: Disk space used run: Get-PSDrive @@ -283,6 +284,7 @@ jobs: attestations: write artifact-metadata: write runs-on: windows-2022 + needs: build-lib strategy: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 @@ -327,6 +329,17 @@ jobs: with: python-version: ${{ matrix.python_version }} + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: open3d-build-windows + path: ${{ env.SRC_DIR }} + + - name: Unpack build artifact + run: | + New-Item -Path ${{ env.BUILD_DIR }} -ItemType Directory -Force + Expand-Archive -Path ${{ env.SRC_DIR }}\open3d_build.zip -DestinationPath "C:\Open3D\" -Force + - name: Install Python dependencies working-directory: ${{ env.SRC_DIR }} run: | @@ -342,23 +355,33 @@ jobs: python -m pip install -U -r open3d_ml/requirements-torch.txt } - - name: Config run: | $ErrorActionPreference = 'Stop' - New-Item -Path ${{ env.BUILD_DIR }} -ItemType Directory cd ${{ env.BUILD_DIR }} if ($Env:DEVELOPER_BUILD -ne "OFF") { $Env:DEVELOPER_BUILD = "ON" } + # Reuse the static build-lib artifact (OFF + STATIC_RUNTIME=ON). Do not + # use cmake --fresh: that would drop CMakeCache while leaving + # ExternalProject stamps that skip re-download of missing headers. + # Match build-lib download dir so VMA/etc. resolve inside the zip. + # Clear cached Python/Torch entries: build-lib cached its own Python + # (BUILD_PYTHON_MODULE=OFF, no torch); re-detect this runner's Python + # (where torch was just installed) so FindPytorch/pybind use it. cmake -G "Visual Studio 17 2022" -A x64 ` + -U "Python3*" -U "PYTHON_*" -U "Pytorch*" -U "Torch*" ` + -DPython3_EXECUTABLE="$((Get-Command python).Source)" ` -DCMAKE_INSTALL_PREFIX="$env:INSTALL_DIR" ` -DDEVELOPER_BUILD="$Env:DEVELOPER_BUILD" ` + -DOPEN3D_THIRD_PARTY_DOWNLOAD_DIR="$env:BUILD_DIR/3rdparty_downloads" ` -DBUILD_SHARED_LIBS=OFF ` -DSTATIC_WINDOWS_RUNTIME=ON ` -DBUILD_COMMON_ISPC_ISAS=ON ` + -DBUILD_PYTHON_MODULE=ON ` -DBUILD_AZURE_KINECT=ON ` -DBUILD_LIBREALSENSE=ON ` + -DBUILD_WEBRTC=ON ` -DBUILD_JUPYTER_EXTENSION=ON ` -DBUILD_PYTORCH_OPS=${{ env.BUILD_PYTORCH_OPS }} ` ${{ env.SRC_DIR }} diff --git a/.gitignore b/.gitignore index 66eaaea98e2..ad67c7fc54b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,9 @@ # Docker build artifacts (wheel, dev package, ccache) docker/*.tar.gz docker/*.tar.xz +open3d-artifacts/*.tar.xz +open3d-artifacts/*.deb +open3d-artifacts/*.whl # Windows build temp cmake file CMakeSettings.json diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index 7a1dc143172..7ac62528ae7 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -489,6 +489,13 @@ if(BUILD_CUDA_MODULE) find_package(CUDAToolkit REQUIRED) endif() +# When building only pybind (+ ML ops) against an installed Open3D devel +# package, skip all heavy ExternalProjects (assimp, embree, vtk, filament, …). +if(OPEN3D_USE_INSTALLED_LIBRARY) + include(${CMAKE_CURRENT_LIST_DIR}/find_dependencies_installed.cmake) + return() +endif() + # Threads open3d_find_package_3rdparty_library(3rdparty_threads REQUIRED @@ -951,7 +958,25 @@ if(NOT USE_SYSTEM_CURL) endif() target_link_libraries(3rdparty_curl INTERFACE 3rdparty_openssl) endif() -list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS_FROM_CUSTOM Open3D::3rdparty_curl Open3D::3rdparty_openssl) +# curl and openssl (BoringSSL) are mutually referential static archives: curl +# needs OpenSSL symbols and, depending on how the final link line gets +# flattened by CMake (e.g. when Open3D itself is a shared library and must +# fully resolve all symbols at build time), they can end up in an order where +# ld's single left-to-right archive scan fails to resolve symbols. Wrap them +# in --start-group and --end-group on UNIX (non-Apple) to ensure GNU ld +# rescans this group of libraries until all symbols resolve, regardless of +# order. We avoid CMake's modern LINK_GROUP RESCAN generator expression here +# because it produces developer warnings when applied to INTERFACE library +# targets (which Open3D::3rdparty_curl/openssl are). +if(UNIX AND NOT APPLE) + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS_FROM_CUSTOM + "-Wl,-(" + Open3D::3rdparty_curl + Open3D::3rdparty_openssl + "-Wl,-)") +else() + list(APPEND Open3D_3RDPARTY_PRIVATE_TARGETS_FROM_CUSTOM Open3D::3rdparty_curl Open3D::3rdparty_openssl) +endif() # PNG if(USE_SYSTEM_PNG) diff --git a/3rdparty/find_dependencies_installed.cmake b/3rdparty/find_dependencies_installed.cmake new file mode 100644 index 00000000000..3c879f86c87 --- /dev/null +++ b/3rdparty/find_dependencies_installed.cmake @@ -0,0 +1,118 @@ +# Minimal 3rd-party setup for OPEN3D_USE_INSTALLED_LIBRARY=ON. +# find_package(Open3D) has already imported Open3D::Open3D and public +# interface targets (e.g. Open3D::3rdparty_eigen3, Open3D::tbb). This file +# only adds what is still needed to compile pybind and optional ML ops: +# pybind11, header-only deps not exported from shared devel packages, and +# libc++ for Filament-linked libOpen3D on Linux. + +message(STATUS "Using slim 3rdparty setup for OPEN3D_USE_INSTALLED_LIBRARY") + +# Pybind11 (not part of the C++ devel package) +if(BUILD_PYTHON_MODULE) + if(USE_SYSTEM_PYBIND11) + find_package(pybind11) + endif() + if(NOT USE_SYSTEM_PYBIND11 OR NOT TARGET pybind11::module) + set(USE_SYSTEM_PYBIND11 OFF) + include(${Open3D_3RDPARTY_DIR}/pybind11/pybind11.cmake) + endif() +endif() + +# Nanoflann is PRIVATE in shared builds, so it is not in Open3DTargets.cmake. +# ML ops still need the headers when BUILD_*_OPS=ON. +if(BUILD_PYTORCH_OPS OR BUILD_TENSORFLOW_OPS) + if(NOT TARGET Open3D::3rdparty_nanoflann) + if(USE_SYSTEM_NANOFLANN) + open3d_find_package_3rdparty_library(3rdparty_nanoflann + PACKAGE nanoflann + VERSION 1.5.0 + TARGETS nanoflann::nanoflann + ) + if(NOT 3rdparty_nanoflann_FOUND) + set(USE_SYSTEM_NANOFLANN OFF) + endif() + endif() + if(NOT USE_SYSTEM_NANOFLANN) + include(${Open3D_3RDPARTY_DIR}/nanoflann/nanoflann.cmake) + open3d_import_3rdparty_library(3rdparty_nanoflann + INCLUDE_DIRS ${NANOFLANN_INCLUDE_DIRS} + DEPENDS ext_nanoflann + ) + endif() + endif() +endif() + +# CUDA header libs for ML ops (same as full find_dependencies, gated on ops) +if(BUILD_CUDA_MODULE AND (BUILD_PYTORCH_OPS OR BUILD_TENSORFLOW_OPS)) + if(CUDAToolkit_VERSION VERSION_LESS "11.0" AND NOT TARGET Open3D::3rdparty_cub) + include(${Open3D_3RDPARTY_DIR}/cub/cub.cmake) + open3d_import_3rdparty_library(3rdparty_cub + INCLUDE_DIRS ${CUB_INCLUDE_DIRS} + DEPENDS ext_cub + ) + endif() + if(NOT TARGET Open3D::3rdparty_cutlass) + if(USE_SYSTEM_CUTLASS) + find_path(3rdparty_cutlass_INCLUDE_DIR NAMES cutlass/cutlass.h) + if(3rdparty_cutlass_INCLUDE_DIR) + add_library(3rdparty_cutlass INTERFACE) + target_include_directories(3rdparty_cutlass INTERFACE + ${3rdparty_cutlass_INCLUDE_DIR}) + add_library(Open3D::3rdparty_cutlass ALIAS 3rdparty_cutlass) + else() + set(USE_SYSTEM_CUTLASS OFF) + endif() + endif() + if(NOT USE_SYSTEM_CUTLASS) + include(${Open3D_3RDPARTY_DIR}/cutlass/cutlass.cmake) + open3d_import_3rdparty_library(3rdparty_cutlass + INCLUDE_DIRS ${CUTLASS_INCLUDE_DIRS} + DEPENDS ext_cutlass + ) + endif() + endif() +endif() + +# Ensure TBB::tbb exists for PYTHON_EXTRA_LIBRARIES and ML ops link lines. +# Devel packages export Open3D::tbb; Open3DConfig may also find_dependency(TBB). +if(NOT TARGET TBB::tbb) + find_package(TBB QUIET) +endif() +if(NOT TARGET TBB::tbb AND TARGET Open3D::tbb) + add_library(TBB::tbb ALIAS Open3D::tbb) +endif() +if(NOT TARGET TBB::tbb) + message(FATAL_ERROR + "TBB::tbb not found. Install TBB or use an Open3D devel package that " + "exports Open3D::tbb / find_dependency(TBB).") +endif() + +# Filament-linked libOpen3D needs matching libc++/libc++abi in the wheel on Linux. +if(BUILD_GUI AND BUILD_PYTHON_MODULE AND UNIX AND NOT APPLE) + if(NOT CPP_LIBRARY OR NOT CPPABI_LIBRARY) + message(STATUS "Searching /usr/lib/llvm-[7..19]/lib/ for libc++ and libc++abi") + foreach(llvm_ver RANGE 7 19) + set(llvm_lib_dir "/usr/lib/llvm-${llvm_ver}/lib") + find_library(CPP_LIBRARY c++ PATHS ${llvm_lib_dir} NO_DEFAULT_PATH) + find_library(CPPABI_LIBRARY c++abi PATHS ${llvm_lib_dir} NO_DEFAULT_PATH) + if(CPP_LIBRARY AND CPPABI_LIBRARY) + message(STATUS "Found libc++ in ${llvm_lib_dir}") + break() + endif() + unset(CPP_LIBRARY CACHE) + unset(CPPABI_LIBRARY CACHE) + endforeach() + endif() + if(NOT CPP_LIBRARY OR NOT CPPABI_LIBRARY) + message(WARNING + "libc++/libc++abi not found; GUI wheels may fail to load Filament " + "symbols from the installed libOpen3D.") + endif() +endif() + +# Keep link-helper lists defined (empty) so open3d_link_3rdparty_libraries is safe +# if any remaining object targets call it. +set(Open3D_3RDPARTY_PUBLIC_TARGETS "") +set(Open3D_3RDPARTY_PRIVATE_TARGETS "") +set(Open3D_3RDPARTY_HEADER_TARGETS "") +set(BUILD_WEBRTC_COMMENT "//") diff --git a/3rdparty/googletest/googletest.cmake b/3rdparty/googletest/googletest.cmake index 50666cc8896..c8d43aaf6c1 100644 --- a/3rdparty/googletest/googletest.cmake +++ b/3rdparty/googletest/googletest.cmake @@ -1,5 +1,11 @@ include(FetchContent) +# Open3D compiles gtest sources into Open3D::3rdparty_googletest; do not install +# the FetchContent targets (shared libgmock.so may be missing / unused). +set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) +set(BUILD_GMOCK ON CACHE BOOL "" FORCE) +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_Declare( ext_googletest PREFIX googletest diff --git a/3rdparty/webrtc/CMakeLists.txt b/3rdparty/webrtc/CMakeLists.txt index 16bc558d0ae..8dda30fadbf 100644 --- a/3rdparty/webrtc/CMakeLists.txt +++ b/3rdparty/webrtc/CMakeLists.txt @@ -32,7 +32,7 @@ cmake_dependent_option(WEBRTC_IS_DEBUG "NOT CMAKE_BUILD_TYPE STREQUAL Debug OR NOT WIN32" ON) option(GLIBCXX_USE_CXX11_ABI "Set -D_GLIBCXX_USE_CXX11_ABI=1" ON) if(MSVC) - option(WEBRTC_STATIC_MSVC_RUNTIME "Use /MT /MTd for WebRTC (Windows)" ON) + option(WEBRTC_STATIC_MSVC_RUNTIME "Use /MT or /MTd for WebRTC (Windows)" ON) endif() # Set paths diff --git a/CMakeLists.txt b/CMakeLists.txt index ba39ca4ef02..2f985ca0c77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,7 @@ endif() include(CMakeDependentOption) # Open3D build options -option(BUILD_SHARED_LIBS "Build shared libraries" OFF) +option(BUILD_SHARED_LIBS "Build shared libraries" ON ) option(BUILD_EXAMPLES "Build Open3D examples programs" ON ) option(BUILD_UNIT_TESTS "Build Open3D unit tests" OFF) option(BUILD_BENCHMARKS "Build the micro benchmarks" OFF) @@ -147,6 +147,13 @@ option(BUILD_TENSORFLOW_OPS "Build ops for TensorFlow" OFF option(BUILD_PYTORCH_OPS "Build ops for PyTorch" OFF) option(BUNDLE_OPEN3D_ML "Includes the Open3D-ML repo in the wheel" OFF) +# Build Python bindings (and optional ML ops) against an installed Open3D +# devel package via find_package(Open3D). Skips compiling the C++ core and +# heavy 3rdparty ExternalProjects. Default OFF keeps the full in-tree build. +option(OPEN3D_USE_INSTALLED_LIBRARY + "Build Python module (and optional ML ops) against find_package(Open3D); skip building the C++ core" + OFF) + # Release build options option(DEVELOPER_BUILD "Add +commit_hash to the project version number" ON ) if (NOT DEVELOPER_BUILD) @@ -251,6 +258,16 @@ endif() if(BUILD_JUPYTER_EXTENSION AND NOT BUILD_PYTHON_MODULE) message(FATAL_ERROR "BUILD_JUPYTER_EXTENSION=ON requires BUILD_PYTHON_MODULE=ON") endif() +if(OPEN3D_USE_INSTALLED_LIBRARY AND NOT BUILD_PYTHON_MODULE) + message(FATAL_ERROR "OPEN3D_USE_INSTALLED_LIBRARY=ON requires BUILD_PYTHON_MODULE=ON") +endif() +if(OPEN3D_USE_INSTALLED_LIBRARY) + # Installed mode only builds pybind (+ optional ML ops). Disable targets + # that require compiling the in-tree C++ core. + set(BUILD_EXAMPLES OFF CACHE BOOL "Build Open3D examples programs" FORCE) + set(BUILD_UNIT_TESTS OFF CACHE BOOL "Build Open3D unit tests" FORCE) + set(BUILD_BENCHMARKS OFF CACHE BOOL "Build the micro benchmarks" FORCE) +endif() # Parse Open3D version number file(STRINGS "cpp/open3d/version.txt" OPEN3D_VERSION_READ) @@ -557,20 +574,35 @@ include(Open3DSYCLTargetSources) # Enumerate all third-party libraries which we need later # This creates the necessary targets and sets the # Open3D_3RDPARTY_*_TARGETS variables we use in open3d_link_3rdparty_libraries +if(OPEN3D_USE_INSTALLED_LIBRARY) + # Prefer an explicit Open3D_ROOT / Open3D_DIR from the caller (CI extracts + # open3d-devel-*.tar.xz and points CMAKE_PREFIX_PATH / Open3D_ROOT at it). + find_package(Open3D CONFIG REQUIRED) + message(STATUS "OPEN3D_USE_INSTALLED_LIBRARY=ON: using ${Open3D_CONFIG}") + get_target_property(_open3d_imported_type Open3D::Open3D TYPE) + if(NOT _open3d_imported_type STREQUAL "SHARED_LIBRARY") + message(FATAL_ERROR + "OPEN3D_USE_INSTALLED_LIBRARY requires a shared libOpen3D " + "(BUILD_SHARED_LIBS=ON when the devel package was built).") + endif() + set(BUILD_SHARED_LIBS ON CACHE BOOL "Build shared libraries" FORCE) +endif() include(3rdparty/find_dependencies.cmake) -# Open3D library +# Open3D library (or pybind-only when OPEN3D_USE_INSTALLED_LIBRARY=ON) add_subdirectory(cpp) -# Examples -add_subdirectory(examples) +if(NOT OPEN3D_USE_INSTALLED_LIBRARY) + # Examples + add_subdirectory(examples) -# Documentation -add_subdirectory(docs) + # Documentation + add_subdirectory(docs) -# Install CMake configuration files -install(EXPORT ${PROJECT_NAME}Targets NAMESPACE ${PROJECT_NAME}:: DESTINATION ${Open3D_INSTALL_CMAKE_DIR}) -export(EXPORT ${PROJECT_NAME}Targets NAMESPACE ${PROJECT_NAME}::) + # Install CMake configuration files + install(EXPORT ${PROJECT_NAME}Targets NAMESPACE ${PROJECT_NAME}:: DESTINATION ${Open3D_INSTALL_CMAKE_DIR}) + export(EXPORT ${PROJECT_NAME}Targets NAMESPACE ${PROJECT_NAME}::) +endif() if (Python3_EXECUTABLE) # `make check-style` checks style for c++/cuda/python/ipynb files @@ -588,7 +620,9 @@ if (Python3_EXECUTABLE) ) endif() -include(Open3DPackaging) +if(NOT OPEN3D_USE_INSTALLED_LIBRARY) + include(Open3DPackaging) +endif() # `make check-cpp-style` checks style for c++/cuda files. # This works outside of python virtualenv. diff --git a/cmake/Open3DPrintConfigurationSummary.cmake b/cmake/Open3DPrintConfigurationSummary.cmake index c12946af62c..4d3c41c6505 100644 --- a/cmake/Open3DPrintConfigurationSummary.cmake +++ b/cmake/Open3DPrintConfigurationSummary.cmake @@ -37,6 +37,7 @@ function(open3d_print_configuration_summary) open3d_aligned_print("Build Unit Tests" "${BUILD_UNIT_TESTS}") open3d_aligned_print("Build Examples" "${BUILD_EXAMPLES}") open3d_aligned_print("Build Python Module" "${BUILD_PYTHON_MODULE}") + open3d_aligned_print("Use Installed Library" "${OPEN3D_USE_INSTALLED_LIBRARY}") open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}") open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}") diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8662fb371c5..79e1fa10a8b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,12 +1,23 @@ -add_subdirectory(open3d) -add_subdirectory(tools) -add_subdirectory(apps) -if (BUILD_UNIT_TESTS) - add_subdirectory(tests) -endif() -if (BUILD_BENCHMARKS) - add_subdirectory(benchmarks) -endif() -if (BUILD_PYTHON_MODULE) - add_subdirectory(pybind) +if(OPEN3D_USE_INSTALLED_LIBRARY) + # Build only Python bindings and optional ML ops against find_package(Open3D). + # Skip in-tree core, tools, apps, and tests (already in the devel package). + if(BUILD_TENSORFLOW_OPS OR BUILD_PYTORCH_OPS) + add_subdirectory(open3d/ml) + endif() + if(BUILD_PYTHON_MODULE) + add_subdirectory(pybind) + endif() +else() + add_subdirectory(open3d) + add_subdirectory(tools) + add_subdirectory(apps) + if(BUILD_UNIT_TESTS) + add_subdirectory(tests) + endif() + if(BUILD_BENCHMARKS) + add_subdirectory(benchmarks) + endif() + if(BUILD_PYTHON_MODULE) + add_subdirectory(pybind) + endif() endif() diff --git a/cpp/open3d/core/Device.cpp b/cpp/open3d/core/Device.cpp index e81c386a67e..e74b622084f 100644 --- a/cpp/open3d/core/Device.cpp +++ b/cpp/open3d/core/Device.cpp @@ -18,42 +18,49 @@ namespace open3d { namespace core { -static Device::DeviceType StringToDeviceType(const std::string& type_colon_id) { +static Device::DeviceType ParseDeviceTypeToken(const std::string& token) { + const std::string device_type_lower = utility::ToLower(token); + if (device_type_lower == "cpu") { + return Device::DeviceType::CPU; + } else if (device_type_lower == "cuda") { + return Device::DeviceType::CUDA; + } else if (device_type_lower == "sycl") { + return Device::DeviceType::SYCL; + } + utility::LogError( + "Invalid device type \"{}\". Expected \"CPU\", \"CUDA\", or " + "\"SYCL\".", + token); +} + +static Device::DeviceType StringToDeviceType(const std::string& device_str) { const std::vector tokens = - utility::SplitString(type_colon_id, ":", true); + utility::SplitString(device_str, ":", true); + if (tokens.size() == 1) { + return ParseDeviceTypeToken(tokens[0]); + } if (tokens.size() == 2) { - std::string device_type_lower = utility::ToLower(tokens[0]); - if (device_type_lower == "cpu") { - return Device::DeviceType::CPU; - } else if (device_type_lower == "cuda") { - return Device::DeviceType::CUDA; - } else if (device_type_lower == "sycl") { - return Device::DeviceType::SYCL; - } else { - utility::LogError( - "Invalid device string {}. Valid device strings are like " - "\"CPU:0\", \"CUDA:1\" or \"SYCL:0\"", - type_colon_id); - } - } else { - utility::LogError( - "Invalid device string {}. Valid device strings are like " - "\"CPU:0\", \"CUDA:1\" or \"SYCL:0\"", - type_colon_id); + return ParseDeviceTypeToken(tokens[0]); } + utility::LogError( + "Invalid device string {}. Valid device strings are like " + "\"CPU\", \"CUDA:0\", or \"SYCL:1\"", + device_str); } -static int StringToDeviceId(const std::string& type_colon_id) { +static int StringToDeviceId(const std::string& device_str) { const std::vector tokens = - utility::SplitString(type_colon_id, ":", true); + utility::SplitString(device_str, ":", true); + if (tokens.size() == 1) { + return 0; + } if (tokens.size() == 2) { return std::stoi(tokens[1]); - } else { - utility::LogError( - "Invalid device string {}. Valid device strings are like " - "\"CPU:0\", \"CUDA:1\" or \"SYCL:0\"", - type_colon_id); } + utility::LogError( + "Invalid device string {}. Valid device strings are like " + "\"CPU\", \"CUDA:0\", or \"SYCL:1\"", + device_str); } Device::Device(DeviceType device_type, int device_id) @@ -68,9 +75,8 @@ Device::Device(DeviceType device_type, int device_id) Device::Device(const std::string& device_type, int device_id) : Device(device_type + ":" + std::to_string(device_id)) {} -Device::Device(const std::string& type_colon_id) - : Device(StringToDeviceType(type_colon_id), - StringToDeviceId(type_colon_id)) {} +Device::Device(const std::string& device_str) + : Device(StringToDeviceType(device_str), StringToDeviceId(device_str)) {} bool Device::operator==(const Device& other) const { return this->device_type_ == other.device_type_ && diff --git a/cpp/open3d/core/Device.h b/cpp/open3d/core/Device.h index d215d16e55e..8c3d33c0a89 100644 --- a/cpp/open3d/core/Device.h +++ b/cpp/open3d/core/Device.h @@ -31,10 +31,12 @@ class Device { explicit Device(DeviceType device_type, int device_id); /// Constructor from device type string and device id. + /// Use ``Device("cuda")`` or ``Device("CUDA:0")`` for id 0. explicit Device(const std::string& device_type, int device_id); - /// Constructor from string, e.g. "CUDA:0". - explicit Device(const std::string& type_colon_id); + /// Constructor from string, e.g. ``"CUDA:0"``, ``"cuda"``, or ``"cpu"``. + /// Bare type names use device id 0. + Device(const std::string& device_str); bool operator==(const Device& other) const; diff --git a/cpp/open3d/ml/CMakeLists.txt b/cpp/open3d/ml/CMakeLists.txt index 35f0b65112a..66f7b6667f1 100644 --- a/cpp/open3d/ml/CMakeLists.txt +++ b/cpp/open3d/ml/CMakeLists.txt @@ -11,4 +11,8 @@ if (BUILD_PYTORCH_OPS) add_subdirectory(pytorch) endif() -add_subdirectory(contrib) +# ml_contrib is an OBJECT library linked into libOpen3D. Skip it when building +# only ops/pybind against an installed Open3D (contrib is already in the .so). +if (NOT OPEN3D_USE_INSTALLED_LIBRARY) + add_subdirectory(contrib) +endif() diff --git a/cpp/open3d/ml/pytorch/CMakeLists.txt b/cpp/open3d/ml/pytorch/CMakeLists.txt index cb47c201f7a..8983e48fabb 100644 --- a/cpp/open3d/ml/pytorch/CMakeLists.txt +++ b/cpp/open3d/ml/pytorch/CMakeLists.txt @@ -124,12 +124,10 @@ endif() # Set output directory according to architecture (cpu/cuda) get_target_property(TORCH_OPS_DIR open3d_torch_ops LIBRARY_OUTPUT_DIRECTORY) -set(TORCH_OPS_ARCH_DIR - "${TORCH_OPS_DIR}/$,cuda,cpu>") set_target_properties(open3d_torch_ops PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${TORCH_OPS_ARCH_DIR}" - ARCHIVE_OUTPUT_DIRECTORY "${TORCH_OPS_ARCH_DIR}" - RUNTIME_OUTPUT_DIRECTORY "${TORCH_OPS_ARCH_DIR}") + LIBRARY_OUTPUT_DIRECTORY "${TORCH_OPS_DIR}" + ARCHIVE_OUTPUT_DIRECTORY "${TORCH_OPS_DIR}" + RUNTIME_OUTPUT_DIRECTORY "${TORCH_OPS_DIR}") # Do not add "lib" prefix set_target_properties(open3d_torch_ops PROPERTIES PREFIX "") diff --git a/cpp/open3d/ml/tensorflow/CMakeLists.txt b/cpp/open3d/ml/tensorflow/CMakeLists.txt index 5e25e3b9cb0..21c7e47ca28 100644 --- a/cpp/open3d/ml/tensorflow/CMakeLists.txt +++ b/cpp/open3d/ml/tensorflow/CMakeLists.txt @@ -126,11 +126,9 @@ open3d_enable_strip(open3d_tf_ops) # Set output directory according to architecture (cpu/cuda) get_target_property(TF_OPS_DIR open3d_tf_ops LIBRARY_OUTPUT_DIRECTORY) -set(TF_OPS_ARCH_DIR - "${TF_OPS_DIR}/$,cuda,cpu>") set_target_properties(open3d_tf_ops PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${TF_OPS_ARCH_DIR}" - ARCHIVE_OUTPUT_DIRECTORY "${TF_OPS_ARCH_DIR}") + LIBRARY_OUTPUT_DIRECTORY "${TF_OPS_DIR}" + ARCHIVE_OUTPUT_DIRECTORY "${TF_OPS_DIR}") # Do not add "lib" prefix set_target_properties(open3d_tf_ops PROPERTIES PREFIX "") diff --git a/cpp/open3d/utility/FileSystem.cpp b/cpp/open3d/utility/FileSystem.cpp index 8e6389a8d84..31c6e470d21 100644 --- a/cpp/open3d/utility/FileSystem.cpp +++ b/cpp/open3d/utility/FileSystem.cpp @@ -23,6 +23,7 @@ #endif #else #include +#include #include #include #include @@ -132,6 +133,56 @@ std::string GetWorkingDirectory() { return std::string(buff); } +std::string GetSelfBinaryDirectory() { +#if defined(__APPLE__) || defined(__linux__) + // dladdr() resolves which shared-library image owns the given code address. + // Passing the address of this very function gives us *this* dylib/so/exe. + // This is POSIX-standard and works identically on macOS and Linux. + ::Dl_info info; + if (::dladdr(reinterpret_cast(&GetSelfBinaryDirectory), &info) && + info.dli_fname && info.dli_fname[0]) { + char resolved[PATH_MAX]; + const char *p = ::realpath(info.dli_fname, resolved) ? resolved + : info.dli_fname; + std::string path(p); + auto slash = path.rfind('/'); + return (slash != std::string::npos) ? path.substr(0, slash) : path; + } + +#elif defined(_WIN32) + // GetModuleHandleExW with FLAG_FROM_ADDRESS retrieves the HMODULE of + // whichever DLL/EXE contains the given virtual address – i.e. this module. + HMODULE hModule = nullptr; + if (::GetModuleHandleExW( + GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&GetSelfBinaryDirectory), &hModule)) { + wchar_t buf[MAX_PATH]; + DWORD len = ::GetModuleFileNameW(hModule, buf, MAX_PATH); + if (len > 0 && len < MAX_PATH) { + // Strip the filename to get the directory. + wchar_t *last_sep = ::wcsrchr(buf, L'\\'); + if (last_sep) *last_sep = L'\0'; + + // Convert UTF-16 directory to UTF-8. + int sz = ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, nullptr, 0, + nullptr, nullptr); + if (sz > 1) { + std::string result(static_cast(sz) - 1, '\0'); + ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, &result[0], sz, + nullptr, nullptr); + // Normalise backslashes for the rest of the codebase. + for (char &c : result) { + if (c == '\\') c = '/'; + } + return result; + } + } + } +#endif + return {}; +} + std::vector GetPathComponents(const std::string &path) { auto SplitByPathSeparators = [](const std::string &path) { std::vector components; diff --git a/cpp/open3d/utility/FileSystem.h b/cpp/open3d/utility/FileSystem.h index f1f56f99a5e..6c7a86e2d1d 100644 --- a/cpp/open3d/utility/FileSystem.h +++ b/cpp/open3d/utility/FileSystem.h @@ -45,6 +45,8 @@ std::string GetRegularizedDirectoryName(const std::string &directory); std::string GetWorkingDirectory(); +std::string GetSelfBinaryDirectory(); + std::vector GetPathComponents(const std::string &path); std::string GetTempDirectoryPath(); diff --git a/cpp/open3d/utility/Logging.cpp b/cpp/open3d/utility/Logging.cpp index 7440212f097..f272fa9efd5 100644 --- a/cpp/open3d/utility/Logging.cpp +++ b/cpp/open3d/utility/Logging.cpp @@ -34,9 +34,6 @@ struct Logger::Impl { // The current print function. std::function print_fcn_; - // The default print function (that prints to console). - static std::function console_print_fcn_; - // Verbosity level. VerbosityLevel verbosity_level_; @@ -57,11 +54,18 @@ struct Logger::Impl { } }; -std::function Logger::Impl::console_print_fcn_ = - [](const std::string &msg) { std::cout << msg << std::endl; }; +namespace { + +const std::function &GetDefaultPrintFunction() { + static const std::function default_fcn = + [](const std::string &msg) { std::cout << msg << std::endl; }; + return default_fcn; +} + +} // namespace Logger::Logger() : impl_(new Logger::Impl()) { - impl_->print_fcn_ = Logger::Impl::console_print_fcn_; + impl_->print_fcn_ = GetDefaultPrintFunction(); impl_->verbosity_level_ = VerbosityLevel::Info; } @@ -118,7 +122,7 @@ const std::function Logger::GetPrintFunction() { } void Logger::ResetPrintFunction() { - impl_->print_fcn_ = impl_->console_print_fcn_; + impl_->print_fcn_ = GetDefaultPrintFunction(); } void Logger::SetVerbosityLevel(VerbosityLevel verbosity_level) { diff --git a/cpp/open3d/visualization/CMakeLists.txt b/cpp/open3d/visualization/CMakeLists.txt index c9c90d1295c..25d72ef7fcf 100644 --- a/cpp/open3d/visualization/CMakeLists.txt +++ b/cpp/open3d/visualization/CMakeLists.txt @@ -108,6 +108,10 @@ if (BUILD_GUI) rendering/gaussian_splat/GaussianSplatVulkanBackend.cpp rendering/gaussian_splat/GaussianSplatVulkanInteropContext.cpp ) + # visualization_impl is an OBJECT library; add_dependencies ensures + # correct build order with ext_vkmemalloc/ext_vmahpp across all + # generators. + add_dependencies(visualization_impl ext_vkmemalloc ext_vmahpp) endif() target_sources(visualization PRIVATE diff --git a/cpp/open3d/visualization/gui/Application.cpp b/cpp/open3d/visualization/gui/Application.cpp index ebc61d5dbc7..0619b180048 100644 --- a/cpp/open3d/visualization/gui/Application.cpp +++ b/cpp/open3d/visualization/gui/Application.cpp @@ -12,12 +12,6 @@ #include // so APIENTRY gets defined and GLFW doesn't define it #endif // _MSC_VER -// Platform headers for GetSelfBinaryDir() -#if defined(__APPLE__) || defined(__linux__) -#include // dladdr (POSIX; libSystem on macOS, libdl on Linux) -#include // PATH_MAX -#endif - #include #include #include // std::getenv @@ -53,69 +47,11 @@ namespace { const double RUNLOOP_DELAY_SEC = 0.010; -// Returns the directory that contains the currently-running binary or shared -// library (whichever loaded this translation unit). This works whether the -// caller is a standalone executable or a Python/app process that loaded the -// Open3D DLL/dylib/.so, making resource discovery robust across deployment -// scenarios. -std::string GetSelfBinaryDir() { - namespace o3dfs = open3d::utility::filesystem; - -#if defined(__APPLE__) || defined(__linux__) - // dladdr() resolves which shared-library image owns the given code address. - // Passing the address of this very function gives us *this* dylib/so/exe. - // This is POSIX-standard and works identically on macOS and Linux. - ::Dl_info info; - if (::dladdr(reinterpret_cast(&GetSelfBinaryDir), &info) && - info.dli_fname && info.dli_fname[0]) { - char resolved[PATH_MAX]; - const char *p = ::realpath(info.dli_fname, resolved) ? resolved - : info.dli_fname; - std::string path(p); - auto slash = path.rfind('/'); - return (slash != std::string::npos) ? path.substr(0, slash) : path; - } - -#elif defined(_WIN32) - // GetModuleHandleExW with FLAG_FROM_ADDRESS retrieves the HMODULE of - // whichever DLL/EXE contains the given virtual address – i.e. this module. - HMODULE hModule = nullptr; - if (::GetModuleHandleExW( - GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - reinterpret_cast(&GetSelfBinaryDir), &hModule)) { - wchar_t buf[MAX_PATH]; - DWORD len = ::GetModuleFileNameW(hModule, buf, MAX_PATH); - if (len > 0 && len < MAX_PATH) { - // Strip the filename to get the directory. - wchar_t *last_sep = ::wcsrchr(buf, L'\\'); - if (last_sep) *last_sep = L'\0'; - - // Convert UTF-16 directory to UTF-8. - int sz = ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, nullptr, 0, - nullptr, nullptr); - if (sz > 1) { - std::string result(static_cast(sz) - 1, '\0'); - ::WideCharToMultiByte(CP_UTF8, 0, buf, -1, &result[0], sz, - nullptr, nullptr); - // Normalise backslashes for the rest of the codebase. - for (char &c : result) { - if (c == '\\') c = '/'; - } - return result; - } - } - } -#endif - - return {}; -} - std::string FindResourcePath() { // Search order (stops at the first hit): // 1. OPEN3D_RESOURCE_PATH environment variable. // 2. Subpaths relative to the directory of this binary or shared library, - // discovered via GetSelfBinaryDir(). + // discovered via GetSelfBinaryDirectory(). namespace o3dfs = open3d::utility::filesystem; // ---- Priority 1: explicit environment variable ------------------------- @@ -130,7 +66,7 @@ std::string FindResourcePath() { } // ---- Priority 2: relative to this binary / shared library -------------- - std::string self_dir = GetSelfBinaryDir(); + std::string self_dir = o3dfs::GetSelfBinaryDirectory(); if (!self_dir.empty()) { #if defined(__APPLE__) // macOS bundle: .../Contents/MacOS/ -> .../Contents/Resources/ diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index 5a3d8b8893e..65b3fbdc375 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -75,9 +75,9 @@ endif() # At `make`: open3d.so (or the equivalents) will be created at # PYTHON_COMPILED_MODULE_DIR. The default location is -# `build/lib/${CMAKE_BUILD_TYPE}/Python/{cpu|cuda}` +# `build/lib/${CMAKE_BUILD_TYPE}/Python` set(PYTHON_COMPILED_MODULE_DIR - "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/Python/$,cuda,cpu>") + "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/Python") if (UNIX AND NOT APPLE) # Use RPATH instead of RUNPATH in pybind so that needed libc++.so can find child dependant libc++abi.so in RPATH @@ -97,22 +97,31 @@ if (BUILD_SHARED_LIBS) else() set(_libopen3d_soname "libOpen3D.so.${OPEN3D_ABI_VERSION}") endif() + # Use Open3D::Open3D so this works for both in-tree and IMPORTED targets. add_custom_command(TARGET pybind POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different - $ $/${_libopen3d_soname} + $ $/${_libopen3d_soname} ) endif() # Include additional libraries that may be absent from the user system # eg: libc++.so, libc++abi.so (needed by filament) for Linux. # libc++.so is a linker script including libc++.so.1 and libc++abi.so, so append 1 to libc++.so -set(PYTHON_EXTRA_LIBRARIES $) +# Prefer TBB::tbb; fall back to Open3D::tbb from an installed devel package. +if (TARGET TBB::tbb) + set(PYTHON_EXTRA_LIBRARIES $) +elseif (TARGET Open3D::tbb) + set(PYTHON_EXTRA_LIBRARIES $) +else() + message(FATAL_ERROR "TBB target required to package Python wheel extras") +endif() if (BUILD_GUI AND CMAKE_SYSTEM_NAME STREQUAL "Linux") - list(APPEND PYTHON_EXTRA_LIBRARIES ${CPP_LIBRARY}.1 ${CPPABI_LIBRARY}) + if (CPP_LIBRARY AND CPPABI_LIBRARY) + list(APPEND PYTHON_EXTRA_LIBRARIES ${CPP_LIBRARY}.1 ${CPPABI_LIBRARY}) + endif() endif() -if (WITH_OPENMP AND APPLE AND NOT BUILD_SHARED_LIBS) +if (WITH_OPENMP AND APPLE) # Package libomp v11.1.0, if it is not installed. Later version cause crash on -# x86_64 if PyTorch is already imported. Case of shared libopen3d.dylib is not -# handled. +# x86_64 if PyTorch is already imported. # https://github.com/microsoft/LightGBM/issues/4229 list(APPEND PYTHON_EXTRA_LIBRARIES ${OpenMP_libomp_LIBRARY}) execute_process(COMMAND brew list --versions libomp diff --git a/cpp/pybind/core/CMakeLists.txt b/cpp/pybind/core/CMakeLists.txt index 511fe4dfdd7..6366dde25c9 100644 --- a/cpp/pybind/core/CMakeLists.txt +++ b/cpp/pybind/core/CMakeLists.txt @@ -13,7 +13,7 @@ target_sources(pybind PRIVATE tensor_accessor.cpp tensor_converter.cpp tensor_function.cpp - tensor_type_caster.cpp + type_caster.cpp tensor.cpp ) diff --git a/cpp/pybind/core/tensor.cpp b/cpp/pybind/core/tensor.cpp index a2d217141da..68ed8bb0d8a 100644 --- a/cpp/pybind/core/tensor.cpp +++ b/cpp/pybind/core/tensor.cpp @@ -21,7 +21,7 @@ #include "open3d/core/TensorKey.h" #include "pybind/core/core.h" #include "pybind/core/tensor_converter.h" -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/docstring.h" #include "pybind/open3d_pybind.h" #include "pybind/pybind_utils.h" diff --git a/cpp/pybind/core/tensor_type_caster.cpp b/cpp/pybind/core/type_caster.cpp similarity index 73% rename from cpp/pybind/core/tensor_type_caster.cpp rename to cpp/pybind/core/type_caster.cpp index d48194414c7..237d086daa5 100644 --- a/cpp/pybind/core/tensor_type_caster.cpp +++ b/cpp/pybind/core/type_caster.cpp @@ -5,7 +5,7 @@ // SPDX-License-Identifier: MIT // ---------------------------------------------------------------------------- -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/core/tensor_converter.h" @@ -32,5 +32,18 @@ bool type_caster::load(handle src, bool convert) { return false; } +bool type_caster::load(handle src, bool convert) { + if (type_caster_base::load(src, convert)) { + return true; + } + if (convert && py::isinstance(src)) { + holder_ = std::make_unique( + py::cast(src)); + value = holder_.get(); + return true; + } + return false; +} + } // namespace detail } // namespace pybind11 diff --git a/cpp/pybind/core/tensor_type_caster.h b/cpp/pybind/core/type_caster.h similarity index 67% rename from cpp/pybind/core/tensor_type_caster.h rename to cpp/pybind/core/type_caster.h index 19455769f2f..8fd4b48dbea 100644 --- a/cpp/pybind/core/tensor_type_caster.h +++ b/cpp/pybind/core/type_caster.h @@ -7,6 +7,7 @@ #pragma once +#include "open3d/core/Device.h" #include "pybind/open3d_pybind.h" namespace open3d { @@ -15,8 +16,8 @@ class Tensor; } } // namespace open3d -// Define type caster allowing implicit conversion to Tensor from common types. -// Needs to be included in each compilation unit. +// Type casters for implicit Python conversions. Include in each compilation +// unit that binds APIs using these types (see open3d_pybind.h). namespace pybind11 { namespace detail { template <> @@ -29,5 +30,15 @@ struct type_caster std::unique_ptr holder_; }; +template <> +struct type_caster + : public type_caster_base { +public: + bool load(handle src, bool convert); + +private: + std::unique_ptr holder_; +}; + } // namespace detail } // namespace pybind11 diff --git a/cpp/pybind/io/rpc.cpp b/cpp/pybind/io/rpc.cpp index 9871e4ec968..b2ee6e87ab0 100644 --- a/cpp/pybind/io/rpc.cpp +++ b/cpp/pybind/io/rpc.cpp @@ -11,7 +11,7 @@ #include "open3d/io/rpc/MessageUtils.h" #include "open3d/io/rpc/RemoteFunctions.h" #include "open3d/io/rpc/ZMQContext.h" -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/docstring.h" #include "pybind/open3d_pybind.h" diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index aa4171414b0..d1f3f94a56d 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -17,19 +17,11 @@ file(COPY ${PYTHON_PACKAGE_SRC_DIR}/ # 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents) # Optionally other modules e.g. open3d_tf_ops.so may be included. -# Folder structure is base_dir/{cpu|cuda}/{pybind*.so|open3d_{torch|tf}_ops.so}, -# so copy base_dir directly to ${PYTHON_PACKAGE_DST_DIR}/open3d +# Copy each compiled extension directly into ${PYTHON_PACKAGE_DST_DIR}/open3d foreach(COMPILED_MODULE_PATH ${COMPILED_MODULE_PATH_LIST}) - get_filename_component(COMPILED_MODULE_NAME ${COMPILED_MODULE_PATH} NAME) - get_filename_component(COMPILED_MODULE_ARCH_DIR ${COMPILED_MODULE_PATH} DIRECTORY) - get_filename_component(COMPILED_MODULE_BASE_DIR ${COMPILED_MODULE_ARCH_DIR} DIRECTORY) - foreach(ARCH cpu cuda) - if(IS_DIRECTORY "${COMPILED_MODULE_BASE_DIR}/${ARCH}") - file(INSTALL "${COMPILED_MODULE_BASE_DIR}/${ARCH}/" DESTINATION - "${PYTHON_PACKAGE_DST_DIR}/open3d/${ARCH}" - FILES_MATCHING PATTERN "${COMPILED_MODULE_NAME}") - endif() - endforeach() + file(COPY ${COMPILED_MODULE_PATH} + DESTINATION ${PYTHON_PACKAGE_DST_DIR}/open3d/ + FOLLOW_SYMLINK_CHAIN) endforeach() # Include additional libraries that may be absent from the user system # eg: libc++.so and libc++abi.so (needed by filament) diff --git a/cpp/pybind/open3d_pybind.h b/cpp/pybind/open3d_pybind.h index 1204dc53c74..0d6ec27209b 100644 --- a/cpp/pybind/open3d_pybind.h +++ b/cpp/pybind/open3d_pybind.h @@ -39,9 +39,9 @@ #include "open3d/pipelines/registration/PoseGraph.h" #include "open3d/utility/Eigen.h" -// We include the type caster for tensor here because it must be included in -// every compilation unit. -#include "pybind/core/tensor_type_caster.h" +// Type casters must be included in every compilation unit that binds these +// types. +#include "pybind/core/type_caster.h" namespace py = pybind11; using namespace py::literals; diff --git a/cpp/pybind/t/geometry/raycasting_scene.cpp b/cpp/pybind/t/geometry/raycasting_scene.cpp index 0d924fc749b..98ac8d22e3e 100644 --- a/cpp/pybind/t/geometry/raycasting_scene.cpp +++ b/cpp/pybind/t/geometry/raycasting_scene.cpp @@ -6,7 +6,7 @@ // ---------------------------------------------------------------------------- #include "open3d/t/geometry/RaycastingScene.h" -#include "pybind/core/tensor_type_caster.h" +#include "pybind/core/type_caster.h" #include "pybind/t/geometry/geometry.h" namespace open3d { diff --git a/cpp/tests/core/Device.cpp b/cpp/tests/core/Device.cpp index 0f54a962d4e..95ca79df2ec 100644 --- a/cpp/tests/core/Device.cpp +++ b/cpp/tests/core/Device.cpp @@ -7,40 +7,59 @@ #include "open3d/core/Device.h" +#include "open3d/core/Tensor.h" #include "tests/Tests.h" namespace open3d { namespace tests { -TEST(Device, DefaultConstructor) { - core::Device device; - EXPECT_EQ(device.GetType(), core::Device::DeviceType::CPU); - EXPECT_EQ(device.GetID(), 0); -} +namespace { -TEST(Device, CPUMustBeID0) { - EXPECT_EQ(core::Device("CPU:0").GetID(), 0); - EXPECT_THROW(core::Device("CPU:1"), std::runtime_error); +void ExpectDevice(const core::Device& device, + core::Device::DeviceType type, + int id) { + EXPECT_EQ(device.GetType(), type); + EXPECT_EQ(device.GetID(), id); } -TEST(Device, SpecifiedConstructor) { - core::Device device(core::Device::DeviceType::CUDA, 1); - EXPECT_EQ(device.GetType(), core::Device::DeviceType::CUDA); - EXPECT_EQ(device.GetID(), 1); -} +} // namespace -TEST(Device, StringConstructor) { - core::Device device("CUDA:1"); - EXPECT_EQ(device.GetType(), core::Device::DeviceType::CUDA); - EXPECT_EQ(device.GetID(), 1); -} +// Default, typed, and string device construction plus parsing. +TEST(Device, ConstructionAndParsing) { + // Default is CPU:0. + ExpectDevice(core::Device(), core::Device::DeviceType::CPU, 0); + // DeviceType + numeric id. + ExpectDevice(core::Device(core::Device::DeviceType::CUDA, 1), + core::Device::DeviceType::CUDA, 1); + // Type name string + id. + ExpectDevice(core::Device("CUDA", 0), core::Device::DeviceType::CUDA, 0); + + const struct { + const char* str; + core::Device::DeviceType type; + int id; + } kCases[] = { + {"CPU:0", core::Device::DeviceType::CPU, 0}, // Type:id form. + {"CUDA:1", core::Device::DeviceType::CUDA, 1}, // Uppercase CUDA. + {"cuda:1", core::Device::DeviceType::CUDA, 1}, // Lowercase type. + {"cuda", core::Device::DeviceType::CUDA, 0}, // Bare type -> id 0. + {"cpu", core::Device::DeviceType::CPU, 0}, // Bare CPU -> id 0. + }; + for (const auto& c : kCases) { + ExpectDevice(core::Device(c.str), c.type, c.id); + } + + // CPU allows only device id 0. + EXPECT_THROW(core::Device("CPU:1"), std::runtime_error); -TEST(Device, StringConstructorLower) { - core::Device device("cuda:1"); - EXPECT_EQ(device.GetType(), core::Device::DeviceType::CUDA); - EXPECT_EQ(device.GetID(), 1); + // Tensor::To accepts canonical and bare CPU strings. + core::Tensor t = + core::Tensor::Ones({2}, core::Float32, core::Device("CPU:0")); + EXPECT_EQ(t.To(core::Device("CPU:0")).GetDevice(), core::Device("CPU:0")); + EXPECT_EQ(t.To(core::Device("cpu")).GetDevice(), core::Device("CPU:0")); } +// Smoke: logs enumerated available devices. TEST(Device, PrintAvailableDevices) { core::Device::PrintAvailableDevices(); } } // namespace tests diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 5c1701396b0..9d4a6bb04f3 100755 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -97,12 +97,15 @@ RUN apt-get update && apt-get install -y \ # which patch level pyenv will install (latest). ENV PYENV_ROOT=/root/.pyenv ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH" -RUN wget -qO- https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash \ +# Install pyenv if it is not already present in BASE_IMAGE. +RUN if [ ! -d "$PYENV_ROOT" ]; then \ + wget -qO- https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash; \ + fi \ && pyenv update \ - && pyenv install $PYTHON_VERSION \ + && pyenv install -s $PYTHON_VERSION \ && pyenv global $PYTHON_VERSION \ && pyenv rehash \ - && ln -s $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; + && ln -sf $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; RUN python --version && pip --version SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -136,8 +139,11 @@ RUN ccache --version \ # Checkout Open3D-ML main branch # TODO: We may add support for local Open3D-ML repo or pinned ML repo tag +# Install Open3D-ML if it is not already present in BASE_IMAGE. ENV OPEN3D_ML_ROOT=/root/Open3D-ML -RUN git clone --depth 1 https://github.com/isl-org/Open3D-ML.git -b main ${OPEN3D_ML_ROOT} +RUN if [ ! -d "${OPEN3D_ML_ROOT}/.git" ]; then \ + git clone --depth 1 https://github.com/isl-org/Open3D-ML.git -b main ${OPEN3D_ML_ROOT}; \ + fi # Open3D repo # Always keep /root/Open3D as the WORKDIR @@ -157,9 +163,10 @@ RUN source util/ci_utils.sh \ && pip install -r python/requirements_test.txt # Open3D Jupyter dependencies +# Avoid gpg prompting to overwrite it interactively. RUN mkdir -p /etc/apt/keyrings \ && wget -qO- https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ - | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + | gpg --yes --dearmor -o /etc/apt/keyrings/nodesource.gpg \ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_25.x nodistro main" \ | tee /etc/apt/sources.list.d/nodesource.list \ && apt-get update \ @@ -180,9 +187,9 @@ RUN \ export CMAKE_C_COMPILER=gcc; \ export BUILD_ISPC_MODULE=ON; \ fi \ - && mkdir build \ + && mkdir -p build \ && cd build \ - && cmake -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} \ + && cmake --fresh -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} \ -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} \ diff --git a/docker/Dockerfile.openblas b/docker/Dockerfile.openblas index 2689b3e14da..f60912dda4e 100644 --- a/docker/Dockerfile.openblas +++ b/docker/Dockerfile.openblas @@ -10,6 +10,11 @@ ARG CONDA_SUFFIX ARG CMAKE_VERSION ARG PYTHON_VERSION ARG DEVELOPER_BUILD +ARG BUILD_PYTHON_MODULE=ON +# OpenBLAS CI builds are static; do not inherit the repo default +# BUILD_SHARED_LIBS=ON (ARM64 has no WebRTC, so shared+static curl/BoringSSL +# currently leaves OpenSSL unresolved in libOpen3D.so). +ARG BUILD_SHARED_LIBS=OFF RUN if [ -z "${CONDA_SUFFIX}" ]; then echo "Error: ARG CONDA_SUFFIX not specified."; exit 1; fi \ && if [ -z "${CMAKE_VERSION}" ]; then echo "Error: ARG CMAKE_VERSION not specified."; exit 1; fi \ && if [ -z "${PYTHON_VERSION}" ]; then echo "Error: ARG PYTHON_VERSION not specified."; exit 1; fi \ @@ -57,12 +62,15 @@ RUN apt-get update && apt-get install -y \ # which patch level pyenv will install (latest). ENV PYENV_ROOT=/root/.pyenv ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH" -RUN curl https://pyenv.run | bash \ +# Install pyenv if it is not already present in BASE_IMAGE. +RUN if [ ! -d "$PYENV_ROOT" ]; then \ + curl https://pyenv.run | bash; \ + fi \ && pyenv update \ - && pyenv install $PYTHON_VERSION \ + && pyenv install -s $PYTHON_VERSION \ && pyenv global $PYTHON_VERSION \ && pyenv rehash \ - && ln -s $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; + && ln -sf $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; RUN python --version && pip --version # CMake @@ -98,16 +106,19 @@ COPY . /root/Open3D WORKDIR /root/Open3D # Build -RUN mkdir build \ +# Use cmake --fresh to clear old configs and ensure correct Python/CMake settings in build/. +RUN mkdir -p build \ && cd build \ - && cmake \ + && cmake --fresh \ -DBUILD_UNIT_TESTS=ON \ + -DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=~/open3d_install \ -DDEVELOPER_BUILD=${DEVELOPER_BUILD} \ + -DBUILD_PYTHON_MODULE=${BUILD_PYTHON_MODULE} \ .. \ && export NPROC=$(($(nproc)+2)) \ && make -j$NPROC \ - && make install-pip-package -j$NPROC \ + && if [ "${BUILD_PYTHON_MODULE}" = "ON" ]; then make install-pip-package -j$NPROC; fi \ && make install -j$NPROC -RUN cp build/lib/python_package/pip_package/*.whl / +RUN if [ "${BUILD_PYTHON_MODULE}" = "ON" ]; then cp build/lib/python_package/pip_package/*.whl /; fi diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index afbc45be5cb..908cf8b5c13 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -2,23 +2,26 @@ ARG BASE_IMAGE=nvidia/cuda:12.6.3-cudnn-devel-ubuntu22.04 FROM ${BASE_IMAGE} -# Customizable build arguments from cuda.yml +# Customizable build arguments ARG DEVELOPER_BUILD ARG CCACHE_TAR_NAME ARG CMAKE_VERSION ARG PYTHON_VERSION ARG BUILD_TENSORFLOW_OPS ARG BUILD_PYTORCH_OPS +ARG BUILD_PYTHON_MODULE +ARG OPEN3D_BUILD_MODE=wheel ARG CI -# Forward all ARG to ENV -# ci_utils.sh requires these environment variables +# Forward all ARG to ENV (ci_utils.sh reads these) ENV DEVELOPER_BUILD=${DEVELOPER_BUILD} ENV CCACHE_TAR_NAME=${CCACHE_TAR_NAME} ENV CMAKE_VERSION=${CMAKE_VERSION} ENV PYTHON_VERSION=${PYTHON_VERSION} ENV BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} ENV BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} +ENV BUILD_PYTHON_MODULE=${BUILD_PYTHON_MODULE} +ENV OPEN3D_BUILD_MODE=${OPEN3D_BUILD_MODE} # Prevent interactive inputs when installing packages ENV DEBIAN_FRONTEND=noninteractive @@ -27,18 +30,10 @@ ENV SUDO=command SHELL ["/bin/bash", "-c"] -# Fix Nvidia repo key rotation issue -# https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212771 -# https://forums.developer.nvidia.com/t/18-04-cuda-docker-image-is-broken/212892/10 -# https://code.visualstudio.com/remote/advancedcontainers/reduce-docker-warnings#:~:text=Warning%3A%20apt%2Dkey%20output%20should,not%20running%20from%20a%20terminal. -RUN export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \ - && apt-key del 7fa2af80 \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub - # Dependencies: basic RUN apt-get update && apt-get install -y \ wget \ + ca-certificates \ ccache \ build-essential \ libssl-dev \ @@ -57,6 +52,11 @@ RUN apt-get update && apt-get install -y \ liblzma-dev \ && rm -rf /var/lib/apt/lists/* +# Install NVIDIA repository keyring package +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb \ + && dpkg -i cuda-keyring_1.1-1_all.deb \ + && rm cuda-keyring_1.1-1_all.deb + # Dependencies: cmake RUN CMAKE_VERSION_NUMBERS=$(echo "${CMAKE_VERSION}" | cut -d"-" -f2) \ && wget -nv https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION_NUMBERS}/${CMAKE_VERSION}.tar.gz \ @@ -65,10 +65,6 @@ RUN CMAKE_VERSION_NUMBERS=$(echo "${CMAKE_VERSION}" | cut -d"-" -f2) \ ENV PATH=${HOME}/${CMAKE_VERSION}/bin:${PATH} # Download ccache from GCS bucket (optional) -# Example directory structure: -# CCACHE_DIR = ~/.cache/ccache -# CCACHE_DIR_NAME = ccache -# CCACHE_DIR_PARENT = ~/.cache RUN ccache --version \ && CCACHE_DIR=$(ccache -p | grep cache_dir | grep -oE "[^ ]+$") \ && CCACHE_DIR_NAME=$(basename ${CCACHE_DIR}) \ @@ -77,72 +73,89 @@ RUN ccache --version \ && cd ${CCACHE_DIR_PARENT} \ && (wget -nv https://storage.googleapis.com/open3d-ci-cache/${CCACHE_TAR_NAME}.tar.xz https://storage.googleapis.com/open3d-ci-cache/${CCACHE_TAR_NAME}.tar.gz || true) \ && { tar -xf ${CCACHE_TAR_NAME}.tar.?z || true; } -# We need to set ccache size explicitly with -M, otherwise the default size is -# *not* determined by ccache's default, but the downloaded ccache file's config. RUN ccache -M 4G \ && ccache -s -# pyenv -# The pyenv python paths are used during docker run, in this way docker run -# does not need to activate the environment again. -# The soft link from the python patch level version to the python mino version -# ensures python wheel commands (i.e. open3d) are in PATH, since we don't know -# which patch level pyenv will install (latest). +# pyenv (needed for wheel builds; lib mode uses a dummy Python version) ENV PYENV_ROOT=/root/.pyenv ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH" -RUN wget -qO- https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash \ +RUN if [ ! -d "$PYENV_ROOT" ]; then \ + wget -qO- https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash; \ + fi \ && pyenv update \ - && pyenv install $PYTHON_VERSION \ + && pyenv install -s $PYTHON_VERSION \ && pyenv global $PYTHON_VERSION \ && pyenv rehash \ - && ln -s $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; + && ln -sf $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; RUN python --version && pip --version -# Checkout Open3D-ML main branch -# TODO: We may add support for local Open3D-ML repo or pinned ML repo tag +# Checkout Open3D-ML (wheel mode may bundle it; lib mode keeps ops OFF) ENV OPEN3D_ML_ROOT=/root/Open3D-ML -RUN git clone --depth 1 https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT} +RUN if [ ! -d "${OPEN3D_ML_ROOT}/.git" ]; then \ + git clone --depth 1 https://github.com/isl-org/Open3D-ML.git ${OPEN3D_ML_ROOT}; \ + fi # Open3D C++ dependencies -# Done before copying the full Open3D directory for better Docker caching COPY ./util/install_deps_ubuntu.sh /root/Open3D/util/ RUN /root/Open3D/util/install_deps_ubuntu.sh assume-yes \ && rm -rf /var/lib/apt/lists/* -# Open3D Python dependencies +# Open3D Python dependencies (wheel mode) COPY ./util/ci_utils.sh /root/Open3D/util/ COPY ./python/requirements.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ COPY ./python/requirements_build.txt /root/Open3D/python/ -RUN source /root/Open3D/util/ci_utils.sh \ - && install_python_dependencies with-jupyter - -# Open3D Jupyter dependencies -RUN mkdir -p /etc/apt/keyrings \ - && wget -qO- https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ - | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ - && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_25.x nodistro main" \ - | tee /etc/apt/sources.list.d/nodesource.list \ - && apt-get update \ - && apt-get install -y nodejs \ - && rm -rf /var/lib/apt/lists/* \ - && node --version \ - && npm install -g yarn \ - && yarn --version +RUN if [ "${OPEN3D_BUILD_MODE}" = "wheel" ]; then \ + source /root/Open3D/util/ci_utils.sh \ + && install_python_dependencies with-jupyter; \ + fi + +# Open3D Jupyter / Node (wheel mode) +RUN if [ "${OPEN3D_BUILD_MODE}" = "wheel" ]; then \ + mkdir -p /etc/apt/keyrings \ + && wget -qO- https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ + | gpg --yes --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_25.x nodistro main" \ + | tee /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ + && apt-get install -y nodejs \ + && rm -rf /var/lib/apt/lists/* \ + && npm install -g yarn; \ + fi + +# Optional pre-extracted devel packages for wheel-from-installed builds +# (copied by docker_build.sh into docker build context as open3d-artifacts/) +ARG OPEN3D_CPU_DEVEL_TAR= +ARG OPEN3D_CUDA_DEVEL_TAR= +COPY open3d-artifacts/ /tmp/open3d-artifacts/ +RUN mkdir -p /opt/open3d-cpu /opt/open3d-cuda /open3d-artifacts \ + && if [ -n "${OPEN3D_CPU_DEVEL_TAR}" ] && [ -f "/tmp/open3d-artifacts/${OPEN3D_CPU_DEVEL_TAR}" ]; then \ + tar -C /opt/open3d-cpu --strip-components=1 -xf "/tmp/open3d-artifacts/${OPEN3D_CPU_DEVEL_TAR}"; \ + fi \ + && if [ -n "${OPEN3D_CUDA_DEVEL_TAR}" ] && [ -f "/tmp/open3d-artifacts/${OPEN3D_CUDA_DEVEL_TAR}" ]; then \ + tar -C /opt/open3d-cuda --strip-components=1 -xf "/tmp/open3d-artifacts/${OPEN3D_CUDA_DEVEL_TAR}"; \ + fi \ + && cp -a /tmp/open3d-artifacts/. /open3d-artifacts/ 2>/dev/null || true + +ENV OPEN3D_CPU_ROOT=/opt/open3d-cpu +ENV OPEN3D_CUDA_ROOT=/opt/open3d-cuda +ENV OPEN3D_ARTIFACTS_DIR=/open3d-artifacts # Open3D repo -# Always keep /root/Open3D as the WORKDIR COPY . /root/Open3D WORKDIR /root/Open3D -# Build python wheel +# Build: lib packages or wheels (logic lives in util/ci_utils.sh) RUN export NPROC=$(($(nproc)+2)) \ - && export BUILD_SHARED_LIBS=OFF \ && source /root/Open3D/util/ci_utils.sh \ - && build_pip_package build_azure_kinect build_jupyter \ - && if [ ${CI:-}a != a ]; then find /root/Open3D/build -mindepth 1 -maxdepth 1 ! -name 'lib' -exec rm -rf {} + ; fi - # remove build folder (except lib) to save CI space on Github + && if [ "${OPEN3D_BUILD_MODE}" = "lib" ]; then \ + build_devel_packages_cpu_cuda; \ + elif [ -d /opt/open3d-cuda/lib/cmake/Open3D ] && [ -d /opt/open3d-cpu/lib/cmake/Open3D ]; then \ + build_pip_package_from_installed build_azure_kinect build_jupyter; \ + else \ + build_pip_package build_azure_kinect build_jupyter; \ + fi # Compress ccache folder, move to / directory RUN ccache -s \ diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 3eb2c188c3a..b9b3a32fcd2 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -57,10 +57,9 @@ OPTION: sycl-static : SYCL (oneAPI) with static lib # ML CIs (Dockerfile.ci) - 2-jammy : CUDA CI, 2-jammy, developer mode - 3-ml-shared-jammy-release : CUDA CI, 3-ml-shared-jammy (cxx11_abi), release mode - 3-ml-shared-jammy : CUDA CI, 3-ml-shared-jammy (cxx11_abi), developer mode - 5-ml-noble : CUDA CI, 5-ml-noble, developer mode + 2-noble : CUDA CI, 2-noble, developer mode + 3-ml-shared-noble-release : CUDA CI, 3-ml-shared-noble (cxx11_abi), release mode + 3-ml-shared-noble : CUDA CI, 3-ml-shared-noble (cxx11_abi), developer mode # CUDA wheels (Dockerfile.wheel) cuda_wheel_py310_dev : CUDA Python 3.10 wheel, developer mode @@ -105,13 +104,13 @@ openblas_export_env() { if [[ "amd64" =~ ^($options)$ ]]; then echo "[openblas_export_env()] platform AMD64" export DOCKER_TAG=open3d-ci:openblas-amd64 - export BASE_IMAGE=ubuntu:22.04 + export BASE_IMAGE=${BASE_IMAGE:-ubuntu:22.04} export CONDA_SUFFIX=x86_64 export CMAKE_VERSION=${CMAKE_VERSION} elif [[ "arm64" =~ ^($options)$ ]]; then echo "[openblas_export_env()] platform ARM64" export DOCKER_TAG=open3d-ci:openblas-arm64 - export BASE_IMAGE=arm64v8/ubuntu:22.04 + export BASE_IMAGE=${BASE_IMAGE:-arm64v8/ubuntu:22.04} export CONDA_SUFFIX=aarch64 export CMAKE_VERSION=${CMAKE_VERSION} else @@ -153,6 +152,12 @@ openblas_export_env() { export BUILD_PYTORCH_OPS=OFF export BUILD_TENSORFLOW_OPS=OFF export BUILD_SYCL_MODULE=OFF + + if [[ "core" =~ ^($options)$ ]]; then + export BUILD_PYTHON_MODULE=OFF + else + export BUILD_PYTHON_MODULE=ON + fi } openblas_build() { @@ -165,18 +170,23 @@ openblas_build() { --build-arg CMAKE_VERSION="${CMAKE_VERSION}" \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ --build-arg DEVELOPER_BUILD="${DEVELOPER_BUILD}" \ + --build-arg BUILD_PYTHON_MODULE="${BUILD_PYTHON_MODULE}" \ + --build-arg BUILD_SHARED_LIBS="${BUILD_SHARED_LIBS}" \ -t "${DOCKER_TAG}" \ -f docker/Dockerfile.openblas . popd - docker run -v "${PWD}:/opt/mount" --rm "${DOCKER_TAG}" \ - bash -c "cp /*.whl /opt/mount \ - && chown $(id -u):$(id -g) /opt/mount/*.whl" + if [ "$BUILD_PYTHON_MODULE" != "OFF" ]; then + docker run -v "${PWD}:/opt/mount" --rm "${DOCKER_TAG}" \ + bash -c "cp /*.whl /opt/mount \ + && chown $(id -u):$(id -g) /opt/mount/*.whl" + fi } cuda_wheel_build() { - BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 + BASE_IMAGE="${BASE_IMAGE:-nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04}" CCACHE_TAR_NAME=open3d-ubuntu-2204-cuda-ci-ccache + DOCKER_TAG="open3d-ci:wheel" options="$(echo "$@" | tr ' ' '|')" echo "[cuda_wheel_build()] options: ${options}" @@ -199,10 +209,54 @@ cuda_wheel_build() { else DEVELOPER_BUILD=OFF fi - echo "[cuda_wheel_build()] PYTHON_VERSION: ${PYTHON_VERSION}" - echo "[cuda_wheel_build()] DEVELOPER_BUILD: ${DEVELOPER_BUILD}" - echo "[cuda_wheel_build()] BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS:?'env var must be set.'}" - echo "[cuda_wheel_build()] BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:?'env var must be set.'}" + + # build-lib: produce CPU/CUDA devel packages + viewer + tests bundle + # (no Python wheel). Otherwise build wheels (from devel tars when present). + local OPEN3D_BUILD_MODE=wheel + local OPEN3D_CPU_DEVEL_TAR="" + local OPEN3D_CUDA_DEVEL_TAR="" + if [[ "build-lib" =~ ^($options)$ ]]; then + OPEN3D_BUILD_MODE=lib + BUILD_PYTHON_MODULE=OFF + # ML ops belong in the wheel stage (Python/Torch/TF ABI). + BUILD_PYTORCH_OPS=OFF + BUILD_TENSORFLOW_OPS=OFF + else + BUILD_PYTHON_MODULE=ON + # Pick up devel tars from OPEN3D_ARTIFACTS_DIR or ./open3d-artifacts. + # When CI already downloaded into open3d-artifacts/, skip the no-op cp + # (cp -a same-file fails with exit 1 under set -e). + local artifacts_src="${OPEN3D_ARTIFACTS_DIR:-${HOST_OPEN3D_ROOT}/open3d-artifacts}" + mkdir -p "${HOST_OPEN3D_ROOT}/open3d-artifacts" + if [[ -d "${artifacts_src}" ]]; then + shopt -s nullglob + OPEN3D_CPU_DEVEL_TAR="" + OPEN3D_CUDA_DEVEL_TAR="" + local artifacts_dst="${HOST_OPEN3D_ROOT}/open3d-artifacts" + local artifacts_src_resolved artifacts_dst_resolved + artifacts_src_resolved="$(cd "${artifacts_src}" && pwd -P)" + artifacts_dst_resolved="$(cd "${artifacts_dst}" && pwd -P)" + for f in "${artifacts_src}"/open3d-devel-*.tar.xz; do + local base + base="$(basename "$f")" + if [[ "${base}" == *"-cuda-"* ]]; then + OPEN3D_CUDA_DEVEL_TAR="${base}" + else + OPEN3D_CPU_DEVEL_TAR="${base}" + fi + if [[ "${artifacts_src_resolved}" != "${artifacts_dst_resolved}" ]]; then + cp -a "$f" "${artifacts_dst}/" + fi + done + shopt -u nullglob + fi + echo "[cuda_wheel_build()] CPU devel tar: ${OPEN3D_CPU_DEVEL_TAR:-}" + echo "[cuda_wheel_build()] CUDA devel tar: ${OPEN3D_CUDA_DEVEL_TAR:-}" + fi + + # Docker COPY requires the path to exist in the build context. + mkdir -p "${HOST_OPEN3D_ROOT}/open3d-artifacts" + touch "${HOST_OPEN3D_ROOT}/open3d-artifacts/.keep" pushd "${HOST_OPEN3D_ROOT}" docker build \ @@ -213,17 +267,29 @@ cuda_wheel_build() { --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ --build-arg BUILD_TENSORFLOW_OPS="${BUILD_TENSORFLOW_OPS}" \ --build-arg BUILD_PYTORCH_OPS="${BUILD_PYTORCH_OPS}" \ + --build-arg BUILD_PYTHON_MODULE="${BUILD_PYTHON_MODULE}" \ + --build-arg OPEN3D_BUILD_MODE="${OPEN3D_BUILD_MODE}" \ + --build-arg OPEN3D_CPU_DEVEL_TAR="${OPEN3D_CPU_DEVEL_TAR}" \ + --build-arg OPEN3D_CUDA_DEVEL_TAR="${OPEN3D_CUDA_DEVEL_TAR}" \ --build-arg CI="${CI:-}" \ - -t open3d-ci:wheel \ + -t "${DOCKER_TAG}" \ -f docker/Dockerfile.wheel . popd - python_package_dir=/root/Open3D/build/lib/python_package - docker run -v "${PWD}:/opt/mount" --rm open3d-ci:wheel \ - bash -c "cp ${python_package_dir}/pip_package/open3d*.whl /opt/mount \ - && cp /${CCACHE_TAR_NAME}.tar.xz /opt/mount \ - && chown $(id -u):$(id -g) /opt/mount/open3d*.whl \ - && chown $(id -u):$(id -g) /opt/mount/${CCACHE_TAR_NAME}.tar.xz" + if [[ "${OPEN3D_BUILD_MODE}" = "lib" ]]; then + docker run -v "${PWD}:/opt/mount" --rm "${DOCKER_TAG}" \ + bash -c "cp -a /open3d-artifacts/open3d-* /opt/mount/ \ + && cp /${CCACHE_TAR_NAME}.tar.xz /opt/mount/ \ + && chown -R $(id -u):$(id -g) /opt/mount/open3d-* \ + && chown $(id -u):$(id -g) /opt/mount/${CCACHE_TAR_NAME}.tar.xz" + else + python_package_dir=/root/Open3D/build/lib/python_package + docker run -v "${PWD}:/opt/mount" --rm open3d-ci:wheel \ + bash -c "cp ${python_package_dir}/pip_package/open3d*.whl /opt/mount \ + && cp /${CCACHE_TAR_NAME}.tar.xz /opt/mount \ + && chown $(id -u):$(id -g) /opt/mount/open3d*.whl \ + && chown $(id -u):$(id -g) /opt/mount/${CCACHE_TAR_NAME}.tar.xz" + fi } ci_build() { @@ -263,12 +329,12 @@ ci_build() { && chown $(id -u):$(id -g) /opt/mount/open3d*" } -2-jammy_export_env() { - export DOCKER_TAG=open3d-ci:2-jammy +2-noble_export_env() { + export DOCKER_TAG=open3d-ci:2-noble export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 export DEVELOPER_BUILD=ON - export CCACHE_TAR_NAME=open3d-ci-2-jammy + export CCACHE_TAR_NAME=open3d-ci-2-noble export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON @@ -278,12 +344,12 @@ ci_build() { export BUILD_SYCL_MODULE=OFF } -3-ml-shared-jammy_export_env() { - export DOCKER_TAG=open3d-ci:3-ml-shared-jammy +3-ml-shared-noble_export_env() { + export DOCKER_TAG=open3d-ci:3-ml-shared-noble export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 export DEVELOPER_BUILD=ON - export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-jammy + export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-noble export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON @@ -293,12 +359,12 @@ ci_build() { export BUILD_SYCL_MODULE=OFF } -3-ml-shared-jammy-release_export_env() { - export DOCKER_TAG=open3d-ci:3-ml-shared-jammy +3-ml-shared-noble-release_export_env() { + export DOCKER_TAG=open3d-ci:3-ml-shared-noble export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04 export DEVELOPER_BUILD=OFF - export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-jammy + export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-noble export PYTHON_VERSION=3.10 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON @@ -308,21 +374,6 @@ ci_build() { export BUILD_SYCL_MODULE=OFF } -5-ml-noble_export_env() { - export DOCKER_TAG=open3d-ci:5-ml-noble - - export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION_LATEST}-devel-ubuntu22.04 - export DEVELOPER_BUILD=ON - export CCACHE_TAR_NAME=open3d-ci-5-ml-noble - export PYTHON_VERSION=3.12 - export BUILD_SHARED_LIBS=OFF - export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON - export BUILD_PYTORCH_OPS=ON - export PACKAGE=OFF - export BUILD_SYCL_MODULE=OFF -} - cpu-static_export_env() { export DOCKER_TAG=open3d-ci:cpu-static @@ -386,17 +437,33 @@ cpu-shared-ml-release_export_env() { sycl-shared_export_env() { export DOCKER_TAG=open3d-ci:sycl-shared + options="$(echo "$@" | tr ' ' '|')" + + # Matches e.g. "py3.10" as passed by the SYCL workflow (py${{ + # matrix.python_version }}, where python_version is dotted, e.g. "3.10"). + if [[ "$options" =~ py3\.(7|8|9|10|11|12|13|14) ]]; then + PYTHON_VERSION=${BASH_REMATCH[0]#py} + export BUILD_PYTHON_MODULE=ON + fi + # https://hub.docker.com/r/intel/oneapi-basekit # https://github.com/intel/oneapi-containers/blob/master/images/docker/basekit/Dockerfile.ubuntu-22.04 - export BASE_IMAGE=intel/cpp-essentials:2025.3.1-0-devel-ubuntu22.04 + export BASE_IMAGE=${BASE_IMAGE:-intel/cpp-essentials:2025.3.1-0-devel-ubuntu22.04} export DEVELOPER_BUILD=${DEVELOPER_BUILD:-ON} export CCACHE_TAR_NAME=open3d-ci-sycl export PYTHON_VERSION=${PYTHON_VERSION:-3.12} export BUILD_SHARED_LIBS=ON + export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=ON export BUILD_PYTORCH_OPS=ON - export PACKAGE=ON + + if [[ "build-lib" =~ ^($options)$ ]]; then + export PACKAGE=ON + else + export PACKAGE=OFF + fi + export BUILD_SYCL_MODULE=ON export IGC_EnableDPEmulation=1 # Enable float64 emulation during compilation @@ -426,7 +493,7 @@ sycl-static_export_env() { } function main() { - if [[ "$#" -ne 1 ]]; then + if [[ "$#" -lt 1 ]]; then echo "Error: invalid number of arguments: $#." >&2 print_usage_and_exit_docker_build fi @@ -536,7 +603,8 @@ function main() { # SYCL CI sycl-shared) - sycl-shared_export_env + shift + sycl-shared_export_env "$@" ci_build ;; sycl-static) @@ -546,51 +614,57 @@ function main() { # CUDA wheels cuda_wheel_py310_dev) - cuda_wheel_build py310 dev + shift + cuda_wheel_build py310 dev "$@" ;; cuda_wheel_py311_dev) - cuda_wheel_build py311 dev + shift + cuda_wheel_build py311 dev "$@" ;; cuda_wheel_py312_dev) - cuda_wheel_build py312 dev + shift + cuda_wheel_build py312 dev "$@" ;; cuda_wheel_py313_dev) - cuda_wheel_build py313 dev + shift + cuda_wheel_build py313 dev "$@" ;; cuda_wheel_py314_dev) - cuda_wheel_build py314 dev + shift + cuda_wheel_build py314 dev "$@" ;; cuda_wheel_py310) - cuda_wheel_build py310 + shift + cuda_wheel_build py310 "$@" ;; cuda_wheel_py311) - cuda_wheel_build py311 + shift + cuda_wheel_build py311 "$@" ;; cuda_wheel_py312) - cuda_wheel_build py312 + shift + cuda_wheel_build py312 "$@" ;; cuda_wheel_py313) - cuda_wheel_build py313 + shift + cuda_wheel_build py313 "$@" ;; cuda_wheel_py314) - cuda_wheel_build py314 + shift + cuda_wheel_build py314 "$@" ;; # ML CIs - 2-jammy) - 2-jammy_export_env - ci_build - ;; - 3-ml-shared-jammy-release) - 3-ml-shared-jammy-release_export_env + 2-noble) + 2-noble_export_env ci_build ;; - 3-ml-shared-jammy) - 3-ml-shared-jammy_export_env + 3-ml-shared-noble-release) + 3-ml-shared-noble-release_export_env ci_build ;; - 5-ml-noble) - 5-ml-noble_export_env + 3-ml-shared-noble) + 3-ml-shared-noble_export_env ci_build ;; *) diff --git a/docker/docker_test.sh b/docker/docker_test.sh index c454f9a96c7..767d2a447ab 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -50,10 +50,9 @@ OPTION: sycl-static : SYCL (oneAPI) with static lib # ML CIs (Dockerfile.ci) - 2-jammy : CUDA CI, 2-jammy, developer mode - 3-ml-shared-jammy-release : CUDA CI, 3-ml-shared-jammy (cxx11_abi), release mode - 3-ml-shared-jammy : CUDA CI, 3-ml-shared-jammy (cxx11_abi), developer mode - 5-ml-noble : CUDA CI, 5-ml-noble, developer mode + 2-noble : CUDA CI, 2-noble, developer mode + 3-ml-shared-noble-release : CUDA CI, 3-ml-shared-noble (cxx11_abi), release mode + 3-ml-shared-noble : CUDA CI, 3-ml-shared-noble (cxx11_abi), developer mode " HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" @@ -348,23 +347,18 @@ sycl-static) ;; # ML CIs -2-jammy) - 2-jammy_export_env +2-noble) + 2-noble_export_env ci_print_env cpp_python_linking_uninstall_test ;; -3-ml-shared-jammy-release) - 3-ml-shared-jammy-release_export_env +3-ml-shared-noble-release) + 3-ml-shared-noble-release_export_env ci_print_env cpp_python_linking_uninstall_test ;; -3-ml-shared-jammy) - 3-ml-shared-jammy_export_env - ci_print_env - cpp_python_linking_uninstall_test - ;; -5-ml-noble) - 5-ml-noble_export_env +3-ml-shared-noble) + 3-ml-shared-noble_export_env ci_print_env cpp_python_linking_uninstall_test ;; diff --git a/docs/compilation.rst b/docs/compilation.rst index fa663c0d72d..3ce492779f7 100644 --- a/docs/compilation.rst +++ b/docs/compilation.rst @@ -25,12 +25,20 @@ System requirements * macOS: Install with Homebrew: ``brew install cmake`` * Windows: Download from: `CMake download page `_ -* CUDA 11.5+ (optional): Open3D supports GPU acceleration of an increasing number - of operations through CUDA on Linux. We recommend using CUDA 12+ for the - best compatibility with recent GPUs and optional external dependencies such - as Tensorflow or PyTorch. Please see the `official documentation - `_ to - install the CUDA toolkit from Nvidia. +* CUDA 11.5+ (optional): Open3D supports GPU acceleration through CUDA on Linux. + Prebuilt wheels statically link the CUDA runtime libraries directly into + ``libOpen3D``, so no separate NVIDIA runtime pip packages are required at + import time. For building from source, install the CUDA toolkit + (``nvidia-smi``, ``nvcc -V``) and configure with ``-DBUILD_CUDA_MODULE=ON`` + (``-DBUILD_WITH_CUDA_STATIC=ON`` by default). We recommend using CUDA 13+ + for the best compatibility with recent GPUs and optional external + dependencies such as Tensorflow or PyTorch. + +* Shared libraries: Open3D builds with ``BUILD_SHARED_LIBS=ON`` by default, + producing a single ``libOpen3D.so`` / ``Open3D.dll`` that contains CPU and one + of CUDA or SYCL code together. Python wheels ship one ``pybind`` extension + module that links against this single library and picks the available + device (CPU + CUDA or SYCL) at runtime. * Ccache 4.0+ (optional, recommended): ccache is a compiler cache that can speed up the compilation process by avoiding recompilation of the same @@ -209,6 +217,42 @@ Finally, verify the Python installation with: Compilation options ------------------- +Build Python against an installed library +````````````````````````````````````````` + +CI builds Linux CUDA wheels by first packaging the C++ library as +``open3d-devel-*.tar.xz``, then compiling only the Python module (and optional +ML ops) against that package. Locally you can use the same path: + +.. code-block:: bash + + # 1) Build and install / package the C++ library (no Python module required) + cmake -S . -B build_lib \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_PYTHON_MODULE=OFF \ + -DBUILD_CUDA_MODULE=ON \ + -DBUILD_PYTORCH_OPS=OFF \ + -DBUILD_TENSORFLOW_OPS=OFF + cmake --build build_lib --target package --parallel + # Extract open3d-devel-*-cuda-*.tar.xz to e.g. /opt/open3d-cuda + + # 2) Build the wheel against the installed prefix + cmake -S . -B build_wheel \ + -DOPEN3D_USE_INSTALLED_LIBRARY=ON \ + -DOpen3D_ROOT=/opt/open3d-cuda \ + -DBUILD_PYTHON_MODULE=ON \ + -DBUILD_CUDA_MODULE=ON \ + -DBUILD_PYTORCH_OPS=ON + cmake --build build_wheel --target pip-package --parallel + +``OPEN3D_USE_INSTALLED_LIBRARY=ON`` skips compiling the C++ core and heavy +third-party ExternalProjects. Torch/TensorFlow ops still compile in this mode +when enabled; their ABI depends on the Python / framework versions, so they are +not part of the shared devel package. + +The default (``OPEN3D_USE_INSTALLED_LIBRARY=OFF``) remains a full in-tree build +with ``make pip-package``. + OpenMP `````` @@ -313,10 +357,14 @@ for all supported ML frameworks and bundling the high level Open3D-ML code. .. code-block:: bash - cmake -DBUILD_CUDA_MODULE=ON -DCMAKE_INSTALL_PREFIX= .. + cmake -DBUILD_CUDA_MODULE=ON -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_INSTALL_PREFIX= .. - Please note that CUDA support is work in progress and experimental. For building - Open3D with CUDA support, ensure that CUDA is properly installed by running following commands: + CUDA runtime libraries are statically linked into ``libOpen3D`` by default + (``-DBUILD_WITH_CUDA_STATIC=ON``), so CUDA wheels do not need any NVIDIA + redistributable pip packages at import time. Pass + ``-DBUILD_WITH_CUDA_STATIC=OFF`` to dynamically link the CUDA runtime + instead. For development, ensure the CUDA toolkit is available: .. code-block:: bash @@ -327,6 +375,89 @@ for all supported ML frameworks and bundling the high level Open3D-ML code. by following the `official documentation. `_ +.. _abi_dependency_compatibility: + +ABI dependency and compatibility +--------------------------------- + +When compiling Open3D from source or using prebuilt wheels with machine learning +(ML) framework bindings (such as PyTorch or TensorFlow) and hardware +acceleration (such as CUDA or SYCL), maintaining Application Binary Interface +(ABI) compatibility across all dependencies is critical. An ABI mismatch +between Open3D, the ML frameworks, and the underlying runtime libraries can lead +to compilation failures, linker errors, or runtime crashes (such as segmentation +faults). + +ABI Dependency Tree +```````````````````` + +The diagram below illustrates how Open3D and its custom operators depend on the +underlying runtime ABI libraries: + +.. code-block:: text + + +-----------------------------------------------------+ + | Runtime ABI Library | + | (glibc, libstdc++, cuda-runtime, sycl-runtime) | + +-----------+--------------+--------------+-----------+ + ^ ^ ^ + | | | + | +-----+-----+ +-----+-----+ + | | PyTorch | |TensorFlow | + | +-----+-----+ +-----+-----+ + | ^ ^ + | | | + | +-----+-----+ +-----+-----+ + | | torch-ops | | tf-ops | + | +-----+-----+ +-----+-----+ + | ^ ^ + | | | + +-----+-----+--------+--------------+-----+ + | Open3D | + +-----------------------------------------+ + +As shown in the diagram: + +* **Open3D** directly links against the core runtime ABI libraries (such as ``glibc``, ``nvidia-rt``, or ``sycl-rt``). CPU, CUDA, and SYCL code all live in the same ``libOpen3D`` shared library. +* When custom ML operators are enabled (``BUILD_PYTORCH_OPS=ON`` or ``BUILD_TENSORFLOW_OPS=ON``), Open3D compiles custom operator libraries (``torch-ops`` and ``tf-ops``). +* These custom operators depend directly on the installed ML frameworks (``pytorch`` and ``tensorflow``). +* Both the ML frameworks and the custom operators must link against the exact same runtime ABI libraries. + +Consequently, the dependency versions must be compatible. Typically, the major +and minor versions of the toolchains and runtime libraries used at build time +must match those used by the installed ML frameworks, and the runtime +environment must satisfy the compatibility guarantees of each library. + +Runtime ABI Libraries and Compatibility Guarantees +````````````````````````````````````````````````````` + +glibc (GNU C Library) +""""""""""""""""""""" + +* **Backward Compatibility**: ``glibc`` guarantees strict backward compatibility. A binary compiled against an older version of ``glibc`` (e.g., ``glibc 2.31`` on Ubuntu 20.04) will run without issues on a system with a newer version of ``glibc`` (e.g., ``glibc 2.35`` on Ubuntu 22.04). It does **not** guarantee forward compatibility. +* **C++ Standard Library ABI (libstdc++)**: + * Open3D compiles with ``GLIBCXX_USE_CXX11_ABI=ON`` by default. + * Modern PyTorch and TensorFlow Linux wheel releases are also built with CXX11 ABI enabled (``_GLIBCXX_USE_CXX11_ABI=1``) by default. + * Open3D's CMake configuration automatically queries and verifies that the ABI configuration of the installed PyTorch and TensorFlow matches Open3D's configuration to prevent linker and runtime errors. + +nvidia-rt (NVIDIA CUDA Runtime and Driver) +"""""""""""""""""""""""""""""""""""""""""" + +* **Static Linking for Safety**: + * To prevent runtime version mismatch issues between different CUDA runtimes, Open3D builds with ``BUILD_WITH_CUDA_STATIC=ON`` by default. + * This statically links CUDA toolkit libraries (such as ``cudart_static``, ``cublas_static``, ``cusolver_static``, ``cusparse_static``, and ``npp*_static``) directly into ``libOpen3D``, isolating Open3D from external CUDA runtime version mismatches. +* **Compatibility Guarantees**: + * **Driver Backward Compatibility**: Newer NVIDIA drivers support older CUDA Toolkit and CUDA Runtime versions. + * **CUDA Minor Version Compatibility**: Starting with CUDA 11, NVIDIA guarantees binary compatibility within the same major version. An application compiled with any CUDA 11.x SDK can run on any driver that supports CUDA 11.0 or later. The same applies to CUDA 12.x and CUDA 13.x. + +sycl-rt (Intel oneAPI DPC++ Runtime) +"""""""""""""""""""""""""""""""""""" + +* **Compatibility Guarantees**: + * **oneAPI Runtime Compatibility**: Intel oneAPI guarantees backward compatibility for the DPC++ runtime (``dpcpp-cpp-rt``). A newer runtime can execute binaries compiled with an older oneAPI compiler. Forward compatibility is not supported. + * **Strict Version Matching**: Due to rapid development in SYCL standards and implementations, it is highly recommended to use matching major and minor versions of the DPC++ compiler and runtime. + * Open3D's SYCL wheels pin the runtime dependency in ``python/requirements_sycl.txt`` (e.g., ``dpcpp-cpp-rt==2025.3.1``). When compiling Open3D with SYCL support, ensure your Intel oneAPI Base Toolkit version is compatible with this runtime. + WebRTC remote visualization ``````````````````````````` diff --git a/open3d-artifacts/.keep b/open3d-artifacts/.keep new file mode 100644 index 00000000000..51f8d718acd --- /dev/null +++ b/open3d-artifacts/.keep @@ -0,0 +1,2 @@ +# Placeholder so Docker COPY open3d-artifacts/ always succeeds. +# CI copies open3d-devel-*.tar.xz here before wheel builds. diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index 87608701d5c..5b9e23454ac 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -15,99 +15,47 @@ # https://github.com/dmlc/xgboost/issues/1715 import os import sys -import re +import site +import warnings os.environ["KMP_DUPLICATE_LIB_OK"] = "True" -# Enable thread composability manager to coordinate Intel OpenMP and TBB threads. Only works with Intel OpenMP. -# TBB must not be already loaded. +# Enable thread composability manager to coordinate Intel OpenMP and TBB +# threads. Only works with Intel OpenMP. TBB must not be already loaded. os.environ["TCM_ENABLE"] = "1" -from ctypes import CDLL -from ctypes.util import find_library from pathlib import Path -import warnings + from open3d._build_config import _build_config -if sys.platform == "win32": # Unix: Use rpath to find libraries - _win32_dll_dir = os.add_dll_directory(str(Path(__file__).parent)) +if sys.platform == "win32": + # Required for CPU wheel (bundled TBB) and SYCL wheel (SYCL runtime, which + # intel-sycl-rt will install in: + # - /Library/bin # (for standard/virtualenv/conda installs) + # - /Library/bin # (for user-level --user installs) + # CUDA runtime is statically linked. + _win32_dll_dirs = [os.add_dll_directory(str(Path(__file__).parent))] + _site_dirs = [*site.PREFIXES, *site.getsitepackages(), site.USER_BASE] + for _site_dir in _site_dirs: + if os.path.isdir(os.path.join(_site_dir, "Library", "bin")): + _win32_dll_dirs.append( + os.add_dll_directory(os.path.join(_site_dir, "Library", "bin"))) + +from open3d.pybind import ( + core, + camera, + data, + geometry, + io, + pipelines, + utility, + t, +) +from open3d import pybind __DEVICE_API__ = "cpu" -if _build_config["BUILD_CUDA_MODULE"]: - # Load CPU pybind dll gracefully without introducing new python variable. - # Do this before loading the CUDA pybind dll to correctly resolve symbols - try: # StopIteration if cpu version not available - CDLL(str(next((Path(__file__).parent / "cpu").glob("pybind*")))) - except StopIteration: - warnings.warn( - "Open3D was built with CUDA support, but Open3D CPU Python " - "bindings were not found. Open3D will not work on systems without" - " CUDA devices.", - ImportWarning, - ) - try: - if sys.platform == "win32" and sys.version_info >= (3, 8): - # Since Python 3.8, the PATH environment variable is not used to find DLLs anymore. - # To allow Windows users to use Open3D with CUDA without running into dependency-problems, - # look for the CUDA bin directory in PATH and explicitly add it to the DLL search path. - cuda_bin_path = None - for path in os.environ['PATH'].split(';'): - # search heuristic: look for a path containing "cuda" and "bin" in this order. - if re.search(r'cuda.*bin', path, re.IGNORECASE): - cuda_bin_path = path - break - - if cuda_bin_path: - os.add_dll_directory(cuda_bin_path) - - # Check CUDA availability without importing CUDA pybind symbols to - # prevent "symbol already registered" errors if first import fails. - _pybind_cuda = CDLL( - str(next((Path(__file__).parent / "cuda").glob("pybind*")))) - if _pybind_cuda.open3d_core_cuda_device_count() > 0: - from open3d.cuda.pybind import ( - core, - camera, - data, - geometry, - io, - pipelines, - utility, - t, - ) - from open3d.cuda import pybind - - __DEVICE_API__ = "cuda" - else: - warnings.warn( - "Open3D was built with CUDA support, but no suitable CUDA " - "devices found. If your system has CUDA devices, check your " - "CUDA drivers and runtime.", - ImportWarning, - ) - except OSError as os_error: - warnings.warn( - f"Open3D was built with CUDA support, but an error ocurred while loading the Open3D CUDA Python bindings. This is usually because the CUDA libraries could not be found. Check your CUDA installation. Falling back to the CPU pybind library. Reported error: {os_error}.", - ImportWarning, - ) - except StopIteration: - warnings.warn( - "Open3D was built with CUDA support, but Open3D CUDA Python " - "binding library not found! Falling back to the CPU Python " - "binding library.", - ImportWarning, - ) - -if __DEVICE_API__ == "cpu": - from open3d.cpu.pybind import ( - core, - camera, - data, - geometry, - io, - pipelines, - utility, - t, - ) - from open3d.cpu import pybind +if core.cuda.is_available(): + __DEVICE_API__ = "cuda" +elif core.sycl.is_available(): + __DEVICE_API__ = "xpu" def _insert_pybind_names(skip_names=()): @@ -115,10 +63,13 @@ def _insert_pybind_names(skip_names=()): python subpackages, since they have a different import mechanism.""" submodules = {} for modname in sys.modules: - if "open3d." + __DEVICE_API__ + ".pybind" in modname: + if "open3d.pybind" in modname: if any("." + skip_name in modname for skip_name in skip_names): continue - subname = modname.replace(__DEVICE_API__ + ".pybind.", "") + # Keep the leading "open3d." so submodules are registered under + # e.g. "open3d.t" rather than a bare "t" (which is not importable + # via `import open3d.t`). + subname = modname.replace("pybind.", "") if subname not in sys.modules: submodules[subname] = sys.modules[modname] sys.modules.update(submodules) @@ -131,7 +82,7 @@ def _insert_pybind_names(skip_names=()): __version__ = "@PROJECT_VERSION@" if int(sys.version_info[0]) < 3: - raise Exception("Open3D only supports Python 3.") + raise RuntimeError("Open3D only supports Python 3.") if (_build_config["BUILD_JUPYTER_EXTENSION"] and os.environ.get( "OPEN3D_DISABLE_WEB_VISUALIZER", "False").lower() != "true"): @@ -210,5 +161,6 @@ def _jupyter_nbextension_paths(): if sys.platform == "win32": - _win32_dll_dir.close() -del os, sys, CDLL, find_library, Path, warnings, _insert_pybind_names + for dll_dir in _win32_dll_dirs: + dll_dir.close() +del os, sys, Path, warnings, _insert_pybind_names diff --git a/python/open3d/core/__init__.py b/python/open3d/core/__init__.py new file mode 100644 index 00000000000..078e436aa30 --- /dev/null +++ b/python/open3d/core/__init__.py @@ -0,0 +1,9 @@ +# ---------------------------------------------------------------------------- +# - Open3D: www.open3d.org - +# ---------------------------------------------------------------------------- +# Copyright (c) 2018-2024 www.open3d.org +# SPDX-License-Identifier: MIT +# ---------------------------------------------------------------------------- + +# Tensor / device API (implemented in the pybind extension). +from open3d.pybind.core import * # noqa: F403 diff --git a/python/open3d/ml/__init__.py b/python/open3d/ml/__init__.py index cf7342e0a27..5834889b534 100644 --- a/python/open3d/ml/__init__.py +++ b/python/open3d/ml/__init__.py @@ -5,12 +5,7 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -import os as _os -import open3d as _open3d -if _open3d.__DEVICE_API__ == 'cuda': - from open3d.cuda.pybind.ml import * -else: - from open3d.cpu.pybind.ml import * +from open3d.pybind.ml import * from . import configs from . import datasets diff --git a/python/open3d/ml/contrib/__init__.py b/python/open3d/ml/contrib/__init__.py index b883e8024fd..a431923712b 100644 --- a/python/open3d/ml/contrib/__init__.py +++ b/python/open3d/ml/contrib/__init__.py @@ -5,8 +5,4 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -import open3d as _open3d -if _open3d.__DEVICE_API__ == 'cuda': - from open3d.cuda.pybind.ml.contrib import * -else: - from open3d.cpu.pybind.ml.contrib import * +from open3d.pybind.ml.contrib import * diff --git a/python/open3d/ml/tf/python/ops/lib.py b/python/open3d/ml/tf/python/ops/lib.py index 4e55e26382d..61a24ebc503 100644 --- a/python/open3d/ml/tf/python/ops/lib.py +++ b/python/open3d/ml/tf/python/ops/lib.py @@ -20,11 +20,8 @@ _package_root = _os.path.join(_this_dir, '..', '..', '..', '..') _lib_ext = {'linux': '.so', 'darwin': '.dylib', 'win32': '.dll'}[_sys.platform] _lib_suffix = '_debug' if _build_config['CMAKE_BUILD_TYPE'] == 'Debug' else '' -_lib_arch = ('cuda', 'cpu') if _build_config["BUILD_CUDA_MODULE"] else ('cpu',) -_lib_path.extend([ - _os.path.join(_package_root, la, 'open3d_tf_ops' + _lib_suffix + _lib_ext) - for la in _lib_arch -]) +_lib_path.append( + _os.path.join(_package_root, 'open3d_tf_ops' + _lib_suffix + _lib_ext)) _load_except = None _loaded = False diff --git a/python/open3d/ml/torch/__init__.py b/python/open3d/ml/torch/__init__.py index 161420fbcb5..76dc74a3cb0 100644 --- a/python/open3d/ml/torch/__init__.py +++ b/python/open3d/ml/torch/__init__.py @@ -18,9 +18,9 @@ if _verp(_torch.__version__).release[:2] != _o3d_torch_version.release[:2]: match_torch_ver = '.'.join( str(v) for v in _o3d_torch_version.release[:2] + ('*',)) - raise Exception('Version mismatch: Open3D needs PyTorch version {}, but ' - 'version {} is installed!'.format(match_torch_ver, - _torch.__version__)) + raise RuntimeError('Version mismatch: Open3D needs PyTorch version {}, but ' + 'version {} is installed!'.format( + match_torch_ver, _torch.__version__)) # Precompiled wheels at # https://github.com/isl-org/open3d_downloads/releases/tag/torch1.8.2 @@ -61,21 +61,15 @@ _package_root = _os.path.join(_this_dir, '..', '..') _lib_ext = {'linux': '.so', 'darwin': '.dylib', 'win32': '.dll'}[_sys.platform] _lib_suffix = '_debug' if _build_config['CMAKE_BUILD_TYPE'] == 'Debug' else '' -_lib_arch = ('cpu',) -if _build_config["BUILD_CUDA_MODULE"] and _torch.cuda.is_available(): - if _torch.version.cuda == _build_config["CUDA_VERSION"]: - _lib_arch = ('cuda', 'cpu') - else: - print("Warning: Open3D was built with CUDA {} but" - "PyTorch was built with CUDA {}. Falling back to CPU for now." - "Otherwise, install PyTorch with CUDA {}.".format( - _build_config["CUDA_VERSION"], _torch.version.cuda, - _build_config["CUDA_VERSION"])) -_lib_path.extend([ - _os.path.join(_package_root, la, - 'open3d_torch_ops' + _lib_suffix + _lib_ext) - for la in _lib_arch -]) +if (_build_config["BUILD_CUDA_MODULE"] and _torch.cuda.is_available() and + _torch.version.cuda != _build_config["CUDA_VERSION"]): + print("Warning: Open3D was built with CUDA {} but" + "PyTorch was built with CUDA {}. Falling back to CPU for now." + "Otherwise, install PyTorch with CUDA {}.".format( + _build_config["CUDA_VERSION"], _torch.version.cuda, + _build_config["CUDA_VERSION"])) +_lib_path.append( + _os.path.join(_package_root, 'open3d_torch_ops' + _lib_suffix + _lib_ext)) _load_except = None _loaded = False diff --git a/python/open3d/visualization/__init__.py b/python/open3d/visualization/__init__.py index cab0cad14ad..90ed21413a1 100644 --- a/python/open3d/visualization/__init__.py +++ b/python/open3d/visualization/__init__.py @@ -6,14 +6,9 @@ # ---------------------------------------------------------------------------- import open3d -if open3d.__DEVICE_API__ == "cuda": - if open3d._build_config["BUILD_GUI"]: - from open3d.cuda.pybind.visualization import gui - from open3d.cuda.pybind.visualization import * -else: - if open3d._build_config["BUILD_GUI"]: - from open3d.cpu.pybind.visualization import gui - from open3d.cpu.pybind.visualization import * +if open3d._build_config["BUILD_GUI"]: + from open3d.pybind.visualization import gui +from open3d.pybind.visualization import * from ._external_visualizer import * from .draw_plotly import get_plotly_fig diff --git a/python/test/core/test_core.py b/python/test/core/test_core.py index 5910a5e012c..51a8d828e0c 100644 --- a/python/test/core/test_core.py +++ b/python/test/core/test_core.py @@ -127,6 +127,20 @@ def test_device(): assert o3c.Device("CUDA", 1).__str__() == "CUDA:1" + device = o3c.Device("cuda") + assert device.get_type() == o3c.Device.DeviceType.CUDA + assert device.get_id() == 0 + + +def test_tensor_to_device_string(): + t = o3c.Tensor([1, 2, 3]) + assert t.to("cpu:0").device == o3c.Device("cpu:0") + assert t.to("cpu").device == o3c.Device("cpu:0") + if o3c.cuda.is_available(): + t_gpu = t.to("cuda:0") + assert t_gpu.device.get_type() == o3c.Device.DeviceType.CUDA + assert t_gpu.to("cpu").device == o3c.Device("cpu:0") + @pytest.mark.parametrize("dtype", list_dtypes()) @pytest.mark.parametrize("device", list_devices(enable_sycl=True)) diff --git a/util/ci_utils.sh b/util/ci_utils.sh old mode 100644 new mode 100755 index 092ae3e103a..b369b1d5de7 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -23,6 +23,7 @@ BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS:-ON} BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:-ON} LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} BUILD_SYCL_MODULE=${BUILD_SYCL_MODULE:-OFF} +BUILD_PYTHON_MODULE=${BUILD_PYTHON_MODULE:-ON} # Dependency versions: # CUDA: see docker/docker_build.sh @@ -120,6 +121,7 @@ build_all() { -DBUILD_UNIT_TESTS=ON -DBUILD_BENCHMARKS=ON -DBUILD_EXAMPLES=OFF + -DBUILD_PYTHON_MODULE="$BUILD_PYTHON_MODULE" ) echo @@ -132,7 +134,9 @@ build_all() { if [[ "$BUILD_SHARED_LIBS" == "ON" ]]; then make package fi - make VERBOSE=1 install-pip-package -j"$NPROC" + if [[ "$BUILD_PYTHON_MODULE" == "ON" ]]; then + make VERBOSE=1 install-pip-package -j"$NPROC" + fi echo } @@ -181,11 +185,9 @@ build_pip_package() { set -u echo - echo Building with CPU only... mkdir -p build - pushd build # PWD=Open3D/build - cmakeOptions=("-DBUILD_SHARED_LIBS=OFF" - "-DDEVELOPER_BUILD=$DEVELOPER_BUILD" + pushd build + cmakeOptions=("-DDEVELOPER_BUILD=$DEVELOPER_BUILD" "-DBUILD_COMMON_ISPC_ISAS=ON" "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT" "-DBUILD_LIBREALSENSE=ON" @@ -199,38 +201,362 @@ build_pip_package() { "-DBUILD_UNIT_TESTS=OFF" "-DBUILD_BENCHMARKS=OFF" "-DBUNDLE_OPEN3D_ML=$BUNDLE_OPEN3D_ML" + "-DBUILD_SHARED_LIBS=ON" ) - set -x # Echo commands on + if [ "$BUILD_CUDA_MODULE" == ON ]; then + install_python_dependencies with-cuda purge-cache + cmakeOptions+=("-DBUILD_CUDA_MODULE=ON" "-DBUILD_COMMON_CUDA_ARCHS=ON") + else + cmakeOptions+=("-DBUILD_CUDA_MODULE=OFF") + fi + set -x + # Always rerun cmake and unset Python cache variables to update cache for + # the current Python and build options. This avoids incorrect/inherited + # Python config and ensures all paths are correct. Do not use --fresh: it + # unnecessarily wipes build objects and wastes disk/CI time. + cmake -U 'Python3*' -U 'PYTHON_*' -U 'Pytorch*' -U 'Torch*' \ + -DBUILD_PYTHON_MODULE="${BUILD_PYTHON_MODULE}" \ + -DPython3_EXECUTABLE="$(command -v python3)" \ + "${cmakeOptions[@]}" .. + set +x + if [ "$BUILD_PYTHON_MODULE" == "OFF" ]; then + echo "Building Open3D C++ Core only..." + make VERBOSE=1 -j"$NPROC" + else + echo "Packaging Open3D pip wheel (single configure)..." + make VERBOSE=1 -j"$NPROC" pip-package + fi + popd + + # A CUDA-enabled build also gets a companion CPU-only "open3d-cpu" wheel + # built alongside it (for users without a CUDA GPU); a CPU-only build IS + # already the CPU wheel, so there is nothing extra to build in that case. + # Prefer build_pip_package_from_installed() in CI (separate devel prefixes). + if [ "$BUILD_PYTHON_MODULE" != "OFF" ] && [ "$BUILD_CUDA_MODULE" == ON ]; then + echo + echo "Building open3d-cpu wheel (reusing single build folder)..." + # 1. Back up the CUDA-enabled wheel(s) to a temporary directory outside build/ + mkdir -p pip_package_backup + cp build/lib/python_package/pip_package/*.whl pip_package_backup/ + + # 2. Reconfigure the existing build directory with CUDA disabled + pushd build + # We must make sure BUILD_CUDA_MODULE=OFF overrides any ON from cmakeOptions. + # Repeating -D left-to-right makes the last one win, so we place it after cmakeOptions. + set -x + cmake "${cmakeOptions[@]}" -DBUILD_CUDA_MODULE=OFF .. + set +x + + # 3. Build the CPU companion wheel + make VERBOSE=1 -j"$NPROC" pip-package + popd + + # 4. Restore the backed-up CUDA wheel(s) into the pip_package directory + mv pip_package_backup/*.whl build/lib/python_package/pip_package/ + rm -rf pip_package_backup + fi + echo +} + +# Selective wipe of Open3D outputs so a CPU→CUDA (or reverse) reconfigure can +# reuse 3rdparty ExternalProjects. Keeps filament/assimp/embree/vtk build trees. +_open3d_wipe_compiled_outputs() { + local build_dir="${1:-build}" + echo "Removing Open3D compiled outputs under ${build_dir} (keeping 3rdparty)" + rm -rf \ + "${build_dir}/bin" \ + "${build_dir}/cpp" \ + "${build_dir}/lib/_build_config.py" \ + "${build_dir}/lib/ml" \ + "${build_dir}/lib/python_package" \ + "${build_dir}/lib/cmake" \ + "${build_dir}/package" \ + "${build_dir}/package-Open3DViewer-deb" \ + "${build_dir}/Open3D" + # Shared lib + static archives produced by the Open3D targets (not 3rdparty) + rm -f "${build_dir}"/lib/Release/libOpen3D.so* \ + "${build_dir}"/lib/Release/*.a \ + "${build_dir}"/lib/libOpen3D.so* 2>/dev/null || true +} + +# Build CPU then CUDA open3d-devel packages (+ viewer deb + C++ tests bundle). +# ML ops are OFF (Python/Torch/TF ABI belongs in the wheel stage). +# Artifacts are copied to OPEN3D_ARTIFACTS_DIR (default /open3d-artifacts). +# Does not run ./bin/tests — use export_cpp_tests_bundle + run_cpp_tests_from_bundle. +build_devel_packages_cpu_cuda() { + local artifacts_dir="${OPEN3D_ARTIFACTS_DIR:-/open3d-artifacts}" + mkdir -p "${artifacts_dir}" + + echo "Building Open3D devel packages (CPU then CUDA) → ${artifacts_dir}" + echo "Using cmake: $(command -v cmake)" + cmake --version + + local cmakeOptions=( + "-DDEVELOPER_BUILD=${DEVELOPER_BUILD}" + "-DBUILD_SHARED_LIBS=ON" + "-DCMAKE_BUILD_TYPE=Release" + "-DBUILD_PYTHON_MODULE=OFF" + "-DBUILD_TENSORFLOW_OPS=OFF" + "-DBUILD_PYTORCH_OPS=OFF" + "-DBUNDLE_OPEN3D_ML=OFF" + "-DBUILD_UNIT_TESTS=ON" + "-DBUILD_BENCHMARKS=OFF" + "-DBUILD_EXAMPLES=OFF" + "-DBUILD_GUI=ON" + "-DBUILD_LIBREALSENSE=ON" + "-DBUILD_AZURE_KINECT=ON" + "-DBUILD_COMMON_ISPC_ISAS=ON" + "-DCMAKE_INSTALL_PREFIX=${OPEN3D_INSTALL_DIR}" + ) + + mkdir -p build + pushd build + + echo + echo "=== CPU devel package ===" + set -x cmake -DBUILD_CUDA_MODULE=OFF "${cmakeOptions[@]}" .. - set +x # Echo commands off + set +x + make VERBOSE=1 -j"$NPROC" + make VERBOSE=1 Open3DViewer -j"$NPROC" + # CPack stages its own install tree; avoid a separate make install that can + # fail on unrelated FetchContent targets (e.g. gmock) while still packaging + # Open3D's install() rules. + make package + make package-Open3DViewer-deb + + # Stage CPU artifacts before the CUDA flip overwrites package/ + shopt -s nullglob + cp -a package/open3d-devel-*.tar.xz "${artifacts_dir}/" + cp -a package-Open3DViewer-deb/open3d-viewer-*-Linux.deb "${artifacts_dir}/" 2>/dev/null \ + || cp -a package-Open3DViewer-deb/*.deb "${artifacts_dir}/" + shopt -u nullglob + + export_cpp_tests_bundle "${artifacts_dir}" + popd + echo + echo "=== CUDA devel package (reusing 3rdparty) ===" + _open3d_wipe_compiled_outputs build + pushd build + set -x + cmake -DBUILD_CUDA_MODULE=ON -DBUILD_COMMON_CUDA_ARCHS=ON "${cmakeOptions[@]}" .. + set +x + make VERBOSE=1 -j"$NPROC" + make package + shopt -s nullglob + cp -a package/open3d-devel-*-cuda-*.tar.xz "${artifacts_dir}/" + shopt -u nullglob + popd + + echo "Devel artifacts:" + ls -lh "${artifacts_dir}" +} + +# Package bin/tests + runtime libs needed to run C++ unit tests out-of-tree. +# Usage: export_cpp_tests_bundle [artifacts_dir] (must run from build/ or pass paths) +export_cpp_tests_bundle() { + local artifacts_dir="${1:-${OPEN3D_ARTIFACTS_DIR:-/open3d-artifacts}}" + local build_dir="${OPEN3D_BUILD_DIR:-}" + if [[ -z "${build_dir}" ]]; then + if [[ -f bin/tests ]]; then + build_dir="." + elif [[ -f build/bin/tests ]]; then + build_dir="build" + else + echo "export_cpp_tests_bundle: bin/tests not found" + exit 1 + fi + fi + mkdir -p "${artifacts_dir}" + local staging + staging="$(mktemp -d)" + mkdir -p "${staging}/bin" "${staging}/lib" + cp -a "${build_dir}/bin/tests" "${staging}/bin/" + # Runtime deps for shared libOpen3D + shopt -s nullglob + cp -a "${build_dir}/lib/Release/libOpen3D.so"* "${staging}/lib/" 2>/dev/null || true + # Bundled TBB next to tests (rpath / LD_LIBRARY_PATH) + if [[ -d "${build_dir}/gnu_"* ]]; then + # shellcheck disable=SC2086 + cp -a ${build_dir}/gnu_*/libtbb.so* "${staging}/lib/" 2>/dev/null || true + fi + find "${build_dir}" -maxdepth 2 -name 'libtbb.so*' -exec cp -a {} "${staging}/lib/" \; 2>/dev/null || true + # GUI resources referenced by some tests + if [[ -d "${build_dir}/bin/resources" ]]; then + cp -a "${build_dir}/bin/resources" "${staging}/bin/" + fi + shopt -u nullglob + local out="${artifacts_dir}/open3d-cpp-tests.tar.xz" + tar -C "${staging}" -cJf "${out}" bin lib + rm -rf "${staging}" + echo "Wrote ${out}" +} + +# Extract a tests bundle and run ./bin/tests. +# Usage: run_cpp_tests_from_bundle /path/to/open3d-cpp-tests.tar.xz [gtest args...] +run_cpp_tests_from_bundle() { + local bundle="$1" + shift || true + local work + work="$(mktemp -d)" + tar -C "${work}" -xf "${bundle}" + export LD_LIBRARY_PATH="${work}/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + pushd "${work}/bin" + echo "Run ./tests $* --gtest_random_seed=SEED to repeat this test sequence." + ./tests "$@" + popd + rm -rf "${work}" +} + +# Build open3d (+ open3d-cpu companion) wheels against extracted devel prefixes. +# Requires: +# OPEN3D_CUDA_ROOT - extracted CUDA open3d-devel prefix +# OPEN3D_CPU_ROOT - extracted CPU open3d-devel prefix +# Optional args: build_azure_kinect build_jupyter (same as build_pip_package) +build_pip_package_from_installed() { + echo "Building Open3D wheels from installed devel packages" + options="$(echo "$@" | tr ' ' '|')" + + if [[ -z "${OPEN3D_CUDA_ROOT:-}" || -z "${OPEN3D_CPU_ROOT:-}" ]]; then + echo "OPEN3D_CUDA_ROOT and OPEN3D_CPU_ROOT must point at extracted devel prefixes" + exit 1 + fi + if [[ ! -d "${OPEN3D_CUDA_ROOT}" || ! -d "${OPEN3D_CPU_ROOT}" ]]; then + echo "Missing devel prefix(es): CUDA=${OPEN3D_CUDA_ROOT} CPU=${OPEN3D_CPU_ROOT}" + exit 1 + fi - echo "Packaging Open3D CPU pip package..." + AARCH="$(uname -m)" + if [[ "$AARCH" == "aarch64" ]]; then + BUILD_FILAMENT_FROM_SOURCE=ON + else + BUILD_FILAMENT_FROM_SOURCE=OFF + fi + set +u + if [[ -f "${OPEN3D_ML_ROOT}/set_open3d_ml_root.sh" ]] && + [[ "$BUILD_TENSORFLOW_OPS" == "ON" || "$BUILD_PYTORCH_OPS" == "ON" ]]; then + echo "Open3D-ML available at ${OPEN3D_ML_ROOT}. Bundling Open3D-ML in wheel." + git -C "${OPEN3D_ML_ROOT}" checkout -b main || true + BUNDLE_OPEN3D_ML=ON + else + echo "Open3D-ML not available." + BUNDLE_OPEN3D_ML=OFF + fi + if [[ "build_azure_kinect" =~ ^($options)$ ]]; then + BUILD_AZURE_KINECT=ON + else + BUILD_AZURE_KINECT=OFF + fi + if [[ "build_jupyter" =~ ^($options)$ ]]; then + BUILD_JUPYTER_EXTENSION=ON + BUILD_WEBRTC_FLAG=ON + else + BUILD_JUPYTER_EXTENSION=OFF + BUILD_WEBRTC_FLAG=OFF + fi + set -u + + # Match main's build_pip_package() ML-ops ABI while building the CPU and CUDA + # wheels in separate dirs. The wheel image already ships CPU torch/tf, so the + # CPU wheel links its ops against CPU torch. Before the CUDA wheel we install + # CUDA torch/tf (see below) so its ops link against CUDA torch. Both wheels + # build torch/tf ops per BUILD_{PYTORCH,TENSORFLOW}_OPS. + if [[ "$BUILD_PYTORCH_OPS" == "ON" || "$BUILD_TENSORFLOW_OPS" == "ON" ]] && + ! python -c "import torch" >/dev/null 2>&1; then + install_python_dependencies purge-cache + fi + + local commonOptions=( + "-DOPEN3D_USE_INSTALLED_LIBRARY=ON" + "-DDEVELOPER_BUILD=${DEVELOPER_BUILD}" + "-DBUILD_SHARED_LIBS=ON" + "-DBUILD_PYTHON_MODULE=ON" + "-DBUILD_AZURE_KINECT=${BUILD_AZURE_KINECT}" + "-DBUILD_LIBREALSENSE=ON" + "-DBUILD_JUPYTER_EXTENSION=${BUILD_JUPYTER_EXTENSION}" + "-DBUILD_WEBRTC=${BUILD_WEBRTC_FLAG}" + "-DBUILD_GUI=ON" + "-DBUILD_UNIT_TESTS=OFF" + "-DBUILD_BENCHMARKS=OFF" + "-DBUILD_EXAMPLES=OFF" + "-DCMAKE_BUILD_TYPE=Release" + "-DPython3_EXECUTABLE=$(command -v python3)" + ) + + local wheel_out="build_wheel_out" + mkdir -p "${wheel_out}" + + # CPU companion wheel first (same torch ABI as the CUDA wheel below). + echo + echo "=== CPU companion wheel (Open3D_ROOT=${OPEN3D_CPU_ROOT}) ===" + mkdir -p build_cpu_wheel + pushd build_cpu_wheel + set -x + cmake -U 'Python3*' -U 'PYTHON_*' -U 'Pytorch*' -U 'Torch*' \ + -DOpen3D_ROOT="${OPEN3D_CPU_ROOT}" \ + -DCMAKE_PREFIX_PATH="${OPEN3D_CPU_ROOT}" \ + -DBUILD_CUDA_MODULE=OFF \ + -DBUILD_TENSORFLOW_OPS="${BUILD_TENSORFLOW_OPS}" \ + -DBUILD_PYTORCH_OPS="${BUILD_PYTORCH_OPS}" \ + -DBUNDLE_OPEN3D_ML="${BUNDLE_OPEN3D_ML}" \ + "${commonOptions[@]}" \ + .. + set +x make VERBOSE=1 -j"$NPROC" pip-package - mv lib/python_package/pip_package/open3d*.whl . # save CPU wheel + cp -a lib/python_package/pip_package/open3d*.whl "../${wheel_out}/" + popd - if [ "$BUILD_CUDA_MODULE" == ON ]; then - echo - echo Installing CUDA versions of TensorFlow and PyTorch... + # Install CUDA torch/tf so the CUDA wheel's ops link against CUDA torch + # (matches main's build_pip_package). The CPU wheel above is already built + # and saved, so replacing torch in the env now is safe. + if [[ "$BUILD_PYTORCH_OPS" == "ON" || "$BUILD_TENSORFLOW_OPS" == "ON" ]]; then + echo "Installing CUDA versions of TensorFlow and PyTorch..." install_python_dependencies with-cuda purge-cache - echo - echo Building with CUDA... - rebuild_list=(bin lib/Release/*.a lib/_build_config.py cpp lib/ml) - echo - echo Removing CPU compiled files / folders: "${rebuild_list[@]}" - rm -r "${rebuild_list[@]}" || true - set -x # Echo commands on - cmake -DBUILD_CUDA_MODULE=ON \ - -DBUILD_COMMON_CUDA_ARCHS=ON \ - "${cmakeOptions[@]}" .. - set +x # Echo commands off fi - echo - echo "Packaging Open3D full pip package..." + echo + echo "=== CUDA wheel (Open3D_ROOT=${OPEN3D_CUDA_ROOT}) ===" + mkdir -p build_cuda_wheel + pushd build_cuda_wheel + set -x + # Unset cached Python/Torch entries: torch changed to the CUDA build above. + cmake -U 'Python3*' -U 'PYTHON_*' -U 'Pytorch*' -U 'Torch*' \ + -DOpen3D_ROOT="${OPEN3D_CUDA_ROOT}" \ + -DCMAKE_PREFIX_PATH="${OPEN3D_CUDA_ROOT}" \ + -DBUILD_CUDA_MODULE=ON \ + -DBUILD_TENSORFLOW_OPS="${BUILD_TENSORFLOW_OPS}" \ + -DBUILD_PYTORCH_OPS="${BUILD_PYTORCH_OPS}" \ + -DBUNDLE_OPEN3D_ML="${BUNDLE_OPEN3D_ML}" \ + "${commonOptions[@]}" \ + .. + set +x + # Guard: installed mode must not rebuild heavy 3rdparty ExternalProjects + if [[ -d assimp || -d embree || -d filament || -d vtk ]]; then + echo "ERROR: 3rdparty ExternalProject dirs present in installed-mode build" + exit 1 + fi make VERBOSE=1 -j"$NPROC" pip-package - mv open3d*.whl lib/python_package/pip_package/ # restore CPU wheel - popd # PWD=Open3D + cp -a lib/python_package/pip_package/open3d*.whl "../${wheel_out}/" + popd + + # Place wheels where docker_build.sh / CI expect them + mkdir -p build/lib/python_package/pip_package + cp -a "${wheel_out}"/*.whl build/lib/python_package/pip_package/ + echo "Wheels built:" + ls -lh build/lib/python_package/pip_package/ +} + +# Extract open3d-devel tar.xz into a prefix directory. +# Usage: extract_open3d_devel /path/to/open3d-devel-....tar.xz /opt/open3d-cuda +extract_open3d_devel() { + local tar_path="$1" + local dest="$2" + mkdir -p "${dest}" + # CPack TXZ typically has a single top-level directory; strip it. + tar -C "${dest}" --strip-components=1 -xf "${tar_path}" + echo "Extracted ${tar_path} → ${dest}" + ls "${dest}/lib/cmake/Open3D" >/dev/null } # Test wheel in blank virtual environment diff --git a/util/run_ci.sh b/util/run_ci.sh index be1e4a1a662..b04e758652b 100755 --- a/util/run_ci.sh +++ b/util/run_ci.sh @@ -18,7 +18,9 @@ echo df -h echo "Running Open3D C++ unit tests..." -run_cpp_unit_tests +if [ "${SKIP_CPP_TESTS:-OFF}" != "ON" ]; then + run_cpp_unit_tests +fi # Run on GPU only. CPU versions run on Github already if nvidia-smi >/dev/null 2>&1; then