From 037c3272bfac23dd8050166c2b73afdc4a735499 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 15:39:28 +0900 Subject: [PATCH 01/19] '-DSPARSEIR_USE_BLAS=ON', # Enable BLAS support --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index 732e39ef..3fdcb749 100644 --- a/python/setup.py +++ b/python/setup.py @@ -112,6 +112,7 @@ def run(self): '-DBUILD_SHARED_LIBS=ON', '-DCMAKE_CXX_STANDARD=11', '-DCMAKE_CXX_STANDARD_REQUIRED=ON', + '-DSPARSEIR_USE_BLAS=ON', # Enable BLAS support ] # Add architecture-specific flags for macOS From 9e837bc90c99e32a1cddb7edc9b253d2663c573a Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 15:41:09 +0900 Subject: [PATCH 02/19] Debug --- .github/workflows/PublishPyPI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 4ac3e56c..150c8b0e 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -3,6 +3,8 @@ name: Python Build & Publish (Trusted) on: push: tags: ["v*"] # 例: v0.1.0 を push で発火 + pull_request: + branches: [main] workflow_dispatch: jobs: From fe888cbd8f353acc7f747bc2057e3b547864b551 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 15:44:07 +0900 Subject: [PATCH 03/19] deps_install: sudo apt update && sudo apt install -y libeigen3-dev libblas-dev liblapack-dev --- .github/workflows/CI_C_API_Python.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI_C_API_Python.yml b/.github/workflows/CI_C_API_Python.yml index 171c24f9..66d1d602 100644 --- a/.github/workflows/CI_C_API_Python.yml +++ b/.github/workflows/CI_C_API_Python.yml @@ -18,6 +18,8 @@ jobs: - "3.12" steps: + - name: Install dependencies + run: sudo apt update && sudo apt install -y libeigen3-dev libblas-dev liblapack-dev - uses: actions/checkout@v4 - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 From 1f0fd9f6f16c6c540f6aef339d3674e03f812e27 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 15:50:25 +0900 Subject: [PATCH 04/19] Support BLAS --- .github/workflows/PublishPyPI.yml | 7 ++- python/setup.py | 76 ++++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 150c8b0e..e5d39fef 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -32,9 +32,14 @@ jobs: CIBW_DEPENDENCY_VERSIONS: "latest" CIBW_BUILD_VERBOSITY: 1 CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" - CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=11.0" + CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=11.0 CMAKE_PREFIX_PATH=/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" + # Install OpenBLAS and development tools before building + CIBW_BEFORE_ALL_LINUX: "yum install -y openblas-devel || (yum install -y epel-release && yum install -y openblas-devel) || dnf install -y openblas-devel || apt-get update && apt-get install -y libopenblas-dev" + CIBW_BEFORE_ALL_MACOS: "brew install openblas" + # Set environment variables to help CMake find OpenBLAS + CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" # CIBW_BUILD: "cp312-*" with: package-dir: ./python diff --git a/python/setup.py b/python/setup.py index 3fdcb749..ac321fa4 100644 --- a/python/setup.py +++ b/python/setup.py @@ -112,9 +112,83 @@ def run(self): '-DBUILD_SHARED_LIBS=ON', '-DCMAKE_CXX_STANDARD=11', '-DCMAKE_CXX_STANDARD_REQUIRED=ON', - '-DSPARSEIR_USE_BLAS=ON', # Enable BLAS support ] + # Only enable BLAS if we can find it or if explicitly requested + enable_blas = os.environ.get('SPARSEIR_USE_BLAS', '').lower() in ('1', 'on', 'true', 'yes') + if not enable_blas: + # Try to detect if BLAS is available + blas_paths = [ + '/usr/lib64/openblas', + '/usr/lib/openblas', + '/usr/local/lib64/openblas', + '/usr/local/lib/openblas', + '/opt/homebrew/opt/openblas/lib', + '/usr/local/opt/openblas/lib' + ] + + for path in blas_paths: + if os.path.exists(path): + enable_blas = True + print(f"Found BLAS library path: {path}", flush=True) + break + + if enable_blas: + cmake_args.append('-DSPARSEIR_USE_BLAS=ON') + print("BLAS support enabled", flush=True) + + # Try to detect and prefer OpenBLAS + openblas_found = False + + # Check common OpenBLAS installation paths + openblas_paths = [] + if platform.system() == 'Darwin': + openblas_paths = [ + '/opt/homebrew/opt/openblas', + '/usr/local/opt/openblas', + '/opt/local' # MacPorts + ] + else: # Linux and others + openblas_paths = [ + '/usr/local', + '/usr', + '/opt/openblas', + '/usr/lib64/openblas', + '/usr/lib/openblas' + ] + + # Look for OpenBLAS + for path in openblas_paths: + if path.endswith('openblas'): + # Direct path to OpenBLAS directory + lib_path = path + include_path = path.replace('lib', 'include').replace('openblas', '') + else: + lib_path = os.path.join(path, 'lib') + include_path = os.path.join(path, 'include') + + # Check for OpenBLAS library + possible_libs = [ + os.path.join(lib_path, 'libopenblas.dylib'), # macOS + os.path.join(lib_path, 'libopenblas.so'), # Linux + os.path.join(lib_path, 'libopenblas.a'), # Static + ] + + if any(os.path.exists(lib) for lib in possible_libs): + print(f"Found OpenBLAS at: {path}", flush=True) + cmake_args.extend([ + f'-DCMAKE_PREFIX_PATH={path}', + f'-DBLAS_ROOT={path}', + ]) + openblas_found = True + break + + if not openblas_found: + print("OpenBLAS not found in standard locations, using system BLAS", flush=True) + else: + print("BLAS support disabled - no BLAS libraries found", flush=True) + openblas_found = False + # Add architecture-specific flags for macOS import platform if platform.system() == 'Darwin': From 1649ec5e3b4bccebae82e5afa3089ac5ad89d864 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 15:52:29 +0900 Subject: [PATCH 05/19] Fix --- python/setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index ac321fa4..ec4a788a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -114,6 +114,9 @@ def run(self): '-DCMAKE_CXX_STANDARD_REQUIRED=ON', ] + # Import platform module here before using it + import platform + # Only enable BLAS if we can find it or if explicitly requested enable_blas = os.environ.get('SPARSEIR_USE_BLAS', '').lower() in ('1', 'on', 'true', 'yes') if not enable_blas: @@ -190,7 +193,6 @@ def run(self): openblas_found = False # Add architecture-specific flags for macOS - import platform if platform.system() == 'Darwin': # Get the target architecture from environment or system target_arch = os.environ.get('ARCHFLAGS', '').replace('-arch ', '') From 17361863f92dbcb52baae615dbc5d99a84a93737 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 15:56:17 +0900 Subject: [PATCH 06/19] Fix --- .github/workflows/PublishPyPI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index e5d39fef..b624156b 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -36,7 +36,7 @@ jobs: CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" # Install OpenBLAS and development tools before building - CIBW_BEFORE_ALL_LINUX: "yum install -y openblas-devel || (yum install -y epel-release && yum install -y openblas-devel) || dnf install -y openblas-devel || apt-get update && apt-get install -y libopenblas-dev" + CIBW_BEFORE_ALL_LINUX: "yum install -y openblas-devel || (yum install -y epel-release && yum install -y openblas-devel) || dnf install -y openblas-devel || (apt-get update && apt-get install -y libopenblas-dev)" CIBW_BEFORE_ALL_MACOS: "brew install openblas" # Set environment variables to help CMake find OpenBLAS CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" From 18ceb96ce687780212ad97cbf92633556fd6fd34 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:00:19 +0900 Subject: [PATCH 07/19] link with BLAS --- .github/workflows/CI_C_API_Python.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI_C_API_Python.yml b/.github/workflows/CI_C_API_Python.yml index 66d1d602..82727f80 100644 --- a/.github/workflows/CI_C_API_Python.yml +++ b/.github/workflows/CI_C_API_Python.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install dependencies - run: sudo apt update && sudo apt install -y libeigen3-dev libblas-dev liblapack-dev + run: sudo apt update && sudo apt install -y libeigen3-dev libopenblas-dev - uses: actions/checkout@v4 - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 @@ -35,8 +35,12 @@ jobs: - name: Run uv sync run: uv sync working-directory: python + env: + SPARSEIR_USE_BLAS: 1 - name: Run tests # For example, using `pytest` run: uv run pytest tests working-directory: python + env: + SPARSEIR_USE_BLAS: 1 From a9570c9500d4eb23bddeec15ef768161cb250ac3 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:01:20 +0900 Subject: [PATCH 08/19] Update python/README.md --- python/README.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/python/README.md b/python/README.md index 0e5feee5..9fea65cf 100644 --- a/python/README.md +++ b/python/README.md @@ -9,6 +9,13 @@ This is a low-level binding for the [libsparseir](https://github.com/SpM-lab/lib - C++11 compatible compiler - numpy +### Optional Dependencies + +- **OpenBLAS** (recommended for better performance) + - macOS: `brew install openblas` + - Ubuntu/Debian: `sudo apt install libopenblas-dev` + - CentOS/RHEL: `sudo yum install openblas-devel` + ## Build ### Install Dependencies and Build @@ -22,6 +29,30 @@ This will: - Build the C++ libsparseir library using CMake - Install the Python package in development mode +### Build with OpenBLAS Support + +To enable OpenBLAS support for improved performance: + +```bash +# Set environment variable to enable BLAS +export SPARSEIR_USE_BLAS=1 +uv sync +``` + +Or for a single build: + +```bash +SPARSEIR_USE_BLAS=1 uv sync +``` + +The build system will automatically detect OpenBLAS if it's installed in standard locations. If OpenBLAS is installed in a custom location, you may need to set additional environment variables: + +```bash +export CMAKE_PREFIX_PATH="/path/to/openblas" +export SPARSEIR_USE_BLAS=1 +uv sync +``` + ### Clean Build Artifacts To remove build artifacts and files copied from the parent directory: @@ -35,3 +66,45 @@ This will remove: - Copied source files: `include/`, `src/`, `fortran/`, `cmake/`, `CMakeLists.txt` - Compiled libraries: `pylibsparseir/*.so`, `pylibsparseir/*.dylib`, `pylibsparseir/*.dll` - Cache directories: `pylibsparseir/__pycache__` + +## Performance Notes + +### BLAS Support + +This package supports BLAS libraries for improved linear algebra performance: + +- **With OpenBLAS**: Significant performance improvements for matrix operations +- **Without BLAS**: Uses Eigen's built-in implementations (still efficient, but slower for large matrices) + +The build system will automatically detect and use OpenBLAS if available. You can verify BLAS support by checking the build output for messages like: + +``` +BLAS support enabled +Found OpenBLAS at: /opt/homebrew/opt/openblas +``` + +### Troubleshooting + +**Build fails with "Could NOT find BLAS":** +```bash +# Install OpenBLAS first +brew install openblas # macOS +sudo apt install libopenblas-dev # Ubuntu + +# Then force BLAS detection +SPARSEIR_USE_BLAS=1 uv sync +``` + +**OpenBLAS not detected automatically:** +```bash +# Set CMake prefix path manually +export CMAKE_PREFIX_PATH="/usr/local/opt/openblas" # or your OpenBLAS path +export SPARSEIR_USE_BLAS=1 +uv sync +``` + +**Verify BLAS support in built package:** +```python +import pylibsparseir +# Check build logs or library dependencies to confirm BLAS linking +``` From 6808f93087591d2561f9234ad123c5a654c00660 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:04:43 +0900 Subject: [PATCH 09/19] Fix macOS OpenBLAS support for both Intel and Apple Silicon - Add matrix variables for deployment target and OpenBLAS paths - Intel Mac: MACOSX_DEPLOYMENT_TARGET=11.0, /usr/local/opt/openblas - Apple Silicon Mac: MACOSX_DEPLOYMENT_TARGET=15.0, /opt/homebrew/opt/openblas - Resolve delocate wheel compatibility issues with OpenBLAS dependencies - Ensure proper OpenBLAS linking for both x86_64 and arm64 architectures --- .github/workflows/PublishPyPI.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index b624156b..6ee4cba1 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -15,12 +15,18 @@ jobs: include: - os: ubuntu-latest cibw_archs: "auto" + deployment_target: "" + openblas_path: "" #- os: windows-latest # cibw_archs: "AMD64" - os: macos-13 # Intel Mac cibw_archs: "x86_64" + deployment_target: "11.0" + openblas_path: "/usr/local/opt/openblas" - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" + deployment_target: "15.0" + openblas_path: "/opt/homebrew/opt/openblas" steps: - uses: actions/checkout@v4 - name: remove python/.gitignore @@ -32,7 +38,7 @@ jobs: CIBW_DEPENDENCY_VERSIONS: "latest" CIBW_BUILD_VERBOSITY: 1 CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" - CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=11.0 CMAKE_PREFIX_PATH=/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment_target || '11.0' }} CMAKE_PREFIX_PATH=${{ matrix.openblas_path }}:/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" # Install OpenBLAS and development tools before building From d385d661b6fce4c45bbf5d0afb9d8b9acbe451ea Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:10:03 +0900 Subject: [PATCH 10/19] Fix Intel Mac deployment target for OpenBLAS compatibility - Change Intel Mac deployment target from 11.0 to 13.0 - Resolves delocate wheel compatibility issues with OpenBLAS dependencies - OpenBLAS and its Fortran/GCC dependencies require macOS 13.0 minimum --- .github/workflows/PublishPyPI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 6ee4cba1..0e89f063 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -21,7 +21,7 @@ jobs: # cibw_archs: "AMD64" - os: macos-13 # Intel Mac cibw_archs: "x86_64" - deployment_target: "11.0" + deployment_target: "13.0" openblas_path: "/usr/local/opt/openblas" - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" From cb20478c124dc66b219935268a35c4c70425610f Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:45:29 +0900 Subject: [PATCH 11/19] Add macOS support to Python C API CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add macOS to test matrix alongside Ubuntu - Install OpenBLAS and Eigen via Homebrew on macOS - Comment out CMAKE_PREFIX_PATH and PKG_CONFIG_PATH for now - Now tests run on 6 environments: Ubuntu + macOS × Python 3.10/3.11/3.12 --- .github/workflows/CI_C_API_Python.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI_C_API_Python.yml b/.github/workflows/CI_C_API_Python.yml index 82727f80..99f76843 100644 --- a/.github/workflows/CI_C_API_Python.yml +++ b/.github/workflows/CI_C_API_Python.yml @@ -9,17 +9,23 @@ on: jobs: test: name: Test - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-latest, macos-latest] python-version: - "3.10" - "3.11" - "3.12" steps: - - name: Install dependencies + - name: Install dependencies (Linux) + if: runner.os == 'Linux' run: sudo apt update && sudo apt install -y libeigen3-dev libopenblas-dev + + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: brew install eigen openblas - uses: actions/checkout@v4 - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 @@ -37,6 +43,8 @@ jobs: working-directory: python env: SPARSEIR_USE_BLAS: 1 + #CMAKE_PREFIX_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas:/opt/homebrew/opt/eigen' || '' }} + #PKG_CONFIG_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas/lib/pkgconfig:/opt/homebrew/opt/eigen/lib/pkgconfig' || '' }} - name: Run tests # For example, using `pytest` @@ -44,3 +52,5 @@ jobs: working-directory: python env: SPARSEIR_USE_BLAS: 1 + #CMAKE_PREFIX_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas:/opt/homebrew/opt/eigen' || '' }} + #PKG_CONFIG_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas/lib/pkgconfig:/opt/homebrew/opt/eigen/lib/pkgconfig' || '' }} From 845a08be87f1b93bf28d8235c9537a14e266af03 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:50:36 +0900 Subject: [PATCH 12/19] Clean up CI config and bump version to 0.1.2 - Remove commented out environment variables from CI_C_API_Python.yml - Bump pylibsparseir version from 0.1.1 to 0.1.2 - Prepare for release with OpenBLAS support --- .github/workflows/CI_C_API_Python.yml | 4 ---- python/pyproject.toml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/CI_C_API_Python.yml b/.github/workflows/CI_C_API_Python.yml index 99f76843..2e0aef65 100644 --- a/.github/workflows/CI_C_API_Python.yml +++ b/.github/workflows/CI_C_API_Python.yml @@ -43,8 +43,6 @@ jobs: working-directory: python env: SPARSEIR_USE_BLAS: 1 - #CMAKE_PREFIX_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas:/opt/homebrew/opt/eigen' || '' }} - #PKG_CONFIG_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas/lib/pkgconfig:/opt/homebrew/opt/eigen/lib/pkgconfig' || '' }} - name: Run tests # For example, using `pytest` @@ -52,5 +50,3 @@ jobs: working-directory: python env: SPARSEIR_USE_BLAS: 1 - #CMAKE_PREFIX_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas:/opt/homebrew/opt/eigen' || '' }} - #PKG_CONFIG_PATH: ${{ runner.os == 'macOS' && '/opt/homebrew/opt/openblas/lib/pkgconfig:/opt/homebrew/opt/eigen/lib/pkgconfig' || '' }} diff --git a/python/pyproject.toml b/python/pyproject.toml index 1f01cae9..e405df2a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pylibsparseir" -version = "0.1.1" +version = "0.1.2" description = "Python bindings for the libsparseir library, providing efficient sparse intermediate representation for many-body physics calculations" readme = "README.md" requires-python = ">=3.10" From f38b872fbea7ce72261593d94a1511dc3eea35e1 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 16:56:19 +0900 Subject: [PATCH 13/19] Enable parallel builds using matrix strategy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Split builds by Python version (3.10, 3.11, 3.12, 3.13) for parallel execution - Each OS + Python version combination runs as independent job (12 total jobs) - Dynamic CIBW_BUILD configuration using matrix.python_version - Unique artifact names for each build combination - Significant build time reduction through parallelization Build matrix: - Linux: 4 parallel jobs (ubuntu-latest × cp310-313) - macOS Intel: 4 parallel jobs (macos-13 × cp310-313) - macOS Apple Silicon: 4 parallel jobs (macos-latest × cp310-313) --- .github/workflows/PublishPyPI.yml | 57 ++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 0e89f063..45d18782 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -13,20 +13,69 @@ jobs: strategy: matrix: include: + # Linux builds - os: ubuntu-latest cibw_archs: "auto" deployment_target: "" openblas_path: "" - #- os: windows-latest - # cibw_archs: "AMD64" + python_version: "cp310-*" + - os: ubuntu-latest + cibw_archs: "auto" + deployment_target: "" + openblas_path: "" + python_version: "cp311-*" + - os: ubuntu-latest + cibw_archs: "auto" + deployment_target: "" + openblas_path: "" + python_version: "cp312-*" + - os: ubuntu-latest + cibw_archs: "auto" + deployment_target: "" + openblas_path: "" + python_version: "cp313-*" + # macOS Intel builds + - os: macos-13 # Intel Mac + cibw_archs: "x86_64" + deployment_target: "13.0" + openblas_path: "/usr/local/opt/openblas" + python_version: "cp310-*" + - os: macos-13 # Intel Mac + cibw_archs: "x86_64" + deployment_target: "13.0" + openblas_path: "/usr/local/opt/openblas" + python_version: "cp311-*" - os: macos-13 # Intel Mac cibw_archs: "x86_64" deployment_target: "13.0" openblas_path: "/usr/local/opt/openblas" + python_version: "cp312-*" + - os: macos-13 # Intel Mac + cibw_archs: "x86_64" + deployment_target: "13.0" + openblas_path: "/usr/local/opt/openblas" + python_version: "cp313-*" + # macOS Apple Silicon builds + - os: macos-latest # Apple Silicon Mac + cibw_archs: "arm64" + deployment_target: "15.0" + openblas_path: "/opt/homebrew/opt/openblas" + python_version: "cp310-*" + - os: macos-latest # Apple Silicon Mac + cibw_archs: "arm64" + deployment_target: "15.0" + openblas_path: "/opt/homebrew/opt/openblas" + python_version: "cp311-*" + - os: macos-latest # Apple Silicon Mac + cibw_archs: "arm64" + deployment_target: "15.0" + openblas_path: "/opt/homebrew/opt/openblas" + python_version: "cp312-*" - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" deployment_target: "15.0" openblas_path: "/opt/homebrew/opt/openblas" + python_version: "cp313-*" steps: - uses: actions/checkout@v4 - name: remove python/.gitignore @@ -38,6 +87,7 @@ jobs: CIBW_DEPENDENCY_VERSIONS: "latest" CIBW_BUILD_VERBOSITY: 1 CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + CIBW_BUILD: "${{ matrix.python_version }}" CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment_target || '11.0' }} CMAKE_PREFIX_PATH=${{ matrix.openblas_path }}:/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" @@ -46,13 +96,12 @@ jobs: CIBW_BEFORE_ALL_MACOS: "brew install openblas" # Set environment variables to help CMake find OpenBLAS CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" - # CIBW_BUILD: "cp312-*" with: package-dir: ./python output-dir: dist - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.cibw_archs }} + name: wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.python_version }} path: dist/* publish-testpypi: From ab94f7dafe275a8340e223a7763bc451555db2ba Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 17:05:36 +0900 Subject: [PATCH 14/19] Fix artifact naming and simplify matrix configuration - Replace invalid '*' character in artifact names with clean py_tag values - Use py_tag field consistently for both CIBW_BUILD and artifact naming - Remove redundant python_version field from matrix - Simplify matrix entries by using py_tag for cleaner configuration - Fix GitHub Actions compliance issues with artifact naming constraints Artifact names now follow pattern: wheels-{os}-{arch}-{py_tag} CIBW_BUILD now uses: {py_tag}-* format for consistent wheel building --- .github/workflows/PublishPyPI.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 45d18782..c0b35147 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -18,64 +18,64 @@ jobs: cibw_archs: "auto" deployment_target: "" openblas_path: "" - python_version: "cp310-*" + py_tag: "cp310" - os: ubuntu-latest cibw_archs: "auto" deployment_target: "" openblas_path: "" - python_version: "cp311-*" + py_tag: "cp311" - os: ubuntu-latest cibw_archs: "auto" deployment_target: "" openblas_path: "" - python_version: "cp312-*" + py_tag: "cp312" - os: ubuntu-latest cibw_archs: "auto" deployment_target: "" openblas_path: "" - python_version: "cp313-*" + py_tag: "cp313" # macOS Intel builds - os: macos-13 # Intel Mac cibw_archs: "x86_64" deployment_target: "13.0" openblas_path: "/usr/local/opt/openblas" - python_version: "cp310-*" + py_tag: "cp310" - os: macos-13 # Intel Mac cibw_archs: "x86_64" deployment_target: "13.0" openblas_path: "/usr/local/opt/openblas" - python_version: "cp311-*" + py_tag: "cp311" - os: macos-13 # Intel Mac cibw_archs: "x86_64" deployment_target: "13.0" openblas_path: "/usr/local/opt/openblas" - python_version: "cp312-*" + py_tag: "cp312" - os: macos-13 # Intel Mac cibw_archs: "x86_64" deployment_target: "13.0" openblas_path: "/usr/local/opt/openblas" - python_version: "cp313-*" + py_tag: "cp313" # macOS Apple Silicon builds - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" deployment_target: "15.0" openblas_path: "/opt/homebrew/opt/openblas" - python_version: "cp310-*" + py_tag: "cp310" - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" deployment_target: "15.0" openblas_path: "/opt/homebrew/opt/openblas" - python_version: "cp311-*" + py_tag: "cp311" - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" deployment_target: "15.0" openblas_path: "/opt/homebrew/opt/openblas" - python_version: "cp312-*" + py_tag: "cp312" - os: macos-latest # Apple Silicon Mac cibw_archs: "arm64" deployment_target: "15.0" openblas_path: "/opt/homebrew/opt/openblas" - python_version: "cp313-*" + py_tag: "cp313" steps: - uses: actions/checkout@v4 - name: remove python/.gitignore @@ -87,7 +87,7 @@ jobs: CIBW_DEPENDENCY_VERSIONS: "latest" CIBW_BUILD_VERBOSITY: 1 CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" - CIBW_BUILD: "${{ matrix.python_version }}" + CIBW_BUILD: "${{ matrix.py_tag }}-*" CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment_target || '11.0' }} CMAKE_PREFIX_PATH=${{ matrix.openblas_path }}:/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" @@ -101,7 +101,7 @@ jobs: output-dir: dist - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.python_version }} + name: wheels-${{ matrix.os }}-${{ matrix.cibw_archs }}-${{ matrix.py_tag }} path: dist/* publish-testpypi: From 1200dbae1a89de6805c2b30eea8908055e103c6d Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 17:14:27 +0900 Subject: [PATCH 15/19] Fix Linux build: Use micromamba to install OpenBLAS in manylinux containers - Replace traditional package managers (yum/dnf/apt-get) with micromamba - Install OpenBLAS from conda-forge channel to /opt/openblas - Update CMAKE_PREFIX_PATH and PKG_CONFIG_PATH to include new installation path - Fixes 'command not found' error for package managers in manylinux_2_28_x86_64 container --- .github/workflows/PublishPyPI.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index c0b35147..817e4b8c 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -91,11 +91,11 @@ jobs: CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment_target || '11.0' }} CMAKE_PREFIX_PATH=${{ matrix.openblas_path }}:/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" - # Install OpenBLAS and development tools before building - CIBW_BEFORE_ALL_LINUX: "yum install -y openblas-devel || (yum install -y epel-release && yum install -y openblas-devel) || dnf install -y openblas-devel || (apt-get update && apt-get install -y libopenblas-dev)" + # Install OpenBLAS using micromamba (conda-forge) + CIBW_BEFORE_ALL_LINUX: "curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" CIBW_BEFORE_ALL_MACOS: "brew install openblas" # Set environment variables to help CMake find OpenBLAS - CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" with: package-dir: ./python output-dir: dist From 0e391111409c4b79ea0b3fd63e9b5685a7457b11 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 17:16:46 +0900 Subject: [PATCH 16/19] Fix micromamba environment creation in Linux build - Add 'micromamba create -y -p /opt/openblas' before installing OpenBLAS - Micromamba requires environment to be created before installing packages - Fixes 'No prefix found at: /opt/openblas' error in manylinux container --- .github/workflows/PublishPyPI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 817e4b8c..21095c53 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -92,7 +92,7 @@ jobs: CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" # Install OpenBLAS using micromamba (conda-forge) - CIBW_BEFORE_ALL_LINUX: "curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" + CIBW_BEFORE_ALL_LINUX: "curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba create -y -p /opt/openblas && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" CIBW_BEFORE_ALL_MACOS: "brew install openblas" # Set environment variables to help CMake find OpenBLAS CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" From 1b667fb0de3f6a6ca8ee217eb37aeae33f66f7f0 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 17:31:37 +0900 Subject: [PATCH 17/19] Fix micromamba installation in musllinux containers - Use CIBW_BEFORE_ALL_MANYLINUX and CIBW_BEFORE_ALL_MUSLLINUX for platform-specific configurations - Install bzip2 in musllinux containers before extracting micromamba - Add CMAKE_PREFIX_PATH and PKG_CONFIG_PATH environment variables for both manylinux and musllinux --- .github/workflows/PublishPyPI.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 21095c53..ec368f4a 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -88,14 +88,16 @@ jobs: CIBW_BUILD_VERBOSITY: 1 CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" CIBW_BUILD: "${{ matrix.py_tag }}-*" - CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment_target || '11.0' }} CMAKE_PREFIX_PATH=${{ matrix.openblas_path }}:/opt/homebrew/opt/openblas:/usr/local/opt/openblas PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:/usr/local/lib/pkgconfig SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.deployment_target || '11.0' }} SPARSEIR_USE_BLAS=1" CIBW_ARCHS: ${{ matrix.cibw_archs }} CIBW_SKIP: "*-manylinux_i686 *-musllinux_i686" - # Install OpenBLAS using micromamba (conda-forge) - CIBW_BEFORE_ALL_LINUX: "curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba create -y -p /opt/openblas && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" + # Install OpenBLAS using micromamba (conda-forge) - separate for manylinux and musllinux + CIBW_BEFORE_ALL_MANYLINUX: "curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba create -y -p /opt/openblas && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" + CIBW_BEFORE_ALL_MUSLLINUX: "apk add --no-cache bzip2 && curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba create -y -p /opt/openblas && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" CIBW_BEFORE_ALL_MACOS: "brew install openblas" # Set environment variables to help CMake find OpenBLAS - CIBW_ENVIRONMENT_LINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_MANYLINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_MUSLLINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" with: package-dir: ./python output-dir: dist From e4fccf7df60bc3ba6e330d4971fe37c470cdd49e Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 17:35:19 +0900 Subject: [PATCH 18/19] Remove redundant environment variable configuration --- .github/workflows/PublishPyPI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index ec368f4a..919cbd9f 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -96,8 +96,8 @@ jobs: CIBW_BEFORE_ALL_MUSLLINUX: "apk add --no-cache bzip2 && curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && ./bin/micromamba create -y -p /opt/openblas && ./bin/micromamba install -y -c conda-forge openblas -p /opt/openblas --no-deps" CIBW_BEFORE_ALL_MACOS: "brew install openblas" # Set environment variables to help CMake find OpenBLAS - CIBW_ENVIRONMENT_MANYLINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" - CIBW_ENVIRONMENT_MUSLLINUX: "CMAKE_PREFIX_PATH=/opt/openblas:/usr/lib64/openblas:/usr/lib/openblas:/usr/local/lib64/openblas:/usr/local/lib/openblas PKG_CONFIG_PATH=/opt/openblas/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/local/lib64/pkgconfig SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_MANYLINUX: "SPARSEIR_USE_BLAS=1" + CIBW_ENVIRONMENT_MUSLLINUX: "SPARSEIR_USE_BLAS=1" with: package-dir: ./python output-dir: dist From 928b2d4f83cdf44488b9b1e4048652dc5ba19b36 Mon Sep 17 00:00:00 2001 From: SatoshiTerasaki Date: Tue, 9 Sep 2025 17:47:51 +0900 Subject: [PATCH 19/19] Do not run PublishPyPI on pull request --- .github/workflows/PublishPyPI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PublishPyPI.yml b/.github/workflows/PublishPyPI.yml index 919cbd9f..ce6ac2b7 100644 --- a/.github/workflows/PublishPyPI.yml +++ b/.github/workflows/PublishPyPI.yml @@ -3,8 +3,8 @@ name: Python Build & Publish (Trusted) on: push: tags: ["v*"] # 例: v0.1.0 を push で発火 - pull_request: - branches: [main] + #pull_request: # for debugging + # branches: [main] # for debugging workflow_dispatch: jobs: