From 95ef11e9cde59dc9894b6b0af03a0b9701439d8e Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 19 Jun 2025 08:02:57 -0700 Subject: [PATCH 01/52] build: allow quick iteration on meta.yaml --- .github/actions/curl-meta-yaml/action.yaml | 30 +++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/actions/curl-meta-yaml/action.yaml b/.github/actions/curl-meta-yaml/action.yaml index 16bda7ab..63a757ee 100644 --- a/.github/actions/curl-meta-yaml/action.yaml +++ b/.github/actions/curl-meta-yaml/action.yaml @@ -5,21 +5,39 @@ runs: steps: - shell: bash -l -e {0} run: | + # Fail if on main branch and conda-recipe/meta.yaml exists + if [[ "${GITHUB_REF_NAME}" == "main" && -f "conda-recipe/meta.yaml" ]]; then + echo "ERROR: conda-recipe/meta.yaml should not be committed to main. Failing build." + exit 1 + fi + set -x mkdir -p conda-recipe + + # If meta.yaml exists, use it directly without modifications + if [[ -f "conda-recipe/meta.yaml" ]]; then + echo "WARN: Using existing conda-recipe/meta.yaml file" + set +x + exit 0 + fi + curl -s -L -o conda-recipe/meta.yaml https://raw.githubusercontent.com/conda-forge/genomekit-feedstock/main/recipe/meta.yaml export GK_VERSION=$(grep "version = " setup.py | awk -F'"' '{print $2}') export OS_TYPE=$(uname) - # - replace the (non-existent) tarball with local path - # - remove the related sha for that tarball - # - build for all locally allowed versions of python (conda-recipe/conda_build_config.yaml) - # - set the version to the local release-please version (for docs and docker publish) if [[ "$OS_TYPE" == "Darwin" ]]; then - sed -i '' -E "s|url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v.*$|path: ../|1; /sha256: /d; /skip: true /d;s/{% set version = \".+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml + SED_CMD="sed -i '' -E" else - sed -i -e "s|url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v.*$|path: ../|1; /sha256: /d; /skip: true /d;s/{% set version = \".\+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml + SED_CMD="sed -i" fi + # replace the (non-existent) tarball with local path + $SED_CMD "s|url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v.*$|path: ../|1" conda-recipe/meta.yaml + # remove the related sha for that tarball + $SED_CMD "/sha256: /d" conda-recipe/meta.yaml + # build for all locally allowed versions of python (conda-recipe/conda_build_config.yaml) + $SED_CMD "/skip: true /d" conda-recipe/meta.yaml + # set the version to the local release-please version (for docs and docker publish) + $SED_CMD "s/{% set version = \".+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml head -10 conda-recipe/meta.yaml set +x From baf56d9f58efd76e58a07253eb757d36215009a5 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 19 Jun 2025 08:16:22 -0700 Subject: [PATCH 02/52] apply meta.yaml sed modifications on feature branch --- .github/actions/curl-meta-yaml/action.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/actions/curl-meta-yaml/action.yaml b/.github/actions/curl-meta-yaml/action.yaml index 63a757ee..eac26480 100644 --- a/.github/actions/curl-meta-yaml/action.yaml +++ b/.github/actions/curl-meta-yaml/action.yaml @@ -14,14 +14,13 @@ runs: set -x mkdir -p conda-recipe - # If meta.yaml exists, use it directly without modifications + # If meta.yaml exists, use it (to allow quick iteration in feature branches) if [[ -f "conda-recipe/meta.yaml" ]]; then - echo "WARN: Using existing conda-recipe/meta.yaml file" - set +x - exit 0 + echo "WARN: Using existing conda-recipe/meta.yaml" + else + curl -s -L -o conda-recipe/meta.yaml https://raw.githubusercontent.com/conda-forge/genomekit-feedstock/main/recipe/meta.yaml fi - curl -s -L -o conda-recipe/meta.yaml https://raw.githubusercontent.com/conda-forge/genomekit-feedstock/main/recipe/meta.yaml export GK_VERSION=$(grep "version = " setup.py | awk -F'"' '{print $2}') export OS_TYPE=$(uname) From 4bc9af5c78b909890b5e30960c67c3042f31a465 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 18 Jun 2025 10:43:22 -0700 Subject: [PATCH 03/52] feat: support python 313 (wip) - fix pyproject.toml to avoid errors --- .github/workflows/build-wheels.yaml | 14 ++++++++------ .github/workflows/run-tests.yaml | 2 ++ _pyproject.toml | 5 ----- docs-src/develop.rst | 6 +++--- genomekit_dev.yml | 6 +++--- pyproject.toml | 7 +++++++ setup.py | 1 + 7 files changed, 24 insertions(+), 17 deletions(-) delete mode 100644 _pyproject.toml create mode 100644 pyproject.toml diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index e849a468..9b553a36 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -46,7 +46,7 @@ jobs: name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} name: Install cibuildwheel @@ -57,7 +57,7 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse && ls -l wheelhouse/ env: GK_BUILD_WHEELS: "1" - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" CIBW_ARCHS_LINUX: ${{ matrix.arch }} CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 @@ -65,11 +65,11 @@ jobs: CIBW_SKIP: "*-musllinux_*" - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} - name: Test wheels across Python 3.9–3.12 + name: Test wheels across Python 3.9–3.13 run: | set -euxo pipefail - for PYVER in 3.9 3.10 3.11 3.12; do + for PYVER in 3.9 3.10 3.11 3.12 3.13; do PYVER_SHORT=${PYVER/./} echo "Testing on Python $PYVER" @@ -119,7 +119,7 @@ jobs: name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: Install cibuildwheel @@ -131,7 +131,7 @@ jobs: env: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*" + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels @@ -154,10 +154,12 @@ jobs: - {"os": "macos-14", "arch": "arm64", "pyver": "3.10", "pyvershort": "310"} - {"os": "macos-14", "arch": "arm64", "pyver": "3.11", "pyvershort": "311"} - {"os": "macos-14", "arch": "arm64", "pyver": "3.12", "pyvershort": "312"} + - {"os": "macos-14", "arch": "arm64", "pyver": "3.13", "pyvershort": "313"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.9", "pyvershort": "39"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.10", "pyvershort": "310"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.11", "pyvershort": "311"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.12", "pyvershort": "312"} + - {"os": "macos-13", "arch": "x86_64", "pyver": "3.13", "pyvershort": "313"} steps: - name: Set up Python diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index e8319d9a..4a259130 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -39,6 +39,8 @@ jobs: - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 diff --git a/_pyproject.toml b/_pyproject.toml deleted file mode 100644 index 5a1436f4..00000000 --- a/_pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -# Specify build system requirements for pip wheel builds. -# This way is incompatible with conda, which is why the file can't have the name pyproject.toml. -# The file is renamed to pyproject.toml in Github Actions before starting the pip wheel build. -[build-system] -requires = ["setuptools>=42", "wheel", "numpy==2"] diff --git a/docs-src/develop.rst b/docs-src/develop.rst index 7581ce00..3ac8f55b 100644 --- a/docs-src/develop.rst +++ b/docs-src/develop.rst @@ -29,7 +29,7 @@ On M1 macs, you might need to set up the environment differently:: Build the package in development mode:: - pip install -e . + PYTHONPATH=. pip install -e . --no-build-isolation This builds the C++ extension and copies it into your source tree (``genome_kit/_cxx.so``). @@ -65,7 +65,7 @@ Making changes If the C/C++ code changed, you must re-run the ``develop`` command:: - pip install -e . + PYTHONPATH=. pip install -e . --no-build-isolation This includes switching branches, merging changes, or editing the C/C++ code yourself. *Forgetting this step may lead to unpredictable behaviour.* @@ -161,7 +161,7 @@ Test data files reside in the source tree under ``tests/data``. To build them, you must have registered your source tree in develop mode:: - pip install -e . + PYTHONPATH=. pip install -e . --no-build-isolation Now that your source tree is the default `genome_kit` import, the ``build`` subcommand will be able to find diff --git a/genomekit_dev.yml b/genomekit_dev.yml index 5ba2f85a..d8b6f33c 100644 --- a/genomekit_dev.yml +++ b/genomekit_dev.yml @@ -4,8 +4,8 @@ channels: - bioconda dependencies: - appdirs>=1.4.0 - - numpy <2.0dev0 - - python>=3.9.0,<3.13.0 + - numpy + - python=3.9.0,<3.14.0 - tqdm - typing-extensions - zlib @@ -16,7 +16,7 @@ dependencies: # dev + test dependencies - ccache - ipython - - pyperformance==1.0.4 + # usage of pyperf was removed internally before open sourcing - pytest - sphinx - sphinx_rtd_theme diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..6069e17c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "setuptools>=61", + "wheel", + "numpy==2" +] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.py b/setup.py index 4f245fa5..8abb9dd9 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ def run(self): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], description="GenomeKit is a Python library for fast and easy access to genomic resources such as sequence, data tracks, and annotations.", long_description=(Path(__file__).parent / "README.md").read_text(), From 88ecac158bf665707e6dfbb14daa99db5ff771d2 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 18 Jun 2025 12:22:16 -0700 Subject: [PATCH 04/52] wip2 --- .github/actions/curl-meta-yaml/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/curl-meta-yaml/action.yaml b/.github/actions/curl-meta-yaml/action.yaml index eac26480..fe577312 100644 --- a/.github/actions/curl-meta-yaml/action.yaml +++ b/.github/actions/curl-meta-yaml/action.yaml @@ -37,6 +37,8 @@ runs: $SED_CMD "/skip: true /d" conda-recipe/meta.yaml # set the version to the local release-please version (for docs and docker publish) $SED_CMD "s/{% set version = \".+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml + # avoid error "No module named 'setup'" due to "from setup import c_ext" in setup.py + $SED_CMD "s/script: {{ PYTHON }} -m pip install --no-deps --ignore-installed ./script: PYTHONPATH=. {{ PYTHON }} -m pip install --no-deps --ignore-installed ./" conda-recipe/meta.yaml head -10 conda-recipe/meta.yaml set +x From de97eaab9f0ebfffbc41efd023de3ed2d6cefa13 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 18 Jun 2025 12:53:57 -0700 Subject: [PATCH 05/52] wip3 --- conda-recipe/conda_build_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 86572dcb..f35afa4d 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -11,3 +11,4 @@ python: - '3.10' - '3.11' - '3.12' +- '3.13' From 24f75ac0b7059d0ece9c177f047930bf799cc296 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 18 Jun 2025 13:18:19 -0700 Subject: [PATCH 06/52] wip4 --- pyproject.toml => _pyproject.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pyproject.toml => _pyproject.toml (100%) diff --git a/pyproject.toml b/_pyproject.toml similarity index 100% rename from pyproject.toml rename to _pyproject.toml From be315e13557671c1d4769bd2594042a0bac1aa53 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 18 Jun 2025 13:40:26 -0700 Subject: [PATCH 07/52] wip5 --- .github/workflows/run-tests.yaml | 18 +++++++++--------- conda-recipe/conda_build_config.yaml | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 4a259130..a2876136 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -31,16 +31,16 @@ jobs: strategy: matrix: include: - - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} - - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} - - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} - - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index f35afa4d..d3682e4c 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -7,8 +7,8 @@ c_stdlib_version: - '2.17' # [linux] - '14.5' # [osx] python: -- '3.9' -- '3.10' -- '3.11' -- '3.12' +#- '3.9' +#- '3.10' +#- '3.11' +#- '3.12' - '3.13' From 815af8aa39e26cd9484e1d779d4226bdb42f8910 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 18 Jun 2025 13:40:32 -0700 Subject: [PATCH 08/52] wip6 --- conda-recipe/conda_build_config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index d3682e4c..fb4a4c0a 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -12,3 +12,8 @@ python: #- '3.11' #- '3.12' - '3.13' + +# if not specified, default numpy version is 1.26 +# but py313 not supported for that version of numpy +numpy: + - 2 # [py>=313] \ No newline at end of file From 69304ea36c36a017416fa740d991106bc3f9919e Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 19 Jun 2025 07:23:12 -0700 Subject: [PATCH 09/52] wip7 --- genomekit_dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genomekit_dev.yml b/genomekit_dev.yml index d8b6f33c..41708cb7 100644 --- a/genomekit_dev.yml +++ b/genomekit_dev.yml @@ -5,7 +5,7 @@ channels: dependencies: - appdirs>=1.4.0 - numpy - - python=3.9.0,<3.14.0 + - python>=3.9.0,<3.14.0 - tqdm - typing-extensions - zlib From 0c80c02b52704e36cc2c338cdfb1c48b88ed6104 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 19 Jun 2025 09:46:40 -0700 Subject: [PATCH 10/52] wip8 --- conda-recipe/meta.yaml | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 conda-recipe/meta.yaml diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml new file mode 100644 index 00000000..42f5b70c --- /dev/null +++ b/conda-recipe/meta.yaml @@ -0,0 +1,68 @@ +{% set version = "7.1.0" %} + +package: + name: genomekit + version: {{ version }} + +source: + url: https://github.com/deepgenomics/GenomeKit/archive/refs/tags/v{{ version }}.tar.gz + sha256: a124ef26234b42da3db8f83d676949b54995b0d2965e4eed22ede3ad8c73506a + +build: + number: 0 + script: {{ PYTHON }} -m pip install --no-deps --ignore-installed . + skip: true # [py>312 or py<39] + +requirements: + build: + - {{ compiler('cxx') }} + - {{ stdlib("c") }} + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + # if not specified, default numpy version is 1.26 + # but py313 not supported for that version of numpy + - numpy ==2 # [py>=313 and build_platform != target_platform] + - numpy # [py<313 and build_platform != target_platform] + - setuptools # [build_platform != target_platform] + + host: + - appdirs + - numpy + - typing-extensions + - zlib + - python {{ python }} + - pip + - setuptools + + run: + - appdirs + - google-cloud-storage + - boto3 + - typing-extensions + - importlib-metadata + - tqdm + - python {{ python }} + +test: + imports: + - genome_kit + requires: + - pytest-cov + - pip + source_files: + - tests + commands: + - pip check + - python -m unittest discover + +about: + doc_url: https://deepgenomics.github.io/GenomeKit/ + home: https://github.com/deepgenomics/GenomeKit + license: "Apache-2.0" + license_file: "LICENSE" + summary: GenomeKit is a Python library for fast and easy access to genomic resources such as sequence, data tracks, and annotations. + +extra: + recipe-maintainers: + - s22chan + - ovesh From df832cdab54ed89507a2f959b689239db723f05f Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 19 Jun 2025 09:54:27 -0700 Subject: [PATCH 11/52] wip9 --- .github/actions/curl-meta-yaml/action.yaml | 2 +- conda-recipe/meta.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/curl-meta-yaml/action.yaml b/.github/actions/curl-meta-yaml/action.yaml index fe577312..f0666a5e 100644 --- a/.github/actions/curl-meta-yaml/action.yaml +++ b/.github/actions/curl-meta-yaml/action.yaml @@ -40,5 +40,5 @@ runs: # avoid error "No module named 'setup'" due to "from setup import c_ext" in setup.py $SED_CMD "s/script: {{ PYTHON }} -m pip install --no-deps --ignore-installed ./script: PYTHONPATH=. {{ PYTHON }} -m pip install --no-deps --ignore-installed ./" conda-recipe/meta.yaml - head -10 conda-recipe/meta.yaml + head -40 conda-recipe/meta.yaml set +x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 42f5b70c..289fbaa0 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -27,7 +27,8 @@ requirements: host: - appdirs - - numpy + - numpy ==2 # [py>=313] + - numpy # [py<313] - typing-extensions - zlib - python {{ python }} From 39ed5ad8c15bc64a4972d6621469672e7f1d2d13 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 19 Jun 2025 10:38:42 -0700 Subject: [PATCH 12/52] wip10 --- conda-recipe/meta.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 289fbaa0..67889804 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -21,14 +21,12 @@ requirements: - cross-python_{{ target_platform }} # [build_platform != target_platform] # if not specified, default numpy version is 1.26 # but py313 not supported for that version of numpy - - numpy ==2 # [py>=313 and build_platform != target_platform] - - numpy # [py<313 and build_platform != target_platform] + - numpy ==2 - setuptools # [build_platform != target_platform] host: - appdirs - - numpy ==2 # [py>=313] - - numpy # [py<313] + - numpy ==2 - typing-extensions - zlib - python {{ python }} From 41127ced93f77138e568ebaf67129a5176399cfe Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Mon, 23 Jun 2025 15:58:01 -0700 Subject: [PATCH 13/52] wip11 --- conda-recipe/meta.yaml | 11 ++- genome_kit/data_manager.py | 178 +++++++++++++++++++------------------ genome_kit/vcf_table.py | 6 +- 3 files changed, 102 insertions(+), 93 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 67889804..20ffaac5 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -11,7 +11,7 @@ source: build: number: 0 script: {{ PYTHON }} -m pip install --no-deps --ignore-installed . - skip: true # [py>312 or py<39] + skip: true # [py>313 or py<39] requirements: build: @@ -21,12 +21,14 @@ requirements: - cross-python_{{ target_platform }} # [build_platform != target_platform] # if not specified, default numpy version is 1.26 # but py313 not supported for that version of numpy - - numpy ==2 + - numpy ==2 # [py>=313 and build_platform != target_platform] + - numpy # [py<313 and build_platform != target_platform] - setuptools # [build_platform != target_platform] host: - appdirs - - numpy ==2 + - numpy ==2 # [py>=313] + - numpy # [py<313] - typing-extensions - zlib - python {{ python }} @@ -35,7 +37,8 @@ requirements: run: - appdirs - - google-cloud-storage + # google-cloud-storage breaks the build on py313 + - google-cloud-storage # [py<313] - boto3 - typing-extensions - importlib-metadata diff --git a/genome_kit/data_manager.py b/genome_kit/data_manager.py index 679ac30d..5f94bc22 100644 --- a/genome_kit/data_manager.py +++ b/genome_kit/data_manager.py @@ -11,7 +11,12 @@ from pathlib import Path from typing import Dict, Any -from google.cloud import storage +_SUPPORT_GCS = True +try: + from google.cloud import storage +except ImportError: + # google.cloud breaks build on py313 + _SUPPORT_GCS = False import boto3 from botocore import UNSIGNED from botocore.config import Config @@ -167,109 +172,110 @@ def FileIO(filename, desc, mode, size, quiet): ) yield decorated -class GCSDataManager(DataManager): - """A minimal data manager implementation that retrieves files from a GCS bucket.""" - def __init__(self, data_dir: str, bucket_name: str): - """ - Args: - data_dir: location where local files are cached - bucket_name: GCS bucket - """ - self._bucket_name = bucket_name - super().__init__(data_dir) +if _SUPPORT_GCS is True: + class GCSDataManager(DataManager): + """A minimal data manager implementation that retrieves files from a GCS bucket.""" + def __init__(self, data_dir: str, bucket_name: str): + """ + Args: + data_dir: location where local files are cached + bucket_name: GCS bucket + """ + self._bucket_name = bucket_name + super().__init__(data_dir) + + @property + def bucket(self): + if not hasattr(self, "_bucket"): + gcloud_client = storage.Client() + try: + self._bucket = gcloud_client.bucket(self._bucket_name, user_project=os.environ.get("GENOMEKIT_GCS_BILLING_PROJECT", None)) + except Exception as e: + # give the user a hint in case of permission errors + print(e, file=sys.stderr) + raise + + return self._bucket + + def get_file(self, filename: str) -> str: + local_path = Path(self.data_dir, filename) + + if local_path.exists(): + return str(local_path) - @property - def bucket(self): - if not hasattr(self, "_bucket"): - gcloud_client = storage.Client() try: - self._bucket = gcloud_client.bucket(self._bucket_name, user_project=os.environ.get("GENOMEKIT_GCS_BILLING_PROJECT", None)) + blob = self.bucket.blob(filename) + if not blob.exists(): + raise FileNotFoundError(f"File '{filename}' not found in the GCS bucket") except Exception as e: - # give the user a hint in case of permission errors - print(e, file=sys.stderr) + if "GENOMEKIT_TRACE" in os.environ: + # give the user a hint in case of permission errors + print(e, file=sys.stderr) raise - return self._bucket - - def get_file(self, filename: str) -> str: - local_path = Path(self.data_dir, filename) + # form a temporary filename to make the download safe + temp_file = tempfile.NamedTemporaryFile(delete=False, mode="wb", dir=self.data_dir, prefix=filename, suffix=".part") + temp_file_path = str(Path(self.data_dir, temp_file.name)) + try: + temp_file.close() + with FileIO(temp_file_path, filename, "wb", blob.size, quiet=False) as f: + blob.download_to_file(f) + except: + os.remove(temp_file_path) + raise + # atomically (on POSIX) rename the file to the real one + os.rename(temp_file_path, local_path) - if local_path.exists(): return str(local_path) - try: - blob = self.bucket.blob(filename) - if not blob.exists(): - raise FileNotFoundError(f"File '{filename}' not found in the GCS bucket") - except Exception as e: - if "GENOMEKIT_TRACE" in os.environ: - # give the user a hint in case of permission errors - print(e, file=sys.stderr) - raise + def _remote_equal(self, blob: storage.Blob, file_path: Path) -> bool: + remote_checksum = base64.b64decode(blob.md5_hash).hex() + with open(file_path, 'rb') as f: + local_checksum = _hashfile(f, hashlib.md5()) + return remote_checksum == local_checksum - # form a temporary filename to make the download safe - temp_file = tempfile.NamedTemporaryFile(delete=False, mode="wb", dir=self.data_dir, prefix=filename, suffix=".part") - temp_file_path = str(Path(self.data_dir, temp_file.name)) - try: - temp_file.close() - with FileIO(temp_file_path, filename, "wb", blob.size, quiet=False) as f: - blob.download_to_file(f) - except: - os.remove(temp_file_path) - raise - # atomically (on POSIX) rename the file to the real one - os.rename(temp_file_path, local_path) - - return str(local_path) - - def _remote_equal(self, blob: storage.Blob, file_path: Path) -> bool: - remote_checksum = base64.b64decode(blob.md5_hash).hex() - with open(file_path, 'rb') as f: - local_checksum = _hashfile(f, hashlib.md5()) - return remote_checksum == local_checksum - - def upload_file(self, filepath: str, filename: str, metadata: Dict[str, str]=None): - blob = self.bucket.blob(filename) - - if blob.exists(): - blob.reload() - if self._remote_equal(blob, Path(filepath)): - logger.info(f"File '{filename}' already exists in the GCS bucket and is identical, skipping.") - return + def upload_file(self, filepath: str, filename: str, metadata: Dict[str, str]=None): + blob = self.bucket.blob(filename) - if "GK_UPLOAD_ALLOW_OVERWRITE" not in os.environ: - raise ValueError(f"File '{filename}' already exists in the GCS bucket." - "Set GK_UPLOAD_ALLOW_OVERWRITE=1 to overwrite.") + if blob.exists(): + blob.reload() + if self._remote_equal(blob, Path(filepath)): + logger.info(f"File '{filename}' already exists in the GCS bucket and is identical, skipping.") + return + + if "GK_UPLOAD_ALLOW_OVERWRITE" not in os.environ: + raise ValueError(f"File '{filename}' already exists in the GCS bucket." + "Set GK_UPLOAD_ALLOW_OVERWRITE=1 to overwrite.") + else: + logger.warning(f"Overwriting '{filename}' on the server.") + + if blob.metadata is None: + blob.metadata = metadata + else: + blob.metadata.update(metadata or {}) else: - logger.warning(f"Overwriting '{filename}' on the server.") - - if blob.metadata is None: blob.metadata = metadata - else: - blob.metadata.update(metadata or {}) - else: - blob.metadata = metadata - with FileIO(filepath, filename, "rb", os.path.getsize(filepath), quiet=False) as f: - blob.upload_from_file(f) + with FileIO(filepath, filename, "rb", os.path.getsize(filepath), quiet=False) as f: + blob.upload_from_file(f) - @lru_cache - def list_available_genomes(self): - names = set() + @lru_cache + def list_available_genomes(self): + names = set() - def get_genomes(filenames): - return ( - name.rpartition(".")[0] - for name in filenames - if name.endswith(".2bit") or name.endswith(".cfg") - ) + def get_genomes(filenames): + return ( + name.rpartition(".")[0] + for name in filenames + if name.endswith(".2bit") or name.endswith(".cfg") + ) - names.update(get_genomes(os.listdir(self.data_dir))) + names.update(get_genomes(os.listdir(self.data_dir))) - blobs = self.bucket.list_blobs() - names.update(get_genomes(blob.name for blob in blobs)) + blobs = self.bucket.list_blobs() + names.update(get_genomes(blob.name for blob in blobs)) - return sorted(names) + return sorted(names) class DefaultDataManager(DataManager): """A minimal data manager implementation that retrieves files from a S3 bucket. diff --git a/genome_kit/vcf_table.py b/genome_kit/vcf_table.py index b8afb4bd..ac590361 100644 --- a/genome_kit/vcf_table.py +++ b/genome_kit/vcf_table.py @@ -5,7 +5,7 @@ import os import subprocess import tempfile -from distutils.spawn import find_executable +import shutil from typing import Optional, Union from typing_extensions import Literal @@ -26,7 +26,7 @@ def _dump_fasta(genome, chrom_mapper): """Dumps a FASTA containing the DNA sequence of refg, indexes it, and returns a context that can be treated as the path to the fasta. The temporaries are deleted once the context is closed.""" - samtools = find_executable("samtools") + samtools = shutil.which("samtools") if not samtools: raise RuntimeError("samtools must be installed in order to normalize VCFs") # pragma: no cover @@ -216,7 +216,7 @@ def open_clinvar(vcfpath): else: if normalize: # Check requirements - bcftools = find_executable("bcftools") + bcftools = shutil.which("bcftools") if not bcftools: raise RuntimeError("bcftools must be installed to normalize VCFs") # pragma: no cover From fd40b03ca45ceef7bce84b0ff9ae768acd86574e Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Mon, 23 Jun 2025 16:19:26 -0700 Subject: [PATCH 14/52] wip12 --- genome_kit/data_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/genome_kit/data_manager.py b/genome_kit/data_manager.py index 5f94bc22..bacde505 100644 --- a/genome_kit/data_manager.py +++ b/genome_kit/data_manager.py @@ -172,7 +172,10 @@ def FileIO(filename, desc, mode, size, quiet): ) yield decorated -if _SUPPORT_GCS is True: +if _SUPPORT_GCS is False: + class GCSDataManager(DataManager): + pass +else: class GCSDataManager(DataManager): """A minimal data manager implementation that retrieves files from a GCS bucket.""" def __init__(self, data_dir: str, bucket_name: str): From 6b601e121c4a2c2f9cabee7810a0727f6b47c912 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Mon, 23 Jun 2025 22:13:43 -0700 Subject: [PATCH 15/52] wip13 --- .github/workflows/run-tests.yaml | 2 +- conda-recipe/meta.yaml | 8 ++------ genome_kit/data_manager.py | 3 ++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index a2876136..f9f31785 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -40,7 +40,7 @@ jobs: # - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} # - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 20ffaac5..c08e1169 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -19,16 +19,12 @@ requirements: - {{ stdlib("c") }} - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - # if not specified, default numpy version is 1.26 - # but py313 not supported for that version of numpy - - numpy ==2 # [py>=313 and build_platform != target_platform] - - numpy # [py<313 and build_platform != target_platform] + - numpy # [build_platform != target_platform] - setuptools # [build_platform != target_platform] host: - appdirs - - numpy ==2 # [py>=313] - - numpy # [py<313] + - numpy - typing-extensions - zlib - python {{ python }} diff --git a/genome_kit/data_manager.py b/genome_kit/data_manager.py index bacde505..96b62279 100644 --- a/genome_kit/data_manager.py +++ b/genome_kit/data_manager.py @@ -174,7 +174,8 @@ def FileIO(filename, desc, mode, size, quiet): if _SUPPORT_GCS is False: class GCSDataManager(DataManager): - pass + def __init__(self, data_dir: str, bucket_name: str): + raise NotImplementedError("GCS support not available on py313") else: class GCSDataManager(DataManager): """A minimal data manager implementation that retrieves files from a GCS bucket.""" From 201809638e92e9558a491a734a034efdf011bbc1 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Mon, 23 Jun 2025 22:18:02 -0700 Subject: [PATCH 16/52] wip14 --- conda-recipe/meta.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index c08e1169..40d58e75 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -19,7 +19,10 @@ requirements: - {{ stdlib("c") }} - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - - numpy # [build_platform != target_platform] + # if not specified, default numpy version is 1.26 + # but py313 not supported for that version of numpy + - numpy ==2 # [py>=313 and build_platform != target_platform] + - numpy # [py<313 and build_platform != target_platform] - setuptools # [build_platform != target_platform] host: From b6b671107df669e0311deabb6ae46d877faa4989 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 07:16:38 -0700 Subject: [PATCH 17/52] wip15 --- conda-recipe/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 40d58e75..e8173c01 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -21,13 +21,13 @@ requirements: - cross-python_{{ target_platform }} # [build_platform != target_platform] # if not specified, default numpy version is 1.26 # but py313 not supported for that version of numpy - - numpy ==2 # [py>=313 and build_platform != target_platform] - - numpy # [py<313 and build_platform != target_platform] + - numpy # [build_platform != target_platform] - setuptools # [build_platform != target_platform] host: - appdirs - - numpy + - numpy ==2 # [py>=313] + - numpy # [py<313] - typing-extensions - zlib - python {{ python }} From c2235ffd70de10ef64f3809a794dd7672e7b1b4e Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 07:36:39 -0700 Subject: [PATCH 18/52] wip16 --- conda-recipe/conda_build_config.yaml | 7 +------ conda-recipe/meta.yaml | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index fb4a4c0a..3a26865c 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -11,9 +11,4 @@ python: #- '3.10' #- '3.11' #- '3.12' -- '3.13' - -# if not specified, default numpy version is 1.26 -# but py313 not supported for that version of numpy -numpy: - - 2 # [py>=313] \ No newline at end of file +- '3.13' \ No newline at end of file diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index e8173c01..e9622012 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -19,13 +19,12 @@ requirements: - {{ stdlib("c") }} - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - # if not specified, default numpy version is 1.26 - # but py313 not supported for that version of numpy - numpy # [build_platform != target_platform] - setuptools # [build_platform != target_platform] host: - appdirs + # I don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 - numpy ==2 # [py>=313] - numpy # [py<313] - typing-extensions From b6e7d90731b8c8998cedd9ff6f8b7a717ac31b90 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 08:50:57 -0700 Subject: [PATCH 19/52] wip17 --- conda-recipe/conda_build_config.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 3a26865c..3a46935a 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -11,4 +11,7 @@ python: #- '3.10' #- '3.11' #- '3.12' -- '3.13' \ No newline at end of file +- '3.13' + +numpy: + - 2 # [py>=313] \ No newline at end of file From c310459f25479de944aad7e31dbd050f4611a3cb Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 10:19:53 -0700 Subject: [PATCH 20/52] wip18 --- conda-recipe/meta.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index e9622012..b74544c4 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -24,9 +24,9 @@ requirements: host: - appdirs - # I don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 - - numpy ==2 # [py>=313] - - numpy # [py<313] + # don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 + - {{ numpy }} # [py>=313] + - numpy # [py<313] - typing-extensions - zlib - python {{ python }} From acd28641449ebc744bc17e767e96ca2422da14b9 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 10:55:48 -0700 Subject: [PATCH 21/52] wip19 --- conda-recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index b74544c4..52db17a6 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -25,7 +25,7 @@ requirements: host: - appdirs # don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 - - {{ numpy }} # [py>=313] + - numpy {{ numpy }} # [py>=313] - numpy # [py<313] - typing-extensions - zlib From 17e9104f7e65bdae10674a283d548f77c3642d1b Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 11:04:03 -0700 Subject: [PATCH 22/52] wip20 --- conda-recipe/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 52db17a6..b5819cd3 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -25,8 +25,8 @@ requirements: host: - appdirs # don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 - - numpy {{ numpy }} # [py>=313] - - numpy # [py<313] + - numpy ==2 # [py>=313] + - numpy # [py<313] - typing-extensions - zlib - python {{ python }} From 1ac0a0f050ce66b7627efeefaa9371e375644b65 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Tue, 24 Jun 2025 13:57:34 -0700 Subject: [PATCH 23/52] wip21 --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index f9f31785..c2fe542e 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -79,7 +79,7 @@ jobs: set -x if [ ! -d "${HOME}/micromamba/envs/test" ]; then micromamba create -yqn test -c conda-forge \ - boa==0.16.0 mamba==1.5.7 conda==24.1.2 coverage + boa==0.16.0 mamba==1.5.7 conda==24.1.2 fi set +x From a5dc958a3959f4dff1febd45155712588090b0f6 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 25 Jun 2025 11:52:03 -0700 Subject: [PATCH 24/52] wip22 --- .github/workflows/run-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index c2fe542e..d0df08c4 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -37,8 +37,8 @@ jobs: # - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} # - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} # - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} -# - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} @@ -79,6 +79,8 @@ jobs: set -x if [ ! -d "${HOME}/micromamba/envs/test" ]; then micromamba create -yqn test -c conda-forge \ + # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed + # (not needed?) boa==0.16.0 mamba==1.5.7 conda==24.1.2 fi set +x From 5fc7559eb7609923a0d7411480525de5092ef209 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 25 Jun 2025 12:17:18 -0700 Subject: [PATCH 25/52] wip23 --- .github/workflows/run-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index d0df08c4..9ad6db92 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -91,6 +91,7 @@ jobs: run: | set -x micromamba activate test + conda --version conda index ~/conda-bld files=(~/conda-bld/*/*py${{ matrix.pyver-short }}*.tar.bz2) if [ ! -e "${files[0]}" ]; then From 736a0f108193dfaa2f41a0ac794bf76774728de0 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 25 Jun 2025 13:40:09 -0700 Subject: [PATCH 26/52] wip24 --- .github/workflows/run-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 9ad6db92..7c49e0dd 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -77,10 +77,10 @@ jobs: shell: bash -l -e {0} run: | set -x + # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed + # (not needed?) if [ ! -d "${HOME}/micromamba/envs/test" ]; then micromamba create -yqn test -c conda-forge \ - # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed - # (not needed?) boa==0.16.0 mamba==1.5.7 conda==24.1.2 fi set +x From c3cce0aaf871a864643e52e3511a0b0074c85e17 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 25 Jun 2025 14:08:27 -0700 Subject: [PATCH 27/52] wip25 --- conda-recipe/conda_build_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 3a46935a..ed09d058 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -10,7 +10,7 @@ python: #- '3.9' #- '3.10' #- '3.11' -#- '3.12' +- '3.12' - '3.13' numpy: From f7f1ab56eaa2b49588795c9282cf88c1289411fe Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 25 Jun 2025 14:28:44 -0700 Subject: [PATCH 28/52] wip26 --- .github/workflows/run-tests.yaml | 13 ++++++------- conda-recipe/conda_build_config.yaml | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7c49e0dd..c2a916c5 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -31,12 +31,12 @@ jobs: strategy: matrix: include: -# - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} -# - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} -# - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} @@ -91,7 +91,6 @@ jobs: run: | set -x micromamba activate test - conda --version conda index ~/conda-bld files=(~/conda-bld/*/*py${{ matrix.pyver-short }}*.tar.bz2) if [ ! -e "${files[0]}" ]; then diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index ed09d058..61c3e60a 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -7,9 +7,9 @@ c_stdlib_version: - '2.17' # [linux] - '14.5' # [osx] python: -#- '3.9' -#- '3.10' -#- '3.11' +- '3.9' +- '3.10' +- '3.11' - '3.12' - '3.13' From 601a50d388761dca216b2fde2adf83252f0d310b Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 25 Jun 2025 14:59:41 -0700 Subject: [PATCH 29/52] wip27 --- .github/workflows/run-tests.yaml | 16 ++++++++-------- conda-recipe/conda_build_config.yaml | 11 ++++------- conda-recipe/meta.yaml | 4 ++-- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index c2a916c5..4c6b39a0 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -31,14 +31,14 @@ jobs: strategy: matrix: include: - - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} - - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} - - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} - - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} - - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} +# - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} +# - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 61c3e60a..d3682e4c 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -7,11 +7,8 @@ c_stdlib_version: - '2.17' # [linux] - '14.5' # [osx] python: -- '3.9' -- '3.10' -- '3.11' -- '3.12' +#- '3.9' +#- '3.10' +#- '3.11' +#- '3.12' - '3.13' - -numpy: - - 2 # [py>=313] \ No newline at end of file diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index b5819cd3..ca485591 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -25,8 +25,8 @@ requirements: host: - appdirs # don't know why, but if not specified, conda selects numpy v1.26, incompatible with py313 - - numpy ==2 # [py>=313] - - numpy # [py<313] + - numpy 2 # [py>=313] + - numpy # [py<313] - typing-extensions - zlib - python {{ python }} From 81a3923f73556420fde520fea934bd7d0c5c4ae6 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 11:56:43 -0700 Subject: [PATCH 30/52] wip28 --- .github/workflows/run-tests.yaml | 4 +--- conda-recipe/meta.yaml | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 4c6b39a0..f9f31785 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -77,11 +77,9 @@ jobs: shell: bash -l -e {0} run: | set -x - # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed - # (not needed?) if [ ! -d "${HOME}/micromamba/envs/test" ]; then micromamba create -yqn test -c conda-forge \ - boa==0.16.0 mamba==1.5.7 conda==24.1.2 + boa==0.16.0 mamba==1.5.7 conda==24.1.2 coverage fi set +x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index ca485591..94d77955 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -47,7 +47,6 @@ test: imports: - genome_kit requires: - - pytest-cov - pip source_files: - tests From 002b0ce229c57c6852681da562600f9effc77885 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 12:42:08 -0700 Subject: [PATCH 31/52] wip29 --- .github/actions/build-gk/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index 731a5f87..d44eca81 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -94,6 +94,6 @@ runs: id: save_gk_pkg uses: actions/cache/save@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld From c41e6ced62fccadf325e8aad1a4bef6e86b9eeff Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 13:12:56 -0700 Subject: [PATCH 32/52] wip30 --- .github/actions/build-gk/action.yaml | 2 +- .github/workflows/dockerize.yaml | 2 +- .github/workflows/publish-docs.yaml | 2 +- .github/workflows/run-tests.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index d44eca81..19c54101 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -24,7 +24,7 @@ runs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld diff --git a/.github/workflows/dockerize.yaml b/.github/workflows/dockerize.yaml index 55780aec..0f4aa8f1 100644 --- a/.github/workflows/dockerize.yaml +++ b/.github/workflows/dockerize.yaml @@ -39,7 +39,7 @@ jobs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld fail-on-cache-miss: true diff --git a/.github/workflows/publish-docs.yaml b/.github/workflows/publish-docs.yaml index 5999bec2..435a66f0 100644 --- a/.github/workflows/publish-docs.yaml +++ b/.github/workflows/publish-docs.yaml @@ -40,7 +40,7 @@ jobs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld fail-on-cache-miss: true diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index f9f31785..4ddad394 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -63,7 +63,7 @@ jobs: id: restore_gk_pkg uses: actions/cache/restore@v4 with: - key: mamba-env-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} + key: gk-tarballs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/workflows/run-tests.yaml', '.github/actions/**', 'conda-recipe/**', 'src/**', 'genome_kit/**', 'setup.py', 'setup/**', 'tests/**') }} path: | ~/conda-bld fail-on-cache-miss: true From 5f98ce03c40ea749d3ad7bd784d512627e7f0052 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 14:40:50 -0700 Subject: [PATCH 33/52] wip31 --- .github/actions/build-gk/action.yaml | 10 +++++++--- .github/workflows/run-tests.yaml | 16 ++++++++-------- conda-recipe/conda_build_config.yaml | 8 ++++---- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index 19c54101..39b2aff1 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -29,18 +29,18 @@ runs: ~/conda-bld - name: install mamba + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} uses: ./.github/actions/install-mamba with: installer-url: https://micro.mamba.pm/api/micromamba/${{ inputs.platform }}/1.5.8 - name: create build env + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} shell: bash -l -e {0} run: | set -x if [ ! -d "${HOME}/micromamba/envs/build" ]; then - micromamba create -yqn build -c conda-forge \ - boa==0.17.0 mamba==1.5.8 conda=24.5 \ - ccache + micromamba create -yqn build -c conda-forge boa==0.17.0 mamba==1.5.8 conda=24.5 ccache fi set +x @@ -50,6 +50,7 @@ runs: env: build - name: save mamba gk ccache + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} id: save_mamba_gk_ccache uses: actions/cache@v4 with: @@ -61,6 +62,7 @@ runs: ~/.ccache - name: set compiler cache size + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} shell: bash -l -e {0} run: | set -x @@ -81,6 +83,7 @@ runs: conda clean -it - name: cache mamba env + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} id: save_mamba_cache uses: actions/cache/save@v4 with: @@ -91,6 +94,7 @@ runs: ~/bin/micromamba - name: cache gk package tarballs + if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} id: save_gk_pkg uses: actions/cache/save@v4 with: diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 4ddad394..40416271 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -31,14 +31,14 @@ jobs: strategy: matrix: include: -# - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} -# - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} -# - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} -# - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} -# - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "39", "python-version": "3.9", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "39", "python-version": "3.9", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "310", "python-version": "3.10", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "310", "python-version": "3.10", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "311", "python-version": "3.11", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "311", "python-version": "3.11", "platform": "osx-64", "runs-on": "macos-latest"} + - {"pyver-short": "312", "python-version": "3.12", "platform": "linux-64", "runs-on": "ubuntu-latest"} + - {"pyver-short": "312", "python-version": "3.12", "platform": "osx-64", "runs-on": "macos-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "linux-64", "runs-on": "ubuntu-latest"} - {"pyver-short": "313", "python-version": "3.13", "platform": "osx-64", "runs-on": "macos-latest"} runs-on: ${{ matrix.runs-on }} diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index d3682e4c..f35afa4d 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -7,8 +7,8 @@ c_stdlib_version: - '2.17' # [linux] - '14.5' # [osx] python: -#- '3.9' -#- '3.10' -#- '3.11' -#- '3.12' +- '3.9' +- '3.10' +- '3.11' +- '3.12' - '3.13' From 1f8a0c7e86b3b87f601f41304c5ee994315701e5 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 15:50:47 -0700 Subject: [PATCH 34/52] wip32 --- .github/actions/build-gk/action.yaml | 2 +- .github/workflows/run-tests.yaml | 5 +++-- conda-recipe/meta.yaml | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index 39b2aff1..c58d4cfa 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -40,7 +40,7 @@ runs: run: | set -x if [ ! -d "${HOME}/micromamba/envs/build" ]; then - micromamba create -yqn build -c conda-forge boa==0.17.0 mamba==1.5.8 conda=24.5 ccache + micromamba create -yqn build -c conda-forge mamba==1.5.8 conda=24.5 ccache fi set +x diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 40416271..47889f54 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -77,9 +77,10 @@ jobs: shell: bash -l -e {0} run: | set -x + # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed + # (not needed?) if [ ! -d "${HOME}/micromamba/envs/test" ]; then - micromamba create -yqn test -c conda-forge \ - boa==0.16.0 mamba==1.5.7 conda==24.1.2 coverage + micromamba create -yqn test -c conda-forge mamba==1.5.7 conda==24.1.2 fi set +x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 94d77955..905528b5 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -47,6 +47,8 @@ test: imports: - genome_kit requires: + # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed + # (not needed?) - pip source_files: - tests From 66041b214c7287cdc69f0fdf6490a1cdc562f74f Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 16:39:03 -0700 Subject: [PATCH 35/52] wip33 --- .github/actions/build-gk/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index c58d4cfa..12ee8d98 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -39,8 +39,9 @@ runs: shell: bash -l -e {0} run: | set -x + # boa required for the mambabuild subcommand if [ ! -d "${HOME}/micromamba/envs/build" ]; then - micromamba create -yqn build -c conda-forge mamba==1.5.8 conda=24.5 ccache + micromamba create -yqn build -c conda-forge boa==0.17.0 mamba==1.5.8 conda=24.5 ccache fi set +x From 35499f193923ad50fa32672f547d82879894c629 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 16:41:24 -0700 Subject: [PATCH 36/52] wip34 --- .github/workflows/run-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 47889f54..871e24ee 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -76,11 +76,12 @@ jobs: - name: create test env shell: bash -l -e {0} run: | + # boa required for the mambabuild subcommand set -x # inconsistently got "coverage 7.9.1 is not supported on this platform" so removed # (not needed?) if [ ! -d "${HOME}/micromamba/envs/test" ]; then - micromamba create -yqn test -c conda-forge mamba==1.5.7 conda==24.1.2 + micromamba create -yqn test -c conda-forge boa==0.16.0 mamba==1.5.7 conda==24.1.2 fi set +x From 1d44e584f18387478fcf0ccc3ca983b9367c1cfe Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Thu, 26 Jun 2025 18:22:01 -0700 Subject: [PATCH 37/52] wip35 --- _pyproject.toml => pyproject.toml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename _pyproject.toml => pyproject.toml (100%) diff --git a/_pyproject.toml b/pyproject.toml similarity index 100% rename from _pyproject.toml rename to pyproject.toml From 2f6b1823bc5b76dc2a1347f4d61e2de339f80403 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Fri, 27 Jun 2025 07:54:47 -0700 Subject: [PATCH 38/52] wip36 --- .github/actions/build-gk/action.yaml | 2 + .github/workflows/build-wheels.yaml | 108 +++++++++++++-------------- 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/actions/build-gk/action.yaml b/.github/actions/build-gk/action.yaml index 12ee8d98..d6656e5c 100644 --- a/.github/actions/build-gk/action.yaml +++ b/.github/actions/build-gk/action.yaml @@ -28,6 +28,8 @@ runs: path: | ~/conda-bld + # skip remaining steps (except configure-conda) on cache hit + - name: install mamba if: ${{ steps.restore_gk_pkg.outputs.cache-hit != 'true' }} uses: ./.github/actions/install-mamba diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 9b553a36..33ab8fa8 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -5,12 +5,14 @@ on: push: # only build+push docker on release-please tags tags: [ "v*" ] + branches: + - feat/py313 paths: - ".github/actions/**" - ".github/workflows/**" - "genome_kit/**" - "setup.py" - - "_pyproject.toml" + - "pyproject.toml" - "setup/**" - "src/**" - "tests/**" @@ -33,15 +35,11 @@ jobs: id: restore_linux_wheels uses: actions/cache/restore@v4 with: - key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit - - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} - name: Rename pyproject.toml - run: mv _pyproject.toml pyproject.toml - - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 @@ -172,7 +170,7 @@ jobs: - name: Download built wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl fail-on-cache-miss: true @@ -214,7 +212,7 @@ jobs: # id: restore_linux_intel_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -222,7 +220,7 @@ jobs: # id: restore_linux_arm_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -230,7 +228,7 @@ jobs: # id: restore_macos_wheels # uses: actions/cache/restore@v4 # with: -# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -248,48 +246,48 @@ jobs: # verbose: 'true' # skip-existing: 'true' - publish-to-pypi: - name: Publish to PyPI - needs: [build-wheel-linux, test-wheel-macos] - runs-on: ubuntu-latest - - environment: - name: pypi - url: https://pypi.org/p/genomekit - - permissions: - id-token: write # mandatory for trusted publishing - - steps: - - uses: actions/checkout@v4 - - - name: Restore the linux intel wheels cache - id: restore_linux_intel_wheels - uses: actions/cache/restore@v4 - with: - key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} - path: wheelhouse/*.whl - fail-on-cache-miss: true - - - name: Restore the linux arm wheels cache - id: restore_linux_arm_wheels - uses: actions/cache/restore@v4 - with: - key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} - path: wheelhouse/*.whl - fail-on-cache-miss: true - - - name: Restore the macos wheels cache - id: restore_macos_wheels - uses: actions/cache/restore@v4 - with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} - path: wheelhouse/*.whl - fail-on-cache-miss: true - - - name: Publish distribution to PyPI - # don't use a version tag with 3rd party actions - uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc - with: - packages-dir: wheelhouse - verbose: 'true' +# publish-to-pypi: +# name: Publish to PyPI +# needs: [build-wheel-linux, test-wheel-macos] +# runs-on: ubuntu-latest +# +# environment: +# name: pypi +# url: https://pypi.org/p/genomekit +# +# permissions: +# id-token: write # mandatory for trusted publishing +# +# steps: +# - uses: actions/checkout@v4 +# +# - name: Restore the linux intel wheels cache +# id: restore_linux_intel_wheels +# uses: actions/cache/restore@v4 +# with: +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# path: wheelhouse/*.whl +# fail-on-cache-miss: true +# +# - name: Restore the linux arm wheels cache +# id: restore_linux_arm_wheels +# uses: actions/cache/restore@v4 +# with: +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# path: wheelhouse/*.whl +# fail-on-cache-miss: true +# +# - name: Restore the macos wheels cache +# id: restore_macos_wheels +# uses: actions/cache/restore@v4 +# with: +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# path: wheelhouse/*.whl +# fail-on-cache-miss: true +# +# - name: Publish distribution to PyPI +# # don't use a version tag with 3rd party actions +# uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc +# with: +# packages-dir: wheelhouse +# verbose: 'true' From 2bc4d725b0d8d7122940fcc0291d9fa146a8cfa2 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Fri, 27 Jun 2025 10:10:05 -0700 Subject: [PATCH 39/52] wip37 --- .github/workflows/build-wheels.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 33ab8fa8..8f8eaec8 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -104,15 +104,11 @@ jobs: id: restore_macos_wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit - - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} - name: Rename pyproject.toml - run: mv _pyproject.toml pyproject.toml - - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 From c07fb296ccc522587c9806d2e477fff96c50b479 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Fri, 27 Jun 2025 11:51:34 -0700 Subject: [PATCH 40/52] wip38 --- .github/workflows/build-wheels.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 8f8eaec8..9f7efdad 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -126,6 +126,8 @@ jobs: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" + CIBW_PIP_EXTRA_ARGS: "--no-deps --ignore-installed" + CIBW_ENVIRONMENT: "PYTHONPATH=." - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels From 6e90291ea966bb266f579945c10cbf6f19affe1c Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Fri, 27 Jun 2025 14:03:22 -0700 Subject: [PATCH 41/52] wip39 --- .github/workflows/build-wheels.yaml | 32 ++++++++++++++++++----------- _pyproject.toml | 6 ++++++ 2 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 _pyproject.toml diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 9f7efdad..bb2b250b 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -12,7 +12,7 @@ on: - ".github/workflows/**" - "genome_kit/**" - "setup.py" - - "pyproject.toml" + - "_pyproject.toml" - "setup/**" - "src/**" - "tests/**" @@ -35,11 +35,15 @@ jobs: id: restore_linux_wheels uses: actions/cache/restore@v4 with: - key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} + key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit + - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} + name: Rename pyproject.toml + run: mv _pyproject.toml pyproject.toml + - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 @@ -104,11 +108,15 @@ jobs: id: restore_macos_wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit + - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} + name: Rename pyproject.toml + run: mv _pyproject.toml pyproject.toml + - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 @@ -126,8 +134,8 @@ jobs: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" - CIBW_PIP_EXTRA_ARGS: "--no-deps --ignore-installed" - CIBW_ENVIRONMENT: "PYTHONPATH=." +# CIBW_PIP_EXTRA_ARGS: "--no-deps --ignore-installed" +# CIBW_ENVIRONMENT: "PYTHONPATH=." - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels @@ -168,7 +176,7 @@ jobs: - name: Download built wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl fail-on-cache-miss: true @@ -210,7 +218,7 @@ jobs: # id: restore_linux_intel_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -218,7 +226,7 @@ jobs: # id: restore_linux_arm_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -226,7 +234,7 @@ jobs: # id: restore_macos_wheels # uses: actions/cache/restore@v4 # with: -# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -263,7 +271,7 @@ jobs: # id: restore_linux_intel_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -271,7 +279,7 @@ jobs: # id: restore_linux_arm_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -279,7 +287,7 @@ jobs: # id: restore_macos_wheels # uses: actions/cache/restore@v4 # with: -# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # diff --git a/_pyproject.toml b/_pyproject.toml new file mode 100644 index 00000000..ce3f3eed --- /dev/null +++ b/_pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = [ + "setuptools>=61", + "wheel", + "numpy==2" +] \ No newline at end of file From 638edf025e9dda599d3c0dea67d8e527c4be9020 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Fri, 27 Jun 2025 14:29:33 -0700 Subject: [PATCH 42/52] Revert "wip39" This reverts commit 6e90291ea966bb266f579945c10cbf6f19affe1c. --- .github/workflows/build-wheels.yaml | 32 +++++++++++------------------ _pyproject.toml | 6 ------ 2 files changed, 12 insertions(+), 26 deletions(-) delete mode 100644 _pyproject.toml diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index bb2b250b..9f7efdad 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -12,7 +12,7 @@ on: - ".github/workflows/**" - "genome_kit/**" - "setup.py" - - "_pyproject.toml" + - "pyproject.toml" - "setup/**" - "src/**" - "tests/**" @@ -35,15 +35,11 @@ jobs: id: restore_linux_wheels uses: actions/cache/restore@v4 with: - key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: linux-wheels-${{ matrix.arch }}-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit - - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} - name: Rename pyproject.toml - run: mv _pyproject.toml pyproject.toml - - if: ${{ steps.restore_linux_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 @@ -108,15 +104,11 @@ jobs: id: restore_macos_wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl # skip remaining steps on cache hit - - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} - name: Rename pyproject.toml - run: mv _pyproject.toml pyproject.toml - - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: Set up Python uses: actions/setup-python@v5 @@ -134,8 +126,8 @@ jobs: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" -# CIBW_PIP_EXTRA_ARGS: "--no-deps --ignore-installed" -# CIBW_ENVIRONMENT: "PYTHONPATH=." + CIBW_PIP_EXTRA_ARGS: "--no-deps --ignore-installed" + CIBW_ENVIRONMENT: "PYTHONPATH=." - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels @@ -176,7 +168,7 @@ jobs: - name: Download built wheels uses: actions/cache/restore@v4 with: - key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} + key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} path: wheelhouse/*.whl fail-on-cache-miss: true @@ -218,7 +210,7 @@ jobs: # id: restore_linux_intel_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -226,7 +218,7 @@ jobs: # id: restore_linux_arm_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -234,7 +226,7 @@ jobs: # id: restore_macos_wheels # uses: actions/cache/restore@v4 # with: -# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -271,7 +263,7 @@ jobs: # id: restore_linux_intel_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-x86_64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -279,7 +271,7 @@ jobs: # id: restore_linux_arm_wheels # uses: actions/cache/restore@v4 # with: -# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: linux-wheels-aarch64-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # @@ -287,7 +279,7 @@ jobs: # id: restore_macos_wheels # uses: actions/cache/restore@v4 # with: -# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', '_pyproject.toml', 'setup.py', 'setup/**') }} +# key: macos-wheels-${{ hashFiles('.github/workflows/build-wheels.yaml', '.github/actions/**', 'src/**', 'genome_kit/**', 'pyproject.toml', 'setup.py', 'setup/**') }} # path: wheelhouse/*.whl # fail-on-cache-miss: true # diff --git a/_pyproject.toml b/_pyproject.toml deleted file mode 100644 index ce3f3eed..00000000 --- a/_pyproject.toml +++ /dev/null @@ -1,6 +0,0 @@ -[build-system] -requires = [ - "setuptools>=61", - "wheel", - "numpy==2" -] \ No newline at end of file From 275fdf9652dd96fe970c8f633111827efa8d21fa Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Fri, 27 Jun 2025 14:30:02 -0700 Subject: [PATCH 43/52] wip40 --- setup.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/setup.py b/setup.py index 8abb9dd9..f9ee7c48 100644 --- a/setup.py +++ b/setup.py @@ -31,17 +31,6 @@ def run(self): if __name__ == "__main__": install_requires = [] - if os.environ.get("GK_BUILD_WHEELS", None) is not None: - install_requires = [ - "appdirs>=1.4.0", - "numpy<2.0dev0", - "google-cloud-storage>=2.10.0", - "boto3", - "tqdm", - "setuptools", - "importlib-metadata", - "typing-extensions", - ] setup( author="Deep Genomics", author_email="info@deepgenomics.com", From be4acf6cc3467e146297a71268b117b35941b24d Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 15:12:00 -0700 Subject: [PATCH 44/52] wip41 --- setup.py | 357 ++++++++++++++++++++++++++++++++++++++++++++++++- setup/c_ext.py | 352 ------------------------------------------------ 2 files changed, 354 insertions(+), 355 deletions(-) delete mode 100644 setup/c_ext.py diff --git a/setup.py b/setup.py index f9ee7c48..bd1072b5 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,19 @@ # Copyright (C) 2016-2023 Deep Genomics Inc. All Rights Reserved. import os +import platform +import sys +import sysconfig +from glob import glob from pathlib import Path +from traceback import extract_stack +from setuptools._distutils import ccompiler +import shutil +from setuptools import Extension from setuptools import setup, find_packages +from setuptools.command.build_ext import build_ext from setuptools.command.egg_info import egg_info -from setup import c_ext + COPYRIGHT_FILE = "COPYRIGHT.txt" LICENSE_FILE = "LICENSE" @@ -29,6 +38,348 @@ def run(self): egg_info.run(self) + + +libname = "genome_kit" + +############################################################## +# Define the C Extension build config +############################################################## + +debug = False # Set to True for C++ debug symbols and extra memory/bounds checks +debug_info = debug +debug_objects = False # Enable printing of Python C++ instances being constructed/destructed +toolset = "msvc" if platform.system() == "Windows" else "gcc" + +gen_dir = os.path.join('build', 'gen') +include_dirs = [ + sys.prefix + "/include", + gen_dir, + ] +library_dirs = [sys.prefix + '/lib'] +runtime_library_dirs = [] + +if not [x for x in extract_stack(limit=20) if 'load_setup_py_data' in x[2]]: + # don't load numpy if we just need the version to fill the conda-build meta.yaml template + import numpy as np + include_dirs.append(np.get_include()) + +sources = glob("src/*.cpp") +headers = glob("src/*.h") +libraries = [] + +define_macros = [ + ("GKPY_LIBNAME", libname), +] + +if debug_objects: + define_macros += [ + ("GKPY_TRACE_MEM", None), # ctor/dtor notifications from C objects + ] + +extra_compile_args = [] +extra_link_args = [] + +if toolset == "gcc": + # Using multiprocessing and ccache massively speeds up incremental builds + ccache = shutil.which('ccache') + if ccache: + os.environ["CC"] = "{} {}".format(ccache, os.environ.get("CC", "gcc")) + # TODO cannot use ccache in CXX since distutils hotpatches into LDSHARED: + # https://github.com/python/cpython/blob/069306312addf87252e2dbf250fc7632fc8b7da3/Lib/distutils/unixccompiler.py#L191 + os.environ["LDSHARED"] = sysconfig.get_config_var("LDSHARED") + else: + print("WARNING: did not find ccache installed; files will be built from scratch every time") + + define_macros += [ + ("_FILE_OFFSET_BITS", 64), + ] + if debug: + define_macros += [ + ("GK_DEBUG", None), # Enable debug assertions etc + ] + + libraries += [ + # python is not linked for conda's python + # https://github.com/ContinuumIO/anaconda-issues/issues/9078#issuecomment-378321357 + "z", + ] + + # GCC flags common to both debug and release modes + extra_compile_args += [ + "-std=c++20", + "-fvisibility=hidden", # reduce symbols for code size/load times + "-fvisibility-inlines-hidden", + "-Wall", + "-Wno-write-strings", # char* in Python API + "-Wno-invalid-offsetof", # offsetof non-POD GK types for Python API + ] + + if debug: + opt_args = [ + "-O0", + "-UNDEBUG", + ] + else: + opt_args = [ + "-O3", + ] + + if debug_info: + opt_args += [ + "-g3", + ] + else: + extra_link_args += [ + "-Wl,-S", + "-Wl,-x", + ] + + if platform.system() == "Darwin": + # >=10.15 required for std::filesystem::remove + osx_sdk = "-mmacosx-version-min={}".format(os.environ.get("MACOSX_DEPLOYMENT_TARGET", "10.15")) + + extra_compile_args += [ + osx_sdk, + "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), + "-stdlib=libc++", + "-Wshorten-64-to-32", # catch implicit truncation as per MSVC + "-Wsign-compare", # match MSVC(/W3)/gcc(-Wall) + "-Wconditional-uninitialized", # gcc does better here but enable for safety + "-Wuninitialized", + "-Wno-unknown-warning-option", + ] + + extra_link_args += [ + osx_sdk, + "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), + ] + define_macros += [ + # https://conda-forge.org/docs/maintainer/knowledge_base/#newer-c-features-with-old-sdk + ("_LIBCPP_DISABLE_AVAILABILITY", None), + ] + + extra_compile_args += opt_args + extra_link_args += opt_args # required for LTO + + +elif toolset == "msvc": + + os.environ["DISTUTILS_USE_SDK"] = "1" + os.environ["MSSdk"] = "1" + + condalib_dir = sys.prefix + "/Library" + condalib_inc = condalib_dir + "/include" + condalib_lib = condalib_dir + "/lib" + include_dirs.append(condalib_inc) + library_dirs.append(condalib_lib) + + define_macros += [ + ("_CRT_SECURE_NO_WARNINGS", None), + ] + + libraries += [ + "zlib", + ] + + # VC flags common to both debug and release modes + extra_compile_args += [ + "/std:c++20", + "/permissive-", + "/Zc:__cplusplus", + "/Zc:strictStrings-", # don"t let strings be written to by default + # compatibility with __VA_OPT__ + # see https://devblogs.microsoft.com/cppblog/announcing-full-support-for-a-c-c-conformant-preprocessor-in-msvc/ + "/Zc:preprocessor", + "/W3", # Warning level 3 + "/EHsc", # Enable C++ and structured exception handling (e.g. catch access violations) + "/wd5033", # Python usage of deprecated register keyword + ] + + extra_link_args += [ + "/PDB:%s\\_cxx.pdb" % libname, # Put it right beside the .pyd/.so file in "develop" mode + ] + + if debug: + # do NOT define _DEBUG; need non-debug runtime to match NDEBUG Python distribution + # define_macros += [ + # ("_DEBUG", None), + # ] + extra_compile_args += [ + "/GS", # Enable buffer overrun checks + "/Zi", # Enable debug information .pdb + "/Od", # Disable optimizations + ] + extra_link_args += [ + "/DEBUG", + ] + else: + extra_compile_args += [ + "/GL", # Enable whole-program optimization + "/Gy", # Enable function-level linking + "/Oy", # Omit frame pointers + "/Oi", # Enable intrinsics + #"/Zi", # Enable debug information .pdb + ] + extra_link_args += [ + "/LTCG", # Enable link-time code generation + #"/DEBUG", + ] + + +class NoCWarningsBuildExt(build_ext): + def build_extensions(self): + for x in ["-Wstrict-prototypes"]: + try: + self.compiler.compiler_so.remove(x) + except (AttributeError, ValueError): + continue + build_ext.build_extensions(self) + + +extension = Extension( + libname + "._cxx", + sources=sources, + depends=headers, + include_dirs=include_dirs, + define_macros=define_macros, + library_dirs=library_dirs, + libraries=libraries, + runtime_library_dirs=runtime_library_dirs, + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + ) + +############################################################## + +# monkey-patch for parallel compilation +# TODO +# taken from http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils + +PARALLEL_JOBS = 4 # number of parallel compilations + + +def gcc_parallel_ccompile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + # those lines are copied from distutils.ccompiler.CCompiler directly + macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + # parallel code + import multiprocessing.pool + + def _single_compile(obj): + try: + src, ext = build[obj] + except KeyError: + return + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + + # convert to list, imap is evaluated on-demand + list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_single_compile, objects)) + return objects + + +def windows_parallel_ccompile(self, + sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info + + from distutils.errors import CompileError + from distutils.errors import DistutilsExecError + + compile_opts = extra_preargs or [] + compile_opts.append("/c") + compile_opts.extend(self.compile_options_debug if debug else self.compile_options) + + def _compile_obj(obj): + try: + src, ext = build[obj] + except KeyError: + return + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + [output_opt] + [input_opt]) + except DistutilsExecError as msg: + raise CompileError(msg) + return + elif ext in self._mc_extensions: + # Compile .MC to .RC file to .RES file. + # * "-h dir" specifies the directory for the + # generated include file + # * "-r dir" specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc] + ["-h", h_dir, "-r", rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + ".rc") + # then compile .RC to .RES file + self.spawn([self.rc] + ["/fo" + obj] + [rc_file]) + + except DistutilsExecError as msg: + raise CompileError(msg) + return + else: + # how to handle this file? + raise CompileError("Don't know how to compile %s to %s" % (src, obj)) + + output_opt = "/Fo" + obj + try: + self.spawn([self.cc] + compile_opts + pp_opts + [input_opt, output_opt] + extra_postargs) + except DistutilsExecError as msg: + raise CompileError(msg) + + import multiprocessing.pool + list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_compile_obj, objects)) + + return objects + + +if sys.platform == "win32": + # TODO from setuptools._distutils._msvccompiler import MSVCCompiler + import distutils._msvccompiler + distutils._msvccompiler.MSVCCompiler.compile = windows_parallel_ccompile +else: + ccompiler.CCompiler.compile = gcc_parallel_ccompile + if __name__ == "__main__": install_requires = [] setup( @@ -56,10 +407,10 @@ def run(self): "Documentation": "https://deepgenomics.github.io/GenomeKit" }, cmdclass={ - 'build_ext': c_ext.NoCWarningsBuildExt, + 'build_ext': NoCWarningsBuildExt, 'egg_info': egg_info_ex }, - ext_modules=[c_ext.extension], + ext_modules=[extension], test_suite="tests", tests_require=tests_require, url=f"https://github.com/deepgenomics/GenomeKit", diff --git a/setup/c_ext.py b/setup/c_ext.py deleted file mode 100644 index 4993c6ae..00000000 --- a/setup/c_ext.py +++ /dev/null @@ -1,352 +0,0 @@ -# Copyright (C) 2016-2023 Deep Genomics Inc. All Rights Reserved. - -import os -import platform -import sys -import sysconfig - -import distutils.ccompiler - -from distutils.spawn import find_executable -from glob import glob -from setuptools.command.build_ext import build_ext -from setuptools import Extension -from traceback import extract_stack - -libname = "genome_kit" - -############################################################## -# Define the C Extension build config -############################################################## - -debug = False # Set to True for C++ debug symbols and extra memory/bounds checks -debug_info = debug -debug_objects = False # Enable printing of Python C++ instances being constructed/destructed -toolset = "msvc" if platform.system() == "Windows" else "gcc" - -gen_dir = os.path.join('build', 'gen') -include_dirs = [ - sys.prefix + "/include", - gen_dir, -] -library_dirs = [sys.prefix + '/lib'] -runtime_library_dirs = [] - -if not [x for x in extract_stack(limit=20) if 'load_setup_py_data' in x[2]]: - # don't load numpy if we just need the version to fill the conda-build meta.yaml template - import numpy as np - include_dirs.append(np.get_include()) - -sources = glob("src/*.cpp") -headers = glob("src/*.h") -libraries = [] - -define_macros = [ - ("GKPY_LIBNAME", libname), -] - -if debug_objects: - define_macros += [ - ("GKPY_TRACE_MEM", None), # ctor/dtor notifications from C objects - ] - -extra_compile_args = [] -extra_link_args = [] - -if toolset == "gcc": - # Using multiprocessing and ccache massively speeds up incremental builds - ccache = find_executable('ccache') - if ccache: - os.environ["CC"] = "{} {}".format(ccache, os.environ.get("CC", "gcc")) - # cannot use ccache in CXX since distutils hotpatches into LDSHARED: - # https://github.com/python/cpython/blob/069306312addf87252e2dbf250fc7632fc8b7da3/Lib/distutils/unixccompiler.py#L191 - os.environ["LDSHARED"] = sysconfig.get_config_var("LDSHARED") - else: - print("WARNING: did not find ccache installed; files will be built from scratch every time") - - define_macros += [ - ("_FILE_OFFSET_BITS", 64), - ] - if debug: - define_macros += [ - ("GK_DEBUG", None), # Enable debug assertions etc - ] - - libraries += [ - # python is not linked for conda's python - # https://github.com/ContinuumIO/anaconda-issues/issues/9078#issuecomment-378321357 - "z", - ] - - # GCC flags common to both debug and release modes - extra_compile_args += [ - "-std=c++20", - "-fvisibility=hidden", # reduce symbols for code size/load times - "-fvisibility-inlines-hidden", - "-Wall", - "-Wno-write-strings", # char* in Python API - "-Wno-invalid-offsetof", # offsetof non-POD GK types for Python API - ] - - if debug: - opt_args = [ - "-O0", - "-UNDEBUG", - ] - else: - opt_args = [ - "-O3", - ] - - if debug_info: - opt_args += [ - "-g3", - ] - else: - extra_link_args += [ - "-Wl,-S", - "-Wl,-x", - ] - - if platform.system() == "Darwin": - # >=10.15 required for std::filesystem::remove - osx_sdk = "-mmacosx-version-min={}".format(os.environ.get("MACOSX_DEPLOYMENT_TARGET", "10.15")) - - extra_compile_args += [ - osx_sdk, - "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), - "-stdlib=libc++", - "-Wshorten-64-to-32", # catch implicit truncation as per MSVC - "-Wsign-compare", # match MSVC(/W3)/gcc(-Wall) - "-Wconditional-uninitialized", # gcc does better here but enable for safety - "-Wuninitialized", - "-Wno-unknown-warning-option", - ] - - extra_link_args += [ - osx_sdk, - "-isysroot{}".format(os.environ.get("CONDA_BUILD_SYSROOT", "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk")), - ] - define_macros += [ - # https://conda-forge.org/docs/maintainer/knowledge_base/#newer-c-features-with-old-sdk - ("_LIBCPP_DISABLE_AVAILABILITY", None), - ] - - extra_compile_args += opt_args - extra_link_args += opt_args # required for LTO - - -elif toolset == "msvc": - - os.environ["DISTUTILS_USE_SDK"] = "1" - os.environ["MSSdk"] = "1" - - condalib_dir = sys.prefix + "/Library" - condalib_inc = condalib_dir + "/include" - condalib_lib = condalib_dir + "/lib" - include_dirs.append(condalib_inc) - library_dirs.append(condalib_lib) - - define_macros += [ - ("_CRT_SECURE_NO_WARNINGS", None), - ] - - libraries += [ - "zlib", - ] - - # VC flags common to both debug and release modes - extra_compile_args += [ - "/std:c++20", - "/permissive-", - "/Zc:__cplusplus", - "/Zc:strictStrings-", # don"t let strings be written to by default - # compatibility with __VA_OPT__ - # see https://devblogs.microsoft.com/cppblog/announcing-full-support-for-a-c-c-conformant-preprocessor-in-msvc/ - "/Zc:preprocessor", - "/W3", # Warning level 3 - "/EHsc", # Enable C++ and structured exception handling (e.g. catch access violations) - "/wd5033", # Python usage of deprecated register keyword - ] - - extra_link_args += [ - "/PDB:%s\\_cxx.pdb" % libname, # Put it right beside the .pyd/.so file in "develop" mode - ] - - if debug: - # do NOT define _DEBUG; need non-debug runtime to match NDEBUG Python distribution - # define_macros += [ - # ("_DEBUG", None), - # ] - extra_compile_args += [ - "/GS", # Enable buffer overrun checks - "/Zi", # Enable debug information .pdb - "/Od", # Disable optimizations - ] - extra_link_args += [ - "/DEBUG", - ] - else: - extra_compile_args += [ - "/GL", # Enable whole-program optimization - "/Gy", # Enable function-level linking - "/Oy", # Omit frame pointers - "/Oi", # Enable intrinsics - #"/Zi", # Enable debug information .pdb - ] - extra_link_args += [ - "/LTCG", # Enable link-time code generation - #"/DEBUG", - ] - - -class NoCWarningsBuildExt(build_ext): - def build_extensions(self): - for x in ["-Wstrict-prototypes"]: - try: - self.compiler.compiler_so.remove(x) - except (AttributeError, ValueError): - continue - build_ext.build_extensions(self) - - -extension = Extension( - libname + "._cxx", - sources=sources, - depends=headers, - include_dirs=include_dirs, - define_macros=define_macros, - library_dirs=library_dirs, - libraries=libraries, - runtime_library_dirs=runtime_library_dirs, - extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, -) - -############################################################## - -# monkey-patch for parallel compilation -# taken from http://stackoverflow.com/questions/11013851/speeding-up-build-process-with-distutils - -PARALLEL_JOBS = 4 # number of parallel compilations - - -def gcc_parallel_ccompile(self, - sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None): - # those lines are copied from distutils.ccompiler.CCompiler directly - macros, objects, extra_postargs, pp_opts, build = self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) - cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) - # parallel code - import multiprocessing.pool - - def _single_compile(obj): - try: - src, ext = build[obj] - except KeyError: - return - self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) - - # convert to list, imap is evaluated on-demand - list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_single_compile, objects)) - return objects - - -def windows_parallel_ccompile(self, - sources, - output_dir=None, - macros=None, - include_dirs=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - depends=None): - if not self.initialized: - self.initialize() - compile_info = self._setup_compile(output_dir, macros, include_dirs, sources, depends, extra_postargs) - macros, objects, extra_postargs, pp_opts, build = compile_info - - from distutils.errors import CompileError - from distutils.errors import DistutilsExecError - - compile_opts = extra_preargs or [] - compile_opts.append("/c") - compile_opts.extend(self.compile_options_debug if debug else self.compile_options) - - def _compile_obj(obj): - try: - src, ext = build[obj] - except KeyError: - return - if debug: - # pass the full pathname to MSVC in debug mode, - # this allows the debugger to find the source file - # without asking the user to browse for it - src = os.path.abspath(src) - - if ext in self._c_extensions: - input_opt = "/Tc" + src - elif ext in self._cpp_extensions: - input_opt = "/Tp" + src - elif ext in self._rc_extensions: - # compile .RC to .RES file - input_opt = src - output_opt = "/fo" + obj - try: - self.spawn([self.rc] + pp_opts + [output_opt] + [input_opt]) - except DistutilsExecError as msg: - raise CompileError(msg) - return - elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. - # * "-h dir" specifies the directory for the - # generated include file - # * "-r dir" specifies the target directory of the - # generated RC file and the binary message resource - # it includes - # - # For now (since there are no options to change this), - # we use the source-directory for the include file and - # the build directory for the RC file and message - # resources. This works at least for win32all. - h_dir = os.path.dirname(src) - rc_dir = os.path.dirname(obj) - try: - # first compile .MC to .RC and .H file - self.spawn([self.mc] + ["-h", h_dir, "-r", rc_dir] + [src]) - base, _ = os.path.splitext(os.path.basename(src)) - rc_file = os.path.join(rc_dir, base + ".rc") - # then compile .RC to .RES file - self.spawn([self.rc] + ["/fo" + obj] + [rc_file]) - - except DistutilsExecError as msg: - raise CompileError(msg) - return - else: - # how to handle this file? - raise CompileError("Don't know how to compile %s to %s" % (src, obj)) - - output_opt = "/Fo" + obj - try: - self.spawn([self.cc] + compile_opts + pp_opts + [input_opt, output_opt] + extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - import multiprocessing.pool - list(multiprocessing.pool.ThreadPool(PARALLEL_JOBS).imap(_compile_obj, objects)) - - return objects - - -if sys.platform == "win32": - import distutils._msvccompiler - distutils._msvccompiler.MSVCCompiler.compile = windows_parallel_ccompile -else: - distutils.ccompiler.CCompiler.compile = gcc_parallel_ccompile From 7c94ba71328b9f74a9c2dff6189d0e2f69a53002 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 15:37:18 -0700 Subject: [PATCH 45/52] wip42 --- .github/actions/curl-meta-yaml/action.yaml | 3 ++- .github/workflows/build-wheels.yaml | 2 -- setup.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/actions/curl-meta-yaml/action.yaml b/.github/actions/curl-meta-yaml/action.yaml index f0666a5e..1f40e81d 100644 --- a/.github/actions/curl-meta-yaml/action.yaml +++ b/.github/actions/curl-meta-yaml/action.yaml @@ -38,7 +38,8 @@ runs: # set the version to the local release-please version (for docs and docker publish) $SED_CMD "s/{% set version = \".+\" %}/{% set version = \"${GK_VERSION}\" %}/" conda-recipe/meta.yaml # avoid error "No module named 'setup'" due to "from setup import c_ext" in setup.py - $SED_CMD "s/script: {{ PYTHON }} -m pip install --no-deps --ignore-installed ./script: PYTHONPATH=. {{ PYTHON }} -m pip install --no-deps --ignore-installed ./" conda-recipe/meta.yaml + # See if this can be removed + # $SED_CMD "s/script: {{ PYTHON }} -m pip install --no-deps --ignore-installed ./script: PYTHONPATH=. {{ PYTHON }} -m pip install --no-deps --ignore-installed ./" conda-recipe/meta.yaml head -40 conda-recipe/meta.yaml set +x diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 9f7efdad..8f8eaec8 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -126,8 +126,6 @@ jobs: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" - CIBW_PIP_EXTRA_ARGS: "--no-deps --ignore-installed" - CIBW_ENVIRONMENT: "PYTHONPATH=." - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels diff --git a/setup.py b/setup.py index bd1072b5..9f8ff1b0 100644 --- a/setup.py +++ b/setup.py @@ -382,6 +382,18 @@ def _compile_obj(obj): if __name__ == "__main__": install_requires = [] + if os.environ.get("GK_BUILD_WHEELS", None) is not None: + install_requires = [ + "appdirs>=1.4.0", + "numpy=2; python_version>='3.13'", + "numpy<2.0dev0; python_version<'3.13'" + "google-cloud-storage>=2.10.0", + "boto3", + "tqdm", + "setuptools", + "importlib-metadata", + "typing-extensions", + ] setup( author="Deep Genomics", author_email="info@deepgenomics.com", From 2c840b67aa1fad99ddc0773a446c03d2f810f42e Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 15:45:55 -0700 Subject: [PATCH 46/52] wip43 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9f8ff1b0..df98c02f 100644 --- a/setup.py +++ b/setup.py @@ -385,7 +385,7 @@ def _compile_obj(obj): if os.environ.get("GK_BUILD_WHEELS", None) is not None: install_requires = [ "appdirs>=1.4.0", - "numpy=2; python_version>='3.13'", + "numpy>=2.0.0; python_version>='3.13'", "numpy<2.0dev0; python_version<'3.13'" "google-cloud-storage>=2.10.0", "boto3", From f540e1e18290f989ed589246e35e262080c59a51 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 15:50:32 -0700 Subject: [PATCH 47/52] wip44 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index df98c02f..48726c54 100644 --- a/setup.py +++ b/setup.py @@ -385,8 +385,8 @@ def _compile_obj(obj): if os.environ.get("GK_BUILD_WHEELS", None) is not None: install_requires = [ "appdirs>=1.4.0", - "numpy>=2.0.0; python_version>='3.13'", - "numpy<2.0dev0; python_version<'3.13'" + "numpy=2; python_version>='3.13'", + "numpy<2.0dev0; python_version<'3.13'", "google-cloud-storage>=2.10.0", "boto3", "tqdm", From 00c86019efe33720ed1d60e1ad01923295862a87 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 15:53:33 -0700 Subject: [PATCH 48/52] wip45 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48726c54..b4871033 100644 --- a/setup.py +++ b/setup.py @@ -385,7 +385,7 @@ def _compile_obj(obj): if os.environ.get("GK_BUILD_WHEELS", None) is not None: install_requires = [ "appdirs>=1.4.0", - "numpy=2; python_version>='3.13'", + "numpy=>2.0.0; python_version>='3.13'", "numpy<2.0dev0; python_version<'3.13'", "google-cloud-storage>=2.10.0", "boto3", From 09ed3a867cc2ce7fff695232e7351dc7928e64c9 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 16:02:51 -0700 Subject: [PATCH 49/52] wip46 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b4871033..55e93ee3 100644 --- a/setup.py +++ b/setup.py @@ -385,7 +385,7 @@ def _compile_obj(obj): if os.environ.get("GK_BUILD_WHEELS", None) is not None: install_requires = [ "appdirs>=1.4.0", - "numpy=>2.0.0; python_version>='3.13'", + "numpy>=2.0.0; python_version>='3.13'", "numpy<2.0dev0; python_version<'3.13'", "google-cloud-storage>=2.10.0", "boto3", From 0e1f3b70e2a1153a8accf8c6537d64905382700c Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 16:26:59 -0700 Subject: [PATCH 50/52] wip47 --- .github/workflows/build-wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 8f8eaec8..4e06dff8 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -124,7 +124,7 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse && ls -l wheelhouse/ env: GK_BUILD_WHEELS: "1" - CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_ARCHS_MACOS: arm64 CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} From c802ec750652f9f298b72aa1c96ee3746ebdc13f Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 21:07:36 -0700 Subject: [PATCH 51/52] wip48 --- .github/workflows/build-wheels.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 4e06dff8..29c3a527 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -124,8 +124,8 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse && ls -l wheelhouse/ env: GK_BUILD_WHEELS: "1" - CIBW_ARCHS_MACOS: arm64 - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-*" + CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-arm64" - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels @@ -153,7 +153,6 @@ jobs: - {"os": "macos-13", "arch": "x86_64", "pyver": "3.10", "pyvershort": "310"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.11", "pyvershort": "311"} - {"os": "macos-13", "arch": "x86_64", "pyver": "3.12", "pyvershort": "312"} - - {"os": "macos-13", "arch": "x86_64", "pyver": "3.13", "pyvershort": "313"} steps: - name: Set up Python From 92634d7074c4cf7997a568b4418e07c08ee1e761 Mon Sep 17 00:00:00 2001 From: Avishai Weissberg Date: Wed, 2 Jul 2025 21:30:50 -0700 Subject: [PATCH 52/52] wip49 --- .github/workflows/build-wheels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yaml b/.github/workflows/build-wheels.yaml index 29c3a527..d54db929 100644 --- a/.github/workflows/build-wheels.yaml +++ b/.github/workflows/build-wheels.yaml @@ -125,7 +125,7 @@ jobs: env: GK_BUILD_WHEELS: "1" CIBW_ARCHS_MACOS: x86_64 arm64 - CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-arm64" + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-macosx_arm64" - if: ${{ steps.restore_macos_wheels.outputs.cache-hit != 'true' }} name: cache macos wheels