diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a9cc7f821..4383a63897 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,43 +17,49 @@ defaults: shell: bash -e {0} # -e to fail on error jobs: - pytest: + get-environments: + runs-on: ubuntu-latest + outputs: + envs: ${{ steps.get-envs.outputs.envs }} + steps: + - uses: actions/checkout@v4 + with: + filter: blob:none + fetch-depth: 0 + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: false + - id: get-envs + run: | + ENVS_JSON=$(NO_COLOR=1 uvx hatch env show --json | jq -c 'to_entries + | map( + select(.key | startswith("hatch-test")) + | { + name: .key, + "test-type": (if (.key | test("pre|min")) then "coverage" else null end), + python: .value.python, + } + )') + echo "envs=${ENVS_JSON}" | tee $GITHUB_OUTPUT + + test: + needs: get-environments runs-on: ubuntu-latest - strategy: matrix: - include: - - python-version: '3.11' - - python-version: '3.13' - - python-version: '3.13' - dependencies-version: min-optional - - python-version: '3.13' - dependencies-version: pre-release - test-type: coverage - - python-version: '3.11' - dependencies-version: minimum - test-type: coverage - - env: # for use codecov’s env_vars tagging - PYTHON: ${{ matrix.python-version }} - DEPS: ${{ matrix.dependencies-version || 'default' }} - TESTS: ${{ matrix.test-type || 'default' }} - + env: ${{ fromJSON(needs.get-environments.outputs.envs) }} + env: # environment variable for use in codecov’s env_vars tagging + ENV_NAME: ${{ matrix.env.name }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 filter: blob:none - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install UV - uses: astral-sh/setup-uv@v5 + - uses: astral-sh/setup-uv@v5 with: enable-cache: true + python-version: ${{ matrix.env.python }} cache-dependency-glob: pyproject.toml - name: Cache downloaded data @@ -63,36 +69,23 @@ jobs: key: pytest - name: Install dependencies - if: matrix.dependencies-version == null - run: uv pip install --system --compile "scanpy[dev,test-full] @ ." - - name: Install dependencies (no optional features) - if: matrix.dependencies-version == 'min-optional' - run: uv pip install --system --compile "scanpy[dev,test-min] @ ." - - name: Install dependencies (minimum versions) - if: matrix.dependencies-version == 'minimum' - run: | - uv pip install --system --compile tomli packaging - deps=$(python3 ci/scripts/min-deps.py pyproject.toml --extra dev test) - uv pip install --system --compile $deps "scanpy @ ." - - name: Install dependencies (pre-release versions) - if: matrix.dependencies-version == 'pre-release' - run: uv pip install -v --system --compile --pre "scanpy[dev,test-full] @ ." "anndata[dev,test] @ git+https://github.com/scverse/anndata.git" + run: uvx hatch -v env create ${{ matrix.env.name }} - - name: Run pytest - if: matrix.test-type == null - run: pytest - - name: Run pytest (coverage) - if: matrix.test-type == 'coverage' - run: coverage run -m pytest --cov --cov-report=xml + - name: Run tests + if: matrix.env.test-type == null + run: uvx hatch run ${{ matrix.env.name }}:run + - name: Run tests (coverage) + if: matrix.env.test-type == 'coverage' + run: uvx hatch run ${{ matrix.env.name }}:run-cov --cov --cov-report=xml - name: Upload coverage data - uses: codecov/codecov-action@v4 - if: matrix.test-type == 'coverage' + uses: codecov/codecov-action@v5 + if: matrix.env.test-type == 'coverage' with: token: ${{ secrets.CODECOV_TOKEN }} - env_vars: "PYTHON,DEPS,TESTS" + env_vars: ENV_NAME fail_ci_if_error: true - file: test-data/coverage.xml + files: test-data/coverage.xml - name: Upload test results # yaml strings can’t start with “!”, so using explicit substitution @@ -100,18 +93,18 @@ jobs: uses: codecov/test-results-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} - env_vars: "PYTHON,DEPS,TESTS" + env_vars: ENV_NAME fail_ci_if_error: true file: test-data/test-results.xml - name: Publish debug artifacts - if: matrix.test-type == 'coverage' + if: matrix.env.test-type == 'coverage' uses: actions/upload-artifact@v4 with: - name: debug-data-${{ matrix.python-version }}-${{ matrix.dependencies-version || 'default' }}-${{ matrix.test-type || 'default' }} + name: debug-data-${{ matrix.env.name }} path: .pytest_cache/d/debug - check-build: + build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -126,3 +119,15 @@ jobs: enable-cache: false - run: uvx --from build pyproject-build --sdist --wheel . - run: uvx twine check dist/* + + check: + if: always() + needs: + - get-environments + - test + - build + runs-on: ubuntu-latest + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.gitignore b/.gitignore index de85b8a6b7..bbe3c82e58 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ # Python build files __pycache__/ -/ci/scanpy-min-deps.txt +/ci/scanpy-low-vers.txt /dist/ /*-env/ /env-*/ diff --git a/ci/scripts/min-deps.py b/ci/scripts/low-vers.py similarity index 91% rename from ci/scripts/min-deps.py rename to ci/scripts/low-vers.py index 595b480b83..bcfe1221cf 100755 --- a/ci/scripts/min-deps.py +++ b/ci/scripts/low-vers.py @@ -3,7 +3,7 @@ # requires-python = ">=3.11" # dependencies = [ "packaging" ] # /// -"""Parse a pyproject.toml file and output a list of minimum dependencies.""" +"""Parse a pyproject.toml file and output a list of minimum dependency versions.""" from __future__ import annotations @@ -31,8 +31,9 @@ def min_dep(req: Requirement) -> Requirement: Example ------- >>> min_dep(Requirement("numpy>=1.0")) - - + + >>> min_dep(Requirement("numpy<3.0")) + """ req_name = req.name if req.extras: @@ -44,7 +45,6 @@ def min_dep(req: Requirement) -> Requirement: if not filter_specs: # TODO: handle markers return Requirement(f"{req_name}{req.specifier}") - min_version = Version("0.0.0.a1") for spec in filter_specs: if spec.operator in {">", ">=", "~="}: @@ -52,16 +52,17 @@ def min_dep(req: Requirement) -> Requirement: elif spec.operator == "==": min_version = Version(spec.version) - return Requirement(f"{req_name}=={min_version}.*") + return Requirement(f"{req_name}=={min_version}") def extract_min_deps( dependencies: Iterable[Requirement], *, pyproject ) -> Generator[Requirement, None, None]: - """Extract minimum dependencies from a list of requirements.""" + """Extract minimum dependency versions from a list of requirements.""" dependencies = deque(dependencies) # We'll be mutating this project_name = pyproject["project"]["name"] + deps = {} while len(dependencies) > 0: req = dependencies.pop() @@ -74,7 +75,11 @@ def extract_min_deps( extra_deps = pyproject["project"]["optional-dependencies"][extra] dependencies += map(Requirement, extra_deps) else: - yield min_dep(req) + if req.name in deps: + req.specifier &= deps[req.name].specifier + req.extras |= deps[req.name].extras + deps[req.name] = min_dep(req) + yield from deps.values() class Args(argparse.Namespace): @@ -100,6 +105,7 @@ def parser(cls) -> argparse.ArgumentParser: prog="min-deps", description=cls.__doc__, usage="pip install `python min-deps.py pyproject.toml`", + allow_abbrev=False, ) parser.add_argument( "_path", diff --git a/hatch.toml b/hatch.toml index df59372512..fe1be3889b 100644 --- a/hatch.toml +++ b/hatch.toml @@ -15,25 +15,27 @@ scripts.clean = "git restore --source=HEAD --staged --worktree -- docs/release-n [envs.hatch-test] default-args = [ ] -features = [ "dev", "test", "dask-ml" ] +features = [ "dev", "test-min" ] extra-dependencies = [ "ipykernel" ] overrides.matrix.deps.env-vars = [ { if = [ "pre" ], key = "UV_PRERELEASE", value = "allow" }, - { if = [ "min" ], key = "UV_CONSTRAINT", value = "ci/scanpy-min-deps.txt" }, + { if = [ "low-vers" ], key = "UV_CONSTRAINT", value = "ci/scanpy-low-vers.txt" }, ] overrides.matrix.deps.pre-install-commands = [ - { if = [ "min" ], value = "uv run ci/scripts/min-deps.py pyproject.toml --all-extras -o ci/scanpy-min-deps.txt" }, + { if = [ + "low-vers", + ], value = "uv run ci/scripts/low-vers.py pyproject.toml --all-extras -o ci/scanpy-low-vers.txt" }, ] overrides.matrix.deps.python = [ - { if = [ "min" ], value = "3.11" }, - { if = [ "stable", "full", "pre" ], value = "3.13" }, -] -overrides.matrix.deps.features = [ - { if = [ "full" ], value = "test-full" }, + { if = [ "low-vers" ], value = "3.11" }, + { if = [ "stable", "pre" ], value = "3.13" }, ] overrides.matrix.deps.extra-dependencies = [ { if = [ "pre" ], value = "anndata[dev,test] @ git+https://github.com/scverse/anndata.git" }, ] +overrides.matrix.deps.features = [ + { if = [ "stable", "pre", "low-vers" ], value = "test" }, +] [[envs.hatch-test.matrix]] -deps = [ "stable", "full", "pre", "min" ] +deps = [ "stable", "pre", "low-vers", "few-extras" ] diff --git a/pyproject.toml b/pyproject.toml index d2d7858ff1..59a3690a60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,27 +46,27 @@ classifiers = [ "Topic :: Scientific/Engineering :: Visualization", ] dependencies = [ - "anndata>=0.9", - "numpy>=1.25", + "anndata>=0.9.2", + "numpy>=1.25.2", "fast-array-utils[accel,sparse]>=1.2.1", - "matplotlib>=3.7", - "pandas >=2.0", - "scipy>=1.11", - "seaborn>=0.13", - "h5py>=3.8", + "matplotlib>=3.7.5", + "pandas >=2.0.3", + "scipy>=1.11.1", + "seaborn>=0.13.2", + "h5py>=3.8.0", "tqdm", - "scikit-learn>=1.1", - "statsmodels>=0.14", + "scikit-learn>=1.1.3", + "statsmodels>=0.14.4", "patsy!=1.0.0", # https://github.com/pydata/patsy/issues/215 - "networkx>=2.8", + "networkx>=2.8.8", "natsort", "joblib", - "numba>=0.58", - "umap-learn>=0.5,!=0.5", - "pynndescent>=0.5", + "numba>=0.58.1", + "umap-learn>=0.5.7", + "pynndescent>=0.5.13", "packaging>=21.3", "session-info2", - "legacy-api-wrap>=1.4", # for positional API deprecations + "legacy-api-wrap>=1.4.1", # for positional API deprecations "typing-extensions; python_version < '3.13'", ] dynamic = [ "version" ] @@ -95,22 +95,15 @@ test-min = [ ] test = [ "scanpy[test-min]", - # Optional but important dependencies - "scanpy[leiden]", - "zarr<3", + # optional storage and processing modes "scanpy[dask]", - "scanpy[scrublet]", -] -test-full = [ - "scanpy[test]", - # optional storage modes "zappy", + "zarr<3", # additional tested algorithms + "scanpy[scrublet]", "scanpy[louvain]", - "scanpy[magic]", + "scanpy[leiden]", "scanpy[skmisc]", - "scanpy[harmony]", - "scanpy[scanorama]", "scanpy[dask-ml]", ] doc = [ @@ -140,14 +133,14 @@ dev = [ ] # Algorithms paga = [ "igraph" ] -louvain = [ "igraph", "louvain>=0.6.0,!=0.6.2" ] # Louvain community detection -leiden = [ "igraph>=0.10", "leidenalg>=0.9.0" ] # Leiden community detection -bbknn = [ "bbknn" ] # Batch balanced KNN (batch correction) -magic = [ "magic-impute>=2.0" ] # MAGIC imputation method -skmisc = [ "scikit-misc>=0.1.3" ] # highly_variable_genes method 'seurat_v3' -harmony = [ "harmonypy" ] # Harmony dataset integration -scanorama = [ "scanorama" ] # Scanorama dataset integration -scrublet = [ "scikit-image>=0.20" ] # Doublet detection with automatic thresholds +louvain = [ "igraph", "louvain>=0.8.2" ] # Louvain community detection +leiden = [ "igraph>=0.10.8", "leidenalg>=0.9.0" ] # Leiden community detection +bbknn = [ "bbknn" ] # Batch balanced KNN (batch correction) +magic = [ "magic-impute>=2.0.4" ] # MAGIC imputation method +skmisc = [ "scikit-misc>=0.5.1" ] # highly_variable_genes method 'seurat_v3' +harmony = [ "harmonypy" ] # Harmony dataset integration +scanorama = [ "scanorama" ] # Scanorama dataset integration +scrublet = [ "scikit-image>=0.20.0" ] # Doublet detection with automatic thresholds # Acceleration rapids = [ "cudf>=0.9", "cuml>=0.9", "cugraph>=0.9" ] # GPU accelerated calculation of neighbors dask = [ "dask[array]>=2023.5.1" ] # Use the Dask parallelization engine