diff --git a/.github/workflows/array-api-tests.yml b/.github/workflows/array-api-tests.yml index 51cc6c41d..f76ba6c48 100644 --- a/.github/workflows/array-api-tests.yml +++ b/.github/workflows/array-api-tests.yml @@ -22,7 +22,6 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.11"] - zarr: ["<3", ">3"] steps: - name: Checkout Cubed @@ -46,9 +45,6 @@ jobs: pushd array-api-tests pip install -e '../cubed[test]' pip install -r requirements.txt # from Array API tests repo - - name: Install zarr${{ matrix.zarr }} - run: | - pip install -U 'zarr${{ matrix.zarr }}' - name: Run Array API tests env: ARRAY_API_TESTS_MODULE: cubed.array_api diff --git a/.github/workflows/slow-tests.yml b/.github/workflows/slow-tests.yml index a0679126a..f7094336b 100644 --- a/.github/workflows/slow-tests.yml +++ b/.github/workflows/slow-tests.yml @@ -19,7 +19,6 @@ jobs: matrix: os: ["ubuntu-latest"] python-version: ["3.11"] - zarr: ["<3"] steps: - name: Checkout source @@ -43,10 +42,6 @@ jobs: run: | python -m pip install -e .[test] memray - - name: Install zarr${{ matrix.zarr }} - run: | - python -m pip install -U 'zarr${{ matrix.zarr }}' - - name: Run tests run: | pytest -vs -k slow --runslow diff --git a/.github/workflows/zarr-v2-tests.yml b/.github/workflows/zarr-v2-tests.yml deleted file mode 100644 index c65eac360..000000000 --- a/.github/workflows/zarr-v2-tests.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Zarr v2 Tests - -on: - push: - branches: - - "main" - pull_request: - workflow_dispatch: - -concurrency: - group: Tests-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - test: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - python-version: ["3.11"] - - steps: - - name: Checkout source - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@v2 - with: - macos-skip-brew-update: 'true' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - - - name: Install - run: | - python -m pip install -e .[test] - python -m pip install -U 'zarr<3' - - - name: Run tests - env: - COVERAGE_CORE: sysmon - run: | - if [ "$RUNNER_OS" == "Windows" ]; then - pytest -v - else - pytest -v --cov=cubed --cov-report=term-missing --cov-fail-under=90 - fi - shell: bash diff --git a/cubed/storage/store.py b/cubed/storage/store.py index dfdac6735..be6086b16 100644 --- a/cubed/storage/store.py +++ b/cubed/storage/store.py @@ -7,29 +7,13 @@ def get_storage_name() -> str: # get storage name from top-level config # e.g. set globally with CUBED_STORAGE_NAME=tensorstore - storage_name = config.get("storage_name", None) - - if storage_name is None: - import zarr - - if zarr.__version__[0] == "3": - storage_name = "zarr-python-v3" - else: - storage_name = "zarr-python" - - return storage_name + return config.get("storage_name", "zarr-python-v3") def is_storage_array(obj) -> bool: storage_name = get_storage_name() - if storage_name == "zarr-python": - import zarr - - from cubed.storage.stores.zarr_python import ZarrArrayGroup # type: ignore - - return isinstance(obj, (zarr.Array, ZarrArrayGroup)) - elif storage_name in ("zarr-python-v3", "zarrs-python"): + if storage_name in ("zarr-python", "zarr-python-v3", "zarrs-python"): import zarr from cubed.storage.stores.zarr_python_v3 import ZarrV3ArrayGroup @@ -58,21 +42,7 @@ def open_storage_array( ): storage_name = get_storage_name() - if storage_name == "zarr-python": - from cubed.storage.stores.zarr_python import open_zarr_array # type: ignore - - open_func = open_zarr_array - - # set object codec if needed - import numpy as np - - if np.dtype(dtype).hasobject and "object_codec" not in kwargs: - import numcodecs - - object_codec = numcodecs.Pickle() - kwargs["object_codec"] = object_codec - - elif storage_name == "zarr-python-v3": + if storage_name in ("zarr-python", "zarr-python-v3"): from cubed.storage.stores.zarr_python_v3 import open_zarr_v3_array open_func = open_zarr_v3_array diff --git a/cubed/storage/stores/zarr_python.py b/cubed/storage/stores/zarr_python.py deleted file mode 100644 index c811ad8f6..000000000 --- a/cubed/storage/stores/zarr_python.py +++ /dev/null @@ -1,72 +0,0 @@ -# type: ignore # Zarr Python v2 types are different to v3 -from typing import Optional, Union - -import zarr -from numcodecs.registry import get_codec - -from cubed.types import T_DType, T_RegularChunks, T_Shape, T_Store -from cubed.utils import join_path - - -class ZarrArrayGroup(dict): - def __init__( - self, - shape: Optional[T_Shape] = None, - dtype: Optional[T_DType] = None, - chunks: Optional[T_RegularChunks] = None, - ): - dict.__init__(self) - self.shape = shape - self.dtype = dtype - self.chunks = chunks - - def __getitem__(self, key): - if isinstance(key, str): - return super().__getitem__(key) - return {field: zarray[key] for field, zarray in self.items()} - - def set_basic_selection(self, selection, value, fields=None): - self[fields][selection] = value - - -def open_zarr_array( - store: T_Store, - mode: str, - *, - shape: Optional[T_Shape] = None, - dtype: Optional[T_DType] = None, - chunks: Optional[T_RegularChunks] = None, - path: Optional[str] = None, - compressor: Union[dict, str, None] = "default", - **kwargs, -): - if isinstance(compressor, dict): - compressor = get_codec(compressor) - - if dtype is None or not hasattr(dtype, "fields") or dtype.fields is None: - return zarr.open_array( - store, - mode=mode, - shape=shape, - dtype=dtype, - chunks=chunks, - path=path, - compressor=compressor, - **kwargs, - ) - else: - ret = ZarrArrayGroup(shape=shape, dtype=dtype, chunks=chunks) - for field in dtype.fields: - field_dtype, _ = dtype.fields[field] - field_path = field if path is None else join_path(path, field) - ret[field] = zarr.open_array( - store, - mode=mode, - shape=shape, - dtype=field_dtype, - chunks=chunks, - path=field_path, - compressor=compressor, - **kwargs, - ) - return ret diff --git a/cubed/tests/storage/test_zarr.py b/cubed/tests/storage/test_zarr.py index 75450e12e..3bc7324c0 100644 --- a/cubed/tests/storage/test_zarr.py +++ b/cubed/tests/storage/test_zarr.py @@ -1,13 +1,10 @@ import pytest -import zarr from numcodecs.registry import get_codec from cubed import config from cubed.storage.store import open_storage_array from cubed.storage.zarr import lazy_zarr_array -ZARR_PYTHON_V3 = zarr.__version__[0] == "3" - def test_lazy_zarr_array(tmp_path): zarr_path = tmp_path / "lazy.zarr" @@ -22,9 +19,6 @@ def test_lazy_zarr_array(tmp_path): arr.open() -@pytest.mark.skipif( - ZARR_PYTHON_V3, reason="setting zarr compressor not yet possible for Zarr Python v3" -) @pytest.mark.parametrize( "compressor", [ @@ -41,7 +35,7 @@ def test_compression(tmp_path, compressor): ) arr.create() - # open with zarr python (for zarr python v2 and tensorstore) + # open with zarr python (for zarr python and tensorstore) with config.set({"storage_name": "zarr-python"}): z = open_storage_array(zarr_path, mode="r") diff --git a/cubed/tests/test_store.py b/cubed/tests/test_store.py index efb103140..6774fc0d3 100644 --- a/cubed/tests/test_store.py +++ b/cubed/tests/test_store.py @@ -7,12 +7,10 @@ import cubed.array_api as xp from cubed.storage.store import get_storage_name -ZARR_PYTHON_V2 = zarr.__version__[0] == "2" - @pytest.mark.skipif( - ZARR_PYTHON_V2 or get_storage_name() == "tensorstore", - reason="setting an arbitrary Zarr store is not supported for Zarr Python v2, or tensorstore", + get_storage_name() == "tensorstore", + reason="setting an arbitrary Zarr store is not supported for tensorstore", ) def test_arbitrary_zarr_store(): store = zarr.storage.MemoryStore() diff --git a/pyproject.toml b/pyproject.toml index d96b358d4..7c25b2c7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "psutil", "tenacity", "toolz", - "zarr != 3.1.2", + "zarr > 3.1.2", ] [project.optional-dependencies]