From 69b0466de82980d51320edfb0fe0ac060bd8f191 Mon Sep 17 00:00:00 2001 From: Peter Dudfield Date: Fri, 15 Aug 2025 09:29:33 +0100 Subject: [PATCH 1/5] update requirements --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ad2bb6b..fed1c1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "hydra-core==1.3.2", "loguru == 0.7.3", "numpy==2.1.2", - "ocf-data-sampler==0.1.4", + "ocf-data-sampler==0.5.5", "pandas==2.2.3", "s3fs==2024.6.1", "safetensors==0.5.2", @@ -36,8 +36,8 @@ dependencies = [ "torch @ https://download.pytorch.org/whl/cpu/torch-2.3.1%2Bcpu-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64'", "torchvision @ https://download.pytorch.org/whl/cpu/torchvision-0.18.1%2Bcpu-cp312-cp312-linux_x86_64.whl ; platform_system == 'Linux' and platform_machine == 'x86_64'", "typer==0.15.1", - "xarray==2025.1.2", - "zarr==2.18.3", + "xarray==2025.7.1", + "zarr==3.1.1", ] [dependency-groups] From d3fa5dd67cc1a0d61fcddc542f985d9a81b2778c Mon Sep 17 00:00:00 2001 From: Peter Dudfield Date: Fri, 15 Aug 2025 09:32:07 +0100 Subject: [PATCH 2/5] upgrade fsspec --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fed1c1f..26a69ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,14 +20,14 @@ classifiers = [ "License :: OSI Approved :: MIT License", ] dependencies = [ - "fsspec==2024.6.1", + "fsspec==2025.7.0", "huggingface-hub==0.28.1", "hydra-core==1.3.2", "loguru == 0.7.3", "numpy==2.1.2", "ocf-data-sampler==0.5.5", "pandas==2.2.3", - "s3fs==2024.6.1", + "s3fs==2025.7.0", "safetensors==0.5.2", "sat_pred @ git+https://github.com/openclimatefix/sat_pred.git@main", # Since torch distributes CPU only packages as wheels, have to specify the target platform in order to pull the wheel compiled for that specific platform From 0dd30df1a31e54f72831787bbb4e84570145d55e Mon Sep 17 00:00:00 2001 From: Peter Dudfield Date: Fri, 15 Aug 2025 09:47:25 +0100 Subject: [PATCH 3/5] upgrade conftest.py --- tests/conftest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 40b65bb..3f2cab8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import pandas as pd import pytest import xarray as xr +import zarr xr.set_options(keep_attrs=True) @@ -17,8 +18,8 @@ def make_sat_data(test_t0, freq_mins): # Load dataset which only contains coordinates, but no data shell_path = f"{os.path.dirname(os.path.abspath(__file__))}/test_data/non_hrv_shell.zarr.zip" - - ds = xr.open_zarr(fsspec.get_mapper(f"zip::{shell_path}")) + with zarr.storage.ZipStore(shell_path, mode="r") as store: + ds = xr.open_zarr(store) # Remove original time dim ds = ds.drop_vars("time") @@ -41,6 +42,14 @@ def make_sat_data(test_t0, freq_mins): ds.data.attrs = ds.attrs["_data_attrs"] del ds.attrs["_data_attrs"] + + # This is important to avoid saving errors + for v in list(ds.coords.keys()): + ds[v].encoding.clear() + + for v in list(ds.variables.keys()): + ds[v].encoding.clear() + return ds From 6fd7f11b17eb0a381d1d6e044063c83e770acc97 Mon Sep 17 00:00:00 2001 From: Peter Dudfield Date: Fri, 15 Aug 2025 10:53:19 +0100 Subject: [PATCH 4/5] fix tests --- tests/conftest.py | 2 +- tests/test_app.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3f2cab8..aa84981 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ @pytest.fixture() def test_t0(): - return pd.Timestamp.now(tz=None).floor("30min") + return pd.Timestamp.now(tz="UTC").replace(tzinfo=None).floor("30min") def make_sat_data(test_t0, freq_mins): diff --git a/tests/test_app.py b/tests/test_app.py index c763ed7..f984ec6 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -32,7 +32,7 @@ def test_app(sat_5_data, tmp_path, test_t0): assert "sat_pred" in ds_y_hat assert ( - list(ds_y_hat.sat_pred.coords)== + sorted(list(ds_y_hat.sat_pred.coords))== ["init_time", "step", "variable", "x_geostationary", "y_geostationary"] ) From eb3498518dab2271760f1694bed22a09a6b28a54 Mon Sep 17 00:00:00 2001 From: Peter Dudfield Date: Fri, 15 Aug 2025 11:16:54 +0100 Subject: [PATCH 5/5] tidy --- tests/conftest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index aa84981..789735d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,10 +43,7 @@ def make_sat_data(test_t0, freq_mins): del ds.attrs["_data_attrs"] - # This is important to avoid saving errors - for v in list(ds.coords.keys()): - ds[v].encoding.clear() - + # # This is important to avoid saving errors for v in list(ds.variables.keys()): ds[v].encoding.clear()