From 07242ce0978d58f9c833299c95c283a2821e56ed Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 22 May 2025 12:43:39 -0400 Subject: [PATCH 1/6] bump to intake-esgf to allow time filtering of files --- pyproject.toml | 15 +++++---------- src/ref_sample_data/data_request/base.py | 6 +++++- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a74c3f13..12bdedb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,7 @@ name = "ref-sample-data" version = "0.6.1" description = "CMIP Rapid Evaluation Framework Sample Data" readme = "README.md" -authors = [ - { name= "Jared Lewis", email="jared.lewis@climate-resource.com" } -] +authors = [{ name = "Jared Lewis", email = "jared.lewis@climate-resource.com" }] requires-python = ">=3.10" [project.license] @@ -18,7 +16,7 @@ platforms = ["osx-arm64", "linux-64", "win-64"] [tool.pixi.dependencies] python = ">=3.10" pooch = ">=1.8.2,<2" -intake-esgf = ">=2025.5.9,<2026" +intake-esgf = ">=2025.5.22,<2026" matplotlib = ">=3.10.3,<4" scipy = ">=1.15.2,<2" xarray = ">=2025.1.0,<2025.2" @@ -64,18 +62,15 @@ exclude_also = [ # Don't complain about missing type checking code: "if TYPE_CHECKING", # Exclude ... literals - "\\.\\.\\." + "\\.\\.\\.", ] [tool.jupytext] formats = "ipynb,py:percent" [tool.pytest.ini_options] -addopts = [ - "--import-mode=importlib", -] -filterwarnings = [ -] +addopts = ["--import-mode=importlib"] +filterwarnings = [] # We currently check for GPL licensed code, but this restriction may be removed [tool.liccheck] diff --git a/src/ref_sample_data/data_request/base.py b/src/ref_sample_data/data_request/base.py index 17e49b26..c20de7ad 100644 --- a/src/ref_sample_data/data_request/base.py +++ b/src/ref_sample_data/data_request/base.py @@ -80,7 +80,11 @@ def fetch_datasets(self) -> pd.DataFrame: """Fetch the datasets from the ESGF.""" cat = ESGFCatalog() - cat.search(**self.facets) + opts = {} + if self.time_span: + opts["file_start"] = self.time_span[0] + opts["file_end"] = self.time_span[1] + cat.search(**(opts | self.facets)) if self.remove_ensembles: cat.remove_ensembles() From d368866160a2232159d8facc13b307978f9910ec Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 22 May 2025 12:47:29 -0400 Subject: [PATCH 2/6] added remaining variables for ilamb/iomb --- scripts/fetch_test_data.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/fetch_test_data.py b/scripts/fetch_test_data.py index 6a438d98..f9279ad5 100755 --- a/scripts/fetch_test_data.py +++ b/scripts/fetch_test_data.py @@ -234,7 +234,7 @@ def process_sample_data_request( facets=dict( source_id="ACCESS-ESM1-5", frequency=["fx", "mon"], - variable_id=["areacella", "sftlf", "gpp", "pr", "tas", "mrro", "mrsos", "cSoil", "lai"], + variable_id=["areacella", "sftlf", "gpp", "pr", "tas", "mrro", "mrsos", "cSoil", "lai", "snc"], experiment_id=["historical"], ), remove_ensembles=True, @@ -256,12 +256,22 @@ def process_sample_data_request( facets=dict( source_id="ACCESS-ESM1-5", frequency=["fx", "mon"], - variable_id=["areacello", "sftof", "tos", "sos", "msftmz"], + variable_id=["areacello", "sftof", "tos", "sos", "msftmz", "thetao"], experiment_id=["historical"], ), remove_ensembles=True, time_span=("2000", "2025"), ), + # Sometimes models have time-dependent volcello + CMIP6Request( + facets=dict( + source_id="ACCESS-ESM1-5", + table_id="Ofx", + variable_id=["volcello"], + experiment_id=["historical"], + ), + remove_ensembles=True, + ), # PMP modes of variability data CMIP6Request( facets=dict( From f84d9ddcefd2fd44f4f4993b9e4d456e61e6a6c9 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 22 May 2025 12:58:23 -0400 Subject: [PATCH 3/6] changelog --- changelog/38.feat.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/38.feat.md diff --git a/changelog/38.feat.md b/changelog/38.feat.md new file mode 100644 index 00000000..16da6c41 --- /dev/null +++ b/changelog/38.feat.md @@ -0,0 +1 @@ +ILAMB snc and IOMB thetao,volcello From e7f84b09b98c04393d50adca1f4580f74a954c60 Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 22 May 2025 13:03:36 -0400 Subject: [PATCH 4/6] didn't know it has to have a time span --- scripts/fetch_test_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/fetch_test_data.py b/scripts/fetch_test_data.py index f9279ad5..0b86bcba 100755 --- a/scripts/fetch_test_data.py +++ b/scripts/fetch_test_data.py @@ -269,6 +269,7 @@ def process_sample_data_request( table_id="Ofx", variable_id=["volcello"], experiment_id=["historical"], + time_span=("2000", "2025"), ), remove_ensembles=True, ), From dcae502c809b1210a903d922002de8ba47fed2af Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 22 May 2025 13:35:28 -0400 Subject: [PATCH 5/6] silly mistake --- scripts/fetch_test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/fetch_test_data.py b/scripts/fetch_test_data.py index 0b86bcba..43e64090 100755 --- a/scripts/fetch_test_data.py +++ b/scripts/fetch_test_data.py @@ -269,9 +269,9 @@ def process_sample_data_request( table_id="Ofx", variable_id=["volcello"], experiment_id=["historical"], - time_span=("2000", "2025"), ), remove_ensembles=True, + time_span=("2000", "2025"), ), # PMP modes of variability data CMIP6Request( From b66fd955fd7231237a1c3d71953dc7b7a9db966d Mon Sep 17 00:00:00 2001 From: Nathan Collier Date: Thu, 22 May 2025 16:37:43 -0400 Subject: [PATCH 6/6] bump intake-esgf again --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 12bdedb8..9269da08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ platforms = ["osx-arm64", "linux-64", "win-64"] [tool.pixi.dependencies] python = ">=3.10" pooch = ">=1.8.2,<2" -intake-esgf = ">=2025.5.22,<2026" +intake-esgf = ">=2025.5.23,<2026" matplotlib = ">=3.10.3,<4" scipy = ">=1.15.2,<2" xarray = ">=2025.1.0,<2025.2"