diff --git a/changelog/399.feature.md b/changelog/399.feature.md new file mode 100644 index 000000000..e7a7cf916 --- /dev/null +++ b/changelog/399.feature.md @@ -0,0 +1 @@ +Add a timerange constraint to ensure the required data is available. diff --git a/packages/climate-ref-core/src/climate_ref_core/constraints.py b/packages/climate-ref-core/src/climate_ref_core/constraints.py index af64d7bc9..c04583db1 100644 --- a/packages/climate-ref-core/src/climate_ref_core/constraints.py +++ b/packages/climate-ref-core/src/climate_ref_core/constraints.py @@ -6,6 +6,8 @@ import warnings from collections import defaultdict from collections.abc import Mapping +from datetime import datetime +from functools import total_ordering from typing import Literal, Protocol, runtime_checkable if sys.version_info < (3, 11): @@ -273,6 +275,123 @@ def from_defaults( return cls(supplementary_facets, **kwargs[source_type]) +@frozen +@total_ordering +class PartialDateTime: + """ + A partial datetime object that can be used to compare datetimes. + + Only the specified fields are used for comparison. + """ + + year: int | None = None + month: int | None = None + day: int | None = None + hour: int | None = None + minute: int | None = None + second: int | None = None + + @property + def _attrs(self) -> dict[str, int]: + """The attributes that are set.""" + return { + a: v + for a in self.__slots__ # type: ignore[attr-defined] + if not a.startswith("_") and (v := getattr(self, a)) is not None + } + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({', '.join(f'{a}={v}' for a, v in self._attrs.items())})" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, datetime): + msg = ( + f"Can only compare PartialDateTime with `datetime.datetime` " + f"objects, got object {other} of type {type(other)}" + ) + raise TypeError(msg) + + for attr, value in self._attrs.items(): + other_value = getattr(other, attr) + if value != other_value: + return False + return True + + def __lt__(self, other: object) -> bool: + if not isinstance(other, datetime): + msg = ( + f"Can only compare PartialDateTime with `datetime.datetime` " + f"objects, got object {other} of type {type(other)}" + ) + raise TypeError(msg) + + for attr, value in self._attrs.items(): + other_value = getattr(other, attr) + if value != other_value: + return value < other_value # type: ignore[no-any-return] + return False + + +@frozen +class RequireTimerange: + """ + A constraint that requires datasets to have a specific timerange. + + Specify the start and/or end of the required timerange using a precision + that matches the frequency of the datasets. + + For example, to ensure that datasets at monthly frequency cover the period + from 2000 to 2010, use start=PartialDateTime(year=2000, month=1) and + end=PartialDateTime(year=2010, month=12). + """ + + group_by: tuple[str, ...] + """ + The fields to group the datasets by. Each group must cover the timerange + to fulfill the constraint. + """ + + start: PartialDateTime | None = None + """ + The start time of the required timerange. If None, no start time is required. + """ + + end: PartialDateTime | None = None + """ + The end time of the required timerange. If None, no end time is required. + """ + + def validate(self, group: pd.DataFrame) -> bool: + """ + Check that all subgroups of the group have a contiguous timerange. + """ + group = group.dropna(subset=["start_time", "end_time"]) + for _, subgroup in group.groupby(list(self.group_by)): + start = subgroup["start_time"].min() + end = subgroup["end_time"].max() + result = True + if self.start is not None and start > self.start: + logger.debug( + f"Constraint {self.__class__.__name__} not satisfied " + f"because start time {start} is after required start time " + f"{self.start} for {', '.join(subgroup['path'])}" + ) + result = False + if self.end is not None and end < self.end: + logger.debug( + f"Constraint {self.__class__.__name__} not satisfied " + f"because end time {end} is before required end time " + f"{self.end} for {', '.join(subgroup['path'])}" + ) + result = False + if result: + result = RequireContiguousTimerange(group_by=self.group_by).validate(subgroup) + if not result: + return False + + return True + + @frozen class RequireContiguousTimerange: """ diff --git a/packages/climate-ref-core/tests/unit/test_constraints.py b/packages/climate-ref-core/tests/unit/test_constraints.py index 26ae66ed6..a11fb32b4 100644 --- a/packages/climate-ref-core/tests/unit/test_constraints.py +++ b/packages/climate-ref-core/tests/unit/test_constraints.py @@ -1,3 +1,5 @@ +import operator +from collections.abc import Callable from datetime import datetime import pandas as pd @@ -7,9 +9,11 @@ AddSupplementaryDataset, GroupOperation, GroupValidator, + PartialDateTime, RequireContiguousTimerange, RequireFacets, RequireOverlappingTimerange, + RequireTimerange, SelectParentExperiment, apply_constraint, ) @@ -144,6 +148,272 @@ def test_apply(self, data_catalog, expected_rows): assert (result == expected).all().all() +class TestPartialDateTime: + @pytest.mark.parametrize( + "pdt, dt, expected", + [ + (PartialDateTime(year=2000), datetime(2000, 1, 1), True), + (PartialDateTime(year=2000), datetime(1999, 12, 31), False), + (PartialDateTime(year=2000, month=6), datetime(2000, 6, 15), True), + (PartialDateTime(year=2000, month=6), datetime(2000, 5, 31), False), + (PartialDateTime(year=2000, month=6, day=15), datetime(2000, 6, 15), True), + (PartialDateTime(year=2000, month=6, day=15), datetime(2000, 6, 14), False), + (PartialDateTime(year=2000, month=6, day=15, hour=12), datetime(2000, 6, 15, 12), True), + (PartialDateTime(year=2000, month=6, day=15, hour=12), datetime(2000, 6, 15, 11), False), + ], + ) + def test_eq(self, pdt: PartialDateTime, dt: datetime, expected: bool) -> None: + assert (pdt == dt) == expected + + @pytest.mark.parametrize( + "pdt, dt, expected", + [ + (PartialDateTime(year=2000), datetime(1999, 1, 1), False), + (PartialDateTime(year=2000), datetime(2000, 12, 1), False), + (PartialDateTime(year=2000), datetime(2001, 12, 31), True), + (PartialDateTime(year=2000, month=6), datetime(2000, 6, 15), False), + (PartialDateTime(year=2000, month=6), datetime(1999, 7, 15), False), + (PartialDateTime(year=2000, month=6), datetime(2000, 7, 31), True), + (PartialDateTime(year=2000, month=6, day=15), datetime(2000, 6, 14), False), + (PartialDateTime(year=2000, month=6, day=15), datetime(2000, 6, 16), True), + (PartialDateTime(year=2000, month=6, day=15, hour=12), datetime(2000, 6, 15, 12), False), + (PartialDateTime(year=2000, month=6, day=15, hour=12), datetime(2000, 6, 15, 13), True), + ], + ) + def test_lt(self, pdt: PartialDateTime, dt: datetime, expected: bool) -> None: + assert (pdt < dt) == expected + + def test_gt(self): + assert (PartialDateTime(year=2000, month=2) > datetime(2000, 1, 1)) == True # noqa: E712 + + @pytest.mark.parametrize("op", [operator.eq, operator.lt, operator.gt]) + def test_not_implemented(self, op: Callable) -> None: + with pytest.raises(TypeError): + assert op(PartialDateTime(year=2000), object()) + + +class TestRequireTimerange: + @pytest.mark.parametrize( + "data, start, end, expected", + [ + ( + pd.DataFrame( + { + "variable_id": [], + "start_time": [], + "end_time": [], + "path": [], + } + ), + PartialDateTime(year=2000), + PartialDateTime(year=2001), + True, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + PartialDateTime(year=2000, month=12), + True, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + None, + True, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + ], + } + ), + None, + PartialDateTime(year=2000, month=5), + True, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas"], + "start_time": [ + datetime(2000, 2, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + None, + False, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + ], + } + ), + None, + PartialDateTime(year=2001, month=1), + False, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas", "tas", "tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + datetime(2001, 1, 16, 12), + datetime(2003, 1, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + datetime(2001, 12, 16, 12), + datetime(2003, 12, 16, 12), + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200112.nc", + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200101-200112.nc", + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200301-200312.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + PartialDateTime(year=2003, month=12), + False, + ), + ( + pd.DataFrame( + { + "variable_id": ["tas", "tas", "areacella"], + "start_time": [ + datetime(2000, 1, 16, 12), + datetime(2001, 1, 16, 12), + None, + ], + "end_time": [ + datetime(2000, 12, 16, 12), + datetime(2001, 12, 16, 12), + None, + ], + "path": [ + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200101-200112.nc", + "areacella_fx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + PartialDateTime(year=2001, month=12), + True, + ), + ( + pd.DataFrame( + { + "variable_id": ["pr", "tas", "tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + datetime(2000, 1, 16, 12), + datetime(2001, 1, 16, 12), + ], + "end_time": [ + datetime(2001, 12, 16, 12), + datetime(2000, 12, 16, 12), + datetime(2001, 12, 16, 12), + ], + "path": [ + "pr_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200112.nc", + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200101-200112.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + PartialDateTime(year=2001, month=12), + True, + ), + ( + pd.DataFrame( + { + "variable_id": ["pr", "tas"], + "start_time": [ + datetime(2000, 1, 16, 12), + datetime(2000, 1, 16, 12), + ], + "end_time": [ + datetime(2000, 12, 16, 12), + datetime(2002, 12, 16, 12), + ], + "path": [ + "pr_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200001-200012.nc", + "tas_Amon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_200101-200212.nc", + ], + } + ), + PartialDateTime(year=2000, month=1), + PartialDateTime(year=2001, month=12), + False, + ), + ], + ) + def test_validate( + self, + data: pd.DataFrame, + start: PartialDateTime, + end: PartialDateTime, + expected: bool, + ) -> None: + constraint = RequireTimerange(group_by=["variable_id"], start=start, end=end) + assert constraint.validate(data) == expected + + class TestContiguousTimerange: constraint = RequireContiguousTimerange(group_by=["variable_id"]) diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py index 4b0e73fb7..464c1366d 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_at_global_warming_levels.py @@ -2,8 +2,9 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, - RequireContiguousTimerange, + PartialDateTime, RequireFacets, + RequireTimerange, ) from climate_ref_core.datasets import FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -26,6 +27,14 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic): "tas", ) + matching_facets = ( + "source_id", + "member_id", + "grid_label", + "table_id", + "variable_id", + ) + data_requirements = ( DataRequirement( source_type=SourceDatasetType.CMIP6, @@ -47,17 +56,15 @@ class ClimateAtGlobalWarmingLevels(ESMValToolDiagnostic): RequireFacets("variable_id", variables), AddSupplementaryDataset( supplementary_facets={"experiment_id": "historical"}, - matching_facets=( - "source_id", - "member_id", - "grid_label", - "table_id", - "variable_id", - ), + matching_facets=matching_facets, optional_matching_facets=tuple(), ), RequireFacets("experiment_id", ("historical",)), - RequireContiguousTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=matching_facets, + start=PartialDateTime(year=1850, month=1), + end=PartialDateTime(year=2100, month=12), + ), AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), ), ), diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py index a6cc2a479..56a72bbce 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/climate_drivers_for_fire.py @@ -2,8 +2,9 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, + PartialDateTime, RequireFacets, - RequireOverlappingTimerange, + RequireTimerange, ) from climate_ref_core.datasets import FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -45,7 +46,11 @@ class ClimateDriversForFire(ESMValToolDiagnostic): group_by=("source_id", "member_id", "grid_label"), constraints=( RequireFacets("variable_id", variables), - RequireOverlappingTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(2013, 1), + end=PartialDateTime(2014, 12), + ), AddSupplementaryDataset.from_defaults("sftlf", SourceDatasetType.CMIP6), ), ), diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py index 10063e715..845d2ccda 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_radiative_effects.py @@ -2,9 +2,10 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, - RequireContiguousTimerange, + PartialDateTime, RequireFacets, RequireOverlappingTimerange, + RequireTimerange, ) from climate_ref_core.datasets import FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -44,7 +45,11 @@ class CloudRadiativeEffects(ESMValToolDiagnostic): group_by=("source_id", "member_id", "grid_label"), constraints=( RequireFacets("variable_id", variables), - RequireContiguousTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1996, 1), + end=PartialDateTime(2014, 12), + ), RequireOverlappingTimerange(group_by=("instance_id",)), AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), ), @@ -58,19 +63,7 @@ def update_recipe(recipe: Recipe, input_files: dict[SourceDatasetType, pandas.Da recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6]) recipe_variables = {k: v for k, v in recipe_variables.items() if k != "areacella"} - # Select a timerange covered by all datasets. - start_times, end_times = [], [] - for variable in recipe_variables.values(): - for dataset in variable["additional_datasets"]: - start, end = dataset["timerange"].split("/") - start_times.append(start) - end_times.append(end) - start_time = max(start_times) - start_time = max(start_time, "20010101T000000") # Earliest observational dataset availability - timerange = f"{start_time}/{min(end_times)}" - datasets = recipe_variables["rsut"]["additional_datasets"] for dataset in datasets: dataset.pop("timerange") recipe["datasets"] = datasets - recipe["timerange_for_models"] = timerange diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py index 600033d9e..238c6d1ff 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/cloud_scatterplots.py @@ -4,9 +4,9 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, - RequireContiguousTimerange, + PartialDateTime, RequireFacets, - RequireOverlappingTimerange, + RequireTimerange, ) from climate_ref_core.datasets import FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -31,9 +31,11 @@ def get_cmip6_data_requirements(variables: tuple[str, ...]) -> tuple[DataRequire group_by=("source_id", "experiment_id", "member_id", "frequency", "grid_label"), constraints=( RequireFacets("variable_id", variables), - RequireContiguousTimerange(group_by=("instance_id",)), - RequireOverlappingTimerange(group_by=("instance_id",)), - # TODO: Add a RequireTimeRange constraint to match reference datasets? + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1996, 1), + end=PartialDateTime(2014, 12), + ), AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), ), ), @@ -47,13 +49,15 @@ def update_recipe( var_y: str, ) -> None: """Update the recipe.""" - recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6], equalize_timerange=True) + recipe_variables = dataframe_to_recipe(input_files[SourceDatasetType.CMIP6]) diagnostics = recipe["diagnostics"] diagnostic_name = f"plot_joint_{var_x}_{var_y}_model" diagnostic = diagnostics.pop(diagnostic_name) diagnostics.clear() diagnostics[diagnostic_name] = diagnostic datasets = next(iter(recipe_variables.values()))["additional_datasets"] + for dataset in datasets: + dataset["timerange"] = "1996/2014" diagnostic["additional_datasets"] = datasets suptitle = "CMIP6 {dataset} {ensemble} {grid} {timerange}".format(**datasets[0]) diagnostic["scripts"]["plot"]["suptitle"] = suptitle @@ -135,7 +139,13 @@ class CloudScatterplotsReference(ESMValToolDiagnostic): ), ), group_by=("instance_id",), - constraints=(RequireContiguousTimerange(group_by=("instance_id",)),), + constraints=( + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(2007, 1), + end=PartialDateTime(2014, 12), + ), + ), # TODO: Add obs4MIPs datasets once available and working: # # obs4MIPs datasets with issues: diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py index 878d4d442..9d44832ee 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/regional_historical_changes.py @@ -7,8 +7,9 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, - RequireContiguousTimerange, + PartialDateTime, RequireFacets, + RequireTimerange, ) from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -130,7 +131,11 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic): group_by=("source_id", "member_id", "grid_label"), constraints=( RequireFacets("variable_id", variables), - RequireContiguousTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1980, 1), + end=PartialDateTime(2009, 12), + ), AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), ), ), @@ -149,7 +154,13 @@ class RegionalHistoricalAnnualCycle(ESMValToolDiagnostic): ), ), group_by=("source_id",), - constraints=(RequireContiguousTimerange(group_by=("instance_id",)),), + constraints=( + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1980, 1), + end=PartialDateTime(2009, 12), + ), + ), # TODO: Add obs4MIPs datasets once available and working: # # obs4MIPs dataset that cannot be ingested (https://github.com/Climate-REF/climate-ref/issues/260): @@ -211,6 +222,15 @@ class RegionalHistoricalTimeSeries(RegionalHistoricalAnnualCycle): name = "Regional historical mean and anomaly of climate variables" slug = "regional-historical-timeseries" base_recipe = "ref/recipe_ref_timeseries_region.yml" + + variables = ( + "hus", + "pr", + "psl", + "tas", + "ua", + ) + series = tuple( SeriesDefinition( file_pattern=f"{diagnostic}-{region}/allplots/*_{var_name}_*.nc", @@ -220,11 +240,67 @@ class RegionalHistoricalTimeSeries(RegionalHistoricalAnnualCycle): index_name="time", attributes=[], ) - for var_name in RegionalHistoricalAnnualCycle.variables + for var_name in variables for region in REGIONS for diagnostic in ["timeseries_abs", "timeseries"] ) + data_requirements = ( + DataRequirement( + source_type=SourceDatasetType.CMIP6, + filters=( + FacetFilter( + facets={ + "variable_id": variables, + "experiment_id": "historical", + "frequency": "mon", + }, + ), + ), + group_by=("source_id", "member_id", "grid_label"), + constraints=( + RequireFacets("variable_id", variables), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1980, 1), + end=PartialDateTime(2014, 12), + ), + AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), + ), + ), + DataRequirement( + source_type=SourceDatasetType.obs4MIPs, + filters=( + FacetFilter( + facets={ + "variable_id": ( + "psl", + "ua", + ), + "source_id": "ERA-5", + "frequency": "mon", + }, + ), + ), + group_by=("source_id",), + constraints=( + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1980, 1), + end=PartialDateTime(2014, 12), + ), + ), + # TODO: Add obs4MIPs datasets once available and working: + # + # obs4MIPs dataset that cannot be ingested (https://github.com/Climate-REF/climate-ref/issues/260): + # - GPCP-V2.3: pr + # + # Not yet available on obs4MIPs: + # - ERA5: hus + # - HadCRUT5_ground_5.0.1.0-analysis: tas + ), + ) + class RegionalHistoricalTrend(ESMValToolDiagnostic): """ @@ -255,7 +331,11 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic): ), group_by=("source_id", "member_id", "grid_label"), constraints=( - RequireContiguousTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1980, 1), + end=PartialDateTime(2009, 12), + ), AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), ), ), @@ -275,7 +355,13 @@ class RegionalHistoricalTrend(ESMValToolDiagnostic): ), ), group_by=("source_id",), - constraints=(RequireContiguousTimerange(group_by=("instance_id",)),), + constraints=( + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1980, 1), + end=PartialDateTime(2009, 12), + ), + ), # TODO: Add obs4MIPs datasets once available and working: # # obs4MIPs dataset that cannot be ingested (https://github.com/Climate-REF/climate-ref/issues/260): diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py index feed84099..0d85ac138 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_area_basic.py @@ -2,7 +2,8 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, - RequireContiguousTimerange, + PartialDateTime, + RequireTimerange, ) from climate_ref_core.datasets import FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -33,7 +34,11 @@ class SeaIceAreaBasic(ESMValToolDiagnostic): ), group_by=("instance_id",), constraints=( - RequireContiguousTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1979, 1), + end=PartialDateTime(2014, 12), + ), AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6), ), ), diff --git a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py index aff516983..4a7dff97b 100644 --- a/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py +++ b/packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/sea_ice_sensitivity.py @@ -5,8 +5,9 @@ from climate_ref_core.constraints import ( AddSupplementaryDataset, - RequireContiguousTimerange, + PartialDateTime, RequireFacets, + RequireTimerange, ) from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType from climate_ref_core.diagnostics import DataRequirement @@ -46,7 +47,11 @@ class SeaIceSensitivity(ESMValToolDiagnostic): constraints=( AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP6), AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6), - RequireContiguousTimerange(group_by=("instance_id",)), + RequireTimerange( + group_by=("instance_id",), + start=PartialDateTime(1979, 1), + end=PartialDateTime(2014, 12), + ), RequireFacets("variable_id", variables), # TODO: Add a constraint to ensure that tas, siconc and areacello # are available for each model or alternatively filter out diff --git a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_radiative_effects.yml b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_radiative_effects.yml index b7b145c69..b671fdc04 100644 --- a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_radiative_effects.yml +++ b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_radiative_effects.yml @@ -18,7 +18,8 @@ datasets: exp: historical grid: gn mip: Amon -timerange_for_models: 20010101T000000/20141216T120000 +timerange_for_models: + timerange: 1996/2014 preprocessors: full_climatology: climate_statistics: diff --git a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_cli_ta.yml b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_cli_ta.yml index 53714ea17..df463a844 100644 --- a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_cli_ta.yml +++ b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_cli_ta.yml @@ -126,7 +126,7 @@ diagnostics: exp: historical grid: gn mip: Amon - timerange: 19960115T120000/20141215T120000 + timerange: 1996/2014 scripts: plot: script: seaborn_jointplot_histogram.py @@ -145,5 +145,5 @@ diagnostics: style: ticks rc: axes.titlepad: 15.0 - suptitle: CMIP6 CESM2 r1i1p1f1 gn 19960115T120000/20141215T120000 - plot_filename: jointplot_cli_ta_CMIP6_CESM2_r1i1p1f1_gn_19960115T120000-20141215T120000 + suptitle: CMIP6 CESM2 r1i1p1f1 gn 1996/2014 + plot_filename: jointplot_cli_ta_CMIP6_CESM2_r1i1p1f1_gn_1996-2014 diff --git a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clivi_lwcre.yml b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clivi_lwcre.yml index 927e51dbf..99e3ccf8b 100644 --- a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clivi_lwcre.yml +++ b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clivi_lwcre.yml @@ -128,7 +128,7 @@ diagnostics: exp: historical grid: gn mip: Amon - timerange: 19960115T120000/20141215T120000 + timerange: 1996/2014 scripts: plot: script: seaborn_jointplot_histogram.py @@ -147,5 +147,5 @@ diagnostics: style: ticks rc: axes.titlepad: 15.0 - suptitle: CMIP6 CESM2 r1i1p1f1 gn 19960115T120000/20141215T120000 - plot_filename: jointplot_clivi_lwcre_CMIP6_CESM2_r1i1p1f1_gn_19960115T120000-20141215T120000 + suptitle: CMIP6 CESM2 r1i1p1f1 gn 1996/2014 + plot_filename: jointplot_clivi_lwcre_CMIP6_CESM2_r1i1p1f1_gn_1996-2014 diff --git a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clt_swcre.yml b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clt_swcre.yml index 5d518e99a..f1fab2ae8 100644 --- a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clt_swcre.yml +++ b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clt_swcre.yml @@ -128,7 +128,7 @@ diagnostics: exp: historical grid: gn mip: Amon - timerange: 19960115T120000/20141215T120000 + timerange: 1996/2014 scripts: plot: script: seaborn_jointplot_histogram.py @@ -147,5 +147,5 @@ diagnostics: style: ticks rc: axes.titlepad: 15.0 - suptitle: CMIP6 CESM2 r1i1p1f1 gn 19960115T120000/20141215T120000 - plot_filename: jointplot_clt_swcre_CMIP6_CESM2_r1i1p1f1_gn_19960115T120000-20141215T120000 + suptitle: CMIP6 CESM2 r1i1p1f1 gn 1996/2014 + plot_filename: jointplot_clt_swcre_CMIP6_CESM2_r1i1p1f1_gn_1996-2014 diff --git a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clwvi_pr.yml b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clwvi_pr.yml index 489fdf869..2cb500318 100644 --- a/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clwvi_pr.yml +++ b/packages/climate-ref-esmvaltool/tests/unit/diagnostics/recipes/recipe_cloud_scatterplots_clwvi_pr.yml @@ -126,7 +126,7 @@ diagnostics: exp: historical grid: gn mip: Amon - timerange: 19960115T120000/20141215T120000 + timerange: 1996/2014 scripts: plot: script: seaborn_jointplot_histogram.py @@ -145,5 +145,5 @@ diagnostics: style: ticks rc: axes.titlepad: 15.0 - suptitle: CMIP6 CESM2 r1i1p1f1 gn 19960115T120000/20141215T120000 - plot_filename: jointplot_clwvi_pr_CMIP6_CESM2_r1i1p1f1_gn_19960115T120000-20141215T120000 + suptitle: CMIP6 CESM2 r1i1p1f1 gn 1996/2014 + plot_filename: jointplot_clwvi_pr_CMIP6_CESM2_r1i1p1f1_gn_1996-2014