Add ESMValTool ENSO diagnostics

changelog/320.feature.md

@@ -0,0 +1 @@
+Added ESMValTool ENSO diagnostics.

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/dataset_registry/data.txt

@@ -1,3 +1,4 @@
+ESMValTool/obs4MIPs/GPCP-V2.3/v20180519/pr_GPCP-SG_L3_v2.3_197901-201710.nc 4dd4678b79ef139446c8406da5aae4fed210abb2f2160ef95f6988bf83e4525b
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rlut_200003-202311.nc ede887cf2d83c848a0d71316799232e4d717662bd2f78d5aa1fc166b41d9953b
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rlutcs_200003-202311.nc e70e3273092edf01527970693271641fc6474d1974887d7d272e7d656bab83c2
 ESMValTool/OBS/Tier2/CERES-EBAF/OBS_CERES-EBAF_sat_Ed4.2_Amon_rsut_200003-202311.nc e31e648886c4fa9c09686672a06ab18fbba687ff0d6de2891616d4c8b74e215d
@@ -84,3 +85,5 @@ ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201101-20111
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201201-201212.nc 86187c3d1174053f2cba6dad010af49ceab77d368aa9314bf53c330b5f2217b9
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201301-201312.nc 8820353570884b2ef182caaffb5986ed6268bbe199fd867f61b56e798ca01f1a
 ESMValTool/OBS/Tier2/OSI-450-sh/OBS_OSI-450-sh_reanaly_v3_OImon_sic_201401-201412.nc 7102d0db3dc02c5b0eb0cfe3535ee50171007ef5b43eb9aae1220ac21b0b98e9
+ESMValTool/OBS/Tier2/TROPFLUX/OBS6_TROPFLUX_reanaly_v1_Amon_tauu_197901-201812.nc bf313e661b42341d5090038b501ed1ff09e58201009c3fccfe45b78e116fdd78
+ESMValTool/OBS/Tier2/TROPFLUX/OBS6_TROPFLUX_reanaly_v1_Omon_tos_197901-201812.nc 5f10a5a2aa47f5d21378ad3178bf8e4b577b0ed72ef8402dc04f5ff6fc99ec07

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/__init__.py

@@ -3,6 +3,7 @@
 from climate_ref_esmvaltool.diagnostics.climate_at_global_warming_levels import ClimateAtGlobalWarmingLevels
 from climate_ref_esmvaltool.diagnostics.cloud_radiative_effects import CloudRadiativeEffects
 from climate_ref_esmvaltool.diagnostics.ecs import EquilibriumClimateSensitivity
+from climate_ref_esmvaltool.diagnostics.enso import ENSOBasicClimatology, ENSOCharacteristics
 from climate_ref_esmvaltool.diagnostics.example import GlobalMeanTimeseries
 from climate_ref_esmvaltool.diagnostics.sea_ice_area_basic import SeaIceAreaBasic
 from climate_ref_esmvaltool.diagnostics.tcr import TransientClimateResponse
@@ -12,6 +13,8 @@
 __all__ = [
     "ClimateAtGlobalWarmingLevels",
     "CloudRadiativeEffects",
+    "ENSOBasicClimatology",
+    "ENSOCharacteristics",
     "EquilibriumClimateSensitivity",
     "GlobalMeanTimeseries",
     "SeaIceAreaBasic",

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/base.py

@@ -101,15 +101,17 @@ def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
         config = {
             "drs": {
                 "CMIP6": "ESGF",
+                "obs4MIPs": "ESGF",
             },
             "output_dir": str(definition.to_output_path("executions")),
             "rootpath": {
-                "default": str(climate_data),
+                "CMIP6": str(climate_data),
+                "obs4MIPs": str(climate_data),
             },
             "search_esgf": "never",
         }
 
-        # Configure the paths to OBS/OBS6/native6 data
+        # Configure the paths to OBS/OBS6/native6 and non-compliant obs4MIPs data
         registry = dataset_registry_manager["esmvaltool"]
         data_dir = registry.abspath / "ESMValTool"  # type: ignore[attr-defined]
         if not data_dir.exists():
@@ -129,10 +131,14 @@ def build_cmd(self, definition: ExecutionDefinition) -> Iterable[str]:
             config["rootpath"].update(  # type: ignore[attr-defined]
                 {
                     "OBS": str(data_dir / "OBS"),
-                    "OBS6": str(data_dir / "OBS6"),
+                    "OBS6": str(data_dir / "OBS"),
                     "native6": str(data_dir / "RAWOBS"),
                 }
             )
+            config["rootpath"]["obs4MIPs"] = [  # type: ignore[index]
+                config["rootpath"]["obs4MIPs"],  # type: ignore[index]
+                str(data_dir),
+            ]
 
         config_dir = definition.to_output_path("config")
         config_dir.mkdir()

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/enso.py

@@ -0,0 +1,155 @@
+from pathlib import Path
+
+import pandas
+import pandas as pd
+
+from climate_ref_core.constraints import (
+    AddSupplementaryDataset,
+    RequireContiguousTimerange,
+    RequireFacets,
+    RequireOverlappingTimerange,
+)
+from climate_ref_core.datasets import ExecutionDatasetCollection, FacetFilter, SourceDatasetType
+from climate_ref_core.diagnostics import DataRequirement
+from climate_ref_core.pycmec.metric import CMECMetric, MetricCV
+from climate_ref_core.pycmec.output import CMECOutput
+from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic
+from climate_ref_esmvaltool.recipe import dataframe_to_recipe
+from climate_ref_esmvaltool.types import MetricBundleArgs, OutputBundleArgs, Recipe
+
+
+class ENSOBasicClimatology(ESMValToolDiagnostic):
+    """
+    Calculate the ENSO CLIVAR metrics - background climatology.
+    """
+
+    name = "ENSO Basic Climatology"
+    slug = "enso-basic-climatology"
+    base_recipe = "ref/recipe_enso_basicclimatology.yml"
+
+    variables = (
+        "pr",
+        "tos",
+        "tauu",
+    )
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": variables,
+                        "experiment_id": "historical",
+                    },
+                ),
+            ),
+            group_by=("source_id", "member_id", "grid_label"),
+            constraints=(
+                RequireFacets("variable_id", variables),
+                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireOverlappingTimerange(group_by=("instance_id",)),
+            ),
+        ),
+    )
+    facets = ()
+
+    @staticmethod
+    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+        """Update the recipe."""
+        recipe_variables = dataframe_to_recipe(input_files)
+        recipe.pop("datasets")
+        for diagnostic in recipe["diagnostics"].values():
+            for variable in diagnostic["variables"].values():
+                variable["additional_datasets"].extend(
+                    recipe_variables[variable["short_name"]]["additional_datasets"]
+                )
+
+
+class ENSOCharacteristics(ESMValToolDiagnostic):
+    """
+    Calculate the ENSO CLIVAR metrics - basic ENSO characteristics.
+    """
+
+    name = "ENSO Characteristics"
+    slug = "enso-characteristics"
+    base_recipe = "ref/recipe_enso_characteristics.yml"
+
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(
+                FacetFilter(
+                    facets={
+                        "variable_id": "tos",
+                        "experiment_id": "historical",
+                    },
+                ),
+            ),
+            group_by=("source_id", "member_id", "grid_label"),
+            constraints=(
+                RequireFacets("variable_id", ("tos",)),
+                RequireContiguousTimerange(group_by=("instance_id",)),
+                RequireOverlappingTimerange(group_by=("instance_id",)),
+                AddSupplementaryDataset.from_defaults("areacello", SourceDatasetType.CMIP6),
+            ),
+        ),
+    )
+    facets = ("grid_label", "member_id", "source_id", "region", "metric")
+
+    @staticmethod
+    def update_recipe(recipe: Recipe, input_files: pandas.DataFrame) -> None:
+        """Update the recipe."""
+        recipe_variables = dataframe_to_recipe(input_files)
+        recipe["datasets"] = recipe_variables["tos"]["additional_datasets"]
+        # TODO: update the observational data requirement once available on ESGF.
+        # Observations - use only one per run
+        recipe["datasets"].append(
+            # {
+            #     "dataset": "NOAA-ERSSTv5",
+            #     "version": "v5",
+            #     "project": "OBS6",
+            #     "type": "reanaly",
+            #     "tier": 2,
+            # }
+            {
+                "dataset": "TROPFLUX",
+                "version": "v1",
+                "project": "OBS6",
+                "type": "reanaly",
+                "tier": 2,
+            }
+        )
+
+    @staticmethod
+    def format_result(
+        result_dir: Path,
+        execution_dataset: ExecutionDatasetCollection,
+        metric_args: MetricBundleArgs,
+        output_args: OutputBundleArgs,
+    ) -> tuple[CMECMetric, CMECOutput]:
+        """Format the result."""
+        metrics = pd.read_csv(
+            result_dir / "work" / "diagnostic_metrics" / "plot_script" / "matrix.csv",
+            names=["dataset", "metric_name", "metric_value"],
+        )
+
+        # Update the diagnostic bundle arguments with the computed diagnostics.
+        metric_args[MetricCV.DIMENSIONS.value] = {
+            "json_structure": [
+                "region",
+                "metric",
+            ],
+            "region": {"global": {}},
+            "metric": {metric: {} for metric in metrics.metric_name},
+        }
+        metric_args[MetricCV.RESULTS.value] = {
+            "global": {
+                metric_name: metric_value
+                for metric_name, metric_value in zip(
+                    metrics.metric_name,
+                    metrics.metric_value,
+                )
+            },
+        }
+
+        return CMECMetric.model_validate(metric_args), CMECOutput.model_validate(output_args)

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/recipes.txt

@@ -4,5 +4,7 @@ recipe_ecs.yml             0cc57034fcb64e32015b4ff949ece5df8cdb8c6f493618b50cede
 recipe_tcr.yml             35f9ef035a4e71aff5cac5dd26c49da2162fc00291bf3b0bd16b661b7b2f606b
 recipe_tcre.yml            48fc9e3baf541bbcef7491853ea3a774053771dca33352b41466425faeaa38af
 recipe_zec.yml             b0af7f789b7610ab3f29a6617124aa40c40866ead958204fc199eaf82863de51
+ref/recipe_enso_basicclimatology.yml 9ea7deb7ee668e39ac44618b96496d898bd82285c22dcee4fce4695e0c9fa82b
+ref/recipe_enso_characteristics.yml 34c2518b138068ac96d212910b979d54a8fcedee2c0089b5acd56a42c41dc3e4
 ref/recipe_ref_cre.yml     4f35d9639f1008be3b5382a5bd8933a855cb5368ccf5d04a1c70227172e2e82c
 ref/recipe_ref_sea_ice_area_basic.yml 7d01a8527880663ca28284772f83a8356d9972fb4f022a4000e50a56ce044b09

packages/climate-ref-esmvaltool/tests/unit/diagnostics/input_files_enso_characteristics.json

@@ -0,0 +1,142 @@
+{
+    "start_time":{
+        "1":"1850-01-16T12:00:00.000",
+        "109":null
+    },
+    "end_time":{
+        "1":"2014-12-16T12:00:00.000",
+        "109":null
+    },
+    "path":{
+        "1":"\/home\/bandela\/climate_data\/CMIP6\/CMIP\/CSIRO\/ACCESS-ESM1-5\/historical\/r1i1p1f1\/Omon\/tos\/gn\/v20191115\/tos_Omon_ACCESS-ESM1-5_historical_r1i1p1f1_gn_185001-201412.nc",
+        "109":"\/home\/bandela\/climate_data\/CMIP6\/CMIP\/CSIRO\/ACCESS-ESM1-5\/historical\/r1i1p1f1\/Ofx\/areacello\/gn\/v20191115\/areacello_Ofx_ACCESS-ESM1-5_historical_r1i1p1f1_gn.nc"
+    },
+    "activity_id":{
+        "1":"CMIP",
+        "109":"CMIP"
+    },
+    "branch_method":{
+        "1":"standard",
+        "109":"standard"
+    },
+    "branch_time_in_child":{
+        "1":0.0,
+        "109":0.0
+    },
+    "branch_time_in_parent":{
+        "1":21915.0,
+        "109":21915.0
+    },
+    "experiment":{
+        "1":"all-forcing simulation of the recent past",
+        "109":"all-forcing simulation of the recent past"
+    },
+    "experiment_id":{
+        "1":"historical",
+        "109":"historical"
+    },
+    "frequency":{
+        "1":"mon",
+        "109":"fx"
+    },
+    "grid":{
+        "1":"native atmosphere N96 grid (145x192 latxlon)",
+        "109":"native atmosphere N96 grid (145x192 latxlon)"
+    },
+    "grid_label":{
+        "1":"gn",
+        "109":"gn"
+    },
+    "institution_id":{
+        "1":"CSIRO",
+        "109":"CSIRO"
+    },
+    "nominal_resolution":{
+        "1":"250 km",
+        "109":"250 km"
+    },
+    "parent_activity_id":{
+        "1":"CMIP",
+        "109":"CMIP"
+    },
+    "parent_experiment_id":{
+        "1":"piControl",
+        "109":"piControl"
+    },
+    "parent_source_id":{
+        "1":"ACCESS-ESM1-5",
+        "109":"ACCESS-ESM1-5"
+    },
+    "parent_time_units":{
+        "1":"days since 0101-1-1",
+        "109":"days since 0101-1-1"
+    },
+    "parent_variant_label":{
+        "1":"r1i1p1f1",
+        "109":"r1i1p1f1"
+    },
+    "product":{
+        "1":"model-output",
+        "109":"model-output"
+    },
+    "realm":{
+        "1":"ocean",
+        "109":"ocean"
+    },
+    "source_id":{
+        "1":"ACCESS-ESM1-5",
+        "109":"ACCESS-ESM1-5"
+    },
+    "source_type":{
+        "1":"AOGCM",
+        "109":"AOGCM"
+    },
+    "sub_experiment":{
+        "1":"none",
+        "109":"none"
+    },
+    "sub_experiment_id":{
+        "1":"none",
+        "109":"none"
+    },
+    "table_id":{
+        "1":"Omon",
+        "109":"Ofx"
+    },
+    "variable_id":{
+        "1":"tos",
+        "109":"areacello"
+    },
+    "variant_label":{
+        "1":"r1i1p1f1",
+        "109":"r1i1p1f1"
+    },
+    "member_id":{
+        "1":"r1i1p1f1",
+        "109":"r1i1p1f1"
+    },
+    "vertical_levels":{
+        "1":1,
+        "109":1
+    },
+    "version":{
+        "1":"v20191115",
+        "109":"v20191115"
+    },
+    "standard_name":{
+        "1":"sea_surface_temperature",
+        "109":"cell_area"
+    },
+    "long_name":{
+        "1":"Sea Surface Temperature",
+        "109":"Grid-Cell Area for Ocean Variables"
+    },
+    "units":{
+        "1":"degC",
+        "109":"m2"
+    },
+    "instance_id":{
+        "1":"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r1i1p1f1.Omon.tos.gn.v20191115",
+        "109":"CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r1i1p1f1.Ofx.areacello.gn.v20191115"
+    }
+}