fix: apply version filtering and fix supplementary dataset selection in regression analysis

changelog/554.fix.md

@@ -0,0 +1 @@
+Applied version filtering to dataset queries and fixed supplementary dataset selection to pick a single dataset per matching group. This eliminates spurious experiment and version entries from regression analysis results.

packages/climate-ref-core/src/climate_ref_core/constraints.py

@@ -248,16 +248,19 @@ def apply(
             facets = matching_facets + list(self.optional_matching_facets)
             datasets = group[facets].drop_duplicates()
             select = pd.Series(False, index=supplementary_group.index)
-            for i in range(len(datasets)):
-                dataset = datasets.iloc[i]
-                # Restrict the supplementary datasets to those that match the main dataset.
+            for _, main_group_df in datasets.groupby(matching_facets):
+                # Restrict the supplementary datasets to those that match this group.
+                first_row = main_group_df.iloc[0]
                 supplementaries = supplementary_group[
-                    (supplementary_group[matching_facets] == dataset[matching_facets]).all(axis="columns")
+                    (supplementary_group[matching_facets] == first_row[matching_facets]).all(axis="columns")
                 ]
                 if not supplementaries.empty:
-                    # Select the best matching supplementary dataset based on the optional matching facets.
-                    scores = (supplementaries[facets] == dataset).sum(axis="columns")
-                    supplementaries = supplementaries[scores == scores.max()]
+                    # Score each candidate against ALL main datasets in this group, take max score.
+                    best_scores = pd.Series(0, index=supplementaries.index, dtype=int)
+                    for i in range(len(main_group_df)):
+                        scores = (supplementaries[facets] == main_group_df.iloc[i]).sum(axis="columns")
+                        best_scores = best_scores.combine(scores, max)
+                    supplementaries = supplementaries[best_scores == best_scores.max()]
                     if "version" in supplementaries.columns:
                         # Select the latest version if there are multiple matches
                         supplementaries = supplementaries[

packages/climate-ref-core/tests/unit/test_constraints.py

@@ -280,7 +280,7 @@ def test_from_defaults_cmip7_matching_facets(self):
                         "version": ["v20210316", "v20220101", "v20210316", "v20210316"],
                     }
                 ),
-                [0, 2, 1, 3],
+                [0, 2, 1],
             ),
         ],
     )
@@ -471,13 +471,8 @@ def test_duplicate_supplementary_index(self):
         group = data_catalog[data_catalog["variable_id"] == "tas"]
         result = self.constraint.apply(group=group, data_catalog=data_catalog)
 
-        # Both tas rows plus all three areacella rows
-        expected = pd.concat(
-            [
-                data_catalog.iloc[[0, 1]],
-                data_catalog.iloc[[2, 3, 4]],
-            ]
-        )
+        # Both tas rows plus the 1pctCO2 areacella rows (best max-score, first-group tiebreak)
+        expected = data_catalog.iloc[[0, 1, 2, 3]]
         assert_frame_equal(result, expected)
 
 

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_climate_drivers_for_fire_.yml

@@ -91,9 +91,7 @@ cmip6_gr1_r1i1p1f1_GFDL-ESM4:
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.hurs.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20211207
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tasmax.gr.v20211207
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Emon.vegFrac.gr.v20211207

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_cloud_scatterplots_clwvi_pr_.yml

@@ -242,7 +242,6 @@ cmip6_historical_mon_gr_r1i1p1f1_EC-Earth3-Veg:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.clwvi.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20211207
 cmip6_historical_mon_gr_r1i1p1f1_EC-Earth3-Veg-LR:
   cmip6:

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_enso_basic_climatology_.yml

@@ -160,7 +160,6 @@ cmip6_gn_r1i1p1f1_NESM3:
   - CMIP6.CMIP.NUIST.NESM3.historical.r1i1p1f1.Omon.tos.gn.v20190703
 cmip6_gn_r1i1p1f1_NorCPM1:
   cmip6:
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20200724
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tauu.gn.v20200724
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Omon.tos.gn.v20200724

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_equilibrium_climate_sensitivity_.yml

@@ -63,7 +63,6 @@ cmip6_gn_r1i1p1f1_ACCESS-CM2:
   - CMIP6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.piControl.r1i1p1f1.Amon.rsdt.gn.v20191112
   - CMIP6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.piControl.r1i1p1f1.Amon.rsut.gn.v20191112
   - CMIP6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.piControl.r1i1p1f1.Amon.tas.gn.v20191112
-  - CMIP6.DAMIP.CSIRO-ARCCSS.ACCESS-CM2.hist-GHG.r1i1p1f1.fx.areacella.gn.v20201120
 cmip6_gn_r1i1p1f1_ACCESS-ESM1-5:
   cmip6:
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.1pctCO2.r1i1p1f1.fx.areacella.gn.v20191115
@@ -75,7 +74,6 @@ cmip6_gn_r1i1p1f1_ACCESS-ESM1-5:
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.piControl.r1i1p1f1.Amon.rsdt.gn.v20210316
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.piControl.r1i1p1f1.Amon.rsut.gn.v20210316
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.piControl.r1i1p1f1.Amon.tas.gn.v20210316
-  - CMIP6.DAMIP.CSIRO.ACCESS-ESM1-5.hist-GHG.r1i1p1f1.fx.areacella.gn.v20200615
 cmip6_gn_r1i1p1f1_AWI-CM-1-1-MR:
   cmip6:
   - CMIP6.CMIP.AWI.AWI-CM-1-1-MR.1pctCO2.r1i1p1f1.fx.areacella.gn.v20191015
@@ -163,7 +161,6 @@ cmip6_gn_r1i1p1f1_CESM2-WACCM:
   - CMIP6.CMIP.NCAR.CESM2-WACCM.piControl.r1i1p1f1.Amon.tas.gn.v20190320
 cmip6_gn_r1i1p1f1_CESM2-WACCM-FV2:
   cmip6:
-  - CMIP6.CMIP.NCAR.CESM2-WACCM-FV2.1pctCO2.r1i1p1f1.fx.areacella.gn.v20200226
   - CMIP6.CMIP.NCAR.CESM2-WACCM-FV2.abrupt-4xCO2.r1i1p1f1.Amon.rlut.gn.v20200403
   - CMIP6.CMIP.NCAR.CESM2-WACCM-FV2.abrupt-4xCO2.r1i1p1f1.Amon.rsdt.gn.v20200403
   - CMIP6.CMIP.NCAR.CESM2-WACCM-FV2.abrupt-4xCO2.r1i1p1f1.Amon.rsut.gn.v20200403
@@ -301,7 +298,6 @@ cmip6_gn_r1i1p1f1_IITM-ESM:
   - CMIP6.CMIP.CCCR-IITM.IITM-ESM.piControl.r1i1p1f1.Amon.tas.gn.v20191120
 cmip6_gn_r1i1p1f1_MIROC6:
   cmip6:
-  - CMIP6.CMIP.MIROC.MIROC6.1pctCO2.r1i1p1f1.fx.areacella.gn.v20190311
   - CMIP6.CMIP.MIROC.MIROC6.abrupt-4xCO2.r1i1p1f1.Amon.rlut.gn.v20190705
   - CMIP6.CMIP.MIROC.MIROC6.abrupt-4xCO2.r1i1p1f1.Amon.rsdt.gn.v20190705
   - CMIP6.CMIP.MIROC.MIROC6.abrupt-4xCO2.r1i1p1f1.Amon.rsut.gn.v20190705
@@ -599,7 +595,6 @@ cmip6_gn_r2i1p1f1_ACCESS-ESM1-5:
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.abrupt-4xCO2.r2i1p1f1.Amon.rsdt.gn.v20200901
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.abrupt-4xCO2.r2i1p1f1.Amon.rsut.gn.v20200901
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.abrupt-4xCO2.r2i1p1f1.Amon.tas.gn.v20200901
-  - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.historical.r2i1p1f1.fx.areacella.gn.v20191128
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.piControl.r1i1p1f1.Amon.rlut.gn.v20210316
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.piControl.r1i1p1f1.Amon.rsdt.gn.v20210316
   - CMIP6.CMIP.CSIRO.ACCESS-ESM1-5.piControl.r1i1p1f1.Amon.rsut.gn.v20210316
@@ -756,7 +751,6 @@ cmip6_gr1_r1i1p1f1_INM-CM4-8:
   - CMIP6.CMIP.INM.INM-CM4-8.piControl.r1i1p1f1.Amon.tas.gr1.v20190605
 cmip6_gr1_r1i1p1f1_INM-CM5-0:
   cmip6:
-  - CMIP6.CMIP.INM.INM-CM5-0.1pctCO2.r1i1p1f1.fx.areacella.gr1.v20190820
   - CMIP6.CMIP.INM.INM-CM5-0.abrupt-4xCO2.r1i1p1f1.Amon.rlut.gr1.v20190610
   - CMIP6.CMIP.INM.INM-CM5-0.abrupt-4xCO2.r1i1p1f1.Amon.rsdt.gr1.v20190610
   - CMIP6.CMIP.INM.INM-CM5-0.abrupt-4xCO2.r1i1p1f1.Amon.rsut.gr1.v20190610
@@ -990,7 +984,6 @@ cmip6_gr_r2i1p1f1_E3SM-2-0:
   - CMIP6.CMIP.E3SM-Project.E3SM-2-0.abrupt-4xCO2.r2i1p1f1.Amon.rsdt.gr.v20220830
   - CMIP6.CMIP.E3SM-Project.E3SM-2-0.abrupt-4xCO2.r2i1p1f1.Amon.rsut.gr.v20220830
   - CMIP6.CMIP.E3SM-Project.E3SM-2-0.abrupt-4xCO2.r2i1p1f1.Amon.tas.gr.v20220830
-  - CMIP6.CMIP.E3SM-Project.E3SM-2-0.historical.r2i1p1f1.fx.areacella.gr.v20220913
   - CMIP6.CMIP.E3SM-Project.E3SM-2-0.piControl.r1i1p1f1.Amon.rlut.gr.v20220913
   - CMIP6.CMIP.E3SM-Project.E3SM-2-0.piControl.r1i1p1f1.Amon.rsdt.gr.v20220913
   - CMIP6.CMIP.E3SM-Project.E3SM-2-0.piControl.r1i1p1f1.Amon.rsut.gr.v20220913
@@ -1050,7 +1043,6 @@ cmip6_gr_r3i1p1f1_IPSL-CM6A-LR:
   - CMIP6.CMIP.IPSL.IPSL-CM6A-LR.piControl.r1i1p1f1.Amon.tas.gr.v20200326
 cmip6_gr_r3i1p1f2_CNRM-ESM2-1:
   cmip6:
-  - CMIP6.CMIP.CNRM-CERFACS.CNRM-ESM2-1.1pctCO2.r1i1p1f2.fx.areacella.gr.v20181018
   - CMIP6.CMIP.CNRM-CERFACS.CNRM-ESM2-1.abrupt-4xCO2.r3i1p1f2.Amon.rlut.gr.v20190208
   - CMIP6.CMIP.CNRM-CERFACS.CNRM-ESM2-1.abrupt-4xCO2.r3i1p1f2.Amon.rsdt.gr.v20190208
   - CMIP6.CMIP.CNRM-CERFACS.CNRM-ESM2-1.abrupt-4xCO2.r3i1p1f2.Amon.rsut.gr.v20190208

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_global_mean_timeseries_.yml

@@ -1023,7 +1023,6 @@ cmip6_historical_gn_r1i1p1f1_NESM3_Amon_tas:
   - CMIP6.CMIP.NUIST.NESM3.historical.r1i1p1f1.Amon.tas.gn.v20190630
 cmip6_historical_gn_r1i1p1f1_NorCPM1_Amon_tas:
   cmip6:
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20200724
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.fx.areacella.gn.v20200724
 cmip6_historical_gn_r1i1p1f1_NorESM2-LM_Amon_tas:
@@ -1079,7 +1078,6 @@ cmip6_historical_gr1_r1i1p1f1_INM-CM5-0_Amon_tas:
   - CMIP6.CMIP.INM.INM-CM5-0.historical.r1i1p1f1.fx.areacella.gr1.v20190610
 cmip6_historical_gr1_r1i1p1f1_KIOST-ESM_Amon_tas:
   cmip6:
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_historical_gr_r1i1p101f1_GISS-E3-G_Amon_tas:
   cmip6:
@@ -1127,7 +1125,6 @@ cmip6_historical_gr_r1i1p1f1_EC-Earth3-Veg-LR_Amon_tas:
 cmip6_historical_gr_r1i1p1f1_EC-Earth3-Veg_Amon_tas:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20211207
 cmip6_historical_gr_r1i1p1f1_EC-Earth3_Amon_tas:
   cmip6:
@@ -1627,7 +1624,6 @@ cmip6_ssp126_gr1_r1i1p1f1_INM-CM5-0_Amon_tas:
   - CMIP6.ScenarioMIP.INM.INM-CM5-0.ssp126.r1i1p1f1.Amon.tas.gr1.v20190619
 cmip6_ssp126_gr1_r1i1p1f1_KIOST-ESM_Amon_tas:
   cmip6:
-  - CMIP6.ScenarioMIP.KIOST.KIOST-ESM.ssp126.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.ScenarioMIP.KIOST.KIOST-ESM.ssp126.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_ssp126_gr_r1i1p1f1_CIESM_Amon_tas:
   cmip6:
@@ -1636,10 +1632,6 @@ cmip6_ssp126_gr_r1i1p1f1_EC-Earth3-Veg-LR_Amon_tas:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg-LR.1pctCO2.r1i1p1f1.fx.areacella.gr.v20220428
   - CMIP6.ScenarioMIP.EC-Earth-Consortium.EC-Earth3-Veg-LR.ssp126.r1i1p1f1.Amon.tas.gr.v20201201
-cmip6_ssp126_gr_r1i1p1f1_EC-Earth3-Veg_Amon_tas:
-  cmip6:
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
-  - CMIP6.ScenarioMIP.EC-Earth-Consortium.EC-Earth3-Veg.ssp126.r1i1p1f1.Amon.tas.gr.v20200225
 cmip6_ssp126_gr_r1i1p1f1_EC-Earth3_Amon_tas:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3.historical.r1i1p1f1.fx.areacella.gr.v20210324
@@ -1650,7 +1642,6 @@ cmip6_ssp126_gr_r1i1p1f1_FGOALS-f3-L_Amon_tas:
 cmip6_ssp126_gr_r1i1p1f1_IPSL-CM5A2-INCA_Amon_tas:
   cmip6:
   - CMIP6.CMIP.IPSL.IPSL-CM5A2-INCA.1pctCO2.r1i1p1f1.fx.areacella.gr.v20201218
-  - CMIP6.ScenarioMIP.IPSL.IPSL-CM5A2-INCA.ssp126.r1i1p1f1.Amon.tas.gr.v20201218
   - CMIP6.ScenarioMIP.IPSL.IPSL-CM5A2-INCA.ssp126.r1i1p1f1.Amon.tas.gr.v20220105
 cmip6_ssp126_gr_r1i1p1f1_IPSL-CM6A-LR_Amon_tas:
   cmip6:
@@ -1788,7 +1779,6 @@ cmip6_ssp245_gr1_r1i1p1f1_INM-CM5-0_Amon_tas:
   - CMIP6.ScenarioMIP.INM.INM-CM5-0.ssp245.r1i1p1f1.Amon.tas.gr1.v20190619
 cmip6_ssp245_gr1_r1i1p1f1_KIOST-ESM_Amon_tas:
   cmip6:
-  - CMIP6.ScenarioMIP.KIOST.KIOST-ESM.ssp245.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.ScenarioMIP.KIOST.KIOST-ESM.ssp245.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_ssp245_gr_r1i1p1f1_CIESM_Amon_tas:
   cmip6:
@@ -2079,8 +2069,6 @@ cmip6_ssp585_gn_r1i1p1f2_MCM-UA-1-0_Amon_tas:
 cmip6_ssp585_gn_r1i1p1f2_MIROC-ES2L_Amon_tas:
   cmip6:
   - CMIP6.C4MIP CDRMIP.MIROC.MIROC-ES2L.esm-1pct-brch-1000PgC.r1i1p1f2.fx.areacella.gn.v20200622
-  - CMIP6.CMIP.MIROC.MIROC-ES2L.1pctCO2.r1i1p1f2.fx.areacella.gn.v20190823
-  - CMIP6.ScenarioMIP.MIROC.MIROC-ES2L.ssp585.r1i1p1f2.Amon.tas.gn.v20190823
   - CMIP6.ScenarioMIP.MIROC.MIROC-ES2L.ssp585.r1i1p1f2.Amon.tas.gn.v20220314
 cmip6_ssp585_gn_r1i1p1f2_UKESM1-0-LL_Amon_tas:
   cmip6:
@@ -2109,7 +2097,6 @@ cmip6_ssp585_gr1_r1i1p1f1_INM-CM5-0_Amon_tas:
   - CMIP6.ScenarioMIP.INM.INM-CM5-0.ssp585.r1i1p1f1.Amon.tas.gr1.v20190724
 cmip6_ssp585_gr1_r1i1p1f1_KIOST-ESM_Amon_tas:
   cmip6:
-  - CMIP6.ScenarioMIP.KIOST.KIOST-ESM.ssp585.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.ScenarioMIP.KIOST.KIOST-ESM.ssp585.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_ssp585_gr_r1i1p1f1_CIESM_Amon_tas:
   cmip6:
@@ -2137,7 +2124,6 @@ cmip6_ssp585_gr_r1i1p1f1_EC-Earth3-Veg-LR_Amon_tas:
 cmip6_ssp585_gr_r1i1p1f1_EC-Earth3-Veg_Amon_tas:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
-  - CMIP6.ScenarioMIP.EC-Earth-Consortium.EC-Earth3-Veg.ssp585.r1i1p1f1.Amon.tas.gr.v20200225
   - CMIP6.ScenarioMIP.EC-Earth-Consortium.EC-Earth3-Veg.ssp585.r1i1p1f1.Amon.tas.gr.v20221112
 cmip6_ssp585_gr_r1i1p1f1_EC-Earth3_Amon_tas:
   cmip6:

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_regional_historical_annual_cycle_.yml

@@ -432,11 +432,8 @@ cmip6_gn_r1i1p1f1_NESM3:
   - CMIP6.CMIP.NUIST.NESM3.historical.r1i1p1f1.Amon.tas.gn.v20190630
 cmip6_gn_r1i1p1f1_NorCPM1:
   cmip6:
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20200724
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.psl.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.psl.gn.v20200724
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20200724
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.fx.areacella.gn.v20200724
 cmip6_gn_r1i1p1f1_NorESM2-LM:
@@ -1157,9 +1154,7 @@ cmip6_gr1_r1i1p1f1_INM-CM5-0:
 cmip6_gr1_r1i1p1f1_KIOST-ESM:
   cmip6:
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.pr.gr1.v20210928
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.psl.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.psl.gr1.v20210601
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_gr_r101i1p1f1_EC-Earth3:
   cmip6:
@@ -1597,11 +1592,8 @@ cmip6_gr_r1i1p1f1_EC-Earth3-CC:
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.psl.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.psl.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20211207
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg-LR:
   cmip6:

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_regional_historical_timeseries_.yml

@@ -431,11 +431,8 @@ cmip6_gn_r1i1p1f1_NESM3:
   - CMIP6.CMIP.NUIST.NESM3.historical.r1i1p1f1.Amon.tas.gn.v20190630
 cmip6_gn_r1i1p1f1_NorCPM1:
   cmip6:
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20200724
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.psl.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.psl.gn.v20200724
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20200724
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.fx.areacella.gn.v20200724
 cmip6_gn_r1i1p1f1_NorESM2-LM:
@@ -1156,9 +1153,7 @@ cmip6_gr1_r1i1p1f1_INM-CM5-0:
 cmip6_gr1_r1i1p1f1_KIOST-ESM:
   cmip6:
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.pr.gr1.v20210928
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.psl.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.psl.gr1.v20210601
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_gr_r101i1p1f1_EC-Earth3:
   cmip6:
@@ -1596,11 +1591,8 @@ cmip6_gr_r1i1p1f1_EC-Earth3-CC:
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.psl.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.psl.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20211207
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg-LR:
   cmip6:

packages/climate-ref-esmvaltool/tests/unit/test_solve_regression/test_solve_regression_regional_historical_trend_.yml

@@ -432,11 +432,8 @@ cmip6_gn_r1i1p1f1_NESM3:
   - CMIP6.CMIP.NUIST.NESM3.historical.r1i1p1f1.Amon.tas.gn.v20190630
 cmip6_gn_r1i1p1f1_NorCPM1:
   cmip6:
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.pr.gn.v20200724
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.psl.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.psl.gn.v20200724
-  - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20190914
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.Amon.tas.gn.v20200724
   - CMIP6.CMIP.NCC.NorCPM1.historical.r1i1p1f1.fx.areacella.gn.v20200724
 cmip6_gn_r1i1p1f1_NorESM2-LM:
@@ -1157,9 +1154,7 @@ cmip6_gr1_r1i1p1f1_INM-CM5-0:
 cmip6_gr1_r1i1p1f1_KIOST-ESM:
   cmip6:
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.pr.gr1.v20210928
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.psl.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.psl.gr1.v20210601
-  - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20191106
   - CMIP6.CMIP.KIOST.KIOST-ESM.historical.r1i1p1f1.Amon.tas.gr1.v20210601
 cmip6_gr_r101i1p1f1_EC-Earth3:
   cmip6:
@@ -1597,11 +1592,8 @@ cmip6_gr_r1i1p1f1_EC-Earth3-CC:
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg:
   cmip6:
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r11i1p1f1.fx.areacella.gr.v20230203
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.pr.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.psl.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.psl.gr.v20211207
-  - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20200225
   - CMIP6.CMIP.EC-Earth-Consortium.EC-Earth3-Veg.historical.r1i1p1f1.Amon.tas.gr.v20211207
 cmip6_gr_r1i1p1f1_EC-Earth3-Veg-LR:
   cmip6: