From 9c98423a85aa7b82bdb8918baaf8045a4c20495b Mon Sep 17 00:00:00 2001 From: Matthew Mizielinski Date: Fri, 10 Apr 2026 16:50:45 +0000 Subject: [PATCH 01/16] #886: First draft of how to add cell measures --- .../mip_convert/requested_variables.py | 10 ++++++ .../mip_convert/save/cmor/cmor_wrapper.py | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 31c07e6c4..9483c6f90 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -125,6 +125,16 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) + # Apply cell_measures if a _cell_measures.json file exists + saver.cmor.apply_cell_measures( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes['region'], + saver.varid) + # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index 56c5a6463..a1dbb372c 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -1,6 +1,8 @@ # (C) British Crown Copyright 2009-2025, Met Office. # Please see LICENSE.md for license details. from collections import OrderedDict +import os +import json import cmor @@ -115,3 +117,32 @@ def zfactor(self, *args, **kwargs): def set_frequency(self, frequency, **kwargs): self._debug_on_args('frequency', [frequency], kwargs) cmor.cmor.set_cur_dataset_attribute('frequency', frequency) + + def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): + # self._debug_on_args('apply_cell_measures', [mip_table_dir]) + """ + Add cell measures read from json file in with mip tables + """ + + cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json') + + if os.path.exists(cell_measures_file): + with open(cell_measures_file) as fh: + cell_measures = json.load(fh) + if 'cell_measures' not in cell_measures: + self.logger.debug(f'"cell_measures" key not found in {cell_measures_file}') + return + cell_measures = cell_measures['cell_measures'] + + root_label, branding = variable.split('_') + key = f'{realm}.{root_label}.{branding}.{frequency}.{region}' + if key in cell_measures: + retval = cmor.cmor.set_variable_attribute( + variable_id, + 'cell_measures', + 'c', + cell_measures[key]) + if retval != 0: + self.logger.debug('cell_measures assignment failed. Check cmor log file for details') + else: + self.logger.debug(f'Cell_measures file "{cell_measures_file}" not found. Continuing') From 22530bb7d99c12bb04b4e31e220eb2542e551bba Mon Sep 17 00:00:00 2001 From: mo-gill Date: Fri, 5 Jun 2026 10:36:37 +0000 Subject: [PATCH 02/16] #886: Fix keyerror. --- mip_convert/mip_convert/requested_variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 9483c6f90..c6c7b52c9 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -132,7 +132,7 @@ def produce_mip_requested_variable( mip_table.id, variable_name, frequency, - user_config.global_attributes['region'], + user_config.global_attributes.get('region', ''), saver.varid) # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' From 3a54a40c1cf018b5e8a48c29749471e0717551d8 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Mon, 8 Jun 2026 09:53:02 +0100 Subject: [PATCH 03/16] #886: Further iteration, fix varid=None bug. --- .../mip_convert/requested_variables.py | 34 ++++++++++++------- .../mip_convert/save/cmor/cmor_wrapper.py | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index c6c7b52c9..2f3996080 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -14,7 +14,7 @@ from mip_convert.configuration.json_config import MIPConfig from mip_convert.configuration.python_config import ModelToMIPMappingConfig -from mip_convert.save import save +from mip_convert.save import create_cmor_variable from mip_convert.save.cmor import cmor_lite from mip_convert.mip_table import get_variable_model_to_mip_mapping, get_variable_mip_metadata @@ -125,23 +125,33 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) - # Apply cell_measures if a _cell_measures.json file exists - saver.cmor.apply_cell_measures( - user_config.mip_era, - user_config.inpath, - mip_table.id, - variable_name, - frequency, - user_config.global_attributes.get('region', ''), - saver.varid) - # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') + cell_measures_applied = False for time_slice in variable.slices_over(period): time_slice.process() logger.debug('MIP output variable contains: {}'.format(time_slice.info)) - save(time_slice, saver) + + # Build the CMOR variable first so the CMOR varid can be created + # before any write, allowing variable attributes to be set safely. + cmor_variable = create_cmor_variable(time_slice) + if not cell_measures_applied: + if saver.varid is None: + saver._getVarId(cmor_variable) + # Apply cell_measures if a _cell_measures.json file exists. + # Must be called before the first write to satisfy CMOR. + saver.cmor.apply_cell_measures( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes.get('region', ''), + saver.varid) + cell_measures_applied = True + + saver(cmor_variable) # Close the 'output netCDF file'. cmor_lite.close(saver.varid) diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index a1dbb372c..1e437d760 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -119,10 +119,10 @@ def set_frequency(self, frequency, **kwargs): cmor.cmor.set_cur_dataset_attribute('frequency', frequency) def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): - # self._debug_on_args('apply_cell_measures', [mip_table_dir]) """ Add cell measures read from json file in with mip tables """ + self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json') From d8b5977f3cda4a9ce4c2405d35b8341b034861ae Mon Sep 17 00:00:00 2001 From: mo-gill Date: Mon, 8 Jun 2026 09:53:32 +0000 Subject: [PATCH 04/16] #886: Refine comments. --- mip_convert/mip_convert/requested_variables.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 2f3996080..f836a25a0 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -134,13 +134,14 @@ def produce_mip_requested_variable( logger.debug('MIP output variable contains: {}'.format(time_slice.info)) # Build the CMOR variable first so the CMOR varid can be created - # before any write, allowing variable attributes to be set safely. + # before any write, allowing variable attributes to be set safely + # (a valid varid is required by apply_cell_measures to set the + # cell_measures attribute via cmor.set_variable_attribute). cmor_variable = create_cmor_variable(time_slice) if not cell_measures_applied: if saver.varid is None: saver._getVarId(cmor_variable) # Apply cell_measures if a _cell_measures.json file exists. - # Must be called before the first write to satisfy CMOR. saver.cmor.apply_cell_measures( user_config.mip_era, user_config.inpath, From 099d59d5e19c1957f237dcf03e7924865294331d Mon Sep 17 00:00:00 2001 From: mo-gill Date: Mon, 8 Jun 2026 10:43:18 +0000 Subject: [PATCH 05/16] #886: Clarify behaviour in docstring. --- mip_convert/mip_convert/requested_variables.py | 3 +-- mip_convert/mip_convert/save/cmor/cmor_wrapper.py | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index f836a25a0..c334e0eb8 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -139,8 +139,7 @@ def produce_mip_requested_variable( # cell_measures attribute via cmor.set_variable_attribute). cmor_variable = create_cmor_variable(time_slice) if not cell_measures_applied: - if saver.varid is None: - saver._getVarId(cmor_variable) + saver._getVarId(cmor_variable) # Apply cell_measures if a _cell_measures.json file exists. saver.cmor.apply_cell_measures( user_config.mip_era, diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index 1e437d760..5640f8ded 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -120,7 +120,13 @@ def set_frequency(self, frequency, **kwargs): def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): """ - Add cell measures read from json file in with mip tables + Set the cell_measures attribute on a CMOR variable using a lookup in + ``{mip_era}_cell_measures.json`` from the MIP tables directory. Returns + silently if the file does not exist. + + The lookup key has the form ``{realm}.{root_label}.{branding}.{frequency}.{region}`` + (variable name split on ``_``), so entries in the JSON must follow the + convention used for CMIP7. """ self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) From fb6e14b7baa6c2e86754f86a999daaa98303a7c1 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Tue, 9 Jun 2026 09:02:23 +0000 Subject: [PATCH 06/16] #886: Move code into time_slice.process. --- mip_convert/mip_convert/new_variable.py | 15 +++++++- .../mip_convert/requested_variables.py | 35 +++++++------------ .../mip_convert/save/cmor/cmor_outputter.py | 12 +++++++ 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index 32d116603..e6fa2682a 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -337,7 +337,7 @@ def ordered_coords(self): return self._ordered_coords - def process(self): + def process(self, saver=None, cell_measures_config=None): """Process the data. The units of the data of the |MIP requested variable| are the @@ -367,6 +367,19 @@ def process(self): self._update_time_units() if hasattr(self.model_to_mip_mapping, 'valid_min'): self._apply_valid_min_correction() + if saver is not None and cell_measures_config is not None: + self.apply_cell_measures(saver, cell_measures_config) + + def apply_cell_measures(self, saver, cell_measures_config): + """Apply cell measures to the CMOR variable if not already applied. Parameters + ---------- + saver: + The CMOR outputter for this variable. + cell_measures_config: tuple + Arguments forwarded to cmor_wrapper.apply_cell_measures, + in the form (mip_era, mip_table_dir, realm, variable, frequency, region). + """ + saver.apply_cell_measures(self, *cell_measures_config) def _remove_alevhalf_bounds(self): """ diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index c334e0eb8..96889d997 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -14,7 +14,7 @@ from mip_convert.configuration.json_config import MIPConfig from mip_convert.configuration.python_config import ModelToMIPMappingConfig -from mip_convert.save import create_cmor_variable +from mip_convert.save import save from mip_convert.save.cmor import cmor_lite from mip_convert.mip_table import get_variable_model_to_mip_mapping, get_variable_mip_metadata @@ -128,30 +128,19 @@ def produce_mip_requested_variable( # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') - cell_measures_applied = False + + cell_measures_config = ( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes.get('region', '')) + for time_slice in variable.slices_over(period): - time_slice.process() + time_slice.process(saver, cell_measures_config) logger.debug('MIP output variable contains: {}'.format(time_slice.info)) - - # Build the CMOR variable first so the CMOR varid can be created - # before any write, allowing variable attributes to be set safely - # (a valid varid is required by apply_cell_measures to set the - # cell_measures attribute via cmor.set_variable_attribute). - cmor_variable = create_cmor_variable(time_slice) - if not cell_measures_applied: - saver._getVarId(cmor_variable) - # Apply cell_measures if a _cell_measures.json file exists. - saver.cmor.apply_cell_measures( - user_config.mip_era, - user_config.inpath, - mip_table.id, - variable_name, - frequency, - user_config.global_attributes.get('region', ''), - saver.varid) - cell_measures_applied = True - - saver(cmor_variable) + save(time_slice, saver) # Close the 'output netCDF file'. cmor_lite.close(saver.varid) diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index b9007455c..795442860 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -38,6 +38,7 @@ from mip_convert import mip_parser import mip_convert.common from mip_convert.common import RelativePathChecker +from mip_convert.save import create_cmor_variable from mip_convert.save.mip_config import MipTableFactory @@ -301,6 +302,17 @@ def _optional_kwargs(self, variable): kwargs['comment'] = self._name_space.namespace_stamp(variable.comment) return kwargs + def apply_cell_measures(self, mip_output_variable, *cell_measures_config): + """Apply cell measures to this CMOR variable. + + Ensures the CMOR variable ID exists (creating it if needed), then + sets the cell_measures attribute via the CMOR wrapper. + """ + if self.varid is None: + cmor_variable = create_cmor_variable(mip_output_variable) + self._getVarId(cmor_variable) + self.cmor.apply_cell_measures(*cell_measures_config, self.varid) + def _close_file(self): self.cmor.close(self.varid, preserve=True) From 23a411c9e609456374f216fd67894e9e56e23e8a Mon Sep 17 00:00:00 2001 From: mo-gill Date: Tue, 9 Jun 2026 09:26:24 +0000 Subject: [PATCH 07/16] #886: Correct indentation. --- mip_convert/mip_convert/save/cmor/cmor_outputter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index 795442860..312ecd009 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -311,7 +311,7 @@ def apply_cell_measures(self, mip_output_variable, *cell_measures_config): if self.varid is None: cmor_variable = create_cmor_variable(mip_output_variable) self._getVarId(cmor_variable) - self.cmor.apply_cell_measures(*cell_measures_config, self.varid) + self.cmor.apply_cell_measures(*cell_measures_config, self.varid) def _close_file(self): self.cmor.close(self.varid, preserve=True) From 30a912a9c435d6f75d11fc68474329d417343316 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Tue, 9 Jun 2026 10:23:19 +0000 Subject: [PATCH 08/16] #886: Switch to using lazy import. --- mip_convert/mip_convert/new_variable.py | 9 +++++++-- mip_convert/mip_convert/save/cmor/cmor_outputter.py | 12 ------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index e6fa2682a..27053d299 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -371,7 +371,8 @@ def process(self, saver=None, cell_measures_config=None): self.apply_cell_measures(saver, cell_measures_config) def apply_cell_measures(self, saver, cell_measures_config): - """Apply cell measures to the CMOR variable if not already applied. Parameters + """Apply cell measures to the CMOR variable if not already applied. + Parameters ---------- saver: The CMOR outputter for this variable. @@ -379,7 +380,11 @@ def apply_cell_measures(self, saver, cell_measures_config): Arguments forwarded to cmor_wrapper.apply_cell_measures, in the form (mip_era, mip_table_dir, realm, variable, frequency, region). """ - saver.apply_cell_measures(self, *cell_measures_config) + if saver.varid is None: + from mip_convert.save import create_cmor_variable # deferred to avoid circular import + cmor_variable = create_cmor_variable(self) + saver._getVarId(cmor_variable) + saver.cmor.apply_cell_measures(*cell_measures_config, saver.varid) def _remove_alevhalf_bounds(self): """ diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index 312ecd009..b9007455c 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -38,7 +38,6 @@ from mip_convert import mip_parser import mip_convert.common from mip_convert.common import RelativePathChecker -from mip_convert.save import create_cmor_variable from mip_convert.save.mip_config import MipTableFactory @@ -302,17 +301,6 @@ def _optional_kwargs(self, variable): kwargs['comment'] = self._name_space.namespace_stamp(variable.comment) return kwargs - def apply_cell_measures(self, mip_output_variable, *cell_measures_config): - """Apply cell measures to this CMOR variable. - - Ensures the CMOR variable ID exists (creating it if needed), then - sets the cell_measures attribute via the CMOR wrapper. - """ - if self.varid is None: - cmor_variable = create_cmor_variable(mip_output_variable) - self._getVarId(cmor_variable) - self.cmor.apply_cell_measures(*cell_measures_config, self.varid) - def _close_file(self): self.cmor.close(self.varid, preserve=True) From 19af1b0b25125443324cacc7e635400b2ecb05f5 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Wed, 10 Jun 2026 10:44:48 +0000 Subject: [PATCH 09/16] #886: Move cell_measures_config into Process. --- mip_convert/mip_convert/new_variable.py | 35 +++++++++++-------- .../mip_convert/requested_variables.py | 18 ++++------ 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index 33c640d63..2cb536556 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -38,7 +38,8 @@ class VariableMetadata(object): def __init__(self, variable_name, stream_id, substream, mip_table_name, mip_metadata, site_information, hybrid_height_information, replacement_coordinates, model_to_mip_mapping, timestep, run_bounds, calendar, base_date, deflate_level, shuffle, ancil_variables, force_coordinate_rotation=False, - reference_time=None, masking=None, removal=None): + reference_time=None, masking=None, removal=None, + mip_era=None, mip_table_dir=None, mip_table_id=None, frequency=None, region=''): """ Parameters ---------- @@ -104,6 +105,11 @@ def __init__(self, variable_name, stream_id, substream, mip_table_name, mip_meta if ancil_variables: self.ancil_variables.extend(ancil_variables) self.force_coordinate_rotation = force_coordinate_rotation + self.mip_era = mip_era + self.mip_table_dir = mip_table_dir + self.mip_table_id = mip_table_id + self.frequency = frequency + self.region = region self._validate_timestep() def _validate_timestep(self): @@ -337,7 +343,7 @@ def ordered_coords(self): return self._ordered_coords - def process(self, saver=None, cell_measures_config=None): + def process(self, saver=None): """Process the data. The units of the data of the |MIP requested variable| are the @@ -367,24 +373,23 @@ def process(self, saver=None, cell_measures_config=None): self._update_time_units() if hasattr(self.model_to_mip_mapping, 'valid_min'): self._apply_valid_min_correction() - if saver is not None and cell_measures_config is not None: - self.apply_cell_measures(saver, cell_measures_config) + if saver is not None: + self.apply_cell_measures(saver) - def apply_cell_measures(self, saver, cell_measures_config): - """Apply cell measures to the CMOR variable if not already applied. - Parameters - ---------- - saver: - The CMOR outputter for this variable. - cell_measures_config: tuple - Arguments forwarded to cmor_wrapper.apply_cell_measures, - in the form (mip_era, mip_table_dir, realm, variable, frequency, region). - """ + def apply_cell_measures(self, saver): + """Apply cell measures to the CMOR variable if not already applied.""" if saver.varid is None: from mip_convert.save import create_cmor_variable # deferred to avoid circular import cmor_variable = create_cmor_variable(self) saver._getVarId(cmor_variable) - saver.cmor.apply_cell_measures(*cell_measures_config, saver.varid) + saver.cmor.apply_cell_measures( + self._variable_metadata.mip_era, + self._variable_metadata.mip_table_dir, + self._variable_metadata.mip_table_id, + self.variable_name, + self._variable_metadata.frequency, + self._variable_metadata.region, + saver.varid) def _remove_alevhalf_bounds(self): """ diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 96889d997..ee54fc05a 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -111,7 +111,12 @@ def produce_mip_requested_variable( user_config.atmos_timestep, user_config.run_bounds, user_config.calendar, user_config.base_date, user_config.deflate_level, user_config.shuffle, user_config.ancil_variables, user_config.force_coordinate_rotation, user_config.reference_time, user_config.masking, - user_config.halo_removals + user_config.halo_removals, + mip_era=user_config.mip_era, + mip_table_dir=user_config.inpath, + mip_table_id=mip_table.id, + frequency=frequency, + region=user_config.global_attributes.get('region', '') ) # Load the data from the 'model output files' and store each 'input variable' in the 'Variable' object @@ -128,17 +133,8 @@ def produce_mip_requested_variable( # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') - - cell_measures_config = ( - user_config.mip_era, - user_config.inpath, - mip_table.id, - variable_name, - frequency, - user_config.global_attributes.get('region', '')) - for time_slice in variable.slices_over(period): - time_slice.process(saver, cell_measures_config) + time_slice.process(saver) logger.debug('MIP output variable contains: {}'.format(time_slice.info)) save(time_slice, saver) From abe53d9b4b166141067bec308de29045131f4368 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Wed, 10 Jun 2026 13:00:02 +0000 Subject: [PATCH 10/16] #886: Restore files to state in commit 22530bb . --- mip_convert/mip_convert/new_variable.py | 45 ++++--------------- .../mip_convert/requested_variables.py | 19 +++++--- .../mip_convert/save/cmor/cmor_wrapper.py | 10 +---- 3 files changed, 22 insertions(+), 52 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index 2cb536556..32d116603 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2015-2026, Met Office. +# (C) British Crown Copyright 2015-2025, Met Office. # Please see LICENSE.md for license details. # pylint: disable=no-member, eval-used, wildcard-import # pylint: disable=unused-wildcard-import @@ -38,8 +38,7 @@ class VariableMetadata(object): def __init__(self, variable_name, stream_id, substream, mip_table_name, mip_metadata, site_information, hybrid_height_information, replacement_coordinates, model_to_mip_mapping, timestep, run_bounds, calendar, base_date, deflate_level, shuffle, ancil_variables, force_coordinate_rotation=False, - reference_time=None, masking=None, removal=None, - mip_era=None, mip_table_dir=None, mip_table_id=None, frequency=None, region=''): + reference_time=None, masking=None, removal=None): """ Parameters ---------- @@ -105,11 +104,6 @@ def __init__(self, variable_name, stream_id, substream, mip_table_name, mip_meta if ancil_variables: self.ancil_variables.extend(ancil_variables) self.force_coordinate_rotation = force_coordinate_rotation - self.mip_era = mip_era - self.mip_table_dir = mip_table_dir - self.mip_table_id = mip_table_id - self.frequency = frequency - self.region = region self._validate_timestep() def _validate_timestep(self): @@ -237,7 +231,7 @@ def slices_over(self, period): # Slice the 'input variable(s)', since they have not yet been processed. message = '-'.join([str(items) for items in date_time]) self.logger.debug('Creating data for "{}"'.format(message)) - sliced_input_variables = self._slice_input_variables(date_time, date_times) + sliced_input_variables = self._slice_input_variables(date_time) yield Variable(sliced_input_variables, self._variable_metadata) else: # Slice the 'MIP output variable'. @@ -343,7 +337,7 @@ def ordered_coords(self): return self._ordered_coords - def process(self, saver=None): + def process(self): """Process the data. The units of the data of the |MIP requested variable| are the @@ -373,23 +367,6 @@ def process(self, saver=None): self._update_time_units() if hasattr(self.model_to_mip_mapping, 'valid_min'): self._apply_valid_min_correction() - if saver is not None: - self.apply_cell_measures(saver) - - def apply_cell_measures(self, saver): - """Apply cell measures to the CMOR variable if not already applied.""" - if saver.varid is None: - from mip_convert.save import create_cmor_variable # deferred to avoid circular import - cmor_variable = create_cmor_variable(self) - saver._getVarId(cmor_variable) - saver.cmor.apply_cell_measures( - self._variable_metadata.mip_era, - self._variable_metadata.mip_table_dir, - self._variable_metadata.mip_table_id, - self.variable_name, - self._variable_metadata.frequency, - self._variable_metadata.region, - saver.varid) def _remove_alevhalf_bounds(self): """ @@ -867,10 +844,10 @@ def _data_dimension(self, coord, axis_direction): raise ValueError(message) return data_dimension - def _slice_input_variables(self, date_time, date_times): + def _slice_input_variables(self, date_time): input_variables = {} - if (len(date_time) > 1 and date_time[1] != 12) and date_time != date_times[-1]: - # don't attach New Year midnight to months that are not december or the final chunk in the slice, + if len(date_time) > 1 and date_time[1] != 12: + # don't attach New Year midnight to other months new_year_midnight = False else: new_year_midnight = True @@ -1251,16 +1228,10 @@ def _mip_table_requested_bounds(bounds): def _setup_time_constraint(date_time, with_new_year_midnight=True): def time_constraint(cell): - """Runs only if with_new_year_midnight evaluates to True. Checks the current cell point and confirms if it - signifies a new year (e.g. 1980-01-01) or a new month (e.g. 1980-04-01). If so, it returns that value and - ensures that the data point is kept, hence maintaining time contiguity between slices.""" - new_month = PartialDateTime(date_time[0], date_time[1] + 1, 1, 0, 0, 0, 0) if len(date_time) > 1 else "" return (PartialDateTime(*date_time) == cell.point or - PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point or - new_month == cell.point) + PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point) def time_constraint2(cell): - # Returns the original input cell point if with_new_year_midnight evaluates to false. return PartialDateTime(*date_time) == cell.point return time_constraint if with_new_year_midnight else time_constraint2 diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index ee54fc05a..c6c7b52c9 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -111,12 +111,7 @@ def produce_mip_requested_variable( user_config.atmos_timestep, user_config.run_bounds, user_config.calendar, user_config.base_date, user_config.deflate_level, user_config.shuffle, user_config.ancil_variables, user_config.force_coordinate_rotation, user_config.reference_time, user_config.masking, - user_config.halo_removals, - mip_era=user_config.mip_era, - mip_table_dir=user_config.inpath, - mip_table_id=mip_table.id, - frequency=frequency, - region=user_config.global_attributes.get('region', '') + user_config.halo_removals ) # Load the data from the 'model output files' and store each 'input variable' in the 'Variable' object @@ -130,11 +125,21 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) + # Apply cell_measures if a _cell_measures.json file exists + saver.cmor.apply_cell_measures( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes.get('region', ''), + saver.varid) + # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') for time_slice in variable.slices_over(period): - time_slice.process(saver) + time_slice.process() logger.debug('MIP output variable contains: {}'.format(time_slice.info)) save(time_slice, saver) diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index 5640f8ded..a1dbb372c 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -119,16 +119,10 @@ def set_frequency(self, frequency, **kwargs): cmor.cmor.set_cur_dataset_attribute('frequency', frequency) def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): + # self._debug_on_args('apply_cell_measures', [mip_table_dir]) """ - Set the cell_measures attribute on a CMOR variable using a lookup in - ``{mip_era}_cell_measures.json`` from the MIP tables directory. Returns - silently if the file does not exist. - - The lookup key has the form ``{realm}.{root_label}.{branding}.{frequency}.{region}`` - (variable name split on ``_``), so entries in the JSON must follow the - convention used for CMIP7. + Add cell measures read from json file in with mip tables """ - self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json') From e57005b48dbebb96e2b4c9c545851de5e9a1790b Mon Sep 17 00:00:00 2001 From: mo-gill Date: Wed, 10 Jun 2026 13:18:39 +0000 Subject: [PATCH 11/16] #886: Match state of main. --- mip_convert/mip_convert/new_variable.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index 32d116603..d82e385e2 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2015-2025, Met Office. +# (C) British Crown Copyright 2015-2026, Met Office. # Please see LICENSE.md for license details. # pylint: disable=no-member, eval-used, wildcard-import # pylint: disable=unused-wildcard-import @@ -231,7 +231,7 @@ def slices_over(self, period): # Slice the 'input variable(s)', since they have not yet been processed. message = '-'.join([str(items) for items in date_time]) self.logger.debug('Creating data for "{}"'.format(message)) - sliced_input_variables = self._slice_input_variables(date_time) + sliced_input_variables = self._slice_input_variables(date_time, date_times) yield Variable(sliced_input_variables, self._variable_metadata) else: # Slice the 'MIP output variable'. @@ -844,10 +844,10 @@ def _data_dimension(self, coord, axis_direction): raise ValueError(message) return data_dimension - def _slice_input_variables(self, date_time): + def _slice_input_variables(self, date_time, date_times): input_variables = {} - if len(date_time) > 1 and date_time[1] != 12: - # don't attach New Year midnight to other months + if (len(date_time) > 1 and date_time[1] != 12) and date_time != date_times[-1]: + # don't attach New Year midnight to months that are not december or the final chunk in the slice, new_year_midnight = False else: new_year_midnight = True @@ -1228,10 +1228,16 @@ def _mip_table_requested_bounds(bounds): def _setup_time_constraint(date_time, with_new_year_midnight=True): def time_constraint(cell): + """Runs only if with_new_year_midnight evaluates to True. Checks the current cell point and confirms if it + signifies a new year (e.g. 1980-01-01) or a new month (e.g. 1980-04-01). If so, it returns that value and + ensures that the data point is kept, hence maintaining time contiguity between slices.""" + new_month = PartialDateTime(date_time[0], date_time[1] + 1, 1, 0, 0, 0, 0) if len(date_time) > 1 else "" return (PartialDateTime(*date_time) == cell.point or - PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point) + PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point or + new_month == cell.point) def time_constraint2(cell): + # Returns the original input cell point if with_new_year_midnight evaluates to false. return PartialDateTime(*date_time) == cell.point return time_constraint if with_new_year_midnight else time_constraint2 From 8993a3e43b44ad79ce5baddfc104a4c72fc60e3c Mon Sep 17 00:00:00 2001 From: mo-gill Date: Wed, 10 Jun 2026 13:26:06 +0000 Subject: [PATCH 12/16] 886: Apply Matts original draft of implementation again. --- mip_convert/mip_convert/new_variable.py | 33 +++++++++++-------- .../mip_convert/save/cmor/cmor_outputter.py | 12 +++++++ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index d82e385e2..e6fa2682a 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2015-2026, Met Office. +# (C) British Crown Copyright 2015-2025, Met Office. # Please see LICENSE.md for license details. # pylint: disable=no-member, eval-used, wildcard-import # pylint: disable=unused-wildcard-import @@ -231,7 +231,7 @@ def slices_over(self, period): # Slice the 'input variable(s)', since they have not yet been processed. message = '-'.join([str(items) for items in date_time]) self.logger.debug('Creating data for "{}"'.format(message)) - sliced_input_variables = self._slice_input_variables(date_time, date_times) + sliced_input_variables = self._slice_input_variables(date_time) yield Variable(sliced_input_variables, self._variable_metadata) else: # Slice the 'MIP output variable'. @@ -337,7 +337,7 @@ def ordered_coords(self): return self._ordered_coords - def process(self): + def process(self, saver=None, cell_measures_config=None): """Process the data. The units of the data of the |MIP requested variable| are the @@ -367,6 +367,19 @@ def process(self): self._update_time_units() if hasattr(self.model_to_mip_mapping, 'valid_min'): self._apply_valid_min_correction() + if saver is not None and cell_measures_config is not None: + self.apply_cell_measures(saver, cell_measures_config) + + def apply_cell_measures(self, saver, cell_measures_config): + """Apply cell measures to the CMOR variable if not already applied. Parameters + ---------- + saver: + The CMOR outputter for this variable. + cell_measures_config: tuple + Arguments forwarded to cmor_wrapper.apply_cell_measures, + in the form (mip_era, mip_table_dir, realm, variable, frequency, region). + """ + saver.apply_cell_measures(self, *cell_measures_config) def _remove_alevhalf_bounds(self): """ @@ -844,10 +857,10 @@ def _data_dimension(self, coord, axis_direction): raise ValueError(message) return data_dimension - def _slice_input_variables(self, date_time, date_times): + def _slice_input_variables(self, date_time): input_variables = {} - if (len(date_time) > 1 and date_time[1] != 12) and date_time != date_times[-1]: - # don't attach New Year midnight to months that are not december or the final chunk in the slice, + if len(date_time) > 1 and date_time[1] != 12: + # don't attach New Year midnight to other months new_year_midnight = False else: new_year_midnight = True @@ -1228,16 +1241,10 @@ def _mip_table_requested_bounds(bounds): def _setup_time_constraint(date_time, with_new_year_midnight=True): def time_constraint(cell): - """Runs only if with_new_year_midnight evaluates to True. Checks the current cell point and confirms if it - signifies a new year (e.g. 1980-01-01) or a new month (e.g. 1980-04-01). If so, it returns that value and - ensures that the data point is kept, hence maintaining time contiguity between slices.""" - new_month = PartialDateTime(date_time[0], date_time[1] + 1, 1, 0, 0, 0, 0) if len(date_time) > 1 else "" return (PartialDateTime(*date_time) == cell.point or - PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point or - new_month == cell.point) + PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point) def time_constraint2(cell): - # Returns the original input cell point if with_new_year_midnight evaluates to false. return PartialDateTime(*date_time) == cell.point return time_constraint if with_new_year_midnight else time_constraint2 diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index b9007455c..312ecd009 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -38,6 +38,7 @@ from mip_convert import mip_parser import mip_convert.common from mip_convert.common import RelativePathChecker +from mip_convert.save import create_cmor_variable from mip_convert.save.mip_config import MipTableFactory @@ -301,6 +302,17 @@ def _optional_kwargs(self, variable): kwargs['comment'] = self._name_space.namespace_stamp(variable.comment) return kwargs + def apply_cell_measures(self, mip_output_variable, *cell_measures_config): + """Apply cell measures to this CMOR variable. + + Ensures the CMOR variable ID exists (creating it if needed), then + sets the cell_measures attribute via the CMOR wrapper. + """ + if self.varid is None: + cmor_variable = create_cmor_variable(mip_output_variable) + self._getVarId(cmor_variable) + self.cmor.apply_cell_measures(*cell_measures_config, self.varid) + def _close_file(self): self.cmor.close(self.varid, preserve=True) From 34cb06211649ad7606b692d28fcf5e282c00188b Mon Sep 17 00:00:00 2001 From: mo-gill Date: Wed, 10 Jun 2026 13:29:25 +0000 Subject: [PATCH 13/16] #886: Revert changes brought in by mistake. --- mip_convert/mip_convert/new_variable.py | 33 ++++++++----------- .../mip_convert/save/cmor/cmor_outputter.py | 12 ------- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index e6fa2682a..d82e385e2 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2015-2025, Met Office. +# (C) British Crown Copyright 2015-2026, Met Office. # Please see LICENSE.md for license details. # pylint: disable=no-member, eval-used, wildcard-import # pylint: disable=unused-wildcard-import @@ -231,7 +231,7 @@ def slices_over(self, period): # Slice the 'input variable(s)', since they have not yet been processed. message = '-'.join([str(items) for items in date_time]) self.logger.debug('Creating data for "{}"'.format(message)) - sliced_input_variables = self._slice_input_variables(date_time) + sliced_input_variables = self._slice_input_variables(date_time, date_times) yield Variable(sliced_input_variables, self._variable_metadata) else: # Slice the 'MIP output variable'. @@ -337,7 +337,7 @@ def ordered_coords(self): return self._ordered_coords - def process(self, saver=None, cell_measures_config=None): + def process(self): """Process the data. The units of the data of the |MIP requested variable| are the @@ -367,19 +367,6 @@ def process(self, saver=None, cell_measures_config=None): self._update_time_units() if hasattr(self.model_to_mip_mapping, 'valid_min'): self._apply_valid_min_correction() - if saver is not None and cell_measures_config is not None: - self.apply_cell_measures(saver, cell_measures_config) - - def apply_cell_measures(self, saver, cell_measures_config): - """Apply cell measures to the CMOR variable if not already applied. Parameters - ---------- - saver: - The CMOR outputter for this variable. - cell_measures_config: tuple - Arguments forwarded to cmor_wrapper.apply_cell_measures, - in the form (mip_era, mip_table_dir, realm, variable, frequency, region). - """ - saver.apply_cell_measures(self, *cell_measures_config) def _remove_alevhalf_bounds(self): """ @@ -857,10 +844,10 @@ def _data_dimension(self, coord, axis_direction): raise ValueError(message) return data_dimension - def _slice_input_variables(self, date_time): + def _slice_input_variables(self, date_time, date_times): input_variables = {} - if len(date_time) > 1 and date_time[1] != 12: - # don't attach New Year midnight to other months + if (len(date_time) > 1 and date_time[1] != 12) and date_time != date_times[-1]: + # don't attach New Year midnight to months that are not december or the final chunk in the slice, new_year_midnight = False else: new_year_midnight = True @@ -1241,10 +1228,16 @@ def _mip_table_requested_bounds(bounds): def _setup_time_constraint(date_time, with_new_year_midnight=True): def time_constraint(cell): + """Runs only if with_new_year_midnight evaluates to True. Checks the current cell point and confirms if it + signifies a new year (e.g. 1980-01-01) or a new month (e.g. 1980-04-01). If so, it returns that value and + ensures that the data point is kept, hence maintaining time contiguity between slices.""" + new_month = PartialDateTime(date_time[0], date_time[1] + 1, 1, 0, 0, 0, 0) if len(date_time) > 1 else "" return (PartialDateTime(*date_time) == cell.point or - PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point) + PartialDateTime(date_time[0] + 1, 1, 1, 0, 0, 0, 0) == cell.point or + new_month == cell.point) def time_constraint2(cell): + # Returns the original input cell point if with_new_year_midnight evaluates to false. return PartialDateTime(*date_time) == cell.point return time_constraint if with_new_year_midnight else time_constraint2 diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index 312ecd009..b9007455c 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -38,7 +38,6 @@ from mip_convert import mip_parser import mip_convert.common from mip_convert.common import RelativePathChecker -from mip_convert.save import create_cmor_variable from mip_convert.save.mip_config import MipTableFactory @@ -302,17 +301,6 @@ def _optional_kwargs(self, variable): kwargs['comment'] = self._name_space.namespace_stamp(variable.comment) return kwargs - def apply_cell_measures(self, mip_output_variable, *cell_measures_config): - """Apply cell measures to this CMOR variable. - - Ensures the CMOR variable ID exists (creating it if needed), then - sets the cell_measures attribute via the CMOR wrapper. - """ - if self.varid is None: - cmor_variable = create_cmor_variable(mip_output_variable) - self._getVarId(cmor_variable) - self.cmor.apply_cell_measures(*cell_measures_config, self.varid) - def _close_file(self): self.cmor.close(self.varid, preserve=True) From fed70745080c27e6aa2a4ebc51191a80e83cf248 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Thu, 11 Jun 2026 08:54:17 +0000 Subject: [PATCH 14/16] #886: Implementation using save. --- mip_convert/mip_convert/requested_variables.py | 8 +++----- mip_convert/mip_convert/save/__init__.py | 9 ++++++++- mip_convert/mip_convert/save/cmor/cmor_wrapper.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index c6c7b52c9..0781a7b10 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -125,15 +125,13 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) - # Apply cell_measures if a _cell_measures.json file exists - saver.cmor.apply_cell_measures( + cell_measures_config = ( user_config.mip_era, user_config.inpath, mip_table.id, variable_name, frequency, - user_config.global_attributes.get('region', ''), - saver.varid) + user_config.global_attributes.get('region', '')) # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. @@ -141,7 +139,7 @@ def produce_mip_requested_variable( for time_slice in variable.slices_over(period): time_slice.process() logger.debug('MIP output variable contains: {}'.format(time_slice.info)) - save(time_slice, saver) + save(time_slice, saver, cell_measures_config=cell_measures_config) # Close the 'output netCDF file'. cmor_lite.close(saver.varid) diff --git a/mip_convert/mip_convert/save/__init__.py b/mip_convert/mip_convert/save/__init__.py index e5127823a..6aa56d34c 100644 --- a/mip_convert/mip_convert/save/__init__.py +++ b/mip_convert/mip_convert/save/__init__.py @@ -19,7 +19,7 @@ make_masked) -def save(mip_output_variable, saver): +def save(mip_output_variable, saver, cell_measures_config=None): """Save the |MIP output variable| to an |output netCDF file| using |CMOR|. @@ -41,10 +41,17 @@ def save(mip_output_variable, saver): The |MIP output variable|. saver: callable A function with the signature ``function(object)`` + cell_measures_config: tuple, optional + Arguments for :meth:`cmor_wrapper.apply_cell_measures` (excluding + ``variable_id``). When provided, cell measures are applied once, + after the CMOR variable is registered but before data is written. """ logger = logging.getLogger(__name__) logger.debug('Saving MIP output variable to an output netCDF file') cmor_variable = create_cmor_variable(mip_output_variable) + if cell_measures_config is not None and saver.varid is None: + saver._getVarId(cmor_variable) + saver.cmor.apply_cell_measures(*cell_measures_config, saver.varid) saver(cmor_variable) diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index a1dbb372c..1e437d760 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -119,10 +119,10 @@ def set_frequency(self, frequency, **kwargs): cmor.cmor.set_cur_dataset_attribute('frequency', frequency) def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): - # self._debug_on_args('apply_cell_measures', [mip_table_dir]) """ Add cell measures read from json file in with mip tables """ + self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json') From a01acf21288851573598299f635da4d2a784ff1a Mon Sep 17 00:00:00 2001 From: mo-gill Date: Thu, 11 Jun 2026 11:34:07 +0000 Subject: [PATCH 15/16] #886: Update functional tests and KGO data to new versions. --- .../test_functional_cmip7/test_cmip7_fx_rootd_ti_u_hxy_lnd.py | 2 +- .../test_cmip7_fx_slthick_ti_sl_hxy_lnd.py | 2 +- .../test_functional_cmip7/test_cmip7_mon_mcd_tavg-alh-hxy-u.py | 2 +- .../test_functional_cmip7/test_cmip7_mon_uas_tavg_h10m_hxy_u.py | 2 +- .../test_cmip7_rlucs4co2_tavg-alh-hxy-u.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_rootd_ti_u_hxy_lnd.py b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_rootd_ti_u_hxy_lnd.py index 83c556a28..b77ba29cd 100644 --- a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_rootd_ti_u_hxy_lnd.py +++ b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_rootd_ti_u_hxy_lnd.py @@ -44,7 +44,7 @@ def get_test_data(self): 'ancil': {'CMIP7_land@fx': 'rootd_ti-u-hxy-lnd'} }, other={ - 'reference_version': 'v5', + 'reference_version': 'v6', 'filenames': ['rootd_ti-u-hxy-lnd_fx_glb_g100_UKESM1-3-LL_1pctCO2_r1i1p1f3.nc'], 'ignore_history': True, } diff --git a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_slthick_ti_sl_hxy_lnd.py b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_slthick_ti_sl_hxy_lnd.py index 2d2a69443..5d8d774ba 100644 --- a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_slthick_ti_sl_hxy_lnd.py +++ b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_fx_slthick_ti_sl_hxy_lnd.py @@ -44,7 +44,7 @@ def get_test_data(self): 'ancil': {'CMIP7_land@fx': 'slthick_ti-sl-hxy-lnd'} }, other={ - 'reference_version': 'v5', + 'reference_version': 'v6', 'filenames': ['slthick_ti-sl-hxy-lnd_fx_glb_g100_UKESM1-3-LL_1pctCO2_r1i1p1f3.nc'], 'ignore_history': True, } diff --git a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_mcd_tavg-alh-hxy-u.py b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_mcd_tavg-alh-hxy-u.py index 25e6d279e..b1210d7fa 100644 --- a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_mcd_tavg-alh-hxy-u.py +++ b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_mcd_tavg-alh-hxy-u.py @@ -41,7 +41,7 @@ def get_test_data(self): 'ap5': {'CMIP7_atmos@mon': 'mcd_tavg-alh-hxy-u'} }, other={ - 'reference_version': 'v3', + 'reference_version': 'v4', 'filenames': ['mcd_tavg-alh-hxy-u_mon_glb_g100_UKESM1-3-LL_1pctCO2_r1i1p1f3_200001-200002.nc'], 'ignore_history': True, } diff --git a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_uas_tavg_h10m_hxy_u.py b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_uas_tavg_h10m_hxy_u.py index b28c9ec8a..95b4bcec4 100644 --- a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_uas_tavg_h10m_hxy_u.py +++ b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_mon_uas_tavg_h10m_hxy_u.py @@ -41,7 +41,7 @@ def get_test_data(self): 'ap5': {'CMIP7_atmos@mon': 'uas_tavg-h10m-hxy-u'} }, other={ - 'reference_version': 'v7', + 'reference_version': 'v8', 'filenames': ['uas_tavg-h10m-hxy-u_mon_glb_g100_UKESM1-3-LL_1pctCO2_r1i1p1f3_196002-196003.nc'], 'ignore_history': True, } diff --git a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_rlucs4co2_tavg-alh-hxy-u.py b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_rlucs4co2_tavg-alh-hxy-u.py index e298cbebf..c957f02cf 100644 --- a/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_rlucs4co2_tavg-alh-hxy-u.py +++ b/mip_convert/mip_convert/tests/test_functional/test_functional_cmip7/test_cmip7_rlucs4co2_tavg-alh-hxy-u.py @@ -47,7 +47,7 @@ def get_test_data(self): }, streams={"ap5": {"CMIP7_atmos@mon": "rlucs4co2_tavg-alh-hxy-u"}}, other={ - "reference_version": "v3", + "reference_version": "v4", "filenames": [ "rlucs4co2_tavg-alh-hxy-u_mon_glb_g100_UKESM1-3-LL_esm-piControl_r1i1p1f1_190001-190002.nc" ], From 190545bd8b93cb2f37b3cd8b3daf7b9b73dea274 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Thu, 11 Jun 2026 12:26:19 +0000 Subject: [PATCH 16/16] #886: Enhance docstrings, typing and formatting. --- .../mip_convert/requested_variables.py | 1 + mip_convert/mip_convert/save/__init__.py | 8 ++++- .../mip_convert/save/cmor/cmor_outputter.py | 3 +- .../mip_convert/save/cmor/cmor_wrapper.py | 34 +++++++++++++++++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 0781a7b10..18f8fe3f7 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -125,6 +125,7 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) + # Assemble the arguments needed to apply cell measures when saving. cell_measures_config = ( user_config.mip_era, user_config.inpath, diff --git a/mip_convert/mip_convert/save/__init__.py b/mip_convert/mip_convert/save/__init__.py index 6aa56d34c..08bcee2bb 100644 --- a/mip_convert/mip_convert/save/__init__.py +++ b/mip_convert/mip_convert/save/__init__.py @@ -6,6 +6,7 @@ import copy import logging from operator import attrgetter +from typing import Optional, Tuple import iris import numpy as np @@ -14,12 +15,17 @@ DEFAULT_FILL_VALUE, has_auxiliary_latitude_longitude, check_values_equal) from mip_convert.load.pp.pp_axis import (BoundedAxis, AxisHybridHeight, TimeSeriesSiteAxis, ReferenceTimeAxis) +from mip_convert.save.cmor.cmor_outputter import AbstractCmorOutputter from mip_convert.variable import Variable as CMORVariable from mip_convert.variable import (CoordinateDomain, PolePoint, TripolarGrid, make_masked) -def save(mip_output_variable, saver, cell_measures_config=None): +def save( + mip_output_variable, + saver: AbstractCmorOutputter, + cell_measures_config: Optional[Tuple[str, str, str, str, Optional[str], str]] = None, +): """Save the |MIP output variable| to an |output netCDF file| using |CMOR|. diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index b9007455c..797267184 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -38,6 +38,7 @@ from mip_convert import mip_parser import mip_convert.common from mip_convert.common import RelativePathChecker +from mip_convert.save.cmor.cmor_wrapper import CmorWrapper from mip_convert.save.mip_config import MipTableFactory @@ -219,7 +220,7 @@ class AbstractCmorOutputter(object): """ def __init__(self, entry, cmor_wrapper): - self.cmor = cmor_wrapper + self.cmor: CmorWrapper = cmor_wrapper self.entry = entry self._name_space = MoNameSpace() self.varid = None diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index 1e437d760..32c2ae4c6 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -119,10 +119,38 @@ def set_frequency(self, frequency, **kwargs): cmor.cmor.set_cur_dataset_attribute('frequency', frequency) def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): + """Set the ``cell_measures`` attribute on a registered CMOR variable. + + Looks up ``{mip_table_dir}/{mip_era}_cell_measures.json`` and, if a + matching entry is found, calls ``cmor.set_variable_attribute`` to + attach the cell-measures string to the variable before any data are + written. If the file does not exist, or no matching key is found, + the method returns silently. + + This method must be called after ``cmor.variable()`` has registered + the variable (so that ``variable_id`` is valid) and before + ``cmor.write()`` is called. + + Parameters + ---------- + mip_era: str + The MIP era (e.g. ``"CMIP7"``). + mip_table_dir: str + Directory containing the MIP tables and the cell-measures JSON. + realm: str + The MIP table identifier (e.g. ``"Amon"``). + variable: str + The variable name in ``{root_label}_{branding}`` form. + frequency: str or None + The output frequency (e.g. ``"mon"``). + region: str + The region string (e.g. ``""`` for global). + variable_id: int + The integer handle returned by ``cmor.variable()``. """ - Add cell measures read from json file in with mip tables - """ - self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) + self._debug_on_args( + "apply_cell_measures", [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {} + ) cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json')