From 9c98423a85aa7b82bdb8918baaf8045a4c20495b Mon Sep 17 00:00:00 2001 From: Matthew Mizielinski Date: Fri, 10 Apr 2026 16:50:45 +0000 Subject: [PATCH 1/7] #886: First draft of how to add cell measures --- .../mip_convert/requested_variables.py | 10 ++++++ .../mip_convert/save/cmor/cmor_wrapper.py | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 31c07e6c4..9483c6f90 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -125,6 +125,16 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) + # Apply cell_measures if a _cell_measures.json file exists + saver.cmor.apply_cell_measures( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes['region'], + saver.varid) + # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index 56c5a6463..a1dbb372c 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -1,6 +1,8 @@ # (C) British Crown Copyright 2009-2025, Met Office. # Please see LICENSE.md for license details. from collections import OrderedDict +import os +import json import cmor @@ -115,3 +117,32 @@ def zfactor(self, *args, **kwargs): def set_frequency(self, frequency, **kwargs): self._debug_on_args('frequency', [frequency], kwargs) cmor.cmor.set_cur_dataset_attribute('frequency', frequency) + + def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): + # self._debug_on_args('apply_cell_measures', [mip_table_dir]) + """ + Add cell measures read from json file in with mip tables + """ + + cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json') + + if os.path.exists(cell_measures_file): + with open(cell_measures_file) as fh: + cell_measures = json.load(fh) + if 'cell_measures' not in cell_measures: + self.logger.debug(f'"cell_measures" key not found in {cell_measures_file}') + return + cell_measures = cell_measures['cell_measures'] + + root_label, branding = variable.split('_') + key = f'{realm}.{root_label}.{branding}.{frequency}.{region}' + if key in cell_measures: + retval = cmor.cmor.set_variable_attribute( + variable_id, + 'cell_measures', + 'c', + cell_measures[key]) + if retval != 0: + self.logger.debug('cell_measures assignment failed. Check cmor log file for details') + else: + self.logger.debug(f'Cell_measures file "{cell_measures_file}" not found. Continuing') From 22530bb7d99c12bb04b4e31e220eb2542e551bba Mon Sep 17 00:00:00 2001 From: mo-gill Date: Fri, 5 Jun 2026 10:36:37 +0000 Subject: [PATCH 2/7] #886: Fix keyerror. --- mip_convert/mip_convert/requested_variables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 9483c6f90..c6c7b52c9 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -132,7 +132,7 @@ def produce_mip_requested_variable( mip_table.id, variable_name, frequency, - user_config.global_attributes['region'], + user_config.global_attributes.get('region', ''), saver.varid) # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' From 3a54a40c1cf018b5e8a48c29749471e0717551d8 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Mon, 8 Jun 2026 09:53:02 +0100 Subject: [PATCH 3/7] #886: Further iteration, fix varid=None bug. --- .../mip_convert/requested_variables.py | 34 ++++++++++++------- .../mip_convert/save/cmor/cmor_wrapper.py | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index c6c7b52c9..2f3996080 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -14,7 +14,7 @@ from mip_convert.configuration.json_config import MIPConfig from mip_convert.configuration.python_config import ModelToMIPMappingConfig -from mip_convert.save import save +from mip_convert.save import create_cmor_variable from mip_convert.save.cmor import cmor_lite from mip_convert.mip_table import get_variable_model_to_mip_mapping, get_variable_mip_metadata @@ -125,23 +125,33 @@ def produce_mip_requested_variable( if frequency: saver.cmor.set_frequency(frequency) - # Apply cell_measures if a _cell_measures.json file exists - saver.cmor.apply_cell_measures( - user_config.mip_era, - user_config.inpath, - mip_table.id, - variable_name, - frequency, - user_config.global_attributes.get('region', ''), - saver.varid) - # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') + cell_measures_applied = False for time_slice in variable.slices_over(period): time_slice.process() logger.debug('MIP output variable contains: {}'.format(time_slice.info)) - save(time_slice, saver) + + # Build the CMOR variable first so the CMOR varid can be created + # before any write, allowing variable attributes to be set safely. + cmor_variable = create_cmor_variable(time_slice) + if not cell_measures_applied: + if saver.varid is None: + saver._getVarId(cmor_variable) + # Apply cell_measures if a _cell_measures.json file exists. + # Must be called before the first write to satisfy CMOR. + saver.cmor.apply_cell_measures( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes.get('region', ''), + saver.varid) + cell_measures_applied = True + + saver(cmor_variable) # Close the 'output netCDF file'. cmor_lite.close(saver.varid) diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index a1dbb372c..1e437d760 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -119,10 +119,10 @@ def set_frequency(self, frequency, **kwargs): cmor.cmor.set_cur_dataset_attribute('frequency', frequency) def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): - # self._debug_on_args('apply_cell_measures', [mip_table_dir]) """ Add cell measures read from json file in with mip tables """ + self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) cell_measures_file = os.path.join(mip_table_dir, f'{mip_era}_cell_measures.json') From d8b5977f3cda4a9ce4c2405d35b8341b034861ae Mon Sep 17 00:00:00 2001 From: mo-gill Date: Mon, 8 Jun 2026 09:53:32 +0000 Subject: [PATCH 4/7] #886: Refine comments. --- mip_convert/mip_convert/requested_variables.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index 2f3996080..f836a25a0 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -134,13 +134,14 @@ def produce_mip_requested_variable( logger.debug('MIP output variable contains: {}'.format(time_slice.info)) # Build the CMOR variable first so the CMOR varid can be created - # before any write, allowing variable attributes to be set safely. + # before any write, allowing variable attributes to be set safely + # (a valid varid is required by apply_cell_measures to set the + # cell_measures attribute via cmor.set_variable_attribute). cmor_variable = create_cmor_variable(time_slice) if not cell_measures_applied: if saver.varid is None: saver._getVarId(cmor_variable) # Apply cell_measures if a _cell_measures.json file exists. - # Must be called before the first write to satisfy CMOR. saver.cmor.apply_cell_measures( user_config.mip_era, user_config.inpath, From 099d59d5e19c1957f237dcf03e7924865294331d Mon Sep 17 00:00:00 2001 From: mo-gill Date: Mon, 8 Jun 2026 10:43:18 +0000 Subject: [PATCH 5/7] #886: Clarify behaviour in docstring. --- mip_convert/mip_convert/requested_variables.py | 3 +-- mip_convert/mip_convert/save/cmor/cmor_wrapper.py | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index f836a25a0..c334e0eb8 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -139,8 +139,7 @@ def produce_mip_requested_variable( # cell_measures attribute via cmor.set_variable_attribute). cmor_variable = create_cmor_variable(time_slice) if not cell_measures_applied: - if saver.varid is None: - saver._getVarId(cmor_variable) + saver._getVarId(cmor_variable) # Apply cell_measures if a _cell_measures.json file exists. saver.cmor.apply_cell_measures( user_config.mip_era, diff --git a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py index 1e437d760..5640f8ded 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_wrapper.py +++ b/mip_convert/mip_convert/save/cmor/cmor_wrapper.py @@ -120,7 +120,13 @@ def set_frequency(self, frequency, **kwargs): def apply_cell_measures(self, mip_era, mip_table_dir, realm, variable, frequency, region, variable_id): """ - Add cell measures read from json file in with mip tables + Set the cell_measures attribute on a CMOR variable using a lookup in + ``{mip_era}_cell_measures.json`` from the MIP tables directory. Returns + silently if the file does not exist. + + The lookup key has the form ``{realm}.{root_label}.{branding}.{frequency}.{region}`` + (variable name split on ``_``), so entries in the JSON must follow the + convention used for CMIP7. """ self._debug_on_args('apply_cell_measures', [mip_era, mip_table_dir, realm, variable, frequency, region, variable_id], {}) From fb6e14b7baa6c2e86754f86a999daaa98303a7c1 Mon Sep 17 00:00:00 2001 From: mo-gill Date: Tue, 9 Jun 2026 09:02:23 +0000 Subject: [PATCH 6/7] #886: Move code into time_slice.process. --- mip_convert/mip_convert/new_variable.py | 15 +++++++- .../mip_convert/requested_variables.py | 35 +++++++------------ .../mip_convert/save/cmor/cmor_outputter.py | 12 +++++++ 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/mip_convert/mip_convert/new_variable.py b/mip_convert/mip_convert/new_variable.py index 32d116603..e6fa2682a 100644 --- a/mip_convert/mip_convert/new_variable.py +++ b/mip_convert/mip_convert/new_variable.py @@ -337,7 +337,7 @@ def ordered_coords(self): return self._ordered_coords - def process(self): + def process(self, saver=None, cell_measures_config=None): """Process the data. The units of the data of the |MIP requested variable| are the @@ -367,6 +367,19 @@ def process(self): self._update_time_units() if hasattr(self.model_to_mip_mapping, 'valid_min'): self._apply_valid_min_correction() + if saver is not None and cell_measures_config is not None: + self.apply_cell_measures(saver, cell_measures_config) + + def apply_cell_measures(self, saver, cell_measures_config): + """Apply cell measures to the CMOR variable if not already applied. Parameters + ---------- + saver: + The CMOR outputter for this variable. + cell_measures_config: tuple + Arguments forwarded to cmor_wrapper.apply_cell_measures, + in the form (mip_era, mip_table_dir, realm, variable, frequency, region). + """ + saver.apply_cell_measures(self, *cell_measures_config) def _remove_alevhalf_bounds(self): """ diff --git a/mip_convert/mip_convert/requested_variables.py b/mip_convert/mip_convert/requested_variables.py index c334e0eb8..96889d997 100644 --- a/mip_convert/mip_convert/requested_variables.py +++ b/mip_convert/mip_convert/requested_variables.py @@ -14,7 +14,7 @@ from mip_convert.configuration.json_config import MIPConfig from mip_convert.configuration.python_config import ModelToMIPMappingConfig -from mip_convert.save import create_cmor_variable +from mip_convert.save import save from mip_convert.save.cmor import cmor_lite from mip_convert.mip_table import get_variable_model_to_mip_mapping, get_variable_mip_metadata @@ -128,30 +128,19 @@ def produce_mip_requested_variable( # Process the data by performing the appropriate 'model to MIP mapping', then save the 'MIP output variable' # to an 'output netCDF file'. period = user_config.slicing.get(stream_id, 'year') - cell_measures_applied = False + + cell_measures_config = ( + user_config.mip_era, + user_config.inpath, + mip_table.id, + variable_name, + frequency, + user_config.global_attributes.get('region', '')) + for time_slice in variable.slices_over(period): - time_slice.process() + time_slice.process(saver, cell_measures_config) logger.debug('MIP output variable contains: {}'.format(time_slice.info)) - - # Build the CMOR variable first so the CMOR varid can be created - # before any write, allowing variable attributes to be set safely - # (a valid varid is required by apply_cell_measures to set the - # cell_measures attribute via cmor.set_variable_attribute). - cmor_variable = create_cmor_variable(time_slice) - if not cell_measures_applied: - saver._getVarId(cmor_variable) - # Apply cell_measures if a _cell_measures.json file exists. - saver.cmor.apply_cell_measures( - user_config.mip_era, - user_config.inpath, - mip_table.id, - variable_name, - frequency, - user_config.global_attributes.get('region', ''), - saver.varid) - cell_measures_applied = True - - saver(cmor_variable) + save(time_slice, saver) # Close the 'output netCDF file'. cmor_lite.close(saver.varid) diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index b9007455c..795442860 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -38,6 +38,7 @@ from mip_convert import mip_parser import mip_convert.common from mip_convert.common import RelativePathChecker +from mip_convert.save import create_cmor_variable from mip_convert.save.mip_config import MipTableFactory @@ -301,6 +302,17 @@ def _optional_kwargs(self, variable): kwargs['comment'] = self._name_space.namespace_stamp(variable.comment) return kwargs + def apply_cell_measures(self, mip_output_variable, *cell_measures_config): + """Apply cell measures to this CMOR variable. + + Ensures the CMOR variable ID exists (creating it if needed), then + sets the cell_measures attribute via the CMOR wrapper. + """ + if self.varid is None: + cmor_variable = create_cmor_variable(mip_output_variable) + self._getVarId(cmor_variable) + self.cmor.apply_cell_measures(*cell_measures_config, self.varid) + def _close_file(self): self.cmor.close(self.varid, preserve=True) From 23a411c9e609456374f216fd67894e9e56e23e8a Mon Sep 17 00:00:00 2001 From: mo-gill Date: Tue, 9 Jun 2026 09:26:24 +0000 Subject: [PATCH 7/7] #886: Correct indentation. --- mip_convert/mip_convert/save/cmor/cmor_outputter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mip_convert/mip_convert/save/cmor/cmor_outputter.py b/mip_convert/mip_convert/save/cmor/cmor_outputter.py index 795442860..312ecd009 100644 --- a/mip_convert/mip_convert/save/cmor/cmor_outputter.py +++ b/mip_convert/mip_convert/save/cmor/cmor_outputter.py @@ -311,7 +311,7 @@ def apply_cell_measures(self, mip_output_variable, *cell_measures_config): if self.varid is None: cmor_variable = create_cmor_variable(mip_output_variable) self._getVarId(cmor_variable) - self.cmor.apply_cell_measures(*cell_measures_config, self.varid) + self.cmor.apply_cell_measures(*cell_measures_config, self.varid) def _close_file(self): self.cmor.close(self.varid, preserve=True)