Skip to content

Support nested variable entries in CMOR tables #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Support nested variable entries in CMOR tables #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
96ccc04
feat: support nested variable entries in CMOR tables
Copilot Apr 17, 2026
54dcf54
fix: harden nested variable entry parsing
Copilot Apr 17, 2026
3323861
refactor: tighten nested variable entry errors
Copilot Apr 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 96 additions & 2 deletions Src/cmor_tables.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,67 @@ int cmor_set_variable_entry(cmor_table_t * table,
return (0);
}

enum cmor_variable_entry_shape {
CMOR_VARIABLE_ENTRY_FLAT = 0,
CMOR_VARIABLE_ENTRY_NESTED = 1,
CMOR_VARIABLE_ENTRY_INVALID = -1
};

static int cmor_get_variable_entry_shape(json_object *json)
{
int has_objects = 0;
int has_non_objects = 0;

json_object_object_foreach(json, attr, value) {
if (attr[0] == '#') {
continue;
}
if (json_object_is_type(value, json_type_object)) {
has_objects = 1;
} else {
has_non_objects = 1;
}
if (has_objects && has_non_objects) {
return (CMOR_VARIABLE_ENTRY_INVALID);
}
}
if (has_objects) {
return (CMOR_VARIABLE_ENTRY_NESTED);
}
return (CMOR_VARIABLE_ENTRY_FLAT);
}

static int cmor_set_nested_variable_entry(cmor_table_t *table,
char *variable_name,
char *brand_name,
json_object *json)
{
extern int cmor_ntables;
int entry_len;
char variable_entry[CMOR_MAX_STRING];

if (brand_name[0] == '\0') {
entry_len = snprintf(variable_entry, CMOR_MAX_STRING, "%s",
variable_name);
} else {
entry_len = snprintf(variable_entry, CMOR_MAX_STRING, "%s_%s",
variable_name, brand_name);
}

if (entry_len < 0 || entry_len >= CMOR_MAX_STRING) {
cmor_handle_error_variadic(
"Variable entry is too long for table: %s",
CMOR_CRITICAL,
table->szTable_id);
if (cmor_ntables >= 0 && table == &cmor_tables[cmor_ntables]) {
cmor_ntables--;
}
return (1);
}

return cmor_set_variable_entry(table, variable_entry, json);
}

/************************************************************************/
/* cmor_set_axis_entry() */
/************************************************************************/
Expand Down Expand Up @@ -1020,15 +1081,48 @@ int cmor_load_table_internal(char szTable[CMOR_MAX_STRING], int *table_id,
done = 1;
} else if (strcmp(key, JSON_KEY_VARIABLE_ENTRY) == 0) {
json_object_object_foreach(value, varname, attributes) {
int variable_entry_shape;

if (varname[0] == '#') {
continue;
}
if (attributes == NULL) {
return (TABLE_ERROR);
}
if (cmor_set_variable_entry(&cmor_tables[cmor_ntables],
varname, attributes) == 1) {
variable_entry_shape = cmor_get_variable_entry_shape(attributes);
if (variable_entry_shape == CMOR_VARIABLE_ENTRY_INVALID) {
cmor_handle_error_variadic(
"Variable entry '%s' mixes nested brand objects with regular attributes",
CMOR_CRITICAL,
varname);
cmor_pop_traceback();
return (TABLE_ERROR);
}
if (json_object_is_type(attributes, json_type_object) &&
variable_entry_shape == CMOR_VARIABLE_ENTRY_NESTED) {
json_object_object_foreach(attributes, brandname,
brand_attributes) {
if (brandname[0] == '#') {
continue;
}
if (brand_attributes == NULL ||
!json_object_is_type(brand_attributes,
json_type_object)) {
cmor_handle_error_variadic(
"Nested variable entry '%s' brand '%s' must be an object",
CMOR_CRITICAL,
varname, brandname);
return (TABLE_ERROR);
}
if (cmor_set_nested_variable_entry(
&cmor_tables[cmor_ntables], varname,
brandname, brand_attributes) == 1) {
cmor_pop_traceback();
return (TABLE_ERROR);
}
}
} else if (cmor_set_variable_entry(&cmor_tables[cmor_ntables],
varname, attributes) == 1) {
cmor_pop_traceback();
return (TABLE_ERROR);
}
Expand Down
76 changes: 76 additions & 0 deletions Test/test_cmor_CMIP7.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def setUp(self):
Write out a simple file using CMOR
"""
self.input_json = Path("Test/input_cmip7.json")
self.nested_ocean_table = Path(CMIP7_TABLES_PATH) / "CMIP7_ocean_nested.json"

# Set up CMOR
cmor.setup(inpath=CMIP7_TABLES_PATH, netcdf_file_action=cmor.CMOR_REPLACE)
Expand All @@ -58,8 +59,32 @@ def setUp(self):
if error_flag:
raise RuntimeError("CMOR dataset_json call failed")

self._write_nested_table(
Path("TestTables/CMIP7_ocean2d.json"),
self.nested_ocean_table,
)

def tearDown(self):
self.input_json.unlink(missing_ok=True)
self.nested_ocean_table.unlink(missing_ok=True)

def _write_nested_table(self, source, destination):
with source.open() as table_handle:
table = json.load(table_handle)

nested_entries = {}
for variable_entry, cfg in table["variable_entry"].items():
out_name = cfg.get("out_name", variable_entry)
brand_name = ""
prefix = f"{out_name}_"
if variable_entry.startswith(prefix):
brand_name = variable_entry[len(prefix):]
nested_entries.setdefault(out_name, {})[brand_name] = cfg

table["variable_entry"] = nested_entries

with destination.open("w") as table_handle:
json.dump(table, table_handle, sort_keys=True, indent=4)

def test_cmip7(self):
data = [27] * (2 * 3 * 4)
Expand Down Expand Up @@ -197,6 +222,57 @@ def test_secondary_modeling_realm(self):

ds.close()

def test_nested_variable_entry(self):
data = [27] * (2 * 3 * 4)
tos = numpy.array(data)
tos.shape = (2, 3, 4)
lat = numpy.array([10, 20, 30])
lat_bnds = numpy.array([5, 15, 25, 35])
lon = numpy.array([0, 90, 180, 270])
lon_bnds = numpy.array([-45, 45,
135,
225,
315
])
time = numpy.array([15.5, 45])
time_bnds = numpy.array([0, 31, 60])
cmor.load_table(self.nested_ocean_table.name)
cmorlat = cmor.axis("latitude",
coord_vals=lat,
cell_bounds=lat_bnds,
units="degrees_north")
cmorlon = cmor.axis("longitude",
coord_vals=lon,
cell_bounds=lon_bnds,
units="degrees_east")
cmortime = cmor.axis("time",
coord_vals=time,
cell_bounds=time_bnds,
units="days since 2018")
axes = [cmortime, cmorlat, cmorlon]
cmortos = cmor.variable("tos_tavg-u-hxy-sea", "degC", axes)
self.assertEqual(cmor.write(cmortos, tos), 0)
filename = cmor.close(cmortos, file_name=True)
self.assertEqual(cmor.close(), 0)

ds = Dataset(filename)
attrs = ds.ncattrs()
test_attrs = {
'branded_variable': 'tos_tavg-u-hxy-sea',
'branding_suffix': 'tavg-u-hxy-sea',
'temporal_label': 'tavg',
'vertical_label': 'u',
'horizontal_label': 'hxy',
'area_label': 'sea',
'realm': 'ocean',
}

for attr, val in test_attrs.items():
self.assertIn(attr, attrs)
self.assertEqual(val, ds.getncattr(attr))

ds.close()


if __name__ == '__main__':
unittest.main()
88 changes: 86 additions & 2 deletions Test/test_python_branded_variable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import cmor
import unittest
import os
from pathlib import Path

from netCDF4 import Dataset

Expand Down Expand Up @@ -46,19 +47,48 @@ def setUp(self):
"""
Write out a simple file using CMOR
"""
self.input_json = Path("Test/input_branded_variable.json")
self.nested_table = Path("TestTables/CMIP6_Omon_branded_variable_nested.json")

# Set up CMOR
cmor.setup(inpath="TestTables", netcdf_file_action=cmor.CMOR_REPLACE,
logfile="cmor.log", create_subdirectories=0)

# Define dataset using DATASET_INFO
with open("Test/input_branded_variable.json", "w") as input_file_handle:
with self.input_json.open("w") as input_file_handle:
json.dump(DATASET_INFO, input_file_handle, sort_keys=True, indent=4)

# read dataset info
error_flag = cmor.dataset_json("Test/input_branded_variable.json")
error_flag = cmor.dataset_json(str(self.input_json))
if error_flag:
raise RuntimeError("CMOR dataset_json call failed")

self._write_nested_table(
Path("TestTables/CMIP6_Omon_branded_variable.json"),
self.nested_table,
)

def tearDown(self):
self.input_json.unlink(missing_ok=True)
self.nested_table.unlink(missing_ok=True)

def _write_nested_table(self, source, destination):
with source.open() as table_handle:
table = json.load(table_handle)

nested_entries = {}
for variable_entry, cfg in table["variable_entry"].items():
out_name = cfg.get("out_name", variable_entry)
brand_name = ""
prefix = f"{out_name}_"
if variable_entry.startswith(prefix):
brand_name = variable_entry[len(prefix):]
nested_entries.setdefault(out_name, {})[brand_name] = cfg

table["variable_entry"] = nested_entries

with destination.open("w") as table_handle:
json.dump(table, table_handle, sort_keys=True, indent=4)

def test_variable_without_branding_suffix(self):
mip_table = "CMIP6_Omon_branded_variable.json"
Expand All @@ -85,6 +115,30 @@ def test_variable_without_branding_suffix(self):
ds.close()
os.remove(filename)

def test_nested_variable_without_branding_suffix(self):
table_id = cmor.load_table(self.nested_table.name)

itim = cmor.axis(
table_entry='time',
units='months since 2010-1-1',
coord_vals=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
cell_bounds=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
ivar = cmor.variable('thetaoga', units='deg_C', axis_ids=[itim, ])

data = [280., ] * 12
cmor.write(ivar, data)
filename = cmor.close(ivar, file_name=True)

ds = Dataset(filename)
attrs = ds.ncattrs()
self.assertTrue('branding_suffix' not in attrs)
self.assertTrue('temporal_label' not in attrs)
self.assertTrue('vertical_label' not in attrs)
self.assertTrue('horizontal_label' not in attrs)
self.assertTrue('area_label' not in attrs)
ds.close()
os.remove(filename)


def test_variable_with_branding_suffix(self):
mip_table = "CMIP6_Omon_branded_variable.json"
Expand Down Expand Up @@ -116,6 +170,36 @@ def test_variable_with_branding_suffix(self):
ds.close()
os.remove(filename)

def test_nested_variable_with_branding_suffix(self):
table_id = cmor.load_table(self.nested_table.name)

itim = cmor.axis(
table_entry='time',
units='months since 2010-1-1',
coord_vals=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
cell_bounds=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
ivar = cmor.variable('thetaoga_w1-x2-y3-z4', units='deg_C',
axis_ids=[itim, ])

data = [280., ] * 12
cmor.write(ivar, data)
filename = cmor.close(ivar, file_name=True)

ds = Dataset(filename)
attrs = ds.ncattrs()
self.assertTrue('branding_suffix' in attrs)
self.assertEqual('w1-x2-y3-z4', ds.getncattr('branding_suffix'))
self.assertTrue('temporal_label' in attrs)
self.assertEqual('w1', ds.getncattr('temporal_label'))
self.assertTrue('vertical_label' in attrs)
self.assertEqual('x2', ds.getncattr('vertical_label'))
self.assertTrue('horizontal_label' in attrs)
self.assertEqual('y3', ds.getncattr('horizontal_label'))
self.assertTrue('area_label' in attrs)
self.assertEqual('z4', ds.getncattr('area_label'))
ds.close()
os.remove(filename)


if __name__ == '__main__':
unittest.main()