diff --git a/fre/yamltools/converters/__init__.py b/fre/yamltools/converters/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/fre/yamltools/converters/__init__.py @@ -0,0 +1 @@ + diff --git a/fre/yamltools/converters/compile/__init__.py b/fre/yamltools/converters/compile/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/fre/yamltools/converters/compile/__init__.py @@ -0,0 +1 @@ + diff --git a/fre/yamltools/converters/compile/agent.md b/fre/yamltools/converters/compile/agent.md new file mode 100644 index 000000000..a6cc93c85 --- /dev/null +++ b/fre/yamltools/converters/compile/agent.md @@ -0,0 +1,200 @@ +--- +name: compile-xml-to-yaml-converter +description: Agent to convert a compile XML experiment to a compile YAML file +--- + +You are an expert at converting FRE (Flexible Runtime Environment) compile XML experiments into compile YAML format. + +## Your roles +- **Converter**: Transform a compile XML experiment into the corresponding YAML structure. +- **Validator**: Validate the produced YAML against the JSON schema at + https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +- **Corrector**: Fix any validation errors so the YAML passes schema validation. + +--- + +## Instructions + +1. Ask the user to provide the compile XML file path. +2. Ask the user which experiment name to convert (e.g. `$(AM5_VERSION)_compile`). The target experiment contains "compile" in its `name` attribute. +3. Read and parse the XML file. Locate the `` element whose `name` attribute contains "compile" and matches the user's choice. Ignore other experiments (e.g. canopy wrapper experiments). +4. Convert each `` inside that experiment to a YAML `src` list entry using the field-by-field rules below. +5. Write the output YAML to a file with the same base name as the XML file but with a `.yaml` extension. For example, `am5_compile.xml` → `am5_compile.yaml`. +6. Validate the YAML against the schema and report any errors to the user. + +--- + +## Top-level YAML structure + +```yaml +compile: + experiment: + container_addlibs: + baremetal_linkerflags: + src: + - component: ... + ... +``` + +--- + +## Field-by-field conversion rules + +### `experiment` +- Source: the `name` attribute of the `` element. +- Convert every `$(VAR)` reference to a YAML anchor `*VAR`. +- If the name contains both literal text and an anchor, use `!join`: + ```yaml + experiment: !join [*AM5_VERSION, "_compile"] + ``` +- If the name is a plain string with no variables, quote it directly. + +--- + +### `component` (the YAML component name / identifier) +- Source: the filename inside `` tags, stripped of the `.git` suffix and whitespace. + - Example: ` FMS.git ` → `"FMS"` + - Example: ` ocean_BGC.git ` → `"ocean_BGC"` +- This name is used as both the YAML `component:` value and the base filename for the `repo:` URL. +- **Exception**: when the codeBase name does not clearly identify the component in context (e.g. `ice_param.git` for the SIS2 component), use the XML `` attribute instead and note the discrepancy. + +--- + +### `repo` +- Constructed by joining the `root` attribute of `` with the `` filename: + `root + "/" + codeBase_filename` +- Always ensure the URL ends with `.git`. +- Examples: + - `root="https://github.com/NOAA-GFDL"` + `FMS.git` → `"https://github.com/NOAA-GFDL/FMS.git"` + - `root="http://gitlab.gfdl.noaa.gov/fms"` + `am5_phys.git` → `"https://gitlab.gfdl.noaa.gov/fms/am5_phys.git"` +- Normalize `http://` to `https://` where the host is known to support HTTPS. + +--- + +### `branch` +- Source: the `version` attribute of ``. +- Always quote the value as a string (version numbers like `2024.01` can be misread as floats). +- Example: `` → `branch: "2024.01_am5"` + +--- + +### `requires` +- Source: the `requires` attribute of ``, which is a space-separated list of XML component names. +- Map each XML component name to its corresponding YAML component name (i.e. the codeBase-derived name used in the `component:` field). +- Output as a YAML list of quoted strings. +- Example: `requires="fms rte-rrtmgp"` → `requires: ["FMS", "rte-rrtmgp"]` +- Omit the field entirely if `requires` is absent. + +--- + +### `paths` +- Source: the `paths` attribute of `` (whitespace-separated, may span multiple lines). +- Output as a YAML list of quoted strings. +- **Expand brace notation** `{a,b,c}` into separate list entries: + - `mom6/src/MOM6/config_src/{infra/FMS2,memory/dynamic_nonsymmetric}` → + ```yaml + paths: ["mom6/src/MOM6/config_src/infra/FMS2", + "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric"] + ``` +- Glob patterns (`*`, `*/*`) are kept as-is in the YAML strings. +- Omit the field if `paths` is absent (some components have no `paths` attribute). + +--- + +### `cppdefs` +- Source: the text content of ``, including content inside ``. +- Strip leading/trailing whitespace from the value. +- Convert `$(VAR)` references: + - If the entire value is a single variable → `cppdefs: *VAR` + - If the value is a mix of a variable and literal flags → use `!join`: + ```yaml + cppdefs: !join [*F2003_FLAGS, " -DSPMD -DCLIMATE_NUDGE"] + ``` + - If no variables are present, quote the string directly: + ```yaml + cppdefs: "-heap-arrays -DRTE_USE_SP" + ``` +- Complex inline expressions such as `"'"\`git-version-string $<\`"'"` should be preserved verbatim as part of the string value — do not attempt to evaluate or simplify them. +- Omit the field if `` is absent. + +--- + +### `makeOverrides` +- Source: the text content of ``. +- Preserve the exact string, quoted with single quotes if it contains double-quote characters. +- Example: `OPENMP=""` → `makeOverrides: 'OPENMP=""'` +- Omit the field if `` is absent. + +--- + +### `doF90Cpp` +- Source: the `doF90Cpp` attribute on the `` element. +- `doF90Cpp="yes"` → `doF90Cpp: True` +- Omit the field if the attribute is absent or not `"yes"`. + +--- + +### `additionalInstructions` +- Source: the content inside `` within a component's ``. +- Output as a `!join` list where each element is either: + - A plain shell command line ending with `"\n"`, or + - A `*VAR` anchor for any `$(VAR)` variable. +- **Splitting rules**: + - Split at actual newlines in the CDATA content. + - Do **not** split inside a single Bash command even if it spans conceptual units; keep each logical line together. + - Append `"\n"` to each line element (except optionally the last). + - Convert `$(VAR)` to `*VAR` anchor references inside the join list. +- Example: + ```xml + + ``` + becomes: + ```yaml + additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", + "pushd mom6\n", + "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n"] + ``` +- Omit the field if no `` element is present. + +--- + +### `otherFlags` +- This field is **not** directly present in the XML; it is derived from the include directory dependencies: + - Components that depend on FMS headers should include `otherFlags: *FMSincludes`. + - Components that also require MOM6 framework headers (e.g. `sis2`, `ocean_BGC`, `coupler`) should include `otherFlags: !join [*FMSincludes, " ", *momIncludes]`. + - The FMS component itself does not need `otherFlags`. +- The anchor definitions for `*FMSincludes` and `*momIncludes` are expected to exist elsewhere in the broader YAML document (e.g. a shared variables section). Do not define them; only reference them. +- If you are unsure whether `otherFlags` applies, leave it out and note it for the user. + +--- + +## Variable anchor conventions + +- XML uses `$(VARNAME)` syntax for make-style variables. +- In YAML, these become anchor references: `*VARNAME`. +- When an anchor appears inside a string with other text, convert the whole value to a `!join` list: + ```yaml + # Instead of: "$(F2003_FLAGS) -DSPMD" + cppdefs: !join [*F2003_FLAGS, " -DSPMD"] + ``` +- Anchor **definitions** (e.g. `AM5_VERSION: &AM5_VERSION "am5_p1"`) come from a separate variables/defaults section of the YAML, not from this conversion. Do not invent anchor definitions. + +--- + +## Ordering of `src` components + +Preserve the order of `` elements as they appear in the XML. The build system may rely on this ordering for dependency resolution. + +--- + +## Validation + +After writing the YAML file: +1. Fetch the schema from https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +2. Validate the output YAML against the schema using `jsonschema` or equivalent. +3. Print all validation errors with their JSON path location. +4. Offer to correct any errors automatically. diff --git a/fre/yamltools/converters/compile/converter.py b/fre/yamltools/converters/compile/converter.py new file mode 100644 index 000000000..3bf06e9aa --- /dev/null +++ b/fre/yamltools/converters/compile/converter.py @@ -0,0 +1,189 @@ +import re + +import xml.etree.ElementTree as ET +import argparse +import yaml + +def parse_component(component: ET.Element) -> dict[str, str | list] | None: + """Parse a single XML element into a YAML-friendly dictionary.""" + + def clean_text(string: str) -> list[str] | None: + clean_string = re.sub(r'\s+', ' ', string.replace('\t', '')).strip() + if clean_string: + return clean_string + return None + + def get_compile_flag(flag: str) -> str | None: + """ + Return text for cppDefs or makeOverrides in compile element block + For example, + + -DDEBUG -Iinclude + -j8 + + returns + "-DDEBUG, -Iinclude" for get_compile_flag("cppDefs") + "-j8" for get_compile_flag("makeOverrides"), or + None if the flag is not defined. + """ + compile_elem = component.find('compile') + if compile_elem is not None: + elem = compile_elem.find(flag) + if elem is not None: + elem_text = clean_text(elem.text) + if elem_text: + return elem_text + return None + + def get_paths() -> list[str] | None: + """ + Return parsed component paths from the paths attribute. + For example, + + returns + ["am5_phys/atmos_param", "am5_phys/atmos_shared"] or + None if paths is not defined. + """ + paths = component.attrib.get('paths') + if paths and paths.strip(): + return [p.strip() for p in paths.split() if p.strip()] + return None + + def get_requires() -> list[str] | None: + """ + Return parsed component dependencies from the requires attribute. + For example, + + returns + ["FMS", "rte-rrtmpgp", "rte-ecckd"] or + None if requires is not defined. + """ + requires = component.attrib.get('requires') + if requires and requires.strip(): + return [r.strip() for r in requires.split() if r.strip()] + return None + + def get_doF90Cpp() -> bool | None: + """ + Parse doF90Cpp from the compile block into a boolean when present. + For example, + + returns True or + None if doF90Cpp is not defined. + """ + compile_elem = component.find('compile') + if compile_elem is not None: + val = compile_elem.attrib.get('doF90Cpp') + map_to_bool = {'yes': True, 'no': False} + if val is not None: + return map_to_bool.get(val.strip().lower()) + + return None + + def get_additional_instructions() -> list[str] | None: + """Extract source/csh instructions as non-empty lines.""" + source_elem = component.find('source') + if source_elem is not None: + csh_elem = source_elem.find('csh') + if csh_elem is not None and csh_elem.text: + cleaned_lines = [] + for line in csh_elem.text.splitlines(): + # Remove tabs and normalize extra spaces + clean_line = clean_text(line) + if clean_line is not None: + cleaned_lines.append(clean_line) + return cleaned_lines if cleaned_lines else None + return None + + def get_repo_and_branch() -> tuple[str | None, str | None]: + """ + Build repo URL and branch/version from source/codeBase tags. + For example, + + FMS.git + + returns + ("https://github.com/NOAA-GFDL/FMS.git", "2026.01") + """ + + repo = None + branch = None + source_elem = component.find('source') + if source_elem is not None: + root = source_elem.attrib.get('root') + codebase_elem = source_elem.find('codeBase') + if root and codebase_elem is not None and codebase_elem.text: + repo = f"{root.rstrip('/')}/{codebase_elem.text.strip().strip()}" + branch = codebase_elem.attrib.get('version') + return repo, branch + + repo, branch = get_repo_and_branch() + component_name = component.attrib.get('name') + if component_name is not None: + component_name = component_name.strip() or None + + d = { + 'component': component_name, + 'repo': repo, + 'branch': branch, + 'paths': get_paths(), + 'requires': get_requires(), + 'cppdefs': get_compile_flag('cppDefs'), + 'makeOverrides': get_compile_flag('makeOverrides'), + 'doF90Cpp': get_doF90Cpp(), + 'additionalInstructions': get_additional_instructions(), + } + # Remove None values + return {k: v for k, v in d.items() if v is not None} + +def parse_experiment(experiment: ET.Element) -> [str, str | list]: + """Parse one element into the compile YAML object.""" + components = [parse_component(c) for c in experiment.findall('component')] + return { + 'experiment': experiment.attrib.get('name'), + 'container_addlibs': '', + 'baremetal_linkerflags': '', + 'src': components if components else [], + } + +def write_yaml(yamldict: dict, yaml_path: str): + """Write the YAML dictionary to a file.""" + with open(yaml_path, 'w', encoding='utf-8') as f: + yaml.safe_dump(yamldict, f, sort_keys=False) + +def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): + """ + Convert compile XML to YAML. + All experiments in the XML will be converted if experiment_name is None + """ + tree = ET.parse(xml_path) + root = tree.getroot() + experiments = root.findall('experiment') + + if experiment_name is not None: + experiments = [exp for exp in experiments if exp.get('name') == experiment_name] + + for exp in experiments: + print(f"Converting experiment '{exp.attrib.get('name')}' to YAML...") + yamldict = {'compile': parse_experiment(exp)} + write_yaml(yamldict, yaml_path) + print(f"Experiment '{exp.attrib.get('name')}' converted to YAML.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert compile XML to YAML") + parser.add_argument("-x", "--xmlfile", required=True, help="Input XML file") + parser.add_argument("-o", "--output", required=True, help="Output YAML file") + parser.add_argument("-e", "--experiment", default=None, help="Experiment name (optional)") + args = parser.parse_args() + + xml_to_yaml(args.xmlfile, args.output, args.experiment) + print(f"\nConverted {args.xmlfile} to {args.output}.") + print(f"Experiment: {args.experiment if args.experiment else 'All experiments'}") + print("_______________") + print("WARNING: THIS CONVERTER OUTPUTS CLOSE-ENOUGH COMPILE YAML") + print("PLEASE CHECK THE FOLLOWING:") + print(" * PATHS") + print(" * CPPDEFS AND OTHER FLAGS") + print(" * ADDITIONALINSTRUCTIONS") + print(" * PLEASE ADD IN THE APPROPRIATE ANCHORS") diff --git a/fre/yamltools/converters/compile/test_converter.py b/fre/yamltools/converters/compile/test_converter.py new file mode 100644 index 000000000..fbf299625 --- /dev/null +++ b/fre/yamltools/converters/compile/test_converter.py @@ -0,0 +1,468 @@ +import pytest +import yaml +import xml.etree.ElementTree as ET +from pathlib import Path + +from . import converter + + +parse_component = converter.parse_component +parse_experiment = converter.parse_experiment +xml_to_yaml = converter.xml_to_yaml + + +class TestGetCompileFlag: + """Test get_compile_flag nested function via parse_component.""" + + def test_cpp_defs_present(self): + """Test parsing cppDefs flag from compile block.""" + xml_str = ''' + + + -DDEBUG -Iinclude + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['cppdefs'] == '-DDEBUG -Iinclude' + + def test_make_overrides_present(self): + """Test parsing makeOverrides flag from compile block.""" + xml_str = ''' + + + -j8 + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['makeOverrides'] == '-j8' + + def test_flag_not_defined(self): + """Test when compile flag is not defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'cppdefs' not in result # Should be filtered out + assert 'makeOverrides' not in result + + +class TestGetPaths: + """Test get_paths nested function via parse_component.""" + + def test_single_path(self): + """Test parsing single path.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['am5_phys/atmos_param'] + + def test_multiple_paths(self): + """Test parsing multiple space-separated paths.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['am5_phys/atmos_param', 'am5_phys/atmos_shared', "am5_phys/madeup"] + + def test_paths_not_defined(self): + """Test when paths attribute is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'paths' not in result + + def test_empty_paths_string(self): + """Test when paths is empty string.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'paths' not in result + + +class TestGetRequires: + """Test get_requires nested function via parse_component.""" + + def test_single_requirement(self): + """Test parsing single requirement.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['FMS'] + + def test_multiple_requirements(self): + """Test parsing multiple space-separated requirements.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['FMS', 'rte-rrtmpgp', 'rte-ecckd'] + + def test_requires_not_defined(self): + """Test when requires attribute is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'requires' not in result + + def test_empty_requires_string(self): + """Test when requires is empty string.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'requires' not in result + + +class TestGetDoF90Cpp: + """Test get_doF90Cpp nested function via parse_component.""" + + def test_doF90Cpp_yes(self): + """Test parsing doF90Cpp as 'yes'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is True + + def test_doF90Cpp_no(self): + """Test parsing doF90Cpp as 'no'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is False + + def test_doF90Cpp_not_defined(self): + """Test when doF90Cpp is not defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'doF90Cpp' not in result + + +class TestGetAdditionalInstructions: + """Test get_additional_instructions nested function via parse_component.""" + + def test_single_instruction(self): + """Test parsing single instruction.""" + xml_str = ''' + + + echo "Building" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "Building"'] + + def test_multiple_instructions(self): + """Test parsing multiple instructions across lines.""" + xml_str = ''' + + + + echo "Line 1" + echo "Line 2" + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "Line 1"', 'echo "Line 2"'] + + def test_no_instructions(self): + """Test when no instructions are defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'additionalInstructions' not in result + + +class TestGetRepoAndBranch: + """Test get_repo_and_branch nested function via parse_component.""" + + def test_valid_repo_and_branch(self): + """Test parsing valid repo URL and branch.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + assert result['branch'] == '2026.01' + + def test_repo_with_trailing_slash(self): + """Test repo root with trailing slash.""" + xml_str = ''' + + + repo.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' + + def test_no_source_element(self): + """Test when source element is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + def test_no_codebase_element(self): + """Test when codeBase element is missing.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + def test_no_root_attribute(self): + """Test when root attribute is missing.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + +class TestParseComponent: + """Test parse_component main function.""" + + def test_complete_component(self): + """Test parsing a complete component with all attributes.""" + xml_str = ''' + + + -DDEBUG + -j8 + + + FMS.git + echo "building" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'am5_phys' + assert result['paths'] == ['path1', 'path2'] + assert result['requires'] == ['FMS'] + assert result['cppdefs'] == '-DDEBUG' + assert result['makeOverrides'] == '-j8' + assert result['doF90Cpp'] is True + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + assert result['branch'] == '2026.01' + assert result['additionalInstructions'] == ['echo "building"'] + + def test_minimal_component(self): + """Test parsing minimal component with only name.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'minimal' + assert len(result) == 1 # Only component field + + def test_component_without_name(self): + """Test parsing component without name attribute.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result == {} # All None values filtered out + + def test_none_values_filtered(self): + """Test that None values are filtered from output.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + # Verify no None values exist in result + for v in result.values(): + assert v is not None + + +class TestParseExperiment: + """Test parse_experiment function.""" + + def test_experiment_with_components(self): + """Test parsing experiment with multiple components.""" + xml_str = ''' + + + + + ''' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] == 'test_exp' + assert result['container_addlibs'] == '' + assert result['baremetal_linkerflags'] == '' + assert len(result['src']) == 2 + assert result['src'][0]['component'] == 'comp1' + assert result['src'][1]['component'] == 'comp2' + + def test_experiment_without_components(self): + """Test parsing experiment without components.""" + xml_str = '' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] == 'empty_exp' + assert result['src'] == [] + + def test_experiment_without_name(self): + """Test parsing experiment without name attribute.""" + xml_str = ''' + + + + ''' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] is None + assert len(result['src']) == 1 + + +class TestXmlToYaml: + """Test xml_to_yaml main conversion function.""" + + def test_basic_xml_to_yaml_conversion(self, tmp_path): + """Test basic XML to YAML conversion.""" + xml_content = ''' + + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert yaml_file.exists() + data = yaml.safe_load(yaml_file.read_text()) + + assert 'compile' in data + assert data['compile']['experiment'] == 'exp1' + assert len(data['compile']['src']) == 1 + + def test_xml_to_yaml_with_specific_experiment(self, tmp_path): + """Test XML to YAML conversion with specific experiment filter.""" + xml_content = ''' + + + + + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file, experiment_name='exp2') + + data = yaml.safe_load(yaml_file.read_text()) + + assert data['compile']['experiment'] == 'exp2' + assert len(data['compile']['src']) == 1 + assert data['compile']['src'][0]['component'] == 'comp2' + + def test_xml_to_yaml_multiple_components(self, tmp_path): + """Test XML to YAML with multiple components.""" + xml_content = ''' + + + + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + data = yaml.safe_load(yaml_file.read_text()) + + assert len(data['compile']['src']) == 3 + assert data['compile']['src'][0]['component'] == 'comp1' + assert data['compile']['src'][1]['component'] == 'comp2' + assert data['compile']['src'][2]['component'] == 'comp3' + + def test_xml_to_yaml_nonexistent_file(self): + """Test error handling for non-existent XML file.""" + with pytest.raises(FileNotFoundError): + xml_to_yaml('/nonexistent/path/test.xml', '/tmp/output.yaml') + + def test_xml_to_yaml_invalid_xml(self, tmp_path): + """Test error handling for invalid XML.""" + xml_file = tmp_path / 'invalid.xml' + yaml_file = tmp_path / 'output.yaml' + + xml_file.write_text('This is not well-formed XML + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert Path(yaml_file).exists() + assert yaml_file.stat().st_size > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/fre/yamltools/converters/reference.xml b/fre/yamltools/converters/reference.xml new file mode 100644 index 000000000..dc4fd40cf --- /dev/null +++ b/fre/yamltools/converters/reference.xml @@ -0,0 +1,148 @@ + + + + + + + + + +Experiment to build executable. See git log for source code provenance. + + + + + FMS.git + + + $(F2003_FLAGS) -Duse_libMPI -Duse_netCDF -Duse_yaml + + + + + + rte-rrtmgp.git + + + -heap-arrays -DRTE_USE_SP + OPENMP="" + + + + + + rte-ecckd.git + + + + + + + am5_phys.git + + + $(F2003_FLAGS) + + + + + GFDL_atmos_cubed_sphere.git + + + USE_R4=$(USE_MIXED_MODE) ISA="-march=core-avx2 -qno-opt-dynamic-align" + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + + atmos_drivers.git + + + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + + + land_lad2.git + + + $(F2003_FLAGS) -nostdinc + + + + + + ocean_BGC.git + + + + + OPENMP="" + + + + + + ice_param.git + + + + + + + + + + coupler.git + + + + + + + + diff --git a/fre/yamltools/converters/reference.yaml b/fre/yamltools/converters/reference.yaml new file mode 100644 index 000000000..54f0d5bc1 --- /dev/null +++ b/fre/yamltools/converters/reference.yaml @@ -0,0 +1,116 @@ +compile: + experiment: !join [*AM5_VERSION, "_compile"] + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + branch: "2026.01" + cppdefs: "-DHAVE_GETTID -DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF -Duse_yaml" + otherFlags: *FMSincludes + - component: "rte-rrtmgp" + requires: ["FMS"] + repo: "http://github.com/NOAA-GFDL/rte-rrtmgp.git" + branch: "2024.01" + paths: ["rte-rrtmgp/rte-frontend", + "rte-rrtmgp/rte-kernels/mo_fluxes_broadband_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_optical_props_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_rte_solver_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_rte_util_array.F90", + "rte-rrtmgp/gas-optics", + "rte-rrtmgp/rrtmgp-frontend/mo_gas_optics_rrtmgp.F90", + "rte-rrtmgp/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90", + "rte-rrtmgp/extensions/mo_heating_rates.F90", + "rte-rrtmgp/examples/mo_load_coefficients.F90"] + cppdefs: "-heap-arrays -DRTE_USE_SP" + makeOverrides: 'OPENMP=""' + otherFlags: *FMSincludes + - component: "rte-ecckd" + requires: ["FMS", "rte-rrtmgp"] + repo: "http://github.com/NOAA-GFDL/rte-ecckd.git" + branch: "2024.03" + paths: ["rte-ecckd/src/gas_optics_ecckd.f90"] + otherFlags: *FMSincludes + - component: "am5_phys" + requires: ["FMS", "rte-rrtmgp", "rte-ecckd"] + repo: "https://gitlab.gfdl.noaa.gov/fms/am5_phys.git" + branch: "2026.01" + paths: ["am5_phys/atmos_param", "am5_phys/atmos_shared"] + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "GFDL_atmos_cubed_sphere" + requires: ["FMS", "am5_phys"] + repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" + branch: "2024.01_am5" + paths: ["GFDL_atmos_cubed_sphere/driver/GFDL", + "GFDL_atmos_cubed_sphere/model", + "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90", + "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", + "GFDL_atmos_cubed_sphere/tools", + "GFDL_atmos_cubed_sphere/GFDL_tools"] + cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" + otherFlags: *FMSincludes + - component: "atmos_drivers" + requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere", "rte-rrtmgp", "rte-ecckd"] + repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" + branch: "2025.03" + paths: ["atmos_drivers/coupled"] + cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" + otherFlags: *FMSincludes + - component: "land_lad2" + requires: ["FMS"] + repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" + branch: "2025.02" + doF90Cpp: True + cppdefs: "-DINTERNAL_FILE_NML -nostdinc" + otherFlags: *FMSincludes + - component: "ocean_BGC" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_BGC.git" + branch: "2023.01" + paths: ["mom6/src/MOM6/config_src/infra/FMS2", + "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", + "mom6/src/MOM6/config_src/drivers/FMS_cap", + "mom6/src/MOM6/config_src/external", + "mom6/src/MOM6/src/*", + "mom6/src/MOM6/src/*/*"] + cppdefs: "-DINTERNAL_FILE_NML -DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2" + otherFlags: !join [*FMSincludes, " ", *momIncludes] + makeOverrides: 'OPENMP=""' + additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", + "pushd mom6\n", + "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n", + "git submodule update --recursive --init src/MOM6 src/SIS2 src/icebergs\n", + "git pull --no-edit https://github.com/uramirez8707/MOM6-examples.git use_diag_yaml \n", + "cd src/MOM6\n", + "git checkout 792a0616c16050dfe74ac5bfc887f99d9c765376 \n", + "cd ../SIS2\n", + "git checkout dec9ee38424eceded0da635671e7c03e5d97d928 \n", + "cd ../icebergs\n", + "git checkout 6a8944fc7544608942de9a42bd3b96f6e56684e0 \n", + "popd \n", + "pushd mom6 \n" + "ln -s", *(GFDL_O_INPUTS), ".datasets \n", + "popd \n", + "test -e mom6/.datasets \n", + "if ($status != 0) then\n", + 'echo ""; echo "" ; echo " WARNING: .datasets link in MOM6 examples directory is invalid"; echo ""; echo ""', + "endif", + "cd .. \n"] + - component: "sis2" + requires: ["FMS", "ocean_BGC"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "2024.02" + paths: ["mom6/src/SIS2/config_src/dynamic_symmetric", + "mom6/src/SIS2/config_src/external/Icepack_interfaces", + "mom6/src/SIS2/src", + "mom6/src/icebergs/src", + "sis2"] + otherFlags: !join [*FMSincludes, " ", *momIncludes] + cppdefs: "-DUSE_FMS2_IO -DINTERNAL_FILE_NML" + - component: "FMScoupler" + requires: ["FMS", "GFDL_atmos_cubed_sphere", "atmos_drivers", "am5_phys", "land_lad2", "sis2", "ocean_BGC"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "2026.01" + paths: ["FMScoupler/full", "FMScoupler/shared"] + otherFlags: !join [*FMSincludes, " ", *momIncludes]