From 46bc9fdf2e6d9c51673c9adc49339d08a824baf6 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 22 Apr 2026 11:30:40 -0400 Subject: [PATCH 01/22] initial code --- fre/yamltools/converters/compile.py | 68 +++++++++++++++++++++++++++++ fre/yamltools/converters/xml.py | 50 +++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 fre/yamltools/converters/compile.py create mode 100644 fre/yamltools/converters/xml.py diff --git a/fre/yamltools/converters/compile.py b/fre/yamltools/converters/compile.py new file mode 100644 index 000000000..64d69f399 --- /dev/null +++ b/fre/yamltools/converters/compile.py @@ -0,0 +1,68 @@ +from pathlib import Path +import yaml + +from bs4 import BeautifulSoup + +from xml import XML + + + +class CompileXML(XML): + + def __init__(self, xmlfile: str|Path, yamlfile: str|Path = None): + + super().__init__(xmlfile) + + self.yamlfile = self.xmlfile.with_suffix(".yaml") if yamlfile is None else Path(yamlfile) + self.yaml = {} + + + def convert(self): + + self.yaml["compile"]: { + "experiment": self.get_tag(xml.soup, "experiment", "name"), + "container_addlibs": "", + "baremetal_linkerflags": "", + "src": self.get_src_list() + } + + + def write_yaml(self): + + with open(self.yamlfile, "w") as openedfile: + yaml.dump(self.yaml, openedfile, sort_keys=False) + + + def get_src_list(self, component_name: str = None): + + components = self.soup.find_all("component") + if component_name is not None: + components = components.find(attrs={"name": component_name}) + if components is None: + return None + + src_list = [] + for component in components: + + component_yaml: ComponentSrcDict = { + "component": self.get_key(component, "name"), + "branch": self.get_tag(component, "codeBase", "version"), + "repo": self.get_tag(component, "source", "root") + "/" + self.get_tag(component, "codeBase"), + "paths": self.make_list(self.get_key(component, "paths")), + "requires": self.make_list(self.get_key(component, "requires")), + "otherFlags": self.make_list(self.get_key(component, "includeDir")), + "cppdefs": self.get_tag(component, "cppDefs"), + "makeOverrides": self.get_tag(component, "makeOverrides"), + "doF90Cpp": self.get_tag(component, "compile", "doF90Cpp"), + "additionalInstructions": self.make_list(self.get_tag(component, "csh", "\n")) + } + + src_list.append( + {key: value for key, value in component_yaml.items() if value is not None} + ) + + return src_list + +xml = CompileXML("./compile.xml") +xml.convert() +xml.write_yaml() diff --git a/fre/yamltools/converters/xml.py b/fre/yamltools/converters/xml.py new file mode 100644 index 000000000..facf5a340 --- /dev/null +++ b/fre/yamltools/converters/xml.py @@ -0,0 +1,50 @@ +from pathlib import Path + +from bs4 import BeautifulSoup +import yaml + + +class XML(): + + def __init__(self, xmlfile: str|Path): + + self.xmlfile = Path(xmlfile) + with open(xmlfile, "r") as openedfile: + self.soup = BeautifulSoup(openedfile, "lxml-xml") + + + def get_key(self, tagobj, tagid): + + tagid_value = tagobj.get(tagid) + + if tagid_value is not None: + tagid_value = tagid_value.strip() + if tagid_value: + return tagid_value + + return None + + + def get_tag(self, soup, tag, tagid: str = None): + + tagobj = soup.find(tag) + + if tagobj is None: + return None + + if tagid is not None: + return self.get_key(tagobj, tagid) + + tagstr = tagobj.text.strip() + if tagstr: + return tagstr + + return None + + + def make_list(self, tag_value: str, split_token: str = None): + + if tag_value is None: + return None + + return [val.strip() for val in tag_value.split(split_token)] From 65865f10692b12b54055e18c97b2766e86827661 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 29 Apr 2026 09:16:41 -0400 Subject: [PATCH 02/22] update, not working --- fre/yamltools/converters/__init__.py | 1 + fre/yamltools/converters/compile.py | 68 ---------------------- fre/yamltools/converters/convert.py | 40 +++++++++++++ fre/yamltools/converters/converters.py | 78 ++++++++++++++++++++++++++ fre/yamltools/converters/xml.py | 48 +++++++++++++--- 5 files changed, 160 insertions(+), 75 deletions(-) create mode 100644 fre/yamltools/converters/__init__.py delete mode 100644 fre/yamltools/converters/compile.py create mode 100644 fre/yamltools/converters/convert.py create mode 100644 fre/yamltools/converters/converters.py diff --git a/fre/yamltools/converters/__init__.py b/fre/yamltools/converters/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/fre/yamltools/converters/__init__.py @@ -0,0 +1 @@ + diff --git a/fre/yamltools/converters/compile.py b/fre/yamltools/converters/compile.py deleted file mode 100644 index 64d69f399..000000000 --- a/fre/yamltools/converters/compile.py +++ /dev/null @@ -1,68 +0,0 @@ -from pathlib import Path -import yaml - -from bs4 import BeautifulSoup - -from xml import XML - - - -class CompileXML(XML): - - def __init__(self, xmlfile: str|Path, yamlfile: str|Path = None): - - super().__init__(xmlfile) - - self.yamlfile = self.xmlfile.with_suffix(".yaml") if yamlfile is None else Path(yamlfile) - self.yaml = {} - - - def convert(self): - - self.yaml["compile"]: { - "experiment": self.get_tag(xml.soup, "experiment", "name"), - "container_addlibs": "", - "baremetal_linkerflags": "", - "src": self.get_src_list() - } - - - def write_yaml(self): - - with open(self.yamlfile, "w") as openedfile: - yaml.dump(self.yaml, openedfile, sort_keys=False) - - - def get_src_list(self, component_name: str = None): - - components = self.soup.find_all("component") - if component_name is not None: - components = components.find(attrs={"name": component_name}) - if components is None: - return None - - src_list = [] - for component in components: - - component_yaml: ComponentSrcDict = { - "component": self.get_key(component, "name"), - "branch": self.get_tag(component, "codeBase", "version"), - "repo": self.get_tag(component, "source", "root") + "/" + self.get_tag(component, "codeBase"), - "paths": self.make_list(self.get_key(component, "paths")), - "requires": self.make_list(self.get_key(component, "requires")), - "otherFlags": self.make_list(self.get_key(component, "includeDir")), - "cppdefs": self.get_tag(component, "cppDefs"), - "makeOverrides": self.get_tag(component, "makeOverrides"), - "doF90Cpp": self.get_tag(component, "compile", "doF90Cpp"), - "additionalInstructions": self.make_list(self.get_tag(component, "csh", "\n")) - } - - src_list.append( - {key: value for key, value in component_yaml.items() if value is not None} - ) - - return src_list - -xml = CompileXML("./compile.xml") -xml.convert() -xml.write_yaml() diff --git a/fre/yamltools/converters/convert.py b/fre/yamltools/converters/convert.py new file mode 100644 index 000000000..dc71adf6a --- /dev/null +++ b/fre/yamltools/converters/convert.py @@ -0,0 +1,40 @@ +import click +from fre.yamltools.converters.converters import CompileConverter + +@click.command("convert", short_help="tool to convert xml to yaml") +@click.option("--xmlfile", "-x", + type=str, + required=True, + help="input XML filename" +) +@click.option("--experiment-name", "-e", + type=str, + default=None, + help=""" + search tag to search for in the experiment name. + something longer coming + """ +) +@click.option("--convert-compile-xml", "-c", + is_flag=True, + default=False, + help="Flag to convert compile xml to yaml. Either the -c or -p option must be provided.") +@click.option("--convert-platform-xml", "-p", + is_flag=True, + default=False, + help="Flag to convert platform xml to yaml. Either the -c or -p option must be provided.") +@click.pass_context +def convert(ctx, xmlfile, experiment_name, convert_compile_xml, convert_platform_xml): + + if convert_compile_xml: + compilexml = CompileConverter(xmlfile=xmlfile) + compilexml.convert(experiment_name) + elif convert_platform_xml: + pass + else: + click.echo("Convert type not specified. Please see below:") + click.echo(ctx.get_help()) + + +if __name__ == "__main__": + convert() diff --git a/fre/yamltools/converters/converters.py b/fre/yamltools/converters/converters.py new file mode 100644 index 000000000..959f31843 --- /dev/null +++ b/fre/yamltools/converters/converters.py @@ -0,0 +1,78 @@ +from pathlib import Path +import yaml + +from bs4 import BeautifulSoup + +from xml import XML + + +class CompileConverter(XML): + + def __init__(self, xmlfile: str|Path, experiment_name: str = None): + + super().__init__(xmlfile) + + self.experiment_name = experiment_name + self.experiments = self.set_experiments(experiment_name) + self.yamldicts = {} + + + def convert(self, experiment_name: str = None): + + for experiment_name, experiment in self.experiments.items(): + compile_yaml = {"compile": { + "experiment": experiment_name, + "container_addlibs": "", + "baremetal_linkerflags": "", + "src": self.get_src_list(experiment)} + } + self.yamldicts[experiment_name] = compile_yaml + + self.write_yaml(compile_yaml, "compile_"+experiment_name +".yaml") + + + def write_yaml(self, yamldict: dict, yamlfile: str|Path): + + with open(yamlfile, "w") as openedfile: + yaml.dump(yamldict, openedfile, sort_keys=False) + + + def set_experiments(self, experiment_name: str = None): + + prettyname = lambda name: name.replace("$(", "").replace(")", "") + + experiments = self.find_all("experiment", search_name=experiment_name) + + if experiments: + return { + prettyname(self.get_key("name", experiment)): experiment for experiment in experiments + } + + raise RuntimeError("Cannot find experiments") + + + def get_src_list(self, experiment, component_name: str = None): + + components = self.find_all("component", experiment, component_name) + + src_list = [] + for component in components: + + component_yaml = { + "component": self.get_key("name", component), + "branch": self.get_tag("codeBase", component, "version"), + "repo": self.get_tag("source", component, "root") + "/" + self.get_tag("codeBase", component), + "paths": self.make_list(self.get_key("paths", component)), + "requires": self.make_list(self.get_key("requires", component)), + "otherFlags": self.make_list(self.get_key("includeDir", component)), + "cppdefs": self.get_tag("cppDefs", component), + "makeOverrides": self.get_tag("makeOverrides", component), + "doF90Cpp": self.get_tag("compile", component, "doF90Cpp"), + "additionalInstructions": self.make_list(self.get_tag("csh", component), "\n") + } + + src_list.append( + {key: value for key, value in component_yaml.items() if value is not None} + ) + + return src_list diff --git a/fre/yamltools/converters/xml.py b/fre/yamltools/converters/xml.py index facf5a340..367abca5b 100644 --- a/fre/yamltools/converters/xml.py +++ b/fre/yamltools/converters/xml.py @@ -9,23 +9,57 @@ class XML(): def __init__(self, xmlfile: str|Path): self.xmlfile = Path(xmlfile) + + if not self.xmlfile.exists(): + raise IOError(f"Cannot find XML file {xmlfile}") + with open(xmlfile, "r") as openedfile: self.soup = BeautifulSoup(openedfile, "lxml-xml") - def get_key(self, tagobj, tagid): + def get_key(self, tagid: str, tagobj= None): - tagid_value = tagobj.get(tagid) + if tagobj is None: + tagobj = self.soup + + tagid = tagobj.get(tagid) - if tagid_value is not None: - tagid_value = tagid_value.strip() - if tagid_value: - return tagid_value + if tagid is not None: + tagid = tagid.strip() + if tagid: + return tagid return None + + def find_all(self, tag: str, soup = None, search_name: str = None): + + if soup is None: + soup = self.soup + + args = [tag] + if search_name is not None: + args.append({"name": search_name}) + + tagobjs = soup.find_all(*args) + + return tagobjs + + + def get_tag(self, soup = None): + + if soup is None: + soup = self.soup - def get_tag(self, soup, tag, tagid: str = None): + tagobjs = soup.find_all(tags) + + if not tagobjs: + return None + + return tagobjs + + + def get_tag(self, tag, soup = None, tagid: str = None): tagobj = soup.find(tag) From e17a30b12bdf5e340d73355f2e9122ef57117ba2 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 13 May 2026 09:27:47 -0400 Subject: [PATCH 03/22] update --- fre/yamltools/converters/converters.py | 150 +++++++++++++++++-------- fre/yamltools/converters/xml.py | 111 +++++++++--------- 2 files changed, 155 insertions(+), 106 deletions(-) diff --git a/fre/yamltools/converters/converters.py b/fre/yamltools/converters/converters.py index 959f31843..e6528a3b9 100644 --- a/fre/yamltools/converters/converters.py +++ b/fre/yamltools/converters/converters.py @@ -1,78 +1,130 @@ from pathlib import Path import yaml -from bs4 import BeautifulSoup - -from xml import XML +try: + from .xml import XML +except ImportError: + from xml import XML class CompileConverter(XML): + """ + Converter class to parse compile experiment XML blocks and convert to YAML dictionaries. + """ def __init__(self, xmlfile: str|Path, experiment_name: str = None): super().__init__(xmlfile) + self.experiments = self.get_experiments(experiment_name) + self.yaml_dicts = {} - self.experiment_name = experiment_name - self.experiments = self.set_experiments(experiment_name) - self.yamldicts = {} - - - def convert(self, experiment_name: str = None): - - for experiment_name, experiment in self.experiments.items(): - compile_yaml = {"compile": { - "experiment": experiment_name, - "container_addlibs": "", - "baremetal_linkerflags": "", - "src": self.get_src_list(experiment)} - } - self.yamldicts[experiment_name] = compile_yaml - - self.write_yaml(compile_yaml, "compile_"+experiment_name +".yaml") - - - def write_yaml(self, yamldict: dict, yamlfile: str|Path): - - with open(yamlfile, "w") as openedfile: - yaml.dump(yamldict, openedfile, sort_keys=False) + def get_experiments(self, experiment_name: str = None): + """ + Parse the XML and save each compile experiment block + """ - def set_experiments(self, experiment_name: str = None): + #prettify name + prettify = lambda name: name.strip().lower().replace("$(", "").replace(")", "") - prettyname = lambda name: name.replace("$(", "").replace(")", "") + # if a particular experiment is not provided, get all compile experiments in xml + experiments = self.get_elements("experiment", self.soup, name=experiment_name) - experiments = self.find_all("experiment", search_name=experiment_name) - if experiments: return { - prettyname(self.get_key("name", experiment)): experiment for experiment in experiments + prettify(self.get_attributes("name", experiment)): experiment for experiment in experiments } raise RuntimeError("Cannot find experiments") - - def get_src_list(self, experiment, component_name: str = None): - components = self.find_all("component", experiment, component_name) + def convert(self): + """ + Convert XML content to YAML dictionary + """ - src_list = [] + for experiment_name, experiment_content in self.experiments.items(): + compile_yaml = {"compile": { + "experiment": experiment_name, + "container_addlibs": "", + "baremetal_linkerflags": "", + "src": self.parse_components(experiment_content)} + } + self.yaml_dicts[experiment_name] = compile_yaml + self.write_yaml(compile_yaml, "compile_"+experiment_name +".yaml") + + + def parse_components(self, xml_content, component_name: str = None): + """ + Parses component blocks to yaml dictionaries. For example, converts + + + + GFDL_atmos_cubed_sphere.git + + + USE_R4=$(USE_MIXED_MODE) ISA="-march=core-avx2 -qno-opt-dynamic-align" + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + to + + { + component: atmos_cubed_sphere + repo: https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git + branch: 2024.01_am5 + paths: [GFDL_atmos_cubed_sphere/model, GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90] + requires: [fms, am5_phys] + otherFlags: [$(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE] + makeOverrides : [USE_R4=$(USE_MIXED_MODE), ISA="-march=core-avx2, -qno-opt-dynamic-align"] + } + """ + + components = self.get_elements("component", xml_content, name=component_name) + if components is None: + return None + + parsed_components = [] for component in components: + codebase = self.get_elements("codeBase", component) + source = self.get_elements("source", component) + cppdefs = self.get_elements("cppDefs", component) + compile_ = self.get_elements("compile", component) + make_overrides = self.get_elements("makeOverrides", component) + csh = self.get_elements("csh", component) + component_yaml = { - "component": self.get_key("name", component), - "branch": self.get_tag("codeBase", component, "version"), - "repo": self.get_tag("source", component, "root") + "/" + self.get_tag("codeBase", component), - "paths": self.make_list(self.get_key("paths", component)), - "requires": self.make_list(self.get_key("requires", component)), - "otherFlags": self.make_list(self.get_key("includeDir", component)), - "cppdefs": self.get_tag("cppDefs", component), - "makeOverrides": self.get_tag("makeOverrides", component), - "doF90Cpp": self.get_tag("compile", component, "doF90Cpp"), - "additionalInstructions": self.make_list(self.get_tag("csh", component), "\n") + "component": self.get_attributes("name", component), + "repo": f"{self.get_attributes('root', source)}/{self.get_values(codebase)}", + "branch": self.get_attributes("version", codebase), + "paths": self.get_attributes("paths", component, tolist=True), + "requires": self.get_attributes("requires", component, tolist=True), + "otherFlags": self.get_attributes("includeDir", component, tolist=True), + "cppdefs": self.get_values(cppdefs), + "makeOverrides": self.get_values(make_overrides), + "doF90Cpp": self.get_attributes("doF90Cpp", compile_), + "additionalInstructions": self.get_values(csh, tolist=True, fieldsep="\n") } - src_list.append( - {key: value for key, value in component_yaml.items() if value is not None} - ) + #remove None + component_yaml = {key: value for key, value in component_yaml.items() if value is not None} + parsed_components.append(component_yaml) - return src_list + return parsed_components + + + def write_yaml(self, yamldict: dict, yamlfile: str|Path): + """ + Write YAML dictionary to file + """ + + with open(yamlfile, "w", encoding="utf-8") as openedfile: + yaml.dump(yamldict, openedfile, sort_keys=False) + + +xml = CompileConverter("compile_experiment.xml") +xml.convert() \ No newline at end of file diff --git a/fre/yamltools/converters/xml.py b/fre/yamltools/converters/xml.py index 367abca5b..2a21a39c6 100644 --- a/fre/yamltools/converters/xml.py +++ b/fre/yamltools/converters/xml.py @@ -1,84 +1,81 @@ from pathlib import Path -from bs4 import BeautifulSoup -import yaml +from bs4 import BeautifulSoup -class XML(): - - def __init__(self, xmlfile: str|Path): +class XML: + """ + Base class to parse XML files and extract elements and key values using BeautifulSoup. + """ - self.xmlfile = Path(xmlfile) + def __init__(self, xmlfile: str | Path): + self.xmlfile = Path(xmlfile) if not self.xmlfile.exists(): raise IOError(f"Cannot find XML file {xmlfile}") - - with open(xmlfile, "r") as openedfile: - self.soup = BeautifulSoup(openedfile, "lxml-xml") + with open(xmlfile, "r", encoding="utf-8") as openedfile: + xml_content = openedfile.read() - def get_key(self, tagid: str, tagobj= None): + self.soup = BeautifulSoup(xml_content, "lxml-xml") - if tagobj is None: - tagobj = self.soup - - tagid = tagobj.get(tagid) - if tagid is not None: - tagid = tagid.strip() - if tagid: - return tagid + def get_attributes(self, attribute: str, element: dict, tolist: bool = False, fieldsep: str = ","): + """ + Get attribute value from an XML element. For example, - return None + element = + get_attributes("root", element) returns "http://github.com/NOAA-GFDL" + element = + get_attributes("requires", element, tolist=True) returns ["fms", "rte-rrtmgp", "rte-ecckd"] + """ - def find_all(self, tag: str, soup = None, search_name: str = None): + value = element.get(attribute) - if soup is None: - soup = self.soup + if value is not None: + value = value.strip() + if value: + if tolist: + return [val.strip() for val in value.split(fieldsep)] + return value - args = [tag] - if search_name is not None: - args.append({"name": search_name}) + return None - tagobjs = soup.find_all(*args) - - return tagobjs - - def get_tag(self, soup = None): + def get_elements(self, element: str, xml_content, name: str = None, find_all: bool = True): + """ + Get XML elements by tag name and optional attribute value. For example, - if soup is None: - soup = self.soup - - tagobjs = soup.find_all(tags) + get_elements("component", soup) returns all blocks in the XML. + get_elements("component", soup, name="am5_phys") returns the element block with name="am5_phys" + """ - if not tagobjs: - return None + search_dict = {} + if name is not None: + search_dict["name"] = name - return tagobjs - - - def get_tag(self, tag, soup = None, tagid: str = None): + find = xml_content.find_all if find_all else xml_content.find + element = find(element, search_dict) - tagobj = soup.find(tag) + return element - if tagobj is None: - return None - - if tagid is not None: - return self.get_key(tagobj, tagid) - - tagstr = tagobj.text.strip() - if tagstr: - return tagstr - return None + def get_values(self, element, tolist: bool = False, fieldsep: str = ","): + """ + Get value for an XML element. For example, + + element = -DDEBUG -Iinclude, + get_values(element, tolist=True) returns ["-DDEBUG", "-Iinclude"] + """ + + value = element.text.strip() - - def make_list(self, tag_value: str, split_token: str = None): + if value: + if tolist: + if fieldsep is None: + return value.split() + return [val.strip() for val in value.split(fieldsep)] + return value - if tag_value is None: - return None - - return [val.strip() for val in tag_value.split(split_token)] + return None \ No newline at end of file From 47261b7fb05be3129d214c246301ba152ac439f8 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 13 May 2026 12:57:00 -0400 Subject: [PATCH 04/22] update --- fre/yamltools/converters/convert.py | 76 +++-- fre/yamltools/converters/converters.py | 393 ++++++++++++++++++++++++- fre/yamltools/converters/xml.py | 26 +- 3 files changed, 444 insertions(+), 51 deletions(-) diff --git a/fre/yamltools/converters/convert.py b/fre/yamltools/converters/convert.py index dc71adf6a..d756e78f7 100644 --- a/fre/yamltools/converters/convert.py +++ b/fre/yamltools/converters/convert.py @@ -1,39 +1,57 @@ import click -from fre.yamltools.converters.converters import CompileConverter +from fre.yamltools.converters.converters import CompileConverter, PlatformConverter -@click.command("convert", short_help="tool to convert xml to yaml") -@click.option("--xmlfile", "-x", - type=str, - required=True, - help="input XML filename" +@click.command("convert", short_help="tool to convert compile and platform xmls to yamls") +@click.option( + "--xmlfile", "-x", + type=str, + required=True, + help="input XML filename" ) -@click.option("--experiment-name", "-e", - type=str, - default=None, - help=""" - search tag to search for in the experiment name. - something longer coming - """ +@click.option( + "--experiment-name", "-e", + type=str, + default=None, + help=""" + experiment name corresponding to the "name" attribute in the xml element + If provided, only the xml body under experiment-name will be converted. + If not provided, all experiments in xmlfile will be converted. + """ +) +@click.option( + "--convert-compile-xml", "-c", + is_flag=True, + default=False, + help=""" + Flag to only convert compile xml to yaml. + If not provided, convert will attempt to convert both compile and platform xmls to platforms. + """ +) +@click.option( + "--convert-platform-xml", "-p", + is_flag=True, + default=False, + help=""" + Flag to only convert platform xml to yaml. + If not provided, convert will attempt to convert both compile and platform xmls to platforms. + """ ) -@click.option("--convert-compile-xml", "-c", - is_flag=True, - default=False, - help="Flag to convert compile xml to yaml. Either the -c or -p option must be provided.") -@click.option("--convert-platform-xml", "-p", - is_flag=True, - default=False, - help="Flag to convert platform xml to yaml. Either the -c or -p option must be provided.") @click.pass_context def convert(ctx, xmlfile, experiment_name, convert_compile_xml, convert_platform_xml): + + """ + Click command to convert compile and platform XML files to YAML files. + """ + + convert_both = not convert_compile_xml and not convert_platform_xml + + if convert_compile_xml or convert_both: + compilexml = CompileConverter(xmlfile=xmlfile, experiment_name=experiment_name) + compilexml.convert() - if convert_compile_xml: - compilexml = CompileConverter(xmlfile=xmlfile) - compilexml.convert(experiment_name) - elif convert_platform_xml: - pass - else: - click.echo("Convert type not specified. Please see below:") - click.echo(ctx.get_help()) + if convert_platform_xml or convert_both: + platformxml = PlatformConverter(xmlfile=xmlfile) + platformxml.convert() if __name__ == "__main__": diff --git a/fre/yamltools/converters/converters.py b/fre/yamltools/converters/converters.py index e6528a3b9..a9de762bc 100644 --- a/fre/yamltools/converters/converters.py +++ b/fre/yamltools/converters/converters.py @@ -1,4 +1,5 @@ from pathlib import Path +import re import yaml try: @@ -25,15 +26,17 @@ def get_experiments(self, experiment_name: str = None): """ #prettify name - prettify = lambda name: name.strip().lower().replace("$(", "").replace(")", "") + prettify = lambda name: name.strip().replace("$(", "").replace(")", "") # if a particular experiment is not provided, get all compile experiments in xml experiments = self.get_elements("experiment", self.soup, name=experiment_name) if experiments: - return { - prettify(self.get_attributes("name", experiment)): experiment for experiment in experiments - } + experiment_dict = {} + for experiment in experiments: + name = prettify(self.get_attributes("name", experiment)) + experiment_dict[name] = experiment + return experiment_dict raise RuntimeError("Cannot find experiments") @@ -45,12 +48,12 @@ def convert(self): for experiment_name, experiment_content in self.experiments.items(): compile_yaml = {"compile": { - "experiment": experiment_name, + "experiment": self.get_attributes("name", experiment_content), "container_addlibs": "", "baremetal_linkerflags": "", "src": self.parse_components(experiment_content)} } - self.yaml_dicts[experiment_name] = compile_yaml + self.yaml_dicts[experiment_name] = compile_yaml self.write_yaml(compile_yaml, "compile_"+experiment_name +".yaml") @@ -90,12 +93,14 @@ def parse_components(self, xml_content, component_name: str = None): parsed_components = [] for component in components: - codebase = self.get_elements("codeBase", component) - source = self.get_elements("source", component) - cppdefs = self.get_elements("cppDefs", component) - compile_ = self.get_elements("compile", component) - make_overrides = self.get_elements("makeOverrides", component) - csh = self.get_elements("csh", component) + codebase = self.get_elements("codeBase", component, find_all=False) + source = self.get_elements("source", component, find_all=False) + cppdefs = self.get_elements("cppDefs", component, find_all=False) + compile_ = self.get_elements("compile", component, find_all=False) + make_overrides = self.get_elements("makeOverrides", component, find_all=False) + csh = self.get_elements("csh", component, find_all=False) + + #"repo": f"{self.get_attributes('root', source)}/{self.get_values(codebase)}", component_yaml = { "component": self.get_attributes("name", component), @@ -126,5 +131,365 @@ def write_yaml(self, yamldict: dict, yamlfile: str|Path): yaml.dump(yamldict, openedfile, sort_keys=False) -xml = CompileConverter("compile_experiment.xml") -xml.convert() \ No newline at end of file +class PlatformConverter(XML): + """ + Converter class to parse platform XML blocks and convert to YAML dictionaries. + """ + + def __init__(self, xmlfile: str | Path): + + super().__init__(xmlfile) + self.platforms = self.get_platforms() + self.yaml_dicts = {} + + + def get_platforms(self): + """ + Parse XML and save each platform block in declaration order. + """ + + platforms = self.get_elements("platform", self.soup) + if platforms is None: + raise RuntimeError("Cannot find platforms") + + platform_dict = {} + for platform in platforms: + platform_name = self.get_attributes("name", platform) + if platform_name is None: + raise RuntimeError("Found element with no name attribute") + if platform_name in platform_dict: + raise RuntimeError(f"Duplicate platform name found: {platform_name}") + platform_dict[platform_name] = platform + + return platform_dict + + + def convert(self): + """ + Convert all XML platform content to one YAML dictionary. + """ + + parsed_platforms = [] + for platform_name in self.platforms: + parsed_platforms.append(self.parse_platform(platform_name)) + + self.yaml_dicts = {"platforms": parsed_platforms} + self.write_yaml(self.yaml_dicts, "platforms.yaml") + + + def parse_platform(self, platform_name: str, visited: set | None = None): + """ + Parse one platform, resolving xi:include references as base content first, + then applying local values as overrides. + """ + + if visited is None: + visited = set() + + if platform_name in visited: + raise RuntimeError(f"Circular xi:include reference detected for {platform_name}") + + if platform_name not in self.platforms: + raise RuntimeError(f"Cannot resolve xi:include reference: {platform_name}") + + visited.add(platform_name) + platform_element = self.platforms[platform_name] + + platform_yaml = {"name": platform_name} + + include_names = self.parse_xincludes(platform_element) + unresolved_includes = [] + for include_name in include_names: + if include_name in self.platforms: + included_platform = self.parse_platform(include_name, visited=visited.copy()) + platform_yaml = self.merge_platform_dicts(platform_yaml, included_platform) + else: + unresolved_includes.append(include_name) + + current_platform = self.parse_platform_body(platform_element) + platform_yaml = self.merge_platform_dicts(platform_yaml, current_platform) + + if include_names: + platform_yaml["includes"] = include_names + if unresolved_includes: + platform_yaml["unresolvedIncludes"] = unresolved_includes + + return self.drop_empty_fields(platform_yaml) + + + def parse_platform_body(self, platform_element): + """ + Parse direct content under a platform block (without include inheritance). + """ + + #fre_version = self.get_values(self.get_elements("freVersion", platform_element, find_all=False)) + project = self.get_values(self.get_elements("project", platform_element, find_all=False)) + compiler = self.parse_compiler(platform_element) + directory = self.parse_directory(platform_element) + properties = self.parse_properties(platform_element) + csh = self.parse_text_block(platform_element, "csh") + + platform_yaml = { + "freVersion": fre_version, + "project": project, + "compiler": compiler.get("compiler"), + "compilerVersion": compiler.get("compilerVersion"), + "compilerType": compiler.get("compilerType"), + "envSetup": self.derive_env_setup(compiler), + "mkTemplate": self.derive_mk_template(platform_element, compiler), + "modelRoot": self.derive_model_root(platform_element, project, directory), + "directory": directory, + "properties": properties, + "csh": csh, + } + + # Preserve known convenience fields used by platform YAML workflows. + platform_yaml.update(self.derive_container_settings(platform_element)) + + return self.drop_empty_fields(platform_yaml) + + + def parse_compiler(self, platform_element): + """ + Parse compiler attributes and normalize a friendly compiler name. + """ + + compiler_element = self.get_elements("compiler", platform_element, find_all=False) + compiler_type = self.get_attributes("type", compiler_element) + compiler_version = self.get_attributes("version", compiler_element) + + compiler_name = None + if compiler_type: + compiler_name = compiler_type.replace("_", "-").split("-")[0].strip() + + return self.drop_empty_fields({ + "compiler": compiler_name, + "compilerType": compiler_type, + "compilerVersion": compiler_version, + }) + + + def parse_directory(self, platform_element): + """ + Parse directory subfields under . + """ + + directory_element = self.get_elements("directory", platform_element, find_all=False) + if directory_element is None: + return None + + directory = { + "stem": self.get_attributes("stem", directory_element), + "include": self.get_values(self.get_elements("include", directory_element, find_all=False)), + "archive": self.get_values(self.get_elements("archive", directory_element, find_all=False)), + "analysis": self.get_values(self.get_elements("analysis", directory_element, find_all=False)), + } + + return self.drop_empty_fields(directory) + + + def parse_properties(self, platform_element): + """ + Parse all blocks into a dictionary. + """ + + properties = self.get_elements("property", platform_element) + if properties is None: + return None + + parsed_properties = {} + for property_element in properties: + property_name = self.get_attributes("name", property_element) + property_value = self.get_attributes("value", property_element) + + if property_name is not None: + parsed_properties[property_name] = property_value if property_value is not None else "" + + return parsed_properties if parsed_properties else None + + + def parse_text_block(self, platform_element, tag_name: str): + """ + Parse free-form text blocks such as . + """ + + return self.get_values(self.get_elements(tag_name, platform_element, find_all=False)) + + + def parse_xincludes(self, platform_element): + """ + Parse xi:include references and extract referenced platform names. + """ + + include_elements = self.get_elements("xi:include", platform_element) + if include_elements is None: + return [] + + include_names = [] + for include_element in include_elements: + xpointer = self.get_attributes("xpointer", include_element) + if xpointer is None: + continue + + include_name = self.extract_platform_name_from_xpointer(xpointer) + if include_name is not None: + include_names.append(include_name) + + return include_names + + + def extract_platform_name_from_xpointer(self, xpointer: str): + """ + Extract platform name from xpointer expression. + """ + + match = re.search(r"@name=['\"]([^'\"]+)['\"]", xpointer) + if match: + return match.group(1).strip() + return None + + + def derive_env_setup(self, compiler: dict): + """ + Build a generic environment setup list from compiler fields. + """ + + if not compiler: + return None + + env_setup = ["source $MODULESHOME/init/sh"] + + compiler_type = compiler.get("compilerType") + compiler_version = compiler.get("compilerVersion") + if compiler_type and compiler_version: + env_setup.append(f"module load {compiler_type}/{compiler_version}") + + return env_setup + + + def derive_mk_template(self, platform_element, compiler: dict): + """ + Build a best-effort mkTemplate path using platform and compiler context. + """ + + platform_name = self.get_attributes("name", platform_element) + if platform_name is None: + return None + + compiler_type = compiler.get("compilerType") if compiler else None + compiler_version = compiler.get("compilerVersion") if compiler else None + + if platform_name.startswith("ncrc") and compiler_type: + site = platform_name.split(".")[0] + compiler_stem = compiler_type.replace("_", "-") + return f"/ncrc/home2/fms/local/opt/fre-commands/bronx-23/site/{site}/{compiler_stem}.mk" + + if platform_name.startswith("hpcme") and compiler and compiler.get("compiler"): + major_version = "" + if compiler_version: + major_version = compiler_version.split(".")[0] + compiler_token = compiler.get("compiler") + major_version + return f"/apps/mkmf/templates/{platform_name.split('.')[0]}-{compiler_token}.mk" + + return None + + + def derive_model_root(self, platform_element, project: str, directory: dict | None): + """ + Build a generic modelRoot path for common site naming conventions. + """ + + platform_name = self.get_attributes("name", platform_element) + if platform_name is None: + return None + + stem = directory.get("stem") if directory else None + + if platform_name.startswith("ncrc5") and project and stem: + return f"/gpfs/f5/{project}/scratch/${{USER}}/{stem}" + + if platform_name.startswith("ncrc6") and project and stem: + return f"/gpfs/f6/{project}/scratch/${{USER}}/{stem}" + + if directory and directory.get("archive"): + return directory["archive"] + + return None + + + def derive_container_settings(self, platform_element): + """ + Add generic container settings for platform names that imply container workflow. + """ + + platform_name = self.get_attributes("name", platform_element) + if platform_name is None: + return {} + + if platform_name.startswith("hpcme"): + return { + "RUNenv": "", + "container": True, + "containerBuild": "podman", + "containerRun": "apptainer", + } + + return {} + + + def merge_platform_dicts(self, base: dict, override: dict): + """ + Merge dictionaries recursively, with override values taking precedence. + """ + + if base is None: + return override if override is not None else {} + if override is None: + return base + + merged = dict(base) + for key, value in override.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = self.merge_platform_dicts(merged[key], value) + else: + merged[key] = value + return merged + + + def drop_empty_fields(self, value): + """ + Remove empty values recursively while keeping boolean False and numeric 0. + """ + + if isinstance(value, dict): + cleaned = {} + for key, item in value.items(): + normalized_item = self.drop_empty_fields(item) + if normalized_item in (None, {}, []): + continue + cleaned[key] = normalized_item + return cleaned + + if isinstance(value, list): + cleaned = [self.drop_empty_fields(item) for item in value] + cleaned = [item for item in cleaned if item not in (None, {}, [])] + return cleaned + + if isinstance(value, str): + normalized = value.strip() + return normalized if normalized else None + + return value + + + def write_yaml(self, yamldict: dict, yamlfile: str | Path): + """ + Write YAML dictionary to file. + """ + + with open(yamlfile, "w", encoding="utf-8") as openedfile: + yaml.dump(yamldict, openedfile, sort_keys=False) + + +#xml = CompileConverter("./compile.xml") +#xml.convert() \ No newline at end of file diff --git a/fre/yamltools/converters/xml.py b/fre/yamltools/converters/xml.py index 2a21a39c6..c63b5be12 100644 --- a/fre/yamltools/converters/xml.py +++ b/fre/yamltools/converters/xml.py @@ -20,7 +20,7 @@ def __init__(self, xmlfile: str | Path): self.soup = BeautifulSoup(xml_content, "lxml-xml") - def get_attributes(self, attribute: str, element: dict, tolist: bool = False, fieldsep: str = ","): + def get_attributes(self, attribute: str, element: dict, tolist: bool = False, fieldsep: str = None): """ Get attribute value from an XML element. For example, @@ -31,6 +31,9 @@ def get_attributes(self, attribute: str, element: dict, tolist: bool = False, fi get_attributes("requires", element, tolist=True) returns ["fms", "rte-rrtmgp", "rte-ecckd"] """ + if element is None: + return None + value = element.get(attribute) if value is not None: @@ -51,17 +54,23 @@ def get_elements(self, element: str, xml_content, name: str = None, find_all: bo get_elements("component", soup, name="am5_phys") returns the element block with name="am5_phys" """ + if xml_content is None: + return None + search_dict = {} if name is not None: search_dict["name"] = name - find = xml_content.find_all if find_all else xml_content.find - element = find(element, search_dict) - - return element + if find_all: + search_results = xml_content.find_all(element, search_dict) + if search_results: + return search_results + return None + else: + return xml_content.find(element, search_dict) - def get_values(self, element, tolist: bool = False, fieldsep: str = ","): + def get_values(self, element, tolist: bool = False, fieldsep: str = None): """ Get value for an XML element. For example, @@ -69,12 +78,13 @@ def get_values(self, element, tolist: bool = False, fieldsep: str = ","): get_values(element, tolist=True) returns ["-DDEBUG", "-Iinclude"] """ + if element is None: + return None + value = element.text.strip() if value: if tolist: - if fieldsep is None: - return value.split() return [val.strip() for val in value.split(fieldsep)] return value From 9ca0ef00e90704517ccf47d92951c68438e8a099 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 07:43:14 -0400 Subject: [PATCH 05/22] update --- fre/yamltools/converters/converters.py | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/fre/yamltools/converters/converters.py b/fre/yamltools/converters/converters.py index a9de762bc..e3880ba32 100644 --- a/fre/yamltools/converters/converters.py +++ b/fre/yamltools/converters/converters.py @@ -2,10 +2,7 @@ import re import yaml -try: - from .xml import XML -except ImportError: - from xml import XML +from xml import XML class CompileConverter(XML): @@ -25,7 +22,7 @@ def get_experiments(self, experiment_name: str = None): Parse the XML and save each compile experiment block """ - #prettify name + # prettify name prettify = lambda name: name.strip().replace("$(", "").replace(")", "") # if a particular experiment is not provided, get all compile experiments in xml @@ -34,7 +31,10 @@ def get_experiments(self, experiment_name: str = None): if experiments: experiment_dict = {} for experiment in experiments: - name = prettify(self.get_attributes("name", experiment)) + raw_name = self.get_attributes("name", experiment) + if raw_name is None: + raise RuntimeError("Found element with no name attribute") + name = prettify(raw_name) experiment_dict[name] = experiment return experiment_dict @@ -47,11 +47,12 @@ def convert(self): """ for experiment_name, experiment_content in self.experiments.items(): + parsed_components = self.parse_components(experiment_content) or [] compile_yaml = {"compile": { "experiment": self.get_attributes("name", experiment_content), "container_addlibs": "", "baremetal_linkerflags": "", - "src": self.parse_components(experiment_content)} + "src": parsed_components} } self.yaml_dicts[experiment_name] = compile_yaml self.write_yaml(compile_yaml, "compile_"+experiment_name +".yaml") @@ -92,6 +93,9 @@ def parse_components(self, xml_content, component_name: str = None): parsed_components = [] for component in components: + component_name_value = self.get_attributes("name", component) + if component_name_value is None: + raise RuntimeError("Found element with no name attribute") codebase = self.get_elements("codeBase", component, find_all=False) source = self.get_elements("source", component, find_all=False) @@ -111,7 +115,7 @@ def parse_components(self, xml_content, component_name: str = None): "otherFlags": self.get_attributes("includeDir", component, tolist=True), "cppdefs": self.get_values(cppdefs), "makeOverrides": self.get_values(make_overrides), - "doF90Cpp": self.get_attributes("doF90Cpp", compile_), + "doF90Cpp": self.parse_bool(self.get_attributes("doF90Cpp", compile_)), "additionalInstructions": self.get_values(csh, tolist=True, fieldsep="\n") } @@ -122,6 +126,24 @@ def parse_components(self, xml_content, component_name: str = None): return parsed_components + @staticmethod + def parse_bool(value: str | None): + """ + Normalize common XML boolean-like strings to Python booleans. + """ + + if value is None: + return None + + normalized = value.strip().lower() + if normalized in {"yes", "true", "1", "on"}: + return True + if normalized in {"no", "false", "0", "off"}: + return False + + return value + + def write_yaml(self, yamldict: dict, yamlfile: str|Path): """ Write YAML dictionary to file From 61447e5a70f196a663a187c9b9a929e669abc60b Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 08:54:23 -0400 Subject: [PATCH 06/22] simpler --- fre/yamltools/converters/compile-converter.py | 174 ++++++ fre/yamltools/converters/convert.py | 58 -- fre/yamltools/converters/converters.py | 517 ------------------ fre/yamltools/converters/xml.py | 91 --- 4 files changed, 174 insertions(+), 666 deletions(-) create mode 100644 fre/yamltools/converters/compile-converter.py delete mode 100644 fre/yamltools/converters/convert.py delete mode 100644 fre/yamltools/converters/converters.py delete mode 100644 fre/yamltools/converters/xml.py diff --git a/fre/yamltools/converters/compile-converter.py b/fre/yamltools/converters/compile-converter.py new file mode 100644 index 000000000..03462b0e4 --- /dev/null +++ b/fre/yamltools/converters/compile-converter.py @@ -0,0 +1,174 @@ +import re + +import xml.etree.ElementTree as ET +import argparse +import yaml + +def parse_component(component: ET.Element) -> dict[str, any]: + """Parse a single XML element into a YAML-friendly dictionary.""" + + def get_compile_flag(flag: str) -> str | None: + """ + Return text for cppDefs or makeOverrides in compile element block + For example, + + -DDEBUG -Iinclude + -j8 + + returns + "-DDEBUG, -Iinclude" for get_compile_flag("cppDefs") + "-j8" for get_compile_flag("makeOverrides"), or + None if the flag is not defined. + """ + compile_elem = component.find('compile') + if compile_elem is not None: + elem = compile_elem.find(flag) + if elem is not None and elem.text: + return elem.text.strip() + return None + + def get_paths() -> list[str] | None: + """ + Return parsed component paths from the paths attribute. + For example, + + returns + ["am5_phys/atmos_param", "am5_phys/atmos_shared"] or + None if paths is not defined. + """ + paths = component.attrib.get('paths') + if paths and paths.strip(): + return [p.strip() for p in paths.split() if p.strip()] + return None + + def get_requires() -> list[str] | None: + """ + Return parsed component dependencies from the requires attribute. + For example, + + returns + ["FMS", "rte-rrtmpgp", "rte-ecckd"] or + None if requires is not defined. + """ + requires = component.attrib.get('requires') + if requires and requires.strip(): + return [r.strip() for r in requires.split() if r.strip()] + return None + + def get_doF90Cpp() -> bool | None: + """ + Parse doF90Cpp from the compile block into a boolean when present. + For example, + + returns True or + None if doF90Cpp is not defined. + """ + compile_elem = component.find('compile') + if compile_elem is not None: + val = compile_elem.attrib.get('doF90Cpp') + if val is not None: + return val.strip().lower() in {'yes', 'true', '1', 'on'} + return None + + def get_additional_instructions() -> list[str] | None: + """Extract source/csh instructions as non-empty lines.""" + source_elem = component.find('source') + if source_elem is not None: + csh_elem = source_elem.find('csh') + if csh_elem is not None and csh_elem.text: + cleaned_lines = [] + for line in csh_elem.text.splitlines(): + # Remove tabs and normalize extra spaces + clean_line = re.sub(r'\s+', ' ', line.replace('\t', '')).strip() + if clean_line: + cleaned_lines.append(clean_line) + return cleaned_lines if cleaned_lines else None + return None + + def get_repo_and_branch() -> tuple[str | None, str | None]: + """ + Build repo URL and branch/version from source/codeBase tags. + For example, + + FMS.git + + returns + ("https://github.com/NOAA-GFDL/FMS.git", "2026.01") + """ + + repo = None + branch = None + source_elem = component.find('source') + if source_elem is not None: + root = source_elem.attrib.get('root') + codebase_elem = source_elem.find('codeBase') + if root and codebase_elem is not None and codebase_elem.text: + repo = f"{root.rstrip('/')}/{codebase_elem.text.strip().lstrip('/')}" + branch = codebase_elem.attrib.get('version') + return repo, branch + + repo, branch = get_repo_and_branch() + component_name = component.attrib.get('name') + if component_name is not None: + component_name = component_name.strip() or None + + d = { + 'component': component_name, + 'repo': repo, + 'branch': branch, + 'paths': get_paths(), + 'requires': get_requires(), + 'cppdefs': get_compile_flag('cppDefs'), + 'makeOverrides': get_compile_flag('makeOverrides'), + 'doF90Cpp': get_doF90Cpp(), + 'additionalInstructions': get_additional_instructions(), + } + # Remove None values + return {k: v for k, v in d.items() if v is not None} + +def parse_experiment(experiment: ET.Element) -> Dict[str, Any]: + """Parse one element into the compile YAML object.""" + components = [parse_component(c) for c in experiment.findall('component')] + return { + 'experiment': experiment.attrib.get('name'), + 'container_addlibs': '', + 'baremetal_linkerflags': '', + 'src': components if components else [], + } + +def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): + """ + Convert compile XML to YAML. + All experiments in the XML will be converted if experiment_name is None + """ + tree = ET.parse(xml_path) + root = tree.getroot() + experiments = root.findall('experiment') + out = {} + for exp in experiments: + if experiment_name and exp.attrib.get('name') != experiment_name: + continue + yamldict = {'compile': parse_experiment(exp)} + if experiment_name: + break + with open(yaml_path, 'w', encoding='utf-8') as f: + yaml.dump(yamldict, f, sort_keys=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert compile XML to YAML") + parser.add_argument("-x", "--xmlfile", required=True, help="Input XML file") + parser.add_argument("-o", "--output", required=True, help="Output YAML file") + parser.add_argument("-e", "--experiment", default=None, help="Experiment name (optional)") + args = parser.parse_args() + + xml_to_yaml(args.xmlfile, args.output, args.experiment) + print(f"\nConverted {args.xmlfile} to {args.output}.") + print(f"Experiment: {args.experiment if args.experiment else 'All experiments'}") + print("_______________") + print("WARNING: THIS CONVERTER OUTPUTS CLOSE-ENOUGH COMPILE YAML") + print("PLEASE CHECK THE FOLLOWING:") + print(" * PATHS") + print(" * CPPDEFS AND OTHER FLAGS") + print(" * ADDITIONALINSTRUCTIONS") + print(" * PLEASE ADD IN THE APPROPRIATE ANCHORS") diff --git a/fre/yamltools/converters/convert.py b/fre/yamltools/converters/convert.py deleted file mode 100644 index d756e78f7..000000000 --- a/fre/yamltools/converters/convert.py +++ /dev/null @@ -1,58 +0,0 @@ -import click -from fre.yamltools.converters.converters import CompileConverter, PlatformConverter - -@click.command("convert", short_help="tool to convert compile and platform xmls to yamls") -@click.option( - "--xmlfile", "-x", - type=str, - required=True, - help="input XML filename" -) -@click.option( - "--experiment-name", "-e", - type=str, - default=None, - help=""" - experiment name corresponding to the "name" attribute in the xml element - If provided, only the xml body under experiment-name will be converted. - If not provided, all experiments in xmlfile will be converted. - """ -) -@click.option( - "--convert-compile-xml", "-c", - is_flag=True, - default=False, - help=""" - Flag to only convert compile xml to yaml. - If not provided, convert will attempt to convert both compile and platform xmls to platforms. - """ -) -@click.option( - "--convert-platform-xml", "-p", - is_flag=True, - default=False, - help=""" - Flag to only convert platform xml to yaml. - If not provided, convert will attempt to convert both compile and platform xmls to platforms. - """ -) -@click.pass_context -def convert(ctx, xmlfile, experiment_name, convert_compile_xml, convert_platform_xml): - - """ - Click command to convert compile and platform XML files to YAML files. - """ - - convert_both = not convert_compile_xml and not convert_platform_xml - - if convert_compile_xml or convert_both: - compilexml = CompileConverter(xmlfile=xmlfile, experiment_name=experiment_name) - compilexml.convert() - - if convert_platform_xml or convert_both: - platformxml = PlatformConverter(xmlfile=xmlfile) - platformxml.convert() - - -if __name__ == "__main__": - convert() diff --git a/fre/yamltools/converters/converters.py b/fre/yamltools/converters/converters.py deleted file mode 100644 index e3880ba32..000000000 --- a/fre/yamltools/converters/converters.py +++ /dev/null @@ -1,517 +0,0 @@ -from pathlib import Path -import re -import yaml - -from xml import XML - - -class CompileConverter(XML): - """ - Converter class to parse compile experiment XML blocks and convert to YAML dictionaries. - """ - - def __init__(self, xmlfile: str|Path, experiment_name: str = None): - - super().__init__(xmlfile) - self.experiments = self.get_experiments(experiment_name) - self.yaml_dicts = {} - - - def get_experiments(self, experiment_name: str = None): - """ - Parse the XML and save each compile experiment block - """ - - # prettify name - prettify = lambda name: name.strip().replace("$(", "").replace(")", "") - - # if a particular experiment is not provided, get all compile experiments in xml - experiments = self.get_elements("experiment", self.soup, name=experiment_name) - - if experiments: - experiment_dict = {} - for experiment in experiments: - raw_name = self.get_attributes("name", experiment) - if raw_name is None: - raise RuntimeError("Found element with no name attribute") - name = prettify(raw_name) - experiment_dict[name] = experiment - return experiment_dict - - raise RuntimeError("Cannot find experiments") - - - def convert(self): - """ - Convert XML content to YAML dictionary - """ - - for experiment_name, experiment_content in self.experiments.items(): - parsed_components = self.parse_components(experiment_content) or [] - compile_yaml = {"compile": { - "experiment": self.get_attributes("name", experiment_content), - "container_addlibs": "", - "baremetal_linkerflags": "", - "src": parsed_components} - } - self.yaml_dicts[experiment_name] = compile_yaml - self.write_yaml(compile_yaml, "compile_"+experiment_name +".yaml") - - - def parse_components(self, xml_content, component_name: str = None): - """ - Parses component blocks to yaml dictionaries. For example, converts - - - - GFDL_atmos_cubed_sphere.git - - - USE_R4=$(USE_MIXED_MODE) ISA="-march=core-avx2 -qno-opt-dynamic-align" - $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE - - - - to - - { - component: atmos_cubed_sphere - repo: https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git - branch: 2024.01_am5 - paths: [GFDL_atmos_cubed_sphere/model, GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90] - requires: [fms, am5_phys] - otherFlags: [$(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE] - makeOverrides : [USE_R4=$(USE_MIXED_MODE), ISA="-march=core-avx2, -qno-opt-dynamic-align"] - } - """ - - components = self.get_elements("component", xml_content, name=component_name) - if components is None: - return None - - parsed_components = [] - for component in components: - component_name_value = self.get_attributes("name", component) - if component_name_value is None: - raise RuntimeError("Found element with no name attribute") - - codebase = self.get_elements("codeBase", component, find_all=False) - source = self.get_elements("source", component, find_all=False) - cppdefs = self.get_elements("cppDefs", component, find_all=False) - compile_ = self.get_elements("compile", component, find_all=False) - make_overrides = self.get_elements("makeOverrides", component, find_all=False) - csh = self.get_elements("csh", component, find_all=False) - - #"repo": f"{self.get_attributes('root', source)}/{self.get_values(codebase)}", - - component_yaml = { - "component": self.get_attributes("name", component), - "repo": f"{self.get_attributes('root', source)}/{self.get_values(codebase)}", - "branch": self.get_attributes("version", codebase), - "paths": self.get_attributes("paths", component, tolist=True), - "requires": self.get_attributes("requires", component, tolist=True), - "otherFlags": self.get_attributes("includeDir", component, tolist=True), - "cppdefs": self.get_values(cppdefs), - "makeOverrides": self.get_values(make_overrides), - "doF90Cpp": self.parse_bool(self.get_attributes("doF90Cpp", compile_)), - "additionalInstructions": self.get_values(csh, tolist=True, fieldsep="\n") - } - - #remove None - component_yaml = {key: value for key, value in component_yaml.items() if value is not None} - parsed_components.append(component_yaml) - - return parsed_components - - - @staticmethod - def parse_bool(value: str | None): - """ - Normalize common XML boolean-like strings to Python booleans. - """ - - if value is None: - return None - - normalized = value.strip().lower() - if normalized in {"yes", "true", "1", "on"}: - return True - if normalized in {"no", "false", "0", "off"}: - return False - - return value - - - def write_yaml(self, yamldict: dict, yamlfile: str|Path): - """ - Write YAML dictionary to file - """ - - with open(yamlfile, "w", encoding="utf-8") as openedfile: - yaml.dump(yamldict, openedfile, sort_keys=False) - - -class PlatformConverter(XML): - """ - Converter class to parse platform XML blocks and convert to YAML dictionaries. - """ - - def __init__(self, xmlfile: str | Path): - - super().__init__(xmlfile) - self.platforms = self.get_platforms() - self.yaml_dicts = {} - - - def get_platforms(self): - """ - Parse XML and save each platform block in declaration order. - """ - - platforms = self.get_elements("platform", self.soup) - if platforms is None: - raise RuntimeError("Cannot find platforms") - - platform_dict = {} - for platform in platforms: - platform_name = self.get_attributes("name", platform) - if platform_name is None: - raise RuntimeError("Found element with no name attribute") - if platform_name in platform_dict: - raise RuntimeError(f"Duplicate platform name found: {platform_name}") - platform_dict[platform_name] = platform - - return platform_dict - - - def convert(self): - """ - Convert all XML platform content to one YAML dictionary. - """ - - parsed_platforms = [] - for platform_name in self.platforms: - parsed_platforms.append(self.parse_platform(platform_name)) - - self.yaml_dicts = {"platforms": parsed_platforms} - self.write_yaml(self.yaml_dicts, "platforms.yaml") - - - def parse_platform(self, platform_name: str, visited: set | None = None): - """ - Parse one platform, resolving xi:include references as base content first, - then applying local values as overrides. - """ - - if visited is None: - visited = set() - - if platform_name in visited: - raise RuntimeError(f"Circular xi:include reference detected for {platform_name}") - - if platform_name not in self.platforms: - raise RuntimeError(f"Cannot resolve xi:include reference: {platform_name}") - - visited.add(platform_name) - platform_element = self.platforms[platform_name] - - platform_yaml = {"name": platform_name} - - include_names = self.parse_xincludes(platform_element) - unresolved_includes = [] - for include_name in include_names: - if include_name in self.platforms: - included_platform = self.parse_platform(include_name, visited=visited.copy()) - platform_yaml = self.merge_platform_dicts(platform_yaml, included_platform) - else: - unresolved_includes.append(include_name) - - current_platform = self.parse_platform_body(platform_element) - platform_yaml = self.merge_platform_dicts(platform_yaml, current_platform) - - if include_names: - platform_yaml["includes"] = include_names - if unresolved_includes: - platform_yaml["unresolvedIncludes"] = unresolved_includes - - return self.drop_empty_fields(platform_yaml) - - - def parse_platform_body(self, platform_element): - """ - Parse direct content under a platform block (without include inheritance). - """ - - #fre_version = self.get_values(self.get_elements("freVersion", platform_element, find_all=False)) - project = self.get_values(self.get_elements("project", platform_element, find_all=False)) - compiler = self.parse_compiler(platform_element) - directory = self.parse_directory(platform_element) - properties = self.parse_properties(platform_element) - csh = self.parse_text_block(platform_element, "csh") - - platform_yaml = { - "freVersion": fre_version, - "project": project, - "compiler": compiler.get("compiler"), - "compilerVersion": compiler.get("compilerVersion"), - "compilerType": compiler.get("compilerType"), - "envSetup": self.derive_env_setup(compiler), - "mkTemplate": self.derive_mk_template(platform_element, compiler), - "modelRoot": self.derive_model_root(platform_element, project, directory), - "directory": directory, - "properties": properties, - "csh": csh, - } - - # Preserve known convenience fields used by platform YAML workflows. - platform_yaml.update(self.derive_container_settings(platform_element)) - - return self.drop_empty_fields(platform_yaml) - - - def parse_compiler(self, platform_element): - """ - Parse compiler attributes and normalize a friendly compiler name. - """ - - compiler_element = self.get_elements("compiler", platform_element, find_all=False) - compiler_type = self.get_attributes("type", compiler_element) - compiler_version = self.get_attributes("version", compiler_element) - - compiler_name = None - if compiler_type: - compiler_name = compiler_type.replace("_", "-").split("-")[0].strip() - - return self.drop_empty_fields({ - "compiler": compiler_name, - "compilerType": compiler_type, - "compilerVersion": compiler_version, - }) - - - def parse_directory(self, platform_element): - """ - Parse directory subfields under . - """ - - directory_element = self.get_elements("directory", platform_element, find_all=False) - if directory_element is None: - return None - - directory = { - "stem": self.get_attributes("stem", directory_element), - "include": self.get_values(self.get_elements("include", directory_element, find_all=False)), - "archive": self.get_values(self.get_elements("archive", directory_element, find_all=False)), - "analysis": self.get_values(self.get_elements("analysis", directory_element, find_all=False)), - } - - return self.drop_empty_fields(directory) - - - def parse_properties(self, platform_element): - """ - Parse all blocks into a dictionary. - """ - - properties = self.get_elements("property", platform_element) - if properties is None: - return None - - parsed_properties = {} - for property_element in properties: - property_name = self.get_attributes("name", property_element) - property_value = self.get_attributes("value", property_element) - - if property_name is not None: - parsed_properties[property_name] = property_value if property_value is not None else "" - - return parsed_properties if parsed_properties else None - - - def parse_text_block(self, platform_element, tag_name: str): - """ - Parse free-form text blocks such as . - """ - - return self.get_values(self.get_elements(tag_name, platform_element, find_all=False)) - - - def parse_xincludes(self, platform_element): - """ - Parse xi:include references and extract referenced platform names. - """ - - include_elements = self.get_elements("xi:include", platform_element) - if include_elements is None: - return [] - - include_names = [] - for include_element in include_elements: - xpointer = self.get_attributes("xpointer", include_element) - if xpointer is None: - continue - - include_name = self.extract_platform_name_from_xpointer(xpointer) - if include_name is not None: - include_names.append(include_name) - - return include_names - - - def extract_platform_name_from_xpointer(self, xpointer: str): - """ - Extract platform name from xpointer expression. - """ - - match = re.search(r"@name=['\"]([^'\"]+)['\"]", xpointer) - if match: - return match.group(1).strip() - return None - - - def derive_env_setup(self, compiler: dict): - """ - Build a generic environment setup list from compiler fields. - """ - - if not compiler: - return None - - env_setup = ["source $MODULESHOME/init/sh"] - - compiler_type = compiler.get("compilerType") - compiler_version = compiler.get("compilerVersion") - if compiler_type and compiler_version: - env_setup.append(f"module load {compiler_type}/{compiler_version}") - - return env_setup - - - def derive_mk_template(self, platform_element, compiler: dict): - """ - Build a best-effort mkTemplate path using platform and compiler context. - """ - - platform_name = self.get_attributes("name", platform_element) - if platform_name is None: - return None - - compiler_type = compiler.get("compilerType") if compiler else None - compiler_version = compiler.get("compilerVersion") if compiler else None - - if platform_name.startswith("ncrc") and compiler_type: - site = platform_name.split(".")[0] - compiler_stem = compiler_type.replace("_", "-") - return f"/ncrc/home2/fms/local/opt/fre-commands/bronx-23/site/{site}/{compiler_stem}.mk" - - if platform_name.startswith("hpcme") and compiler and compiler.get("compiler"): - major_version = "" - if compiler_version: - major_version = compiler_version.split(".")[0] - compiler_token = compiler.get("compiler") + major_version - return f"/apps/mkmf/templates/{platform_name.split('.')[0]}-{compiler_token}.mk" - - return None - - - def derive_model_root(self, platform_element, project: str, directory: dict | None): - """ - Build a generic modelRoot path for common site naming conventions. - """ - - platform_name = self.get_attributes("name", platform_element) - if platform_name is None: - return None - - stem = directory.get("stem") if directory else None - - if platform_name.startswith("ncrc5") and project and stem: - return f"/gpfs/f5/{project}/scratch/${{USER}}/{stem}" - - if platform_name.startswith("ncrc6") and project and stem: - return f"/gpfs/f6/{project}/scratch/${{USER}}/{stem}" - - if directory and directory.get("archive"): - return directory["archive"] - - return None - - - def derive_container_settings(self, platform_element): - """ - Add generic container settings for platform names that imply container workflow. - """ - - platform_name = self.get_attributes("name", platform_element) - if platform_name is None: - return {} - - if platform_name.startswith("hpcme"): - return { - "RUNenv": "", - "container": True, - "containerBuild": "podman", - "containerRun": "apptainer", - } - - return {} - - - def merge_platform_dicts(self, base: dict, override: dict): - """ - Merge dictionaries recursively, with override values taking precedence. - """ - - if base is None: - return override if override is not None else {} - if override is None: - return base - - merged = dict(base) - for key, value in override.items(): - if isinstance(value, dict) and isinstance(merged.get(key), dict): - merged[key] = self.merge_platform_dicts(merged[key], value) - else: - merged[key] = value - return merged - - - def drop_empty_fields(self, value): - """ - Remove empty values recursively while keeping boolean False and numeric 0. - """ - - if isinstance(value, dict): - cleaned = {} - for key, item in value.items(): - normalized_item = self.drop_empty_fields(item) - if normalized_item in (None, {}, []): - continue - cleaned[key] = normalized_item - return cleaned - - if isinstance(value, list): - cleaned = [self.drop_empty_fields(item) for item in value] - cleaned = [item for item in cleaned if item not in (None, {}, [])] - return cleaned - - if isinstance(value, str): - normalized = value.strip() - return normalized if normalized else None - - return value - - - def write_yaml(self, yamldict: dict, yamlfile: str | Path): - """ - Write YAML dictionary to file. - """ - - with open(yamlfile, "w", encoding="utf-8") as openedfile: - yaml.dump(yamldict, openedfile, sort_keys=False) - - -#xml = CompileConverter("./compile.xml") -#xml.convert() \ No newline at end of file diff --git a/fre/yamltools/converters/xml.py b/fre/yamltools/converters/xml.py deleted file mode 100644 index c63b5be12..000000000 --- a/fre/yamltools/converters/xml.py +++ /dev/null @@ -1,91 +0,0 @@ -from pathlib import Path - -from bs4 import BeautifulSoup - - -class XML: - """ - Base class to parse XML files and extract elements and key values using BeautifulSoup. - """ - - def __init__(self, xmlfile: str | Path): - - self.xmlfile = Path(xmlfile) - if not self.xmlfile.exists(): - raise IOError(f"Cannot find XML file {xmlfile}") - - with open(xmlfile, "r", encoding="utf-8") as openedfile: - xml_content = openedfile.read() - - self.soup = BeautifulSoup(xml_content, "lxml-xml") - - - def get_attributes(self, attribute: str, element: dict, tolist: bool = False, fieldsep: str = None): - """ - Get attribute value from an XML element. For example, - - element = - get_attributes("root", element) returns "http://github.com/NOAA-GFDL" - - element = - get_attributes("requires", element, tolist=True) returns ["fms", "rte-rrtmgp", "rte-ecckd"] - """ - - if element is None: - return None - - value = element.get(attribute) - - if value is not None: - value = value.strip() - if value: - if tolist: - return [val.strip() for val in value.split(fieldsep)] - return value - - return None - - - def get_elements(self, element: str, xml_content, name: str = None, find_all: bool = True): - """ - Get XML elements by tag name and optional attribute value. For example, - - get_elements("component", soup) returns all blocks in the XML. - get_elements("component", soup, name="am5_phys") returns the element block with name="am5_phys" - """ - - if xml_content is None: - return None - - search_dict = {} - if name is not None: - search_dict["name"] = name - - if find_all: - search_results = xml_content.find_all(element, search_dict) - if search_results: - return search_results - return None - else: - return xml_content.find(element, search_dict) - - - def get_values(self, element, tolist: bool = False, fieldsep: str = None): - """ - Get value for an XML element. For example, - - element = -DDEBUG -Iinclude, - get_values(element, tolist=True) returns ["-DDEBUG", "-Iinclude"] - """ - - if element is None: - return None - - value = element.text.strip() - - if value: - if tolist: - return [val.strip() for val in value.split(fieldsep)] - return value - - return None \ No newline at end of file From 7ae7210e90c33b7361a003b36587965dd5d7ef93 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 09:06:30 -0400 Subject: [PATCH 07/22] add test --- fre/yamltools/converters/test.py | 611 +++++++++++++++++++++++++++++++ 1 file changed, 611 insertions(+) create mode 100644 fre/yamltools/converters/test.py diff --git a/fre/yamltools/converters/test.py b/fre/yamltools/converters/test.py new file mode 100644 index 000000000..e68f18812 --- /dev/null +++ b/fre/yamltools/converters/test.py @@ -0,0 +1,611 @@ +import pytest +import tempfile +import os +import yaml +import xml.etree.ElementTree as ET +from pathlib import Path +import importlib.util + +# Import functions from compile-converter (hyphenated module name) +spec = importlib.util.spec_from_file_location("compile_converter", "compile-converter.py") +compile_converter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(compile_converter) + +parse_component = compile_converter.parse_component +parse_experiment = compile_converter.parse_experiment +xml_to_yaml = compile_converter.xml_to_yaml + + +class TestGetCompileFlag: + """Test get_compile_flag nested function via parse_component.""" + + def test_cpp_defs_present(self): + """Test parsing cppDefs flag from compile block.""" + xml_str = ''' + + + -DDEBUG -Iinclude + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['cppdefs'] == '-DDEBUG -Iinclude' + + def test_make_overrides_present(self): + """Test parsing makeOverrides flag from compile block.""" + xml_str = ''' + + + -j8 + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['makeOverrides'] == '-j8' + + def test_flag_not_defined(self): + """Test when compile flag is not defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'cppdefs' not in result # Should be filtered out + assert 'makeOverrides' not in result + + def test_flag_with_whitespace(self): + """Test flag with leading/trailing whitespace.""" + xml_str = ''' + + + -DDEBUG -Iinclude + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['cppdefs'] == '-DDEBUG -Iinclude' + + +class TestGetPaths: + """Test get_paths nested function via parse_component.""" + + def test_single_path(self): + """Test parsing single path.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['am5_phys/atmos_param'] + + def test_multiple_paths(self): + """Test parsing multiple space-separated paths.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['am5_phys/atmos_param', 'am5_phys/atmos_shared'] + + def test_paths_with_extra_whitespace(self): + """Test parsing paths with extra whitespace.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['path1', 'path2', 'path3'] + + def test_paths_not_defined(self): + """Test when paths attribute is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'paths' not in result + + def test_empty_paths_string(self): + """Test when paths is empty string.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'paths' not in result + + +class TestGetRequires: + """Test get_requires nested function via parse_component.""" + + def test_single_requirement(self): + """Test parsing single requirement.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['FMS'] + + def test_multiple_requirements(self): + """Test parsing multiple space-separated requirements.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['FMS', 'rte-rrtmpgp', 'rte-ecckd'] + + def test_requires_with_extra_whitespace(self): + """Test parsing requires with extra whitespace.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['req1', 'req2', 'req3'] + + def test_requires_not_defined(self): + """Test when requires attribute is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'requires' not in result + + def test_empty_requires_string(self): + """Test when requires is empty string.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'requires' not in result + + +class TestGetDoF90Cpp: + """Test get_doF90Cpp nested function via parse_component.""" + + def test_doF90Cpp_yes(self): + """Test parsing doF90Cpp as 'yes'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is True + + def test_doF90Cpp_no(self): + """Test parsing doF90Cpp as 'no'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + # False values should be included (only None values are filtered) + assert result['doF90Cpp'] is False + + def test_doF90Cpp_true(self): + """Test parsing doF90Cpp as 'true'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is True + + def test_doF90Cpp_one(self): + """Test parsing doF90Cpp as '1'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is True + + def test_doF90Cpp_on(self): + """Test parsing doF90Cpp as 'on'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is True + + def test_doF90Cpp_not_defined(self): + """Test when doF90Cpp is not defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'doF90Cpp' not in result + + +class TestGetAdditionalInstructions: + """Test get_additional_instructions nested function via parse_component.""" + + def test_single_instruction(self): + """Test parsing single instruction.""" + xml_str = ''' + + + echo "Building" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "Building"'] + + def test_multiple_instructions(self): + """Test parsing multiple instructions across lines.""" + xml_str = ''' + + + +echo "Line 1" +echo "Line 2" + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "Line 1"', 'echo "Line 2"'] + + def test_instructions_with_tabs(self): + """Test parsing instructions with tabs (should be normalized).""" + xml_str = ''' + + + echo "test" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + # Tabs should be removed and multiple spaces normalized to single space + assert result['additionalInstructions'] == ['echo "test"'] + + def test_instructions_with_extra_spaces(self): + """Test parsing instructions with extra spaces.""" + xml_str = ''' + + + echo "extra" spaces + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "extra" spaces'] + + def test_no_instructions(self): + """Test when no instructions are defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'additionalInstructions' not in result + + +class TestGetRepoAndBranch: + """Test get_repo_and_branch nested function via parse_component.""" + + def test_valid_repo_and_branch(self): + """Test parsing valid repo URL and branch.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + assert result['branch'] == '2026.01' + + def test_repo_with_trailing_slash(self): + """Test repo root with trailing slash.""" + xml_str = ''' + + + repo.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' + + def test_codebse_with_leading_slash(self): + """Test codeBase with leading slash.""" + xml_str = ''' + + + /repo.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' + + def test_codebase_with_whitespace(self): + """Test codeBase with surrounding whitespace.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + + def test_no_source_element(self): + """Test when source element is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + def test_no_codebase_element(self): + """Test when codeBase element is missing.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + def test_no_root_attribute(self): + """Test when root attribute is missing.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + + +class TestParseComponent: + """Test parse_component main function.""" + + def test_complete_component(self): + """Test parsing a complete component with all attributes.""" + xml_str = ''' + + + -DDEBUG + -j8 + + + FMS.git + echo "building" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'am5_phys' + assert result['paths'] == ['path1', 'path2'] + assert result['requires'] == ['FMS'] + assert result['cppdefs'] == '-DDEBUG' + assert result['makeOverrides'] == '-j8' + assert result['doF90Cpp'] is True + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + assert result['branch'] == '2026.01' + assert result['additionalInstructions'] == ['echo "building"'] + + def test_minimal_component(self): + """Test parsing minimal component with only name.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'minimal' + assert len(result) == 1 # Only component field + + def test_component_without_name(self): + """Test parsing component without name attribute.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result == {} # All None values filtered out + + def test_component_name_with_whitespace(self): + """Test component name with whitespace.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'test_name' + + def test_none_values_filtered(self): + """Test that None values are filtered from output.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + # Verify no None values exist in result + for v in result.values(): + assert v is not None + + +class TestParseExperiment: + """Test parse_experiment function.""" + + def test_experiment_with_components(self): + """Test parsing experiment with multiple components.""" + xml_str = ''' + + + + + ''' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] == 'test_exp' + assert result['container_addlibs'] == '' + assert result['baremetal_linkerflags'] == '' + assert len(result['src']) == 2 + assert result['src'][0]['component'] == 'comp1' + assert result['src'][1]['component'] == 'comp2' + + def test_experiment_without_components(self): + """Test parsing experiment without components.""" + xml_str = '' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] == 'empty_exp' + assert result['src'] == [] + + def test_experiment_without_name(self): + """Test parsing experiment without name attribute.""" + xml_str = ''' + + + + ''' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] is None + assert len(result['src']) == 1 + + +class TestXmlToYaml: + """Test xml_to_yaml main conversion function.""" + + def test_basic_xml_to_yaml_conversion(self): + """Test basic XML to YAML conversion.""" + xml_content = ''' + + + + + + ''' + + with tempfile.TemporaryDirectory() as tmpdir: + xml_file = os.path.join(tmpdir, 'test.xml') + yaml_file = os.path.join(tmpdir, 'test.yaml') + + with open(xml_file, 'w') as f: + f.write(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert os.path.exists(yaml_file) + with open(yaml_file, 'r') as f: + data = yaml.safe_load(f) + + assert 'compile' in data + assert data['compile']['experiment'] == 'exp1' + assert len(data['compile']['src']) == 1 + + def test_xml_to_yaml_with_specific_experiment(self): + """Test XML to YAML conversion with specific experiment filter.""" + xml_content = ''' + + + + + + + + + ''' + + with tempfile.TemporaryDirectory() as tmpdir: + xml_file = os.path.join(tmpdir, 'test.xml') + yaml_file = os.path.join(tmpdir, 'test.yaml') + + with open(xml_file, 'w') as f: + f.write(xml_content) + + xml_to_yaml(xml_file, yaml_file, experiment_name='exp2') + + with open(yaml_file, 'r') as f: + data = yaml.safe_load(f) + + assert data['compile']['experiment'] == 'exp2' + assert len(data['compile']['src']) == 1 + assert data['compile']['src'][0]['component'] == 'comp2' + + def test_xml_to_yaml_multiple_components(self): + """Test XML to YAML with multiple components.""" + xml_content = ''' + + + + + + + + ''' + + with tempfile.TemporaryDirectory() as tmpdir: + xml_file = os.path.join(tmpdir, 'test.xml') + yaml_file = os.path.join(tmpdir, 'test.yaml') + + with open(xml_file, 'w') as f: + f.write(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + with open(yaml_file, 'r') as f: + data = yaml.safe_load(f) + + assert len(data['compile']['src']) == 3 + assert data['compile']['src'][0]['component'] == 'comp1' + assert data['compile']['src'][1]['component'] == 'comp2' + assert data['compile']['src'][2]['component'] == 'comp3' + + def test_xml_to_yaml_nonexistent_file(self): + """Test error handling for non-existent XML file.""" + with pytest.raises(FileNotFoundError): + xml_to_yaml('/nonexistent/path/test.xml', '/tmp/output.yaml') + + def test_xml_to_yaml_invalid_xml(self): + """Test error handling for invalid XML.""" + with tempfile.TemporaryDirectory() as tmpdir: + xml_file = os.path.join(tmpdir, 'invalid.xml') + yaml_file = os.path.join(tmpdir, 'output.yaml') + + with open(xml_file, 'w') as f: + f.write('This is not well-formed XML + + + + + ''' + + with tempfile.TemporaryDirectory() as tmpdir: + xml_file = os.path.join(tmpdir, 'test.xml') + yaml_file = os.path.join(tmpdir, 'test.yaml') + + with open(xml_file, 'w') as f: + f.write(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert Path(yaml_file).exists() + assert os.path.getsize(yaml_file) > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 7ed694e3c3370d113986f708bb18bf303dff65d0 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 09:59:39 -0400 Subject: [PATCH 08/22] update test --- fre/yamltools/converters/compile-converter.py | 6 +- fre/yamltools/converters/test.py | 149 ++---------------- pyproject.toml | 1 + 3 files changed, 19 insertions(+), 137 deletions(-) diff --git a/fre/yamltools/converters/compile-converter.py b/fre/yamltools/converters/compile-converter.py index 03462b0e4..09ae544f3 100644 --- a/fre/yamltools/converters/compile-converter.py +++ b/fre/yamltools/converters/compile-converter.py @@ -66,8 +66,10 @@ def get_doF90Cpp() -> bool | None: compile_elem = component.find('compile') if compile_elem is not None: val = compile_elem.attrib.get('doF90Cpp') + map_to_bool = {'yes': True, 'no': False} if val is not None: - return val.strip().lower() in {'yes', 'true', '1', 'on'} + return map_to_bool.get(val.strip().lower()) + return None def get_additional_instructions() -> list[str] | None: @@ -103,7 +105,7 @@ def get_repo_and_branch() -> tuple[str | None, str | None]: root = source_elem.attrib.get('root') codebase_elem = source_elem.find('codeBase') if root and codebase_elem is not None and codebase_elem.text: - repo = f"{root.rstrip('/')}/{codebase_elem.text.strip().lstrip('/')}" + repo = f"{root.rstrip('/')}/{codebase_elem.text.strip().strip()}" branch = codebase_elem.attrib.get('version') return repo, branch diff --git a/fre/yamltools/converters/test.py b/fre/yamltools/converters/test.py index e68f18812..523ed6872 100644 --- a/fre/yamltools/converters/test.py +++ b/fre/yamltools/converters/test.py @@ -24,7 +24,7 @@ def test_cpp_defs_present(self): xml_str = ''' - -DDEBUG -Iinclude + -DDEBUG -Iinclude ''' @@ -53,19 +53,6 @@ def test_flag_not_defined(self): assert 'cppdefs' not in result # Should be filtered out assert 'makeOverrides' not in result - def test_flag_with_whitespace(self): - """Test flag with leading/trailing whitespace.""" - xml_str = ''' - - - -DDEBUG -Iinclude - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['cppdefs'] == '-DDEBUG -Iinclude' - class TestGetPaths: """Test get_paths nested function via parse_component.""" @@ -79,18 +66,11 @@ def test_single_path(self): def test_multiple_paths(self): """Test parsing multiple space-separated paths.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['paths'] == ['am5_phys/atmos_param', 'am5_phys/atmos_shared'] - - def test_paths_with_extra_whitespace(self): - """Test parsing paths with extra whitespace.""" - xml_str = '' + xml_str = '' component = ET.fromstring(xml_str) result = parse_component(component) - assert result['paths'] == ['path1', 'path2', 'path3'] - + assert result['paths'] == ['am5_phys/atmos_param', 'am5_phys/atmos_shared', "am5_phys/madeup"] + def test_paths_not_defined(self): """Test when paths attribute is missing.""" xml_str = '' @@ -118,18 +98,11 @@ def test_single_requirement(self): def test_multiple_requirements(self): """Test parsing multiple space-separated requirements.""" - xml_str = '' + xml_str = '' component = ET.fromstring(xml_str) result = parse_component(component) assert result['requires'] == ['FMS', 'rte-rrtmpgp', 'rte-ecckd'] - - def test_requires_with_extra_whitespace(self): - """Test parsing requires with extra whitespace.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['requires'] == ['req1', 'req2', 'req3'] - + def test_requires_not_defined(self): """Test when requires attribute is missing.""" xml_str = '' @@ -168,42 +141,8 @@ def test_doF90Cpp_no(self): ''' component = ET.fromstring(xml_str) result = parse_component(component) - # False values should be included (only None values are filtered) assert result['doF90Cpp'] is False - - def test_doF90Cpp_true(self): - """Test parsing doF90Cpp as 'true'.""" - xml_str = ''' - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['doF90Cpp'] is True - - def test_doF90Cpp_one(self): - """Test parsing doF90Cpp as '1'.""" - xml_str = ''' - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['doF90Cpp'] is True - - def test_doF90Cpp_on(self): - """Test parsing doF90Cpp as 'on'.""" - xml_str = ''' - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['doF90Cpp'] is True - + def test_doF90Cpp_not_defined(self): """Test when doF90Cpp is not defined.""" xml_str = '' @@ -234,8 +173,8 @@ def test_multiple_instructions(self): -echo "Line 1" -echo "Line 2" + echo "Line 1" + echo "Line 2" @@ -243,34 +182,7 @@ def test_multiple_instructions(self): component = ET.fromstring(xml_str) result = parse_component(component) assert result['additionalInstructions'] == ['echo "Line 1"', 'echo "Line 2"'] - - def test_instructions_with_tabs(self): - """Test parsing instructions with tabs (should be normalized).""" - xml_str = ''' - - - echo "test" - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - # Tabs should be removed and multiple spaces normalized to single space - assert result['additionalInstructions'] == ['echo "test"'] - - def test_instructions_with_extra_spaces(self): - """Test parsing instructions with extra spaces.""" - xml_str = ''' - - - echo "extra" spaces - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['additionalInstructions'] == ['echo "extra" spaces'] - + def test_no_instructions(self): """Test when no instructions are defined.""" xml_str = '' @@ -287,7 +199,7 @@ def test_valid_repo_and_branch(self): xml_str = ''' - FMS.git + FMS.git ''' @@ -308,33 +220,7 @@ def test_repo_with_trailing_slash(self): component = ET.fromstring(xml_str) result = parse_component(component) assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' - - def test_codebse_with_leading_slash(self): - """Test codeBase with leading slash.""" - xml_str = ''' - - - /repo.git - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' - - def test_codebase_with_whitespace(self): - """Test codeBase with surrounding whitespace.""" - xml_str = ''' - - - FMS.git - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' - + def test_no_source_element(self): """Test when source element is missing.""" xml_str = '' @@ -367,6 +253,7 @@ def test_no_root_attribute(self): component = ET.fromstring(xml_str) result = parse_component(component) assert 'repo' not in result + assert 'branch' not in result class TestParseComponent: @@ -375,7 +262,7 @@ class TestParseComponent: def test_complete_component(self): """Test parsing a complete component with all attributes.""" xml_str = ''' - + -DDEBUG -j8 @@ -416,14 +303,6 @@ def test_component_without_name(self): assert result == {} # All None values filtered out - def test_component_name_with_whitespace(self): - """Test component name with whitespace.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - - assert result['component'] == 'test_name' - def test_none_values_filtered(self): """Test that None values are filtered from output.""" xml_str = '' diff --git a/pyproject.toml b/pyproject.toml index 2664168dd..549bcae56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ keywords = [ dependencies = [ 'analysis_scripts', + 'beautifulsoup4', 'catalogbuilder', 'cdo', 'cftime', From c552427aee183c282e58aa7f9c966cb064f9edba Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:16:16 -0400 Subject: [PATCH 09/22] use tmp_path --- fre/yamltools/converters/test.py | 137 +++++++++++++------------------ 1 file changed, 59 insertions(+), 78 deletions(-) diff --git a/fre/yamltools/converters/test.py b/fre/yamltools/converters/test.py index 523ed6872..81ca9a4a0 100644 --- a/fre/yamltools/converters/test.py +++ b/fre/yamltools/converters/test.py @@ -1,6 +1,4 @@ import pytest -import tempfile -import os import yaml import xml.etree.ElementTree as ET from pathlib import Path @@ -361,7 +359,7 @@ def test_experiment_without_name(self): class TestXmlToYaml: """Test xml_to_yaml main conversion function.""" - def test_basic_xml_to_yaml_conversion(self): + def test_basic_xml_to_yaml_conversion(self, tmp_path): """Test basic XML to YAML conversion.""" xml_content = ''' @@ -370,25 +368,21 @@ def test_basic_xml_to_yaml_conversion(self): ''' - - with tempfile.TemporaryDirectory() as tmpdir: - xml_file = os.path.join(tmpdir, 'test.xml') - yaml_file = os.path.join(tmpdir, 'test.yaml') - - with open(xml_file, 'w') as f: - f.write(xml_content) - - xml_to_yaml(xml_file, yaml_file) - - assert os.path.exists(yaml_file) - with open(yaml_file, 'r') as f: - data = yaml.safe_load(f) - - assert 'compile' in data - assert data['compile']['experiment'] == 'exp1' - assert len(data['compile']['src']) == 1 + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert yaml_file.exists() + data = yaml.safe_load(yaml_file.read_text()) + + assert 'compile' in data + assert data['compile']['experiment'] == 'exp1' + assert len(data['compile']['src']) == 1 - def test_xml_to_yaml_with_specific_experiment(self): + def test_xml_to_yaml_with_specific_experiment(self, tmp_path): """Test XML to YAML conversion with specific experiment filter.""" xml_content = ''' @@ -400,24 +394,20 @@ def test_xml_to_yaml_with_specific_experiment(self): ''' - - with tempfile.TemporaryDirectory() as tmpdir: - xml_file = os.path.join(tmpdir, 'test.xml') - yaml_file = os.path.join(tmpdir, 'test.yaml') - - with open(xml_file, 'w') as f: - f.write(xml_content) - - xml_to_yaml(xml_file, yaml_file, experiment_name='exp2') - - with open(yaml_file, 'r') as f: - data = yaml.safe_load(f) - - assert data['compile']['experiment'] == 'exp2' - assert len(data['compile']['src']) == 1 - assert data['compile']['src'][0]['component'] == 'comp2' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file, experiment_name='exp2') + + data = yaml.safe_load(yaml_file.read_text()) + + assert data['compile']['experiment'] == 'exp2' + assert len(data['compile']['src']) == 1 + assert data['compile']['src'][0]['component'] == 'comp2' - def test_xml_to_yaml_multiple_components(self): + def test_xml_to_yaml_multiple_components(self, tmp_path): """Test XML to YAML with multiple components.""" xml_content = ''' @@ -428,42 +418,36 @@ def test_xml_to_yaml_multiple_components(self): ''' - - with tempfile.TemporaryDirectory() as tmpdir: - xml_file = os.path.join(tmpdir, 'test.xml') - yaml_file = os.path.join(tmpdir, 'test.yaml') - - with open(xml_file, 'w') as f: - f.write(xml_content) - - xml_to_yaml(xml_file, yaml_file) - - with open(yaml_file, 'r') as f: - data = yaml.safe_load(f) - - assert len(data['compile']['src']) == 3 - assert data['compile']['src'][0]['component'] == 'comp1' - assert data['compile']['src'][1]['component'] == 'comp2' - assert data['compile']['src'][2]['component'] == 'comp3' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + data = yaml.safe_load(yaml_file.read_text()) + + assert len(data['compile']['src']) == 3 + assert data['compile']['src'][0]['component'] == 'comp1' + assert data['compile']['src'][1]['component'] == 'comp2' + assert data['compile']['src'][2]['component'] == 'comp3' def test_xml_to_yaml_nonexistent_file(self): """Test error handling for non-existent XML file.""" with pytest.raises(FileNotFoundError): xml_to_yaml('/nonexistent/path/test.xml', '/tmp/output.yaml') - def test_xml_to_yaml_invalid_xml(self): + def test_xml_to_yaml_invalid_xml(self, tmp_path): """Test error handling for invalid XML.""" - with tempfile.TemporaryDirectory() as tmpdir: - xml_file = os.path.join(tmpdir, 'invalid.xml') - yaml_file = os.path.join(tmpdir, 'output.yaml') - - with open(xml_file, 'w') as f: - f.write('This is not well-formed XMLThis is not well-formed XML @@ -472,18 +456,15 @@ def test_xml_to_yaml_output_file_created(self): ''' - - with tempfile.TemporaryDirectory() as tmpdir: - xml_file = os.path.join(tmpdir, 'test.xml') - yaml_file = os.path.join(tmpdir, 'test.yaml') - - with open(xml_file, 'w') as f: - f.write(xml_content) - - xml_to_yaml(xml_file, yaml_file) - - assert Path(yaml_file).exists() - assert os.path.getsize(yaml_file) > 0 + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert Path(yaml_file).exists() + assert yaml_file.stat().st_size > 0 if __name__ == "__main__": From a67c6af43f3be01a7d3d56ce9d18ce784151e326 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:29:45 -0400 Subject: [PATCH 10/22] untracked files --- fre/yamltools/converters/compile-converter.py | 176 ------- fre/yamltools/converters/test.py | 471 ------------------ 2 files changed, 647 deletions(-) delete mode 100644 fre/yamltools/converters/compile-converter.py delete mode 100644 fre/yamltools/converters/test.py diff --git a/fre/yamltools/converters/compile-converter.py b/fre/yamltools/converters/compile-converter.py deleted file mode 100644 index 09ae544f3..000000000 --- a/fre/yamltools/converters/compile-converter.py +++ /dev/null @@ -1,176 +0,0 @@ -import re - -import xml.etree.ElementTree as ET -import argparse -import yaml - -def parse_component(component: ET.Element) -> dict[str, any]: - """Parse a single XML element into a YAML-friendly dictionary.""" - - def get_compile_flag(flag: str) -> str | None: - """ - Return text for cppDefs or makeOverrides in compile element block - For example, - - -DDEBUG -Iinclude - -j8 - - returns - "-DDEBUG, -Iinclude" for get_compile_flag("cppDefs") - "-j8" for get_compile_flag("makeOverrides"), or - None if the flag is not defined. - """ - compile_elem = component.find('compile') - if compile_elem is not None: - elem = compile_elem.find(flag) - if elem is not None and elem.text: - return elem.text.strip() - return None - - def get_paths() -> list[str] | None: - """ - Return parsed component paths from the paths attribute. - For example, - - returns - ["am5_phys/atmos_param", "am5_phys/atmos_shared"] or - None if paths is not defined. - """ - paths = component.attrib.get('paths') - if paths and paths.strip(): - return [p.strip() for p in paths.split() if p.strip()] - return None - - def get_requires() -> list[str] | None: - """ - Return parsed component dependencies from the requires attribute. - For example, - - returns - ["FMS", "rte-rrtmpgp", "rte-ecckd"] or - None if requires is not defined. - """ - requires = component.attrib.get('requires') - if requires and requires.strip(): - return [r.strip() for r in requires.split() if r.strip()] - return None - - def get_doF90Cpp() -> bool | None: - """ - Parse doF90Cpp from the compile block into a boolean when present. - For example, - - returns True or - None if doF90Cpp is not defined. - """ - compile_elem = component.find('compile') - if compile_elem is not None: - val = compile_elem.attrib.get('doF90Cpp') - map_to_bool = {'yes': True, 'no': False} - if val is not None: - return map_to_bool.get(val.strip().lower()) - - return None - - def get_additional_instructions() -> list[str] | None: - """Extract source/csh instructions as non-empty lines.""" - source_elem = component.find('source') - if source_elem is not None: - csh_elem = source_elem.find('csh') - if csh_elem is not None and csh_elem.text: - cleaned_lines = [] - for line in csh_elem.text.splitlines(): - # Remove tabs and normalize extra spaces - clean_line = re.sub(r'\s+', ' ', line.replace('\t', '')).strip() - if clean_line: - cleaned_lines.append(clean_line) - return cleaned_lines if cleaned_lines else None - return None - - def get_repo_and_branch() -> tuple[str | None, str | None]: - """ - Build repo URL and branch/version from source/codeBase tags. - For example, - - FMS.git - - returns - ("https://github.com/NOAA-GFDL/FMS.git", "2026.01") - """ - - repo = None - branch = None - source_elem = component.find('source') - if source_elem is not None: - root = source_elem.attrib.get('root') - codebase_elem = source_elem.find('codeBase') - if root and codebase_elem is not None and codebase_elem.text: - repo = f"{root.rstrip('/')}/{codebase_elem.text.strip().strip()}" - branch = codebase_elem.attrib.get('version') - return repo, branch - - repo, branch = get_repo_and_branch() - component_name = component.attrib.get('name') - if component_name is not None: - component_name = component_name.strip() or None - - d = { - 'component': component_name, - 'repo': repo, - 'branch': branch, - 'paths': get_paths(), - 'requires': get_requires(), - 'cppdefs': get_compile_flag('cppDefs'), - 'makeOverrides': get_compile_flag('makeOverrides'), - 'doF90Cpp': get_doF90Cpp(), - 'additionalInstructions': get_additional_instructions(), - } - # Remove None values - return {k: v for k, v in d.items() if v is not None} - -def parse_experiment(experiment: ET.Element) -> Dict[str, Any]: - """Parse one element into the compile YAML object.""" - components = [parse_component(c) for c in experiment.findall('component')] - return { - 'experiment': experiment.attrib.get('name'), - 'container_addlibs': '', - 'baremetal_linkerflags': '', - 'src': components if components else [], - } - -def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): - """ - Convert compile XML to YAML. - All experiments in the XML will be converted if experiment_name is None - """ - tree = ET.parse(xml_path) - root = tree.getroot() - experiments = root.findall('experiment') - out = {} - for exp in experiments: - if experiment_name and exp.attrib.get('name') != experiment_name: - continue - yamldict = {'compile': parse_experiment(exp)} - if experiment_name: - break - with open(yaml_path, 'w', encoding='utf-8') as f: - yaml.dump(yamldict, f, sort_keys=False) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Convert compile XML to YAML") - parser.add_argument("-x", "--xmlfile", required=True, help="Input XML file") - parser.add_argument("-o", "--output", required=True, help="Output YAML file") - parser.add_argument("-e", "--experiment", default=None, help="Experiment name (optional)") - args = parser.parse_args() - - xml_to_yaml(args.xmlfile, args.output, args.experiment) - print(f"\nConverted {args.xmlfile} to {args.output}.") - print(f"Experiment: {args.experiment if args.experiment else 'All experiments'}") - print("_______________") - print("WARNING: THIS CONVERTER OUTPUTS CLOSE-ENOUGH COMPILE YAML") - print("PLEASE CHECK THE FOLLOWING:") - print(" * PATHS") - print(" * CPPDEFS AND OTHER FLAGS") - print(" * ADDITIONALINSTRUCTIONS") - print(" * PLEASE ADD IN THE APPROPRIATE ANCHORS") diff --git a/fre/yamltools/converters/test.py b/fre/yamltools/converters/test.py deleted file mode 100644 index 81ca9a4a0..000000000 --- a/fre/yamltools/converters/test.py +++ /dev/null @@ -1,471 +0,0 @@ -import pytest -import yaml -import xml.etree.ElementTree as ET -from pathlib import Path -import importlib.util - -# Import functions from compile-converter (hyphenated module name) -spec = importlib.util.spec_from_file_location("compile_converter", "compile-converter.py") -compile_converter = importlib.util.module_from_spec(spec) -spec.loader.exec_module(compile_converter) - -parse_component = compile_converter.parse_component -parse_experiment = compile_converter.parse_experiment -xml_to_yaml = compile_converter.xml_to_yaml - - -class TestGetCompileFlag: - """Test get_compile_flag nested function via parse_component.""" - - def test_cpp_defs_present(self): - """Test parsing cppDefs flag from compile block.""" - xml_str = ''' - - - -DDEBUG -Iinclude - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['cppdefs'] == '-DDEBUG -Iinclude' - - def test_make_overrides_present(self): - """Test parsing makeOverrides flag from compile block.""" - xml_str = ''' - - - -j8 - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['makeOverrides'] == '-j8' - - def test_flag_not_defined(self): - """Test when compile flag is not defined.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'cppdefs' not in result # Should be filtered out - assert 'makeOverrides' not in result - - -class TestGetPaths: - """Test get_paths nested function via parse_component.""" - - def test_single_path(self): - """Test parsing single path.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['paths'] == ['am5_phys/atmos_param'] - - def test_multiple_paths(self): - """Test parsing multiple space-separated paths.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['paths'] == ['am5_phys/atmos_param', 'am5_phys/atmos_shared', "am5_phys/madeup"] - - def test_paths_not_defined(self): - """Test when paths attribute is missing.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'paths' not in result - - def test_empty_paths_string(self): - """Test when paths is empty string.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'paths' not in result - - -class TestGetRequires: - """Test get_requires nested function via parse_component.""" - - def test_single_requirement(self): - """Test parsing single requirement.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['requires'] == ['FMS'] - - def test_multiple_requirements(self): - """Test parsing multiple space-separated requirements.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['requires'] == ['FMS', 'rte-rrtmpgp', 'rte-ecckd'] - - def test_requires_not_defined(self): - """Test when requires attribute is missing.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'requires' not in result - - def test_empty_requires_string(self): - """Test when requires is empty string.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'requires' not in result - - -class TestGetDoF90Cpp: - """Test get_doF90Cpp nested function via parse_component.""" - - def test_doF90Cpp_yes(self): - """Test parsing doF90Cpp as 'yes'.""" - xml_str = ''' - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['doF90Cpp'] is True - - def test_doF90Cpp_no(self): - """Test parsing doF90Cpp as 'no'.""" - xml_str = ''' - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['doF90Cpp'] is False - - def test_doF90Cpp_not_defined(self): - """Test when doF90Cpp is not defined.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'doF90Cpp' not in result - - -class TestGetAdditionalInstructions: - """Test get_additional_instructions nested function via parse_component.""" - - def test_single_instruction(self): - """Test parsing single instruction.""" - xml_str = ''' - - - echo "Building" - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['additionalInstructions'] == ['echo "Building"'] - - def test_multiple_instructions(self): - """Test parsing multiple instructions across lines.""" - xml_str = ''' - - - - echo "Line 1" - echo "Line 2" - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['additionalInstructions'] == ['echo "Line 1"', 'echo "Line 2"'] - - def test_no_instructions(self): - """Test when no instructions are defined.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'additionalInstructions' not in result - - -class TestGetRepoAndBranch: - """Test get_repo_and_branch nested function via parse_component.""" - - def test_valid_repo_and_branch(self): - """Test parsing valid repo URL and branch.""" - xml_str = ''' - - - FMS.git - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' - assert result['branch'] == '2026.01' - - def test_repo_with_trailing_slash(self): - """Test repo root with trailing slash.""" - xml_str = ''' - - - repo.git - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' - - def test_no_source_element(self): - """Test when source element is missing.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'repo' not in result - assert 'branch' not in result - - def test_no_codebase_element(self): - """Test when codeBase element is missing.""" - xml_str = ''' - - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'repo' not in result - assert 'branch' not in result - - def test_no_root_attribute(self): - """Test when root attribute is missing.""" - xml_str = ''' - - - FMS.git - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - assert 'repo' not in result - assert 'branch' not in result - - -class TestParseComponent: - """Test parse_component main function.""" - - def test_complete_component(self): - """Test parsing a complete component with all attributes.""" - xml_str = ''' - - - -DDEBUG - -j8 - - - FMS.git - echo "building" - - - ''' - component = ET.fromstring(xml_str) - result = parse_component(component) - - assert result['component'] == 'am5_phys' - assert result['paths'] == ['path1', 'path2'] - assert result['requires'] == ['FMS'] - assert result['cppdefs'] == '-DDEBUG' - assert result['makeOverrides'] == '-j8' - assert result['doF90Cpp'] is True - assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' - assert result['branch'] == '2026.01' - assert result['additionalInstructions'] == ['echo "building"'] - - def test_minimal_component(self): - """Test parsing minimal component with only name.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - - assert result['component'] == 'minimal' - assert len(result) == 1 # Only component field - - def test_component_without_name(self): - """Test parsing component without name attribute.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - - assert result == {} # All None values filtered out - - def test_none_values_filtered(self): - """Test that None values are filtered from output.""" - xml_str = '' - component = ET.fromstring(xml_str) - result = parse_component(component) - - # Verify no None values exist in result - for v in result.values(): - assert v is not None - - -class TestParseExperiment: - """Test parse_experiment function.""" - - def test_experiment_with_components(self): - """Test parsing experiment with multiple components.""" - xml_str = ''' - - - - - ''' - experiment = ET.fromstring(xml_str) - result = parse_experiment(experiment) - - assert result['experiment'] == 'test_exp' - assert result['container_addlibs'] == '' - assert result['baremetal_linkerflags'] == '' - assert len(result['src']) == 2 - assert result['src'][0]['component'] == 'comp1' - assert result['src'][1]['component'] == 'comp2' - - def test_experiment_without_components(self): - """Test parsing experiment without components.""" - xml_str = '' - experiment = ET.fromstring(xml_str) - result = parse_experiment(experiment) - - assert result['experiment'] == 'empty_exp' - assert result['src'] == [] - - def test_experiment_without_name(self): - """Test parsing experiment without name attribute.""" - xml_str = ''' - - - - ''' - experiment = ET.fromstring(xml_str) - result = parse_experiment(experiment) - - assert result['experiment'] is None - assert len(result['src']) == 1 - - -class TestXmlToYaml: - """Test xml_to_yaml main conversion function.""" - - def test_basic_xml_to_yaml_conversion(self, tmp_path): - """Test basic XML to YAML conversion.""" - xml_content = ''' - - - - - - ''' - xml_file = tmp_path / 'test.xml' - yaml_file = tmp_path / 'test.yaml' - - xml_file.write_text(xml_content) - - xml_to_yaml(xml_file, yaml_file) - - assert yaml_file.exists() - data = yaml.safe_load(yaml_file.read_text()) - - assert 'compile' in data - assert data['compile']['experiment'] == 'exp1' - assert len(data['compile']['src']) == 1 - - def test_xml_to_yaml_with_specific_experiment(self, tmp_path): - """Test XML to YAML conversion with specific experiment filter.""" - xml_content = ''' - - - - - - - - - ''' - xml_file = tmp_path / 'test.xml' - yaml_file = tmp_path / 'test.yaml' - - xml_file.write_text(xml_content) - - xml_to_yaml(xml_file, yaml_file, experiment_name='exp2') - - data = yaml.safe_load(yaml_file.read_text()) - - assert data['compile']['experiment'] == 'exp2' - assert len(data['compile']['src']) == 1 - assert data['compile']['src'][0]['component'] == 'comp2' - - def test_xml_to_yaml_multiple_components(self, tmp_path): - """Test XML to YAML with multiple components.""" - xml_content = ''' - - - - - - - - ''' - xml_file = tmp_path / 'test.xml' - yaml_file = tmp_path / 'test.yaml' - - xml_file.write_text(xml_content) - - xml_to_yaml(xml_file, yaml_file) - - data = yaml.safe_load(yaml_file.read_text()) - - assert len(data['compile']['src']) == 3 - assert data['compile']['src'][0]['component'] == 'comp1' - assert data['compile']['src'][1]['component'] == 'comp2' - assert data['compile']['src'][2]['component'] == 'comp3' - - def test_xml_to_yaml_nonexistent_file(self): - """Test error handling for non-existent XML file.""" - with pytest.raises(FileNotFoundError): - xml_to_yaml('/nonexistent/path/test.xml', '/tmp/output.yaml') - - def test_xml_to_yaml_invalid_xml(self, tmp_path): - """Test error handling for invalid XML.""" - xml_file = tmp_path / 'invalid.xml' - yaml_file = tmp_path / 'output.yaml' - - xml_file.write_text('This is not well-formed XML - - - - - ''' - xml_file = tmp_path / 'test.xml' - yaml_file = tmp_path / 'test.yaml' - - xml_file.write_text(xml_content) - - xml_to_yaml(xml_file, yaml_file) - - assert Path(yaml_file).exists() - assert yaml_file.stat().st_size > 0 - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) From 47fd3491d7c2dc78fe72ad605fc99ca5ea84d1fe Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:29:54 -0400 Subject: [PATCH 11/22] untracked dir --- fre/yamltools/converters/compile/agent.md | 71 +++ fre/yamltools/converters/compile/converter.py | 176 +++++++ .../converters/compile/reference.xml | 148 ++++++ .../converters/compile/reference.yaml | 110 ++++ fre/yamltools/converters/compile/test.py | 471 ++++++++++++++++++ 5 files changed, 976 insertions(+) create mode 100644 fre/yamltools/converters/compile/agent.md create mode 100644 fre/yamltools/converters/compile/converter.py create mode 100644 fre/yamltools/converters/compile/reference.xml create mode 100644 fre/yamltools/converters/compile/reference.yaml create mode 100644 fre/yamltools/converters/compile/test.py diff --git a/fre/yamltools/converters/compile/agent.md b/fre/yamltools/converters/compile/agent.md new file mode 100644 index 000000000..4fc6fd8fe --- /dev/null +++ b/fre/yamltools/converters/compile/agent.md @@ -0,0 +1,71 @@ +### Compile XML to YAML Converter + +QUESTION: +1. Is it better to print out a generic platform.yaml than convert it? + It seems like the content of platform.yaml is very different from platform.xml +2. How to handle csh blocks? +3. How to handle CDATA? +4. How to handle yaml anchors? + +You are a xml-to-yaml converter that converts files in xml format to yaml format. + +** Example ** +Study this example: + +For this xml: + + Experiment to build executable. See git log for source code provenance. + + + ocean_BGC.git + + + + + OPENMP="" + + + +the converted yaml looks like: + +compile: + experiment: !join [*AM5_VERSION, "_compile"] + container_addlibs: + baremetal_linkerflags: + src: + - component: "mom6" + repo: "https://github.com/NOAA-GFDL/ocean_GBC.git" + branch: 2023.01 + paths: ["mom6/src/MOM6/config_src/infra/FMS2", + "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", + "mom6/src/MOM6/config_src/drivers/FMS_cap", + "mom6/src/MOM6/config_src/external", + "mom6/src/MOM6/src/*", + "mom6/src/MOM6/src/*/*"] + cppdefs: " + +** yaml format ** +The xml must be converted to a yaml file that follows the json schema in + +https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json + + +** xml format ** +Study this example xml: + + +There may be multiple sections in the xml file. +There may be multiple sections under a + +** guide ** +Here's how the xml elements and attributes map to yaml keys + + + + diff --git a/fre/yamltools/converters/compile/converter.py b/fre/yamltools/converters/compile/converter.py new file mode 100644 index 000000000..09ae544f3 --- /dev/null +++ b/fre/yamltools/converters/compile/converter.py @@ -0,0 +1,176 @@ +import re + +import xml.etree.ElementTree as ET +import argparse +import yaml + +def parse_component(component: ET.Element) -> dict[str, any]: + """Parse a single XML element into a YAML-friendly dictionary.""" + + def get_compile_flag(flag: str) -> str | None: + """ + Return text for cppDefs or makeOverrides in compile element block + For example, + + -DDEBUG -Iinclude + -j8 + + returns + "-DDEBUG, -Iinclude" for get_compile_flag("cppDefs") + "-j8" for get_compile_flag("makeOverrides"), or + None if the flag is not defined. + """ + compile_elem = component.find('compile') + if compile_elem is not None: + elem = compile_elem.find(flag) + if elem is not None and elem.text: + return elem.text.strip() + return None + + def get_paths() -> list[str] | None: + """ + Return parsed component paths from the paths attribute. + For example, + + returns + ["am5_phys/atmos_param", "am5_phys/atmos_shared"] or + None if paths is not defined. + """ + paths = component.attrib.get('paths') + if paths and paths.strip(): + return [p.strip() for p in paths.split() if p.strip()] + return None + + def get_requires() -> list[str] | None: + """ + Return parsed component dependencies from the requires attribute. + For example, + + returns + ["FMS", "rte-rrtmpgp", "rte-ecckd"] or + None if requires is not defined. + """ + requires = component.attrib.get('requires') + if requires and requires.strip(): + return [r.strip() for r in requires.split() if r.strip()] + return None + + def get_doF90Cpp() -> bool | None: + """ + Parse doF90Cpp from the compile block into a boolean when present. + For example, + + returns True or + None if doF90Cpp is not defined. + """ + compile_elem = component.find('compile') + if compile_elem is not None: + val = compile_elem.attrib.get('doF90Cpp') + map_to_bool = {'yes': True, 'no': False} + if val is not None: + return map_to_bool.get(val.strip().lower()) + + return None + + def get_additional_instructions() -> list[str] | None: + """Extract source/csh instructions as non-empty lines.""" + source_elem = component.find('source') + if source_elem is not None: + csh_elem = source_elem.find('csh') + if csh_elem is not None and csh_elem.text: + cleaned_lines = [] + for line in csh_elem.text.splitlines(): + # Remove tabs and normalize extra spaces + clean_line = re.sub(r'\s+', ' ', line.replace('\t', '')).strip() + if clean_line: + cleaned_lines.append(clean_line) + return cleaned_lines if cleaned_lines else None + return None + + def get_repo_and_branch() -> tuple[str | None, str | None]: + """ + Build repo URL and branch/version from source/codeBase tags. + For example, + + FMS.git + + returns + ("https://github.com/NOAA-GFDL/FMS.git", "2026.01") + """ + + repo = None + branch = None + source_elem = component.find('source') + if source_elem is not None: + root = source_elem.attrib.get('root') + codebase_elem = source_elem.find('codeBase') + if root and codebase_elem is not None and codebase_elem.text: + repo = f"{root.rstrip('/')}/{codebase_elem.text.strip().strip()}" + branch = codebase_elem.attrib.get('version') + return repo, branch + + repo, branch = get_repo_and_branch() + component_name = component.attrib.get('name') + if component_name is not None: + component_name = component_name.strip() or None + + d = { + 'component': component_name, + 'repo': repo, + 'branch': branch, + 'paths': get_paths(), + 'requires': get_requires(), + 'cppdefs': get_compile_flag('cppDefs'), + 'makeOverrides': get_compile_flag('makeOverrides'), + 'doF90Cpp': get_doF90Cpp(), + 'additionalInstructions': get_additional_instructions(), + } + # Remove None values + return {k: v for k, v in d.items() if v is not None} + +def parse_experiment(experiment: ET.Element) -> Dict[str, Any]: + """Parse one element into the compile YAML object.""" + components = [parse_component(c) for c in experiment.findall('component')] + return { + 'experiment': experiment.attrib.get('name'), + 'container_addlibs': '', + 'baremetal_linkerflags': '', + 'src': components if components else [], + } + +def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): + """ + Convert compile XML to YAML. + All experiments in the XML will be converted if experiment_name is None + """ + tree = ET.parse(xml_path) + root = tree.getroot() + experiments = root.findall('experiment') + out = {} + for exp in experiments: + if experiment_name and exp.attrib.get('name') != experiment_name: + continue + yamldict = {'compile': parse_experiment(exp)} + if experiment_name: + break + with open(yaml_path, 'w', encoding='utf-8') as f: + yaml.dump(yamldict, f, sort_keys=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert compile XML to YAML") + parser.add_argument("-x", "--xmlfile", required=True, help="Input XML file") + parser.add_argument("-o", "--output", required=True, help="Output YAML file") + parser.add_argument("-e", "--experiment", default=None, help="Experiment name (optional)") + args = parser.parse_args() + + xml_to_yaml(args.xmlfile, args.output, args.experiment) + print(f"\nConverted {args.xmlfile} to {args.output}.") + print(f"Experiment: {args.experiment if args.experiment else 'All experiments'}") + print("_______________") + print("WARNING: THIS CONVERTER OUTPUTS CLOSE-ENOUGH COMPILE YAML") + print("PLEASE CHECK THE FOLLOWING:") + print(" * PATHS") + print(" * CPPDEFS AND OTHER FLAGS") + print(" * ADDITIONALINSTRUCTIONS") + print(" * PLEASE ADD IN THE APPROPRIATE ANCHORS") diff --git a/fre/yamltools/converters/compile/reference.xml b/fre/yamltools/converters/compile/reference.xml new file mode 100644 index 000000000..dc4fd40cf --- /dev/null +++ b/fre/yamltools/converters/compile/reference.xml @@ -0,0 +1,148 @@ + + + + + + + + + +Experiment to build executable. See git log for source code provenance. + + + + + FMS.git + + + $(F2003_FLAGS) -Duse_libMPI -Duse_netCDF -Duse_yaml + + + + + + rte-rrtmgp.git + + + -heap-arrays -DRTE_USE_SP + OPENMP="" + + + + + + rte-ecckd.git + + + + + + + am5_phys.git + + + $(F2003_FLAGS) + + + + + GFDL_atmos_cubed_sphere.git + + + USE_R4=$(USE_MIXED_MODE) ISA="-march=core-avx2 -qno-opt-dynamic-align" + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + + atmos_drivers.git + + + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + + + land_lad2.git + + + $(F2003_FLAGS) -nostdinc + + + + + + ocean_BGC.git + + + + + OPENMP="" + + + + + + ice_param.git + + + + + + + + + + coupler.git + + + + + + + + diff --git a/fre/yamltools/converters/compile/reference.yaml b/fre/yamltools/converters/compile/reference.yaml new file mode 100644 index 000000000..706bf9e0b --- /dev/null +++ b/fre/yamltools/converters/compile/reference.yaml @@ -0,0 +1,110 @@ +compile: + experiment: !join [*AM5_VERSION, "_compile"] + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + branch: "2026.01" + cppdefs: "-DHAVE_GETTID -DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF -Duse_yaml" + otherFlags: *FMSincludes + - component: "rte-rrtmgp" + requires: ["FMS"] + repo: "http://github.com/NOAA-GFDL/rte-rrtmgp.git" + branch: "2024.01" + paths: ["rte-rrtmgp/rte-frontend", + "rte-rrtmgp/rte-kernels/mo_fluxes_broadband_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_optical_props_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_rte_solver_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_rte_util_array.F90", + "rte-rrtmgp/gas-optics", + "rte-rrtmgp/rrtmgp-frontend/mo_gas_optics_rrtmgp.F90", + "rte-rrtmgp/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90", + "rte-rrtmgp/extensions/mo_heating_rates.F90", + "rte-rrtmgp/examples/mo_load_coefficients.F90"] + cppdefs: "-heap-arrays -DRTE_USE_SP" + makeOverrides: 'OPENMP=""' + otherFlags: *FMSincludes + - component: "rte-ecckd" + requires: ["FMS", "rte-rrtmgp"] + repo: "http://github.com/NOAA-GFDL/rte-ecckd.git" + branch: "2024.03" + paths: ["rte-ecckd/src/gas_optics_ecckd.f90"] + otherFlags: *FMSincludes + - component: "am5_phys" + requires: ["FMS", "rte-rrtmgp", "rte-ecckd"] + repo: "https://gitlab.gfdl.noaa.gov/fms/am5_phys.git" + branch: "2026.01" + paths: ["am5_phys/atmos_param", "am5_phys/atmos_shared"] + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "GFDL_atmos_cubed_sphere" + requires: ["FMS", "am5_phys"] + repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" + branch: "2024.01_am5" + paths: ["GFDL_atmos_cubed_sphere/driver/GFDL", + "GFDL_atmos_cubed_sphere/model", + "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90", + "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", + "GFDL_atmos_cubed_sphere/tools", + "GFDL_atmos_cubed_sphere/GFDL_tools"] + cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" + otherFlags: *FMSincludes + - component: "atmos_drivers" + requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere", "rte-rrtmgp", "rte-ecckd"] + repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" + branch: "2025.03" + paths: ["atmos_drivers/coupled"] + cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" + otherFlags: *FMSincludes + - component: "land_lad2" + requires: ["FMS"] + repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" + branch: "2025.02" + doF90Cpp: True + cppdefs: "-DINTERNAL_FILE_NML -nostdinc" + otherFlags: *FMSincludes + - component: "ocean_BGC" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_BGC.git" + branch: "2023.01" + paths: ["mom6/src/MOM6/config_src/infra/FMS2", + "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", + "mom6/src/MOM6/config_src/drivers/FMS_cap", + "mom6/src/MOM6/config_src/external", + "mom6/src/MOM6/src/*", + "mom6/src/MOM6/src/*/*"] + cppdefs: "-DINTERNAL_FILE_NML -DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2" + otherFlags: !join [*FMSincludes, " ", *momIncludes] + makeOverrides: 'OPENMP=""' + additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", + "cd mom6\n", + "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n", + "git submodule update --recursive --init src/MOM6 src/SIS2 src/icebergs\n", + "git pull --no-edit https://github.com/uramirez8707/MOM6-examples.git use_diag_yaml \n", + "cd src/MOM6\n", + "git checkout 792a0616c16050dfe74ac5bfc887f99d9c765376 \n", + "cd ../SIS2\n", + "git checkout dec9ee38424eceded0da635671e7c03e5d97d928 \n", + "cd ../icebergs\n", + "git checkout 6a8944fc7544608942de9a42bd3b96f6e56684e0 \n", + "cd ../.. \n", + "ln -s /gpfs/f5/gfdl_o/world-shared/datasets .datasets \n", + "cd .. \n"] + - component: "sis2" + requires: ["FMS", "ocean_BGC"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "2024.02" + paths: ["mom6/src/SIS2/config_src/dynamic_symmetric", + "mom6/src/SIS2/config_src/external/Icepack_interfaces", + "mom6/src/SIS2/src", + "mom6/src/icebergs/src", + "sis2"] + otherFlags: !join [*FMSincludes, " ", *momIncludes] + cppdefs: "-DUSE_FMS2_IO -DINTERNAL_FILE_NML" + - component: "FMScoupler" + requires: ["FMS", "GFDL_atmos_cubed_sphere", "atmos_drivers", "am5_phys", "land_lad2", "sis2", "ocean_BGC"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "2026.01" + paths: ["FMScoupler/full", "FMScoupler/shared"] + otherFlags: !join [*FMSincludes, " ", *momIncludes] diff --git a/fre/yamltools/converters/compile/test.py b/fre/yamltools/converters/compile/test.py new file mode 100644 index 000000000..81ca9a4a0 --- /dev/null +++ b/fre/yamltools/converters/compile/test.py @@ -0,0 +1,471 @@ +import pytest +import yaml +import xml.etree.ElementTree as ET +from pathlib import Path +import importlib.util + +# Import functions from compile-converter (hyphenated module name) +spec = importlib.util.spec_from_file_location("compile_converter", "compile-converter.py") +compile_converter = importlib.util.module_from_spec(spec) +spec.loader.exec_module(compile_converter) + +parse_component = compile_converter.parse_component +parse_experiment = compile_converter.parse_experiment +xml_to_yaml = compile_converter.xml_to_yaml + + +class TestGetCompileFlag: + """Test get_compile_flag nested function via parse_component.""" + + def test_cpp_defs_present(self): + """Test parsing cppDefs flag from compile block.""" + xml_str = ''' + + + -DDEBUG -Iinclude + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['cppdefs'] == '-DDEBUG -Iinclude' + + def test_make_overrides_present(self): + """Test parsing makeOverrides flag from compile block.""" + xml_str = ''' + + + -j8 + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['makeOverrides'] == '-j8' + + def test_flag_not_defined(self): + """Test when compile flag is not defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'cppdefs' not in result # Should be filtered out + assert 'makeOverrides' not in result + + +class TestGetPaths: + """Test get_paths nested function via parse_component.""" + + def test_single_path(self): + """Test parsing single path.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['am5_phys/atmos_param'] + + def test_multiple_paths(self): + """Test parsing multiple space-separated paths.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['paths'] == ['am5_phys/atmos_param', 'am5_phys/atmos_shared', "am5_phys/madeup"] + + def test_paths_not_defined(self): + """Test when paths attribute is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'paths' not in result + + def test_empty_paths_string(self): + """Test when paths is empty string.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'paths' not in result + + +class TestGetRequires: + """Test get_requires nested function via parse_component.""" + + def test_single_requirement(self): + """Test parsing single requirement.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['FMS'] + + def test_multiple_requirements(self): + """Test parsing multiple space-separated requirements.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['requires'] == ['FMS', 'rte-rrtmpgp', 'rte-ecckd'] + + def test_requires_not_defined(self): + """Test when requires attribute is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'requires' not in result + + def test_empty_requires_string(self): + """Test when requires is empty string.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'requires' not in result + + +class TestGetDoF90Cpp: + """Test get_doF90Cpp nested function via parse_component.""" + + def test_doF90Cpp_yes(self): + """Test parsing doF90Cpp as 'yes'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is True + + def test_doF90Cpp_no(self): + """Test parsing doF90Cpp as 'no'.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['doF90Cpp'] is False + + def test_doF90Cpp_not_defined(self): + """Test when doF90Cpp is not defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'doF90Cpp' not in result + + +class TestGetAdditionalInstructions: + """Test get_additional_instructions nested function via parse_component.""" + + def test_single_instruction(self): + """Test parsing single instruction.""" + xml_str = ''' + + + echo "Building" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "Building"'] + + def test_multiple_instructions(self): + """Test parsing multiple instructions across lines.""" + xml_str = ''' + + + + echo "Line 1" + echo "Line 2" + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['additionalInstructions'] == ['echo "Line 1"', 'echo "Line 2"'] + + def test_no_instructions(self): + """Test when no instructions are defined.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'additionalInstructions' not in result + + +class TestGetRepoAndBranch: + """Test get_repo_and_branch nested function via parse_component.""" + + def test_valid_repo_and_branch(self): + """Test parsing valid repo URL and branch.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + assert result['branch'] == '2026.01' + + def test_repo_with_trailing_slash(self): + """Test repo root with trailing slash.""" + xml_str = ''' + + + repo.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert result['repo'] == 'https://github.com/NOAA-GFDL/repo.git' + + def test_no_source_element(self): + """Test when source element is missing.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + def test_no_codebase_element(self): + """Test when codeBase element is missing.""" + xml_str = ''' + + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + def test_no_root_attribute(self): + """Test when root attribute is missing.""" + xml_str = ''' + + + FMS.git + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + assert 'repo' not in result + assert 'branch' not in result + + +class TestParseComponent: + """Test parse_component main function.""" + + def test_complete_component(self): + """Test parsing a complete component with all attributes.""" + xml_str = ''' + + + -DDEBUG + -j8 + + + FMS.git + echo "building" + + + ''' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'am5_phys' + assert result['paths'] == ['path1', 'path2'] + assert result['requires'] == ['FMS'] + assert result['cppdefs'] == '-DDEBUG' + assert result['makeOverrides'] == '-j8' + assert result['doF90Cpp'] is True + assert result['repo'] == 'https://github.com/NOAA-GFDL/FMS.git' + assert result['branch'] == '2026.01' + assert result['additionalInstructions'] == ['echo "building"'] + + def test_minimal_component(self): + """Test parsing minimal component with only name.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result['component'] == 'minimal' + assert len(result) == 1 # Only component field + + def test_component_without_name(self): + """Test parsing component without name attribute.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + assert result == {} # All None values filtered out + + def test_none_values_filtered(self): + """Test that None values are filtered from output.""" + xml_str = '' + component = ET.fromstring(xml_str) + result = parse_component(component) + + # Verify no None values exist in result + for v in result.values(): + assert v is not None + + +class TestParseExperiment: + """Test parse_experiment function.""" + + def test_experiment_with_components(self): + """Test parsing experiment with multiple components.""" + xml_str = ''' + + + + + ''' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] == 'test_exp' + assert result['container_addlibs'] == '' + assert result['baremetal_linkerflags'] == '' + assert len(result['src']) == 2 + assert result['src'][0]['component'] == 'comp1' + assert result['src'][1]['component'] == 'comp2' + + def test_experiment_without_components(self): + """Test parsing experiment without components.""" + xml_str = '' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] == 'empty_exp' + assert result['src'] == [] + + def test_experiment_without_name(self): + """Test parsing experiment without name attribute.""" + xml_str = ''' + + + + ''' + experiment = ET.fromstring(xml_str) + result = parse_experiment(experiment) + + assert result['experiment'] is None + assert len(result['src']) == 1 + + +class TestXmlToYaml: + """Test xml_to_yaml main conversion function.""" + + def test_basic_xml_to_yaml_conversion(self, tmp_path): + """Test basic XML to YAML conversion.""" + xml_content = ''' + + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert yaml_file.exists() + data = yaml.safe_load(yaml_file.read_text()) + + assert 'compile' in data + assert data['compile']['experiment'] == 'exp1' + assert len(data['compile']['src']) == 1 + + def test_xml_to_yaml_with_specific_experiment(self, tmp_path): + """Test XML to YAML conversion with specific experiment filter.""" + xml_content = ''' + + + + + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file, experiment_name='exp2') + + data = yaml.safe_load(yaml_file.read_text()) + + assert data['compile']['experiment'] == 'exp2' + assert len(data['compile']['src']) == 1 + assert data['compile']['src'][0]['component'] == 'comp2' + + def test_xml_to_yaml_multiple_components(self, tmp_path): + """Test XML to YAML with multiple components.""" + xml_content = ''' + + + + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + data = yaml.safe_load(yaml_file.read_text()) + + assert len(data['compile']['src']) == 3 + assert data['compile']['src'][0]['component'] == 'comp1' + assert data['compile']['src'][1]['component'] == 'comp2' + assert data['compile']['src'][2]['component'] == 'comp3' + + def test_xml_to_yaml_nonexistent_file(self): + """Test error handling for non-existent XML file.""" + with pytest.raises(FileNotFoundError): + xml_to_yaml('/nonexistent/path/test.xml', '/tmp/output.yaml') + + def test_xml_to_yaml_invalid_xml(self, tmp_path): + """Test error handling for invalid XML.""" + xml_file = tmp_path / 'invalid.xml' + yaml_file = tmp_path / 'output.yaml' + + xml_file.write_text('This is not well-formed XML + + + + + ''' + xml_file = tmp_path / 'test.xml' + yaml_file = tmp_path / 'test.yaml' + + xml_file.write_text(xml_content) + + xml_to_yaml(xml_file, yaml_file) + + assert Path(yaml_file).exists() + assert yaml_file.stat().st_size > 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From c8a04510c02945e2adea8f7eae3e2a48f170d762 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:30:16 -0400 Subject: [PATCH 12/22] init --- fre/yamltools/converters/compile/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 fre/yamltools/converters/compile/__init__.py diff --git a/fre/yamltools/converters/compile/__init__.py b/fre/yamltools/converters/compile/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/fre/yamltools/converters/compile/__init__.py @@ -0,0 +1 @@ + From 7de4c914bd457ddd91c5c8f3b48d8224b970ee3e Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:31:35 -0400 Subject: [PATCH 13/22] cleanup --- fre/yamltools/converters/compile/{test.py => test_converter.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fre/yamltools/converters/compile/{test.py => test_converter.py} (100%) diff --git a/fre/yamltools/converters/compile/test.py b/fre/yamltools/converters/compile/test_converter.py similarity index 100% rename from fre/yamltools/converters/compile/test.py rename to fre/yamltools/converters/compile/test_converter.py From 8704821e8502d85f21913a39ca447e0a086817a7 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:32:09 -0400 Subject: [PATCH 14/22] untracked files --- .../converters/compile/reference.xml | 148 ------------------ .../converters/compile/reference.yaml | 110 ------------- 2 files changed, 258 deletions(-) delete mode 100644 fre/yamltools/converters/compile/reference.xml delete mode 100644 fre/yamltools/converters/compile/reference.yaml diff --git a/fre/yamltools/converters/compile/reference.xml b/fre/yamltools/converters/compile/reference.xml deleted file mode 100644 index dc4fd40cf..000000000 --- a/fre/yamltools/converters/compile/reference.xml +++ /dev/null @@ -1,148 +0,0 @@ - - - - - - - - - -Experiment to build executable. See git log for source code provenance. - - - - - FMS.git - - - $(F2003_FLAGS) -Duse_libMPI -Duse_netCDF -Duse_yaml - - - - - - rte-rrtmgp.git - - - -heap-arrays -DRTE_USE_SP - OPENMP="" - - - - - - rte-ecckd.git - - - - - - - am5_phys.git - - - $(F2003_FLAGS) - - - - - GFDL_atmos_cubed_sphere.git - - - USE_R4=$(USE_MIXED_MODE) ISA="-march=core-avx2 -qno-opt-dynamic-align" - $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE - - - - - atmos_drivers.git - - - $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE - - - - - - land_lad2.git - - - $(F2003_FLAGS) -nostdinc - - - - - - ocean_BGC.git - - - - - OPENMP="" - - - - - - ice_param.git - - - - - - - - - - coupler.git - - - - - - - - diff --git a/fre/yamltools/converters/compile/reference.yaml b/fre/yamltools/converters/compile/reference.yaml deleted file mode 100644 index 706bf9e0b..000000000 --- a/fre/yamltools/converters/compile/reference.yaml +++ /dev/null @@ -1,110 +0,0 @@ -compile: - experiment: !join [*AM5_VERSION, "_compile"] - container_addlibs: - baremetal_linkerflags: - src: - - component: "FMS" - repo: "https://github.com/NOAA-GFDL/FMS.git" - branch: "2026.01" - cppdefs: "-DHAVE_GETTID -DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF -Duse_yaml" - otherFlags: *FMSincludes - - component: "rte-rrtmgp" - requires: ["FMS"] - repo: "http://github.com/NOAA-GFDL/rte-rrtmgp.git" - branch: "2024.01" - paths: ["rte-rrtmgp/rte-frontend", - "rte-rrtmgp/rte-kernels/mo_fluxes_broadband_kernels.F90", - "rte-rrtmgp/rte-kernels/mo_optical_props_kernels.F90", - "rte-rrtmgp/rte-kernels/mo_rte_solver_kernels.F90", - "rte-rrtmgp/rte-kernels/mo_rte_util_array.F90", - "rte-rrtmgp/gas-optics", - "rte-rrtmgp/rrtmgp-frontend/mo_gas_optics_rrtmgp.F90", - "rte-rrtmgp/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90", - "rte-rrtmgp/extensions/mo_heating_rates.F90", - "rte-rrtmgp/examples/mo_load_coefficients.F90"] - cppdefs: "-heap-arrays -DRTE_USE_SP" - makeOverrides: 'OPENMP=""' - otherFlags: *FMSincludes - - component: "rte-ecckd" - requires: ["FMS", "rte-rrtmgp"] - repo: "http://github.com/NOAA-GFDL/rte-ecckd.git" - branch: "2024.03" - paths: ["rte-ecckd/src/gas_optics_ecckd.f90"] - otherFlags: *FMSincludes - - component: "am5_phys" - requires: ["FMS", "rte-rrtmgp", "rte-ecckd"] - repo: "https://gitlab.gfdl.noaa.gov/fms/am5_phys.git" - branch: "2026.01" - paths: ["am5_phys/atmos_param", "am5_phys/atmos_shared"] - cppdefs: "-DINTERNAL_FILE_NML" - otherFlags: *FMSincludes - - component: "GFDL_atmos_cubed_sphere" - requires: ["FMS", "am5_phys"] - repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" - branch: "2024.01_am5" - paths: ["GFDL_atmos_cubed_sphere/driver/GFDL", - "GFDL_atmos_cubed_sphere/model", - "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90", - "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", - "GFDL_atmos_cubed_sphere/tools", - "GFDL_atmos_cubed_sphere/GFDL_tools"] - cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" - otherFlags: *FMSincludes - - component: "atmos_drivers" - requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere", "rte-rrtmgp", "rte-ecckd"] - repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" - branch: "2025.03" - paths: ["atmos_drivers/coupled"] - cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" - otherFlags: *FMSincludes - - component: "land_lad2" - requires: ["FMS"] - repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" - branch: "2025.02" - doF90Cpp: True - cppdefs: "-DINTERNAL_FILE_NML -nostdinc" - otherFlags: *FMSincludes - - component: "ocean_BGC" - requires: ["FMS"] - repo: "https://github.com/NOAA-GFDL/ocean_BGC.git" - branch: "2023.01" - paths: ["mom6/src/MOM6/config_src/infra/FMS2", - "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", - "mom6/src/MOM6/config_src/drivers/FMS_cap", - "mom6/src/MOM6/config_src/external", - "mom6/src/MOM6/src/*", - "mom6/src/MOM6/src/*/*"] - cppdefs: "-DINTERNAL_FILE_NML -DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2" - otherFlags: !join [*FMSincludes, " ", *momIncludes] - makeOverrides: 'OPENMP=""' - additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", - "cd mom6\n", - "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n", - "git submodule update --recursive --init src/MOM6 src/SIS2 src/icebergs\n", - "git pull --no-edit https://github.com/uramirez8707/MOM6-examples.git use_diag_yaml \n", - "cd src/MOM6\n", - "git checkout 792a0616c16050dfe74ac5bfc887f99d9c765376 \n", - "cd ../SIS2\n", - "git checkout dec9ee38424eceded0da635671e7c03e5d97d928 \n", - "cd ../icebergs\n", - "git checkout 6a8944fc7544608942de9a42bd3b96f6e56684e0 \n", - "cd ../.. \n", - "ln -s /gpfs/f5/gfdl_o/world-shared/datasets .datasets \n", - "cd .. \n"] - - component: "sis2" - requires: ["FMS", "ocean_BGC"] - repo: "https://github.com/NOAA-GFDL/ice_param.git" - branch: "2024.02" - paths: ["mom6/src/SIS2/config_src/dynamic_symmetric", - "mom6/src/SIS2/config_src/external/Icepack_interfaces", - "mom6/src/SIS2/src", - "mom6/src/icebergs/src", - "sis2"] - otherFlags: !join [*FMSincludes, " ", *momIncludes] - cppdefs: "-DUSE_FMS2_IO -DINTERNAL_FILE_NML" - - component: "FMScoupler" - requires: ["FMS", "GFDL_atmos_cubed_sphere", "atmos_drivers", "am5_phys", "land_lad2", "sis2", "ocean_BGC"] - repo: "https://github.com/NOAA-GFDL/FMScoupler.git" - branch: "2026.01" - paths: ["FMScoupler/full", "FMScoupler/shared"] - otherFlags: !join [*FMSincludes, " ", *momIncludes] From d1d11c325b65b473084b66657885b699c0dc4111 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:38:57 -0400 Subject: [PATCH 15/22] fix type hinting --- fre/yamltools/converters/compile/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/yamltools/converters/compile/converter.py b/fre/yamltools/converters/compile/converter.py index 09ae544f3..0ace27b8d 100644 --- a/fre/yamltools/converters/compile/converter.py +++ b/fre/yamltools/converters/compile/converter.py @@ -4,7 +4,7 @@ import argparse import yaml -def parse_component(component: ET.Element) -> dict[str, any]: +def parse_component(component: ET.Element) -> dict[str, str | list] | None: """Parse a single XML element into a YAML-friendly dictionary.""" def get_compile_flag(flag: str) -> str | None: @@ -128,7 +128,7 @@ def get_repo_and_branch() -> tuple[str | None, str | None]: # Remove None values return {k: v for k, v in d.items() if v is not None} -def parse_experiment(experiment: ET.Element) -> Dict[str, Any]: +def parse_experiment(experiment: ET.Element) -> [str, str | list]: """Parse one element into the compile YAML object.""" components = [parse_component(c) for c in experiment.findall('component')] return { From 41e6f2356f3923e48bf2b2cd67064fb0562d865c Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:51:49 -0400 Subject: [PATCH 16/22] add clean line --- fre/yamltools/converters/compile/converter.py | 16 ++++++++++++---- .../converters/compile/test_converter.py | 13 +++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/fre/yamltools/converters/compile/converter.py b/fre/yamltools/converters/compile/converter.py index 0ace27b8d..fcf2b0afd 100644 --- a/fre/yamltools/converters/compile/converter.py +++ b/fre/yamltools/converters/compile/converter.py @@ -7,6 +7,12 @@ def parse_component(component: ET.Element) -> dict[str, str | list] | None: """Parse a single XML element into a YAML-friendly dictionary.""" + def clean_text(string: str) -> list[str] | None: + clean_string = re.sub(r'\s+', ' ', string.replace('\t', '')).strip() + if clean_string: + return clean_string + return None + def get_compile_flag(flag: str) -> str | None: """ Return text for cppDefs or makeOverrides in compile element block @@ -23,8 +29,10 @@ def get_compile_flag(flag: str) -> str | None: compile_elem = component.find('compile') if compile_elem is not None: elem = compile_elem.find(flag) - if elem is not None and elem.text: - return elem.text.strip() + if elem is not None: + elem_text = clean_text(elem.text) + if elem_text: + return elem_text return None def get_paths() -> list[str] | None: @@ -81,8 +89,8 @@ def get_additional_instructions() -> list[str] | None: cleaned_lines = [] for line in csh_elem.text.splitlines(): # Remove tabs and normalize extra spaces - clean_line = re.sub(r'\s+', ' ', line.replace('\t', '')).strip() - if clean_line: + clean_line = clean_text(line) + if clean_line is not None: cleaned_lines.append(clean_line) return cleaned_lines if cleaned_lines else None return None diff --git a/fre/yamltools/converters/compile/test_converter.py b/fre/yamltools/converters/compile/test_converter.py index 81ca9a4a0..fbf299625 100644 --- a/fre/yamltools/converters/compile/test_converter.py +++ b/fre/yamltools/converters/compile/test_converter.py @@ -2,16 +2,13 @@ import yaml import xml.etree.ElementTree as ET from pathlib import Path -import importlib.util -# Import functions from compile-converter (hyphenated module name) -spec = importlib.util.spec_from_file_location("compile_converter", "compile-converter.py") -compile_converter = importlib.util.module_from_spec(spec) -spec.loader.exec_module(compile_converter) +from . import converter -parse_component = compile_converter.parse_component -parse_experiment = compile_converter.parse_experiment -xml_to_yaml = compile_converter.xml_to_yaml + +parse_component = converter.parse_component +parse_experiment = converter.parse_experiment +xml_to_yaml = converter.xml_to_yaml class TestGetCompileFlag: From 2ae7db4f9f093ddf8f1c83e20977208fcbc66b11 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 11:53:49 -0400 Subject: [PATCH 17/22] restore pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 549bcae56..2664168dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,6 @@ keywords = [ dependencies = [ 'analysis_scripts', - 'beautifulsoup4', 'catalogbuilder', 'cdo', 'cftime', From 4916eb6b9abbbef4cccebfcf57f70d1d175f325d Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 14:19:35 -0400 Subject: [PATCH 18/22] fixed converter --- fre/yamltools/converters/compile/converter.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/fre/yamltools/converters/compile/converter.py b/fre/yamltools/converters/compile/converter.py index fcf2b0afd..506d62074 100644 --- a/fre/yamltools/converters/compile/converter.py +++ b/fre/yamltools/converters/compile/converter.py @@ -146,6 +146,11 @@ def parse_experiment(experiment: ET.Element) -> [str, str | list]: 'src': components if components else [], } +def write_yaml(yamldict: dict, yaml_path: str): + """Write the YAML dictionary to a file.""" + with open(yaml_path, 'w', encoding='utf-8') as f: + yaml.dump(yamldict, f, sort_keys=False) + def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): """ Convert compile XML to YAML. @@ -154,15 +159,15 @@ def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): tree = ET.parse(xml_path) root = tree.getroot() experiments = root.findall('experiment') - out = {} + + if experiment_name is not None: + experiments = [exp for exp in experiments if exp.get('name') == experiment_name] + for exp in experiments: - if experiment_name and exp.attrib.get('name') != experiment_name: - continue + print(f"Converting experiment '{exp.attrib.get('name')}' to YAML...") yamldict = {'compile': parse_experiment(exp)} - if experiment_name: - break - with open(yaml_path, 'w', encoding='utf-8') as f: - yaml.dump(yamldict, f, sort_keys=False) + write_yaml(yamldict, yaml_path) + print(f"Experiment '{exp.attrib.get('name')}' converted to YAML.") if __name__ == "__main__": From c02db35f33c76f628b310d898dc06abf68028a64 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 15:53:21 -0400 Subject: [PATCH 19/22] agent and dump to safe_dump --- fre/yamltools/converters/compile/agent.md | 90 ++++++++++++------- fre/yamltools/converters/compile/converter.py | 2 +- 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/fre/yamltools/converters/compile/agent.md b/fre/yamltools/converters/compile/agent.md index 4fc6fd8fe..9c5f9ced7 100644 --- a/fre/yamltools/converters/compile/agent.md +++ b/fre/yamltools/converters/compile/agent.md @@ -1,18 +1,23 @@ -### Compile XML to YAML Converter +---- +name: compile-xml-to-yaml-converter +description: Agent to convert compile xml to compile yaml +--- -QUESTION: -1. Is it better to print out a generic platform.yaml than convert it? - It seems like the content of platform.yaml is very different from platform.xml -2. How to handle csh blocks? -3. How to handle CDATA? -4. How to handle yaml anchors? +You are an expert in compile xml files and compile yaml files -You are a xml-to-yaml converter that converts files in xml format to yaml format. -** Example ** -Study this example: +## Your role +- You convert a compile xml file to a yaml file. A compile xml contains the tag +where the name contains the string "compile" +- You are also a yaml validator that validates compile yaml against the json schema in +https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +- You are also a yaml corrector that corrects yaml files so that it passes the validation test when +using the json schema in https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json + -For this xml: +## Background XML knowledge +This is a sample xml +``` Experiment to build executable. See git log for source code provenance. @@ -22,7 +27,7 @@ For this xml: git clone -b dev/gfdl https://github.com/NOAA-GFDL/MOM6-examples.git mom6 test -e mom6/.datasets if ($status != 0) then - echo ""; echo "" ; echo " WARNING: .datasets link in MOM6 examples directory is invalid"; echo ""; echo "" + echo ""; echo "" ; echo " WARNING: datasets link in MOM6 examples directory is invalid"; echo ""; echo "" endif ]]> @@ -30,10 +35,24 @@ For this xml: OPENMP="" + + + + FMS.git + + + $(F2003_FLAGS) -Duse_libMPI -Duse_netCDF -Duse_yaml + -the converted yaml looks like: + +``` +- There can be multiple in the xml file. +- Each experiment has multiple components +## Background YAML knowledge +This is a sample yaml +``` compile: experiment: !join [*AM5_VERSION, "_compile"] container_addlibs: @@ -41,30 +60,41 @@ compile: src: - component: "mom6" repo: "https://github.com/NOAA-GFDL/ocean_GBC.git" - branch: 2023.01 + branch: "2023.01" paths: ["mom6/src/MOM6/config_src/infra/FMS2", "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", "mom6/src/MOM6/config_src/drivers/FMS_cap", "mom6/src/MOM6/config_src/external", "mom6/src/MOM6/src/*", "mom6/src/MOM6/src/*/*"] - cppdefs: " - -** yaml format ** -The xml must be converted to a yaml file that follows the json schema in - -https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json - - -** xml format ** -Study this example xml: - - -There may be multiple sections in the xml file. -There may be multiple sections under a + cppdefs: "" + doF90Cpp: True + makeOverrides: "OPENMP=''" + - component: "fms" + repo: "https://github.com/NOAA-GFDL/FMS.git" + branch: 2026.01 + cppdefs: !join [*F2003_FLAGS, "-Duse_libMPI", "-Duse_netCDF", "-Duse_yaml"] +``` + +## json schema for YAML validation. +- The schema is in https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json + +## Guidelines +- There can be more than one tag in the xml file. +- Convert all variables $(variable) to yaml anchors *variable. +- If the yaml string contains an anchor, convert the string into a list and appropriately use join [*variable] +- For additionalInstructions, convert the +- Only use the content in ![CDATA[content]]. Treat content as a string +- For additionalInstructions, split the string into an array. Split by new line unless the string content is part of a Bash command. + +## Instructions +- Ask the user to provide the compile xml file to convert +- Ask the user the experiment name to convert. +- Read and study the compile xml file +- Convert the compile xml file to yaml format and write it to a file with the same name as the compile xml file except with the yaml extension. + For example, name the yaml file 'this_compile_experiment.yaml' if the xml file is 'this_compile_experiment.xml' +- Validate the yaml file to the json schema and print the errors for the user to analyze. -** guide ** -Here's how the xml elements and attributes map to yaml keys diff --git a/fre/yamltools/converters/compile/converter.py b/fre/yamltools/converters/compile/converter.py index 506d62074..3bf06e9aa 100644 --- a/fre/yamltools/converters/compile/converter.py +++ b/fre/yamltools/converters/compile/converter.py @@ -149,7 +149,7 @@ def parse_experiment(experiment: ET.Element) -> [str, str | list]: def write_yaml(yamldict: dict, yaml_path: str): """Write the YAML dictionary to a file.""" with open(yaml_path, 'w', encoding='utf-8') as f: - yaml.dump(yamldict, f, sort_keys=False) + yaml.safe_dump(yamldict, f, sort_keys=False) def xml_to_yaml(xml_path: str, yaml_path: str, experiment_name: str = None): """ From 9bd061fcae292f239d0ba67336a483f3aaabf87a Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 15:53:55 -0400 Subject: [PATCH 20/22] tmp files will remove later --- fre/yamltools/converters/reference.xml | 148 ++++++++++++++++++++++++ fre/yamltools/converters/reference.yaml | 110 ++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 fre/yamltools/converters/reference.xml create mode 100644 fre/yamltools/converters/reference.yaml diff --git a/fre/yamltools/converters/reference.xml b/fre/yamltools/converters/reference.xml new file mode 100644 index 000000000..dc4fd40cf --- /dev/null +++ b/fre/yamltools/converters/reference.xml @@ -0,0 +1,148 @@ + + + + + + + + + +Experiment to build executable. See git log for source code provenance. + + + + + FMS.git + + + $(F2003_FLAGS) -Duse_libMPI -Duse_netCDF -Duse_yaml + + + + + + rte-rrtmgp.git + + + -heap-arrays -DRTE_USE_SP + OPENMP="" + + + + + + rte-ecckd.git + + + + + + + am5_phys.git + + + $(F2003_FLAGS) + + + + + GFDL_atmos_cubed_sphere.git + + + USE_R4=$(USE_MIXED_MODE) ISA="-march=core-avx2 -qno-opt-dynamic-align" + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + + atmos_drivers.git + + + $(F2003_FLAGS) -DSPMD -DCLIMATE_NUDGE + + + + + + land_lad2.git + + + $(F2003_FLAGS) -nostdinc + + + + + + ocean_BGC.git + + + + + OPENMP="" + + + + + + ice_param.git + + + + + + + + + + coupler.git + + + + + + + + diff --git a/fre/yamltools/converters/reference.yaml b/fre/yamltools/converters/reference.yaml new file mode 100644 index 000000000..706bf9e0b --- /dev/null +++ b/fre/yamltools/converters/reference.yaml @@ -0,0 +1,110 @@ +compile: + experiment: !join [*AM5_VERSION, "_compile"] + container_addlibs: + baremetal_linkerflags: + src: + - component: "FMS" + repo: "https://github.com/NOAA-GFDL/FMS.git" + branch: "2026.01" + cppdefs: "-DHAVE_GETTID -DINTERNAL_FILE_NML -Duse_libMPI -Duse_netCDF -Duse_yaml" + otherFlags: *FMSincludes + - component: "rte-rrtmgp" + requires: ["FMS"] + repo: "http://github.com/NOAA-GFDL/rte-rrtmgp.git" + branch: "2024.01" + paths: ["rte-rrtmgp/rte-frontend", + "rte-rrtmgp/rte-kernels/mo_fluxes_broadband_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_optical_props_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_rte_solver_kernels.F90", + "rte-rrtmgp/rte-kernels/mo_rte_util_array.F90", + "rte-rrtmgp/gas-optics", + "rte-rrtmgp/rrtmgp-frontend/mo_gas_optics_rrtmgp.F90", + "rte-rrtmgp/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90", + "rte-rrtmgp/extensions/mo_heating_rates.F90", + "rte-rrtmgp/examples/mo_load_coefficients.F90"] + cppdefs: "-heap-arrays -DRTE_USE_SP" + makeOverrides: 'OPENMP=""' + otherFlags: *FMSincludes + - component: "rte-ecckd" + requires: ["FMS", "rte-rrtmgp"] + repo: "http://github.com/NOAA-GFDL/rte-ecckd.git" + branch: "2024.03" + paths: ["rte-ecckd/src/gas_optics_ecckd.f90"] + otherFlags: *FMSincludes + - component: "am5_phys" + requires: ["FMS", "rte-rrtmgp", "rte-ecckd"] + repo: "https://gitlab.gfdl.noaa.gov/fms/am5_phys.git" + branch: "2026.01" + paths: ["am5_phys/atmos_param", "am5_phys/atmos_shared"] + cppdefs: "-DINTERNAL_FILE_NML" + otherFlags: *FMSincludes + - component: "GFDL_atmos_cubed_sphere" + requires: ["FMS", "am5_phys"] + repo: "https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere.git" + branch: "2024.01_am5" + paths: ["GFDL_atmos_cubed_sphere/driver/GFDL", + "GFDL_atmos_cubed_sphere/model", + "GFDL_atmos_cubed_sphere/driver/SHiELD/cloud_diagnosis.F90", + "GFDL_atmos_cubed_sphere/driver/SHiELD/gfdl_cloud_microphys.F90", + "GFDL_atmos_cubed_sphere/tools", + "GFDL_atmos_cubed_sphere/GFDL_tools"] + cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" + otherFlags: *FMSincludes + - component: "atmos_drivers" + requires: ["FMS", "am5_phys", "GFDL_atmos_cubed_sphere", "rte-rrtmgp", "rte-ecckd"] + repo: "https://github.com/NOAA-GFDL/atmos_drivers.git" + branch: "2025.03" + paths: ["atmos_drivers/coupled"] + cppdefs: "-DINTERNAL_FILE_NML -DSPMD -DCLIMATE_NUDGE" + otherFlags: *FMSincludes + - component: "land_lad2" + requires: ["FMS"] + repo: "https://gitlab.gfdl.noaa.gov/FMS/land_lad2.git" + branch: "2025.02" + doF90Cpp: True + cppdefs: "-DINTERNAL_FILE_NML -nostdinc" + otherFlags: *FMSincludes + - component: "ocean_BGC" + requires: ["FMS"] + repo: "https://github.com/NOAA-GFDL/ocean_BGC.git" + branch: "2023.01" + paths: ["mom6/src/MOM6/config_src/infra/FMS2", + "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", + "mom6/src/MOM6/config_src/drivers/FMS_cap", + "mom6/src/MOM6/config_src/external", + "mom6/src/MOM6/src/*", + "mom6/src/MOM6/src/*/*"] + cppdefs: "-DINTERNAL_FILE_NML -DMAX_FIELDS_=100 -DNOT_SET_AFFINITY -D_USE_MOM6_DIAG -D_USE_GENERIC_TRACER -DUSE_PRECISION=2" + otherFlags: !join [*FMSincludes, " ", *momIncludes] + makeOverrides: 'OPENMP=""' + additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", + "cd mom6\n", + "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n", + "git submodule update --recursive --init src/MOM6 src/SIS2 src/icebergs\n", + "git pull --no-edit https://github.com/uramirez8707/MOM6-examples.git use_diag_yaml \n", + "cd src/MOM6\n", + "git checkout 792a0616c16050dfe74ac5bfc887f99d9c765376 \n", + "cd ../SIS2\n", + "git checkout dec9ee38424eceded0da635671e7c03e5d97d928 \n", + "cd ../icebergs\n", + "git checkout 6a8944fc7544608942de9a42bd3b96f6e56684e0 \n", + "cd ../.. \n", + "ln -s /gpfs/f5/gfdl_o/world-shared/datasets .datasets \n", + "cd .. \n"] + - component: "sis2" + requires: ["FMS", "ocean_BGC"] + repo: "https://github.com/NOAA-GFDL/ice_param.git" + branch: "2024.02" + paths: ["mom6/src/SIS2/config_src/dynamic_symmetric", + "mom6/src/SIS2/config_src/external/Icepack_interfaces", + "mom6/src/SIS2/src", + "mom6/src/icebergs/src", + "sis2"] + otherFlags: !join [*FMSincludes, " ", *momIncludes] + cppdefs: "-DUSE_FMS2_IO -DINTERNAL_FILE_NML" + - component: "FMScoupler" + requires: ["FMS", "GFDL_atmos_cubed_sphere", "atmos_drivers", "am5_phys", "land_lad2", "sis2", "ocean_BGC"] + repo: "https://github.com/NOAA-GFDL/FMScoupler.git" + branch: "2026.01" + paths: ["FMScoupler/full", "FMScoupler/shared"] + otherFlags: !join [*FMSincludes, " ", *momIncludes] From 78716564e56404907ddfaa3e5d41536c6e6da973 Mon Sep 17 00:00:00 2001 From: mlee03 Date: Wed, 20 May 2026 16:04:18 -0400 Subject: [PATCH 21/22] icouldhavedonethisonmylaptop --- fre/yamltools/converters/reference.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fre/yamltools/converters/reference.yaml b/fre/yamltools/converters/reference.yaml index 706bf9e0b..54f0d5bc1 100644 --- a/fre/yamltools/converters/reference.yaml +++ b/fre/yamltools/converters/reference.yaml @@ -78,7 +78,7 @@ compile: otherFlags: !join [*FMSincludes, " ", *momIncludes] makeOverrides: 'OPENMP=""' additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", - "cd mom6\n", + "pushd mom6\n", "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n", "git submodule update --recursive --init src/MOM6 src/SIS2 src/icebergs\n", "git pull --no-edit https://github.com/uramirez8707/MOM6-examples.git use_diag_yaml \n", @@ -88,8 +88,14 @@ compile: "git checkout dec9ee38424eceded0da635671e7c03e5d97d928 \n", "cd ../icebergs\n", "git checkout 6a8944fc7544608942de9a42bd3b96f6e56684e0 \n", - "cd ../.. \n", - "ln -s /gpfs/f5/gfdl_o/world-shared/datasets .datasets \n", + "popd \n", + "pushd mom6 \n" + "ln -s", *(GFDL_O_INPUTS), ".datasets \n", + "popd \n", + "test -e mom6/.datasets \n", + "if ($status != 0) then\n", + 'echo ""; echo "" ; echo " WARNING: .datasets link in MOM6 examples directory is invalid"; echo ""; echo ""', + "endif", "cd .. \n"] - component: "sis2" requires: ["FMS", "ocean_BGC"] From 0bafaaaa117cff5e4640e53e00b50e88d6041d43 Mon Sep 17 00:00:00 2001 From: Miso Potstickers Date: Wed, 20 May 2026 16:10:46 -0400 Subject: [PATCH 22/22] update agent.md --- fre/yamltools/converters/compile/agent.md | 265 +++++++++++++++------- 1 file changed, 182 insertions(+), 83 deletions(-) diff --git a/fre/yamltools/converters/compile/agent.md b/fre/yamltools/converters/compile/agent.md index 9c5f9ced7..a6cc93c85 100644 --- a/fre/yamltools/converters/compile/agent.md +++ b/fre/yamltools/converters/compile/agent.md @@ -1,101 +1,200 @@ ----- +--- name: compile-xml-to-yaml-converter -description: Agent to convert compile xml to compile yaml +description: Agent to convert a compile XML experiment to a compile YAML file --- -You are an expert in compile xml files and compile yaml files +You are an expert at converting FRE (Flexible Runtime Environment) compile XML experiments into compile YAML format. +## Your roles +- **Converter**: Transform a compile XML experiment into the corresponding YAML structure. +- **Validator**: Validate the produced YAML against the JSON schema at + https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +- **Corrector**: Fix any validation errors so the YAML passes schema validation. -## Your role -- You convert a compile xml file to a yaml file. A compile xml contains the tag -where the name contains the string "compile" -- You are also a yaml validator that validates compile yaml against the json schema in -https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json -- You are also a yaml corrector that corrects yaml files so that it passes the validation test when -using the json schema in https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +--- +## Instructions -## Background XML knowledge -This is a sample xml -``` - - Experiment to build executable. See git log for source code provenance. - - - ocean_BGC.git - - - - - OPENMP="" - - - - - FMS.git - - - $(F2003_FLAGS) -Duse_libMPI -Duse_netCDF -Duse_yaml - - - - -``` -- There can be multiple in the xml file. -- Each experiment has multiple components +1. Ask the user to provide the compile XML file path. +2. Ask the user which experiment name to convert (e.g. `$(AM5_VERSION)_compile`). The target experiment contains "compile" in its `name` attribute. +3. Read and parse the XML file. Locate the `` element whose `name` attribute contains "compile" and matches the user's choice. Ignore other experiments (e.g. canopy wrapper experiments). +4. Convert each `` inside that experiment to a YAML `src` list entry using the field-by-field rules below. +5. Write the output YAML to a file with the same base name as the XML file but with a `.yaml` extension. For example, `am5_compile.xml` → `am5_compile.yaml`. +6. Validate the YAML against the schema and report any errors to the user. -## Background YAML knowledge -This is a sample yaml -``` +--- + +## Top-level YAML structure + +```yaml compile: - experiment: !join [*AM5_VERSION, "_compile"] - container_addlibs: - baremetal_linkerflags: - src: - - component: "mom6" - repo: "https://github.com/NOAA-GFDL/ocean_GBC.git" - branch: "2023.01" - paths: ["mom6/src/MOM6/config_src/infra/FMS2", - "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric", - "mom6/src/MOM6/config_src/drivers/FMS_cap", - "mom6/src/MOM6/config_src/external", - "mom6/src/MOM6/src/*", - "mom6/src/MOM6/src/*/*"] - cppdefs: "" - doF90Cpp: True - makeOverrides: "OPENMP=''" - - component: "fms" - repo: "https://github.com/NOAA-GFDL/FMS.git" - branch: 2026.01 - cppdefs: !join [*F2003_FLAGS, "-Duse_libMPI", "-Duse_netCDF", "-Duse_yaml"] + experiment: + container_addlibs: + baremetal_linkerflags: + src: + - component: ... + ... ``` -## json schema for YAML validation. -- The schema is in https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +--- -## Guidelines -- There can be more than one tag in the xml file. -- Convert all variables $(variable) to yaml anchors *variable. -- If the yaml string contains an anchor, convert the string into a list and appropriately use join [*variable] -- For additionalInstructions, convert the -- Only use the content in ![CDATA[content]]. Treat content as a string -- For additionalInstructions, split the string into an array. Split by new line unless the string content is part of a Bash command. +## Field-by-field conversion rules -## Instructions -- Ask the user to provide the compile xml file to convert -- Ask the user the experiment name to convert. -- Read and study the compile xml file -- Convert the compile xml file to yaml format and write it to a file with the same name as the compile xml file except with the yaml extension. - For example, name the yaml file 'this_compile_experiment.yaml' if the xml file is 'this_compile_experiment.xml' -- Validate the yaml file to the json schema and print the errors for the user to analyze. +### `experiment` +- Source: the `name` attribute of the `` element. +- Convert every `$(VAR)` reference to a YAML anchor `*VAR`. +- If the name contains both literal text and an anchor, use `!join`: + ```yaml + experiment: !join [*AM5_VERSION, "_compile"] + ``` +- If the name is a plain string with no variables, quote it directly. + +--- + +### `component` (the YAML component name / identifier) +- Source: the filename inside `` tags, stripped of the `.git` suffix and whitespace. + - Example: ` FMS.git ` → `"FMS"` + - Example: ` ocean_BGC.git ` → `"ocean_BGC"` +- This name is used as both the YAML `component:` value and the base filename for the `repo:` URL. +- **Exception**: when the codeBase name does not clearly identify the component in context (e.g. `ice_param.git` for the SIS2 component), use the XML `` attribute instead and note the discrepancy. + +--- + +### `repo` +- Constructed by joining the `root` attribute of `` with the `` filename: + `root + "/" + codeBase_filename` +- Always ensure the URL ends with `.git`. +- Examples: + - `root="https://github.com/NOAA-GFDL"` + `FMS.git` → `"https://github.com/NOAA-GFDL/FMS.git"` + - `root="http://gitlab.gfdl.noaa.gov/fms"` + `am5_phys.git` → `"https://gitlab.gfdl.noaa.gov/fms/am5_phys.git"` +- Normalize `http://` to `https://` where the host is known to support HTTPS. + +--- + +### `branch` +- Source: the `version` attribute of ``. +- Always quote the value as a string (version numbers like `2024.01` can be misread as floats). +- Example: `` → `branch: "2024.01_am5"` + +--- + +### `requires` +- Source: the `requires` attribute of ``, which is a space-separated list of XML component names. +- Map each XML component name to its corresponding YAML component name (i.e. the codeBase-derived name used in the `component:` field). +- Output as a YAML list of quoted strings. +- Example: `requires="fms rte-rrtmgp"` → `requires: ["FMS", "rte-rrtmgp"]` +- Omit the field entirely if `requires` is absent. + +--- + +### `paths` +- Source: the `paths` attribute of `` (whitespace-separated, may span multiple lines). +- Output as a YAML list of quoted strings. +- **Expand brace notation** `{a,b,c}` into separate list entries: + - `mom6/src/MOM6/config_src/{infra/FMS2,memory/dynamic_nonsymmetric}` → + ```yaml + paths: ["mom6/src/MOM6/config_src/infra/FMS2", + "mom6/src/MOM6/config_src/memory/dynamic_nonsymmetric"] + ``` +- Glob patterns (`*`, `*/*`) are kept as-is in the YAML strings. +- Omit the field if `paths` is absent (some components have no `paths` attribute). + +--- + +### `cppdefs` +- Source: the text content of ``, including content inside ``. +- Strip leading/trailing whitespace from the value. +- Convert `$(VAR)` references: + - If the entire value is a single variable → `cppdefs: *VAR` + - If the value is a mix of a variable and literal flags → use `!join`: + ```yaml + cppdefs: !join [*F2003_FLAGS, " -DSPMD -DCLIMATE_NUDGE"] + ``` + - If no variables are present, quote the string directly: + ```yaml + cppdefs: "-heap-arrays -DRTE_USE_SP" + ``` +- Complex inline expressions such as `"'"\`git-version-string $<\`"'"` should be preserved verbatim as part of the string value — do not attempt to evaluate or simplify them. +- Omit the field if `` is absent. + +--- + +### `makeOverrides` +- Source: the text content of ``. +- Preserve the exact string, quoted with single quotes if it contains double-quote characters. +- Example: `OPENMP=""` → `makeOverrides: 'OPENMP=""'` +- Omit the field if `` is absent. + +--- + +### `doF90Cpp` +- Source: the `doF90Cpp` attribute on the `` element. +- `doF90Cpp="yes"` → `doF90Cpp: True` +- Omit the field if the attribute is absent or not `"yes"`. + +--- + +### `additionalInstructions` +- Source: the content inside `` within a component's ``. +- Output as a `!join` list where each element is either: + - A plain shell command line ending with `"\n"`, or + - A `*VAR` anchor for any `$(VAR)` variable. +- **Splitting rules**: + - Split at actual newlines in the CDATA content. + - Do **not** split inside a single Bash command even if it spans conceptual units; keep each logical line together. + - Append `"\n"` to each line element (except optionally the last). + - Convert `$(VAR)` to `*VAR` anchor references inside the join list. +- Example: + ```xml + + ``` + becomes: + ```yaml + additionalInstructions: !join ["git clone https://github.com/NOAA-GFDL/MOM6-examples.git mom6 \n", + "pushd mom6\n", + "git checkout ", *MOM6_EXAMPLES_GIT_TAG, "\n"] + ``` +- Omit the field if no `` element is present. + +--- + +### `otherFlags` +- This field is **not** directly present in the XML; it is derived from the include directory dependencies: + - Components that depend on FMS headers should include `otherFlags: *FMSincludes`. + - Components that also require MOM6 framework headers (e.g. `sis2`, `ocean_BGC`, `coupler`) should include `otherFlags: !join [*FMSincludes, " ", *momIncludes]`. + - The FMS component itself does not need `otherFlags`. +- The anchor definitions for `*FMSincludes` and `*momIncludes` are expected to exist elsewhere in the broader YAML document (e.g. a shared variables section). Do not define them; only reference them. +- If you are unsure whether `otherFlags` applies, leave it out and note it for the user. + +--- + +## Variable anchor conventions + +- XML uses `$(VARNAME)` syntax for make-style variables. +- In YAML, these become anchor references: `*VARNAME`. +- When an anchor appears inside a string with other text, convert the whole value to a `!join` list: + ```yaml + # Instead of: "$(F2003_FLAGS) -DSPMD" + cppdefs: !join [*F2003_FLAGS, " -DSPMD"] + ``` +- Anchor **definitions** (e.g. `AM5_VERSION: &AM5_VERSION "am5_p1"`) come from a separate variables/defaults section of the YAML, not from this conversion. Do not invent anchor definitions. +--- + +## Ordering of `src` components +Preserve the order of `` elements as they appear in the XML. The build system may rely on this ordering for dependency resolution. +--- +## Validation +After writing the YAML file: +1. Fetch the schema from https://github.com/NOAA-GFDL/gfdl_msd_schemas/blob/main/FRE/fre_make.json +2. Validate the output YAML against the schema using `jsonschema` or equivalent. +3. Print all validation errors with their JSON path location. +4. Offer to correct any errors automatically.