From edd3f561e5b8ad9c478450a75ef29b5c1b973e17 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 11 Mar 2026 15:21:51 -0400 Subject: [PATCH 001/109] Map NHF hydrofabric to old field names --- src/geopackage/read.cpp | 195 ++++++++++++++++++++-------------------- 1 file changed, 100 insertions(+), 95 deletions(-) diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index c4031a00a4..1dc91eb6e0 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -27,12 +27,6 @@ std::shared_ptr ngen::geopackage::read( // Check for malicious/invalid layer input check_table_name(layer); std::vector features; - if (ids.size() > 0) - features.reserve(ids.size()); - double min_x = std::numeric_limits::infinity(); - double min_y = std::numeric_limits::infinity(); - double max_x = -std::numeric_limits::infinity(); - double max_y = -std::numeric_limits::infinity(); LOG(LogLevel::DEBUG, "Establishing connection to geopackage %s.", gpkg_path.c_str()); ngen::sqlite::database db{gpkg_path}; @@ -57,103 +51,114 @@ std::shared_ptr ngen::geopackage::read( Logger::logMsgAndThrowError(errmsg); } - // Introspect if the layer is divides to see which ID field is in use - std::string id_column = "id"; - if(layer == "divides"){ - try { - //TODO: A bit primitive. Actually introspect the schema somehow? https://www.sqlite.org/c3ref/funclist.html - auto query_get_first_row = db.query("SELECT divide_id FROM " + layer + " LIMIT 1"); - id_column = "divide_id"; - } - catch (const std::exception& e){ - #ifndef NGEN_QUIET - // output debug info on what is read exactly - read_ss << "WARN: Using legacy ID column \"id\" in layer " << layer << " is DEPRECATED and may stop working at any time." << std::endl; - LOG(read_ss.str(), LogLevel::WARNING); read_ss.str(""); - #endif - } + std::string id_column; + std::string feature_query; + if (layer == "divides") { + id_column = "div_id"; + feature_query = + "SELECT " + "('cat-' || divides.div_id) AS id, " + "('nex-' || flowpaths.dn_nex_id) AS toid, " + "flowpaths.slope AS So, " + "divides.geom AS geom " + "FROM divides " + "LEFT JOIN flowpaths " + "ON divides.div_id = flowpaths.div_id"; + } else if (layer == "nexus") { + id_column = "nex_id"; + feature_query = + "SELECT " + "('nex-' || nexus.nex_id) AS id, " + "IIF(flowpaths.div_id, ('cat-' || flowpaths.div_id), 'terminal') AS toid, " + "flowpaths.slope AS So, " + "nexus.geom AS geom " + "FROM nexus " + "LEFT JOIN flowpaths " + "ON nexus.dn_fp_id = flowpaths.fp_id"; + } else { + Logger::logMsgAndThrowError("Geopackage read only accepts layers `divides` and `nexus`. The layer entered was " + layer); } - // execute sub-queries if the number of IDs gets too long or once if ids.size() == 0 - int bind_limit = 900; - boost::span id_span(ids); - for (int i = 0; i < ids.size() || (i == 0 && ids.size() == 0); i += bind_limit) { - int span_size = (i + bind_limit >= ids.size()) ? (ids.size() - i) : bind_limit; - boost::span sub_ids = id_span.subspan(i, span_size); - - // Layer exists, getting statement for it - // - // this creates a string in the form: - // WHERE id IN (?, ?, ?, ...) - // so that it can be bound by SQLite. - // This is safer than trying to concatenate - // the IDs together. - std::string joined_ids = ""; - if (!sub_ids.empty()) { - joined_ids = " WHERE "+id_column+" IN (?"; - for (size_t i = 1; i < sub_ids.size(); i++) { - joined_ids += ", ?"; + std::string joined_ids = ""; + if (!ids.empty()) { + std::stringstream filter; + filter << " WHERE " << layer << '.' << id_column << " IN ("; + for (size_t i = 0; i < ids.size(); ++i) { + if (i != 0) + filter << ','; + auto &filter_id = ids[i]; + size_t sep_index = filter_id.find('-'); + if (sep_index == std::string::npos) { + sep_index = 0; + } else { + sep_index++; } - joined_ids += ")"; + int id_num = std::atoi(filter_id.c_str() + sep_index); + if (id_num <= 0) + Logger::logMsgAndThrowError("Could not convert input " + layer + " ID into a number: " + filter_id); + filter << id_num; } + filter << ')'; + joined_ids = filter.str(); + } - // Get number of features - auto query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer + joined_ids, sub_ids); - query_get_layer_count.next(); - const int layer_feature_count = query_get_layer_count.get(0); - - #ifndef NGEN_QUIET - // output debug info on what is read exactly - read_ss << "Reading " << layer_feature_count << " features from layer " << layer << " using ID column `"<< id_column << "`"; - if (!sub_ids.empty()) { - read_ss << " (id subset:"; - for (auto& id : sub_ids) { - read_ss << " " << id; - } - read_ss << ")"; - } - read_ss << std::endl; - LOG(read_ss.str(), LogLevel::DEBUG); read_ss.str(""); - #endif - - // Get layer feature metadata (geometry column name + type) - auto query_get_layer_geom_meta = db.query("SELECT column_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); - query_get_layer_geom_meta.next(); - const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); - - // Get layer - LOG(LogLevel::DEBUG, "Reading %d features from layer %s.", layer_feature_count, layer.c_str()); - auto query_get_layer = db.query("SELECT * FROM " + layer + joined_ids, sub_ids); - query_get_layer.next(); - - // build features out of layer query - if (ids.size() == 0) - features.reserve(layer_feature_count); - while(!query_get_layer.done()) { - geojson::Feature feature = build_feature( - query_get_layer, - id_column, - layer_geometry_column - ); - - features.push_back(feature); - query_get_layer.next(); - } + // Get number of features + auto query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer + joined_ids); + query_get_layer_count.next(); + const int layer_feature_count = query_get_layer_count.get(0); + features.reserve(layer_feature_count); + if (!ids.empty() && ids.size() != layer_feature_count) { + LOG(LogLevel::WARNING, "The number of input IDs (%d) does not equal the number of features with those IDs in the geopackage (%d) for layer %s.", + ids.size(), layer_feature_count, layer.c_str()); + } - // get layer bounding box from features - // - // GeoPackage contains a bounding box in the SQLite DB, - // however, it is in the SRS of the GPKG. By creating - // the bbox after the features are built, the projection - // is already done. This also should be fairly cheap to do. - for (const auto& feature : features) { - const auto& bbox = feature->get_bounding_box(); - min_x = bbox[0] < min_x ? bbox[0] : min_x; - min_y = bbox[1] < min_y ? bbox[1] : min_y; - max_x = bbox[2] > max_x ? bbox[2] : max_x; - max_y = bbox[3] > max_y ? bbox[3] : max_y; + #ifndef NGEN_QUIET + // output debug info on what is read exactly + read_ss << "Reading " << layer_feature_count << " features from layer " << layer << " using ID column `"<< id_column << "`"; + if (!ids.empty()) { + read_ss << " (id subset:"; + for (auto& id : ids) { + read_ss << " " << id; } + read_ss << ")"; + } + read_ss << std::endl; + LOG(read_ss.str(), LogLevel::DEBUG); read_ss.str(""); + #endif + + // Get layer + LOG(LogLevel::DEBUG, "Reading %d features from layer %s.", layer_feature_count, layer.c_str()); + auto query_get_layer = db.query(feature_query + joined_ids); + query_get_layer.next(); + + // build features out of layer query + while(!query_get_layer.done()) { + geojson::Feature feature = build_feature( + query_get_layer, + "id", + "geom" + ); + + features.push_back(feature); + query_get_layer.next(); + } + // get layer bounding box from features + // + // GeoPackage contains a bounding box in the SQLite DB, + // however, it is in the SRS of the GPKG. By creating + // the bbox after the features are built, the projection + // is already done. This also should be fairly cheap to do. + double min_x = std::numeric_limits::infinity(); + double min_y = std::numeric_limits::infinity(); + double max_x = -std::numeric_limits::infinity(); + double max_y = -std::numeric_limits::infinity(); + for (const auto& feature : features) { + const auto& bbox = feature->get_bounding_box(); + min_x = bbox[0] < min_x ? bbox[0] : min_x; + min_y = bbox[1] < min_y ? bbox[1] : min_y; + max_x = bbox[2] > max_x ? bbox[2] : max_x; + max_y = bbox[3] > max_y ? bbox[3] : max_y; } auto fc = std::make_shared( From b04007e59214bfce0ca4d5a9f38d684f43f40fd3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Mar 2026 11:10:12 -0400 Subject: [PATCH 002/109] Improved messaging for geopackage error handling --- src/NGen.cpp | 10 ++++++---- src/partitionGenerator.cpp | 12 +++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 9fefc64468..eaeb99bb03 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -446,11 +446,12 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { #if NGEN_WITH_SQLITE3 try { nexus_collection = ngen::geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); - } catch (...) { + } catch (std::exception &e) { // Handle all exceptions std::string msg = "Geopackage error occurred reading nexuses: " + nexusDataFile; LOG(msg,LogLevel::FATAL); - throw std::runtime_error(msg); + LOG(LogLevel::FATAL, e.what()); + throw; } #else Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); @@ -477,11 +478,12 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { try { catchment_collection = ngen::geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); - } catch (...) { + } catch (std::exception &e) { // Handle all exceptions std::string msg = "Geopackage error occurred reading divides: " + catchmentDataFile; LOG(msg,LogLevel::FATAL); - throw std::runtime_error(msg); + LOG(LogLevel::FATAL, e.what()); + throw; } #else diff --git a/src/partitionGenerator.cpp b/src/partitionGenerator.cpp index f8b48ee616..d724c1623a 100644 --- a/src/partitionGenerator.cpp +++ b/src/partitionGenerator.cpp @@ -433,11 +433,12 @@ int main(int argc, char* argv[]) #if NGEN_WITH_SQLITE3 try { catchment_collection = ngen::geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); - } catch (...) { + } catch (std::exception &e) { // Handle all exceptions std::string msg = "Geopackage error occurred reading divides: " + catchmentDataFile; - LOG(msg,LogLevel::FATAL); - throw std::runtime_error(msg); + LOG(msg, LogLevel::FATAL); + LOG(LogLevel::FATAL, e.what()); + throw; } #else Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); @@ -470,11 +471,12 @@ int main(int argc, char* argv[]) #if NGEN_WITH_SQLITE3 try { global_nexus_collection = ngen::geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); - } catch (...) { + } catch (std::exception &e) { // Handle all exceptions std::string msg = "Geopackage error occurred reading nexuses: " + nexusDataFile; LOG(msg,LogLevel::FATAL); - throw std::runtime_error(msg); + LOG(LogLevel::FATAL, e.what()); + throw; } #else Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); From f1d9bcaff22e7971426a12dfe9a6ed80d161f3ad Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Mar 2026 14:18:20 -0400 Subject: [PATCH 003/109] Account for realization config json not including feature type prefix --- .../realizations/catchment/Bmi_Multi_Formulation.hpp | 2 +- .../realizations/catchment/Catchment_Formulation.hpp | 6 ++++++ .../realizations/catchment/Formulation_Manager.hpp | 8 +++++--- src/geopackage/read.cpp | 1 + src/realizations/catchment/Catchment_Formulation.cpp | 11 +++++++++++ 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 64ee733db0..b750234cff 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -620,7 +620,7 @@ namespace realization { // Since this is a nested formulation, support usage of the '{{id}}' syntax for init config file paths. Catchment_Formulation::config_pattern_substitution(properties, BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG, - "{{id}}", id); + "{{id}}", Catchment_Formulation::config_pattern_id_replacement(id)); // Call create_formulation to perform the rest of the typical initialization steps for the formulation. mod->create_formulation(properties); diff --git a/include/realizations/catchment/Catchment_Formulation.hpp b/include/realizations/catchment/Catchment_Formulation.hpp index 5a777c6857..06d4ef5f7a 100644 --- a/include/realizations/catchment/Catchment_Formulation.hpp +++ b/include/realizations/catchment/Catchment_Formulation.hpp @@ -32,6 +32,12 @@ namespace realization { static void config_pattern_substitution(geojson::PropertyMap &properties, const std::string &key, const std::string &pattern, const std::string &replacement); + /**Remove leading non-numeric characters from the ID string. + * + * This may be needed to correct NGEN adding an identifying prefix to the ID with system file names without the prefix. + */ + static std::string config_pattern_id_replacement(const std::string &id); + /** * Get a header line appropriate for a file made up of entries from this type's implementation of * ``get_output_line_for_timestep``. diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 4006409231..1865eec9a0 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -200,7 +200,7 @@ namespace realization { this->add_formulation( this->construct_formulation_from_config( simulation_time_config, - catchment_config.first, + "cat-" + catchment_config.first, catchment_formulation, output_stream ) @@ -553,7 +553,7 @@ namespace realization { global_copy.formulation.parameters, BMI_REALIZATION_CFG_PARAM_REQ__INIT_CONFIG, "{{id}}", - identifier + Catchment_Formulation::config_pattern_id_replacement(identifier) ); } else { ss.str(""); ss << "init_config is present but empty for identifier: " << identifier << std::endl; @@ -665,7 +665,9 @@ namespace realization { // Replace {{id}} if present if (id_index != std::string::npos) { - filepattern = filepattern.replace(id_index, sizeof("{{id}}") - 1, identifier); + // account generate the regex to search for the ID with or without a prefix + std::string pattern_id = Catchment_Formulation::config_pattern_id_replacement(identifier); + filepattern = filepattern.replace(id_index, sizeof("{{id}}") - 1, pattern_id); } // Compile the file pattern as a regex diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 1dc91eb6e0..2b3ecf7409 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -60,6 +60,7 @@ std::shared_ptr ngen::geopackage::read( "('cat-' || divides.div_id) AS id, " "('nex-' || flowpaths.dn_nex_id) AS toid, " "flowpaths.slope AS So, " + "divides.area_sqkm AS areasqkm, " // faster for later code to rename the field here "divides.geom AS geom " "FROM divides " "LEFT JOIN flowpaths " diff --git a/src/realizations/catchment/Catchment_Formulation.cpp b/src/realizations/catchment/Catchment_Formulation.cpp index 00b22e0cf0..71c2972d0d 100644 --- a/src/realizations/catchment/Catchment_Formulation.cpp +++ b/src/realizations/catchment/Catchment_Formulation.cpp @@ -59,6 +59,17 @@ namespace realization { // LOG(ss.str(), LogLevel::DEBUG); } + std::string Catchment_Formulation::config_pattern_id_replacement(const std::string &id) { + size_t index = id.find_last_of('-'); + if (index != std::string::npos && ++index < id.length()) { + // check if first character after the last hyphen is a number + if (static_cast(id[index]) - static_cast('0') <= 9) { + return id.substr(index); + } + } + return id; + } + std::string Catchment_Formulation::get_output_header_line(std::string delimiter) const { return "Total Discharge"; } From c56c235c35fe6fd90a98e6364cadec28a155342f Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 25 Mar 2026 11:31:54 -0400 Subject: [PATCH 004/109] Change IIF to CASE for sqlite 3.26 support --- src/geopackage/read.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 2b3ecf7409..2b41cbb4e0 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -70,8 +70,14 @@ std::shared_ptr ngen::geopackage::read( feature_query = "SELECT " "('nex-' || nexus.nex_id) AS id, " - "IIF(flowpaths.div_id, ('cat-' || flowpaths.div_id), 'terminal') AS toid, " - "flowpaths.slope AS So, " + "CASE " + "WHEN flowpaths.div_id IS NULL THEN 'terminal' " + "ELSE ('cat-' || flowpaths.div_id) " + "END AS toid, " + "CASE " + "WHEN flowpaths.slope IS NULL THEN 0.0 " + "ELSE flowpaths.slope " + "END AS So, " "nexus.geom AS geom " "FROM nexus " "LEFT JOIN flowpaths " From eceb4f5645aa6504ffdcf34dcdc338bb8ec74256 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 09:53:50 -0800 Subject: [PATCH 005/109] Restore support for older-format NetCDF forcings files that have Time variables with dimensions (catchment, time) --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index ad936ca5d0..ef1a8a6cd1 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -147,8 +147,19 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat std::vector raw_time(num_times); try { - time_var.getVar(raw_time.data()); - } catch(const netCDF::exceptions::NcException& e) { + auto dim_count = time_var.getDimCount(); + // Old-format files have dimensions (catchment, time), new-format + // files generated by the forcings engine have just (time) + if (dim_count == 2) { + time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); + } else if (dim_count == 1) { + time_var.getVar({0ul}, {num_times}, raw_time.data()); + } else { + throw std::runtime_error("Unexpected " + std::to_string(dim_count) + + " dimensions on Time variable in NetCDF file '" + + input_path + "'"); + } + } catch(const std::exception& e) { netcdf_ss << "Error reading time variable: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); throw; From e48284fa9643f8bae50c5783a80b6893fea40c23 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 09:56:35 -0800 Subject: [PATCH 006/109] Clean up comments and formatting --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index ef1a8a6cd1..03d18bfd7a 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -135,8 +135,6 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat // correct string release nc_free_string(num_ids,&string_buffers[0]); -// Modified code to handle units, epoch start, and reading all time values correctly - KSL - // Get the time variable - getVar collects all values at once and stores in memory // Extremely large timespans could be problematic, but for ngen use cases, this should not be a problem auto time_var = nc_file->getVar("Time"); @@ -168,7 +166,6 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat std::string time_units; try { time_var.getAtt("units").getValues(time_units); - } catch(const netCDF::exceptions::NcException& e) { netcdf_ss << "Error reading time units: " << e.what() << std::endl; LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); @@ -180,9 +177,9 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat double time_scale_factor = 1.0; std::time_t epoch_start_time = 0; - //The following makes some assumptions that NetCDF output from the forcing engine will be relatively uniform - //Specifically, it assumes time values are in units since the Unix Epoch. - //If the forcings engine outputs additional unit formats, this will need to be expanded + // The following makes some assumptions that NetCDF output from the forcing engine will be relatively uniform + // Specifically, it assumes time values are in units since the Unix Epoch. + // If the forcings engine outputs additional unit formats, this will need to be expanded if (time_units.find("minutes since") != std::string::npos) { time_unit = TIME_MINUTES; time_scale_factor = 60.0; @@ -193,14 +190,13 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat time_unit = TIME_SECONDS; time_scale_factor = 1.0; } - //This is also based on the NetCDF from the forcings engine, and may not be super flexible + // This is also based on the NetCDF from the forcings engine, and may not be super flexible std::string datetime_str = time_units.substr(time_units.find("since") + 6); std::tm tm = {}; std::istringstream ss(datetime_str); - ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); //This may be particularly inflexible - epoch_start_time = timegm(&tm); //timegm may not be available in all environments/OSes ie: Windows + ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); // This may be particularly inflexible + epoch_start_time = timegm(&tm); // timegm may not be available in all environments/OSes ie: Windows time_vals.resize(raw_time.size()); -// End modification - KSL std::transform(raw_time.begin(), raw_time.end(), time_vals.begin(), [&](const auto& n) { From 6130bef7508752f7c5c204423e62cd5783902eb3 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 11:45:31 -0800 Subject: [PATCH 007/109] Handle more units, with and without a specified reference epoch --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 50 +++++++++++++++----- 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index 03d18bfd7a..ccecc2e0b2 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -177,25 +177,42 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat double time_scale_factor = 1.0; std::time_t epoch_start_time = 0; - // The following makes some assumptions that NetCDF output from the forcing engine will be relatively uniform - // Specifically, it assumes time values are in units since the Unix Epoch. - // If the forcings engine outputs additional unit formats, this will need to be expanded - if (time_units.find("minutes since") != std::string::npos) { + std::string time_base_unit; + auto since_index = time_units.find("since"); + if (since_index != std::string::npos) { + time_base_unit = time_units.substr(0, since_index - 1); + + std::string datetime_str = time_units.substr(since_index + 6); + std::tm tm = {}; + std::istringstream ss(datetime_str); + ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); // This may be particularly inflexible + epoch_start_time = timegm(&tm); // timegm may not be available in all environments/OSes ie: Windows + } else { + time_base_unit = time_units; + } + + if (time_base_unit == "minutes") { time_unit = TIME_MINUTES; time_scale_factor = 60.0; - } else if (time_units.find("hours since") != std::string::npos) { + } else if (time_base_unit == "hours") { time_unit = TIME_HOURS; time_scale_factor = 3600.0; - } else { + } else if (time_base_unit == "seconds" || time_base_unit == "s") { time_unit = TIME_SECONDS; time_scale_factor = 1.0; + } else if (time_base_unit == "milliseconds" || time_base_unit == "ms") { + time_unit = TIME_MILLISECONDS; + time_scale_factor = 1.0e-3; + } else if (time_base_unit == "microseconds" || time_base_unit == "us") { + time_unit = TIME_MICROSECONDS; + time_scale_factor = 1.0e-6; + } else if (time_base_unit == "nanoseconds" || time_base_unit == "ns") { + time_unit = TIME_NANOSECONDS; + time_scale_factor = 1.0e-9; + } else { + Logger::logMsgAndThrowError("In NetCDF file '" + input_path + "', time unit '" + time_base_unit + "' could not be converted"); } - // This is also based on the NetCDF from the forcings engine, and may not be super flexible - std::string datetime_str = time_units.substr(time_units.find("since") + 6); - std::tm tm = {}; - std::istringstream ss(datetime_str); - ss >> std::get_time(&tm, "%Y-%m-%d %H:%M:%S"); // This may be particularly inflexible - epoch_start_time = timegm(&tm); // timegm may not be available in all environments/OSes ie: Windows + time_vals.resize(raw_time.size()); std::transform(raw_time.begin(), raw_time.end(), time_vals.begin(), @@ -221,13 +238,20 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat #endif netcdf_ss << "All time intervals are constant within tolerance." << std::endl; - LOG(netcdf_ss.str(), LogLevel::SEVERE); netcdf_ss.str(""); + LOG(netcdf_ss.str(), LogLevel::DEBUG); netcdf_ss.str(""); // determine start_time and stop_time; start_time = time_vals[0]; stop_time = time_vals.back() + time_stride; sim_to_data_time_offset = sim_start_date_time_epoch - start_time; + + netcdf_ss << "NetCDF Forcing from file '" << input_path << "'" + << "Start time " << (time_t)start_time + << ", Stop time " << (time_t)stop_time + << ", sim_start_date_time_epoch " << sim_start_date_time_epoch + ; + LOG(netcdf_ss.str(), LogLevel::DEBUG); netcdf_ss.str(""); } NetCDFPerFeatureDataProvider::~NetCDFPerFeatureDataProvider() = default; From 6e7bf292ebdc2d9e6c4b7355a027aeb596ac7fca Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Fri, 26 Dec 2025 13:39:31 -0800 Subject: [PATCH 008/109] Handle different orderings of variable dimensions --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 51 ++++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index ccecc2e0b2..bb8507145d 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -149,6 +149,9 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat // Old-format files have dimensions (catchment, time), new-format // files generated by the forcings engine have just (time) if (dim_count == 2) { + if (time_var.getDim(0).getName() != "catchment-id" || time_var.getDim(1).getName() != "time") { + Logger::logMsgAndThrowError("In NetCDF file '" + input_path + "', 'Time' variable dimensions don't match expectations"); + } time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); } else if (dim_count == 1) { time_var.getVar({0ul}, {num_times}, raw_time.data()); @@ -335,7 +338,8 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& auto stride = idx2 - idx1; - std::vector start, count; + std::vector start(2), count(2); + std::vector var_index_map(2); auto cat_pos = id_pos[selector.get_id()]; @@ -356,16 +360,29 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& //TODO: Currently assuming a whole variable cache slice across all catchments for a single timestep...but some stuff here to support otherwise. // For reference: https://stackoverflow.com/a/72030286 -//Modified to work with NetCDF dimension shapes and fix errors - KSL size_t cache_slices_t_n = (read_len + cache_slice_t_size - 1) / cache_slice_t_size; // Ceiling division to ensure remainders have a slice - - //Explicitly setting dimension shapes auto dims = ncvar.getDims(); - size_t catchment_dim_size = dims[1].getSize(); - size_t time_dim_size = dims[0].getSize(); - //Cache slicing - modified to work with dimensions structure + int dim_time, dim_catchment; + if (dims.size() != 2) { + Logger::logMsgAndThrowError("Variable dimension count isn't 2"); + } + if (dims[0].getName() == "time" && dims[1].getName() == "catchment-id") { + // Forcings Engine NetCDF output case + dim_time = 0; + dim_catchment = 1; + } else if (dims[1].getName() == "time" && dims[0].getName() == "catchment-id") { + // Classic NetCDF file case + dim_time = 1; + dim_catchment = 0; + } else { + Logger::logMsgAndThrowError("Variable dimensions aren't 'time' and 'catchment-id'"); + } + + size_t time_dim_size = dims[dim_time].getSize(); + size_t catchment_dim_size = dims[dim_catchment].getSize(); + for( size_t i = 0; i < cache_slices_t_n; i++ ) { std::shared_ptr> cached; size_t cache_t_idx = idx1 + i * cache_slice_t_size; @@ -376,14 +393,18 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& cached = value_cache.get(key).get(); } else { cached = std::make_shared>(catchment_dim_size * slice_size); - start.clear(); - start.push_back(cache_t_idx); // start from correct time index - start.push_back(0); // Start from the first catchment - count.clear(); - count.push_back(slice_size); // Read the calculated slice size for time - count.push_back(catchment_dim_size); // Read all catchments + start[dim_time] = cache_t_idx; // start from correct time index + start[dim_catchment] = 0; // Start from the first catchment + count[dim_time] = slice_size; // Read the calculated slice size for time + count[dim_catchment] = catchment_dim_size; // Read all catchments + // Whichever order the file stores the data in, the + // resulting array should have all catchments for a given + // time step contiguous + var_index_map[dim_time] = catchment_dim_size; + var_index_map[dim_catchment] = 1; + try { - ncvar.getVar(start,count,&(*cached)[0]); + ncvar.getVar(start,count, {1l, 1l}, var_index_map, cached->data()); value_cache.insert(key, cached); } catch (netCDF::exceptions::NcException& e) { netcdf_ss << "NetCDF exception: " << e.what() << std::endl; @@ -408,7 +429,7 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& } } } -// End modification + rvalue = 0.0; double a , b = 0.0; From 7871dcd87b2bcdd8ea79c9efba352c01aac41b79 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 15:55:21 -0800 Subject: [PATCH 009/109] Flesh out API for state saving and restoring, with adjusted use in Bmi_Module_Formulation --- .../catchment/Bmi_Formulation.hpp | 10 ++ .../catchment/Bmi_Module_Formulation.hpp | 9 +- .../state_save_restore/State_Save_Restore.hpp | 125 ++++++++++++++++++ .../catchment/Bmi_Module_Formulation.cpp | 8 +- 4 files changed, 142 insertions(+), 10 deletions(-) create mode 100644 include/state_save_restore/State_Save_Restore.hpp diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index f2a13074e8..40cb1df399 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -41,6 +41,8 @@ class Bmi_Formulation_Test; class Bmi_C_Formulation_Test; class Bmi_C_Pet_IT; +class State_Snapshot_Saver; + namespace realization { /** @@ -68,6 +70,14 @@ namespace realization { virtual ~Bmi_Formulation() {}; + /** + * Passes a serialized representation of the model's state to ``saver`` + * + * Asks the model to serialize its state, queries the pointer + * and length, passes that to saver, and then releases it + */ + virtual void save_state(std::shared_ptr saver) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index cd6dda4969..2b09ea9f75 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -7,7 +7,6 @@ #include "Bmi_Adapter.hpp" #include #include "bmi_utilities.hpp" -#include #include #include "bmi/protocols.hpp" @@ -48,13 +47,7 @@ namespace realization { void create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global = nullptr) override; void create_formulation(geojson::PropertyMap properties) override; - /** - * Passes a serialized representation of the model's state to ``saver`` - * - * Asks the model to serialize its state, queries the pointer - * and length, passes that to saver, and then releases it - */ - void save_state(std::shared_ptr saver) const; + void save_state(std::shared_ptr saver) const override; /** * Get the collection of forcing output property names this instance can provide. diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp new file mode 100644 index 0000000000..2871865dc9 --- /dev/null +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -0,0 +1,125 @@ +#ifndef NGEN_STATE_SAVE_RESTORE_HPP +#define NGEN_STATE_SAVE_RESTORE_HPP + +#include + +#include +#include +#include + +class State_Snapshot_Saver; + +class State_Saver +{ +public: + using snapshot_time_t = std::chrono::time_point; + + // Flag type to indicate whether state saving needs to ensure + // stability of saved data wherever it is stored before returning + // success + enum class State_Durability { relaxed, strict }; + + State_Saver() = default; + virtual ~State_Saver() = default; + + /** + * Return an object suitable for saving a simulation state as of a + * particular moment in time, @param epoch + * + * @param durability indicates whether callers expect all + * potential errors to be checked and reported before finalize() + * and/or State_Snapshot_Saver::finish_saving() return + */ + virtual std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) = 0; + + /** + * Execute any logic necessary to cleanly finish usage, and + * potentially report errors, before destructors would + * execute. E.g. closing files opened in parallel across MPI + * ranks. + */ + virtual void finalize() = 0; +}; + +class State_Snapshot_Saver +{ +public: + State_Snapshot_Saver() = delete; + State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + virtual ~State_Snapshot_Saver() = default; + + /** + * Capture the data from a single unit of the simulation + */ + virtual void save_unit(std::string const& unit_name, boost::span data) = 0; + + /** + * Execute logic to complete the saving process + * + * Data may be flushed here, and delayed errors may be detected + * and reported here. With relaxed durability, error reports may + * not come until the parent State_Saver::finalize() call is made, + * or ever. + */ + virtual void finish_saving() = 0; + +protected: + State_Saver::snapshot_time_t epoch_; + State_Saver::State_Durability durability_; +}; + +class State_Snapshot_Loader; + +class State_Loader +{ +public: + State_Loader() = default; + virtual ~State_Loader() = default; + + /** + * Return an object suitable for loading a simulation state as of + * a particular moment in time, @param epoch + */ + virtual std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) = 0; + + /** + * Execute any logic necessary to cleanly finish usage, and + * potentially report errors, before destructors would + * execute. E.g. closing files opened in parallel across MPI + * ranks. + */ + virtual void finalize() = 0; +}; + +class State_Unit_Loader; + +class State_Snapshot_Loader +{ +public: + State_Snapshot_Loader() = default; + virtual ~State_Snapshot_Loader() = default; + + /** + * Load data from whatever source, and pass it to @param unit_loader->load() + */ + virtual void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) = 0; + + /** + * Execute logic to complete the saving process + * + * Data may be flushed here, and delayed errors may be detected + * and reported here. With relaxed durability, error reports may + * not come until the parent State_Saver::finalize() call is made, + * or ever. + */ + virtual void finish_saving() = 0; +}; + +class State_Unit_Loader +{ + State_Unit_Loader() = default; + virtual ~State_Unit_Loader() = default; + virtual void load(boost::span data) = 0; +}; + +#endif // NGEN_STATE_SAVE_RESTORE_HPP diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index da478a294e..af50ad92e5 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -2,6 +2,7 @@ #include "utilities/logging_utils.h" #include #include "Logger.hpp" +#include std::stringstream bmiform_ss; @@ -15,7 +16,7 @@ namespace realization { inner_create_formulation(properties, true); } - void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { + void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { auto model = get_bmi_model(); size_t size = 1; @@ -25,7 +26,10 @@ namespace realization { auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); boost::span data(serialization_state, size); - saver->save(data); + // Rely on Formulation_Manager also using this->get_id() + // as a unique key for the individual catchment + // formulations + saver->save_unit(this->get_id(), data); model->SetValue("serialization_free", &size); } From 9bb5c3ba024891769bb999f7b496493063053adf Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 17:06:24 -0800 Subject: [PATCH 010/109] WIP Wiring all the pieces together --- include/core/Layer.hpp | 4 ++++ include/core/NgenSimulation.hpp | 4 ++++ src/core/Layer.cpp | 12 ++++++++++ src/core/NgenSimulation.cpp | 10 +++++++++ .../catchment/Bmi_Multi_Formulation.cpp | 22 +++++++++++++++++++ 5 files changed, 52 insertions(+) diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 5c3c4481fa..6e5fe707e8 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -16,6 +16,8 @@ namespace hy_features class HY_Features_MPI; } +class State_Snapshot_Saver; + namespace ngen { @@ -110,6 +112,8 @@ namespace ngen std::unordered_map &nexus_indexes, int current_step); + virtual void save_state_snapshot(std::shared_ptr snapshot_saver); + protected: const LayerDescription description; diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 00e5ef49eb..0e164b664a 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -12,6 +12,8 @@ namespace hy_features class HY_Features_MPI; } +class State_Snapshot_Saver; + #include #include #include @@ -62,6 +64,8 @@ class NgenSimulation private: void advance_models_one_output_step(); + void save_state_snapshot(std::shared_ptr snapshot_saver); + int simulation_step_; std::shared_ptr sim_time_; diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index aac9ced099..cbde74e8fd 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -1,5 +1,6 @@ #include #include +#include #if NGEN_WITH_MPI #include "HY_Features_MPI.hpp" @@ -92,3 +93,14 @@ void ngen::Layer::update_models(boost::span catchment_outflows, simulation_time.advance_timestep(); } } + +void ngen::Layer::save_state_snapshot(std::shared_ptr snapshot_saver) +{ + // XXX Handle any of this class's own state as a meta-data unit + + for (auto const& id : processing_units) { + auto r = features.catchment_at(id); + auto r_c = std::dynamic_pointer_cast(r); + r_c->save_state(snapshot_saver); + } +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c5cce69103..c2f9bdf04a 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -108,6 +108,16 @@ void NgenSimulation::advance_models_one_output_step() } +void NgenSimulation::save_state_snapshot(std::shared_ptr snapshot_saver) +{ + + // XXX Handle self, then recursively pass responsibility to Layers + for (auto& layer : layers_) { + layer->save_state_snapshot(snapshot_saver); + } +} + + int NgenSimulation::get_nexus_index(std::string const& nexus_id) const { auto iter = nexus_indexes_.find(nexus_id); diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 145b7dfb73..b934166068 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -13,8 +13,30 @@ #include "Bmi_Py_Formulation.hpp" #include "Logger.hpp" +#include + using namespace realization; +void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { +#if 0 + auto model = get_bmi_model(); + + size_t size = 1; + model->SetValue("serialization_create", &size); + model->GetValue("serialization_size", &size); + + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + boost::span data(serialization_state, size); + + // Rely on Formulation_Manager also using this->get_id() + // as a unique key for the individual catchment + // formulations + saver->save_unit(this->get_id(), data); + + model->SetValue("serialization_free", &size); +#endif +} + void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap properties, bool needs_param_validation) { if (needs_param_validation) { validate_parameters(properties); From 170e9c7a1d064d3c292971c3b6370f18f4844876 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Mon, 8 Dec 2025 17:53:19 -0800 Subject: [PATCH 011/109] Add logic and structures for parsing state saving configuration from realization config --- CMakeLists.txt | 1 + data/example_state_saving_config.json | 109 ++++++++++++++++++ .../state_save_restore/State_Save_Restore.hpp | 28 +++++ src/NGen.cpp | 5 +- src/state_save_restore/CMakeLists.txt | 17 +++ src/state_save_restore/State_Save_Restore.cpp | 64 ++++++++++ 6 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 data/example_state_saving_config.json create mode 100644 src/state_save_restore/CMakeLists.txt create mode 100644 src/state_save_restore/State_Save_Restore.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c96c93b27..a4ebf877c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,6 +318,7 @@ add_subdirectory("src/geojson") add_subdirectory("src/bmi") add_subdirectory("src/realizations/catchment") add_subdirectory("src/forcing") +add_subdirectory("src/state_save_restore") add_subdirectory("src/utilities") add_subdirectory("src/utilities/mdarray") add_subdirectory("src/utilities/mdframe") diff --git a/data/example_state_saving_config.json b/data/example_state_saving_config.json new file mode 100644 index 0000000000..cba48afcdb --- /dev/null +++ b/data/example_state_saving_config.json @@ -0,0 +1,109 @@ +{ + "global": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "file_pattern": ".*{{id}}.*.csv", + "path": "./data/forcing/" + } + }, + "state_saving": { + "label": "end", + "path": "state_end", + "type": "FilePerUnit", + "when": "EndOfRun" + }, + "time": { + "start_time": "2015-12-01 00:00:00", + "end_time": "2015-12-30 23:00:00", + "output_interval": 3600 + }, + "output_root": "./output_dir/", + "catchments": { + "cat-27": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "path": "./data/forcing/cat-27_2015-12-01 00_00_00_2015-12-30 23_00_00.csv" + } + }, + "cat-52": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "path": "./data/forcing/cat-52_2015-12-01 00_00_00_2015-12-30 23_00_00.csv" + } + }, + "cat-67": { + "formulations": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "test_bmi_cpp", + "library_file": "./extern/test_bmi_cpp/cmake_build/libtestbmicppmodel.so", + "init_config": "./data/bmi/c/test/test_bmi_c_config.ini", + "main_output_variable": "OUTPUT_VAR_2", + "variables_names_map" : { + "INPUT_VAR_2": "TMP_2maboveground", + "INPUT_VAR_1": "precip_rate" + }, + "create_function": "bmi_model_create", + "destroy_function": "bmi_model_destroy", + "uses_forcing_file": false + } + } + ], + "forcing": { + "path": "./data/forcing/cat-67_2015-12-01 00_00_00_2015-12-30 23_00_00.csv" + } + } + } +} diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 2871865dc9..a7226a5df8 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -3,9 +3,37 @@ #include +#include +#include + #include #include #include +#include +#include + +class State_Save_Config +{ +public: + /** + * Expects the tree @param config that potentially contains a "state_saving" key + * + * + */ + State_Save_Config(boost::property_tree::ptree const& config); + + struct instance + { + instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); + + std::string label_; + std::string path_; + std::string mechanism_; + std::string timing_; + }; + + std::vector instances_; +}; class State_Snapshot_Saver; diff --git a/src/NGen.cpp b/src/NGen.cpp index eaeb99bb03..ce6d705fa9 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -54,6 +54,8 @@ #include #include +#include + void ngen::exec_info::runtime_summary(std::ostream& stream) noexcept { stream << "Runtime configuration summary:\n"; @@ -516,9 +518,10 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } auto simulation_time_config = realization::config::Time(*possible_simulation_time).make_params(); - sim_time = std::make_shared(simulation_time_config); + auto state_saving_config = State_Save_Config(realization_config); + ss << "Initializing formulations" << std::endl; LOG(ss.str(), LogLevel::INFO); ss.str(""); diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt new file mode 100644 index 0000000000..7a3db83e44 --- /dev/null +++ b/src/state_save_restore/CMakeLists.txt @@ -0,0 +1,17 @@ +include(${PROJECT_SOURCE_DIR}/cmake/dynamic_sourced_library.cmake) +dynamic_sourced_cxx_library(state_save_restore "${CMAKE_CURRENT_SOURCE_DIR}") + +add_library(NGen::state_save_restore ALIAS state_save_restore) +target_link_libraries(state_save_restore PUBLIC + NGen::config_header + Boost::boost # Headers-only Boost + ) + +#target_link_libraries(core +# PRIVATE +# ) + +target_include_directories(state_save_restore PUBLIC + ${PROJECT_SOURCE_DIR}/include + ) + diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp new file mode 100644 index 0000000000..1b53fb79fe --- /dev/null +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -0,0 +1,64 @@ +#include + +#include + +#include +#include + +#include + +State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) +{ + auto maybe = tree.get_child_optional("state_saving"); + + // Default initialization will represent the "not enabled" case + if (!maybe) { + LOG("State saving not configured", LogLevel::INFO); + return; + } + + auto subtree = *maybe; + + try { + auto single_what = subtree.get("label"); + auto single_where = subtree.get("path"); + auto single_how = subtree.get("type"); + auto single_when = subtree.get("when"); + + instance i{single_what, single_where, single_how, single_when}; + instances_.push_back(i); + } catch (...) { + LOG("Bad state saving config", LogLevel::WARNING); + throw; + } + + LOG("State saving configured", LogLevel::INFO); +} + +State_Save_Config::instance::instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) + : label_(label) + , path_(path) + , mechanism_(mechanism) + , timing_(timing) +{ + if (mechanism_ == "FilePerUnit") { + // nothing to do here + } else { + Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism_ + "'"); + } + + if (timing_ == "EndOfRun") { + // nothing to do here + } else if (timing_ == "FirstOfMonth") { + // nothing to do here + } else { + Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing_ + "'"); + } +} + +State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) + : epoch_(epoch) + , durability_(durability) +{ + +} From dd0458df925466b4c12eb1024161866f18fd4ea0 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Wed, 10 Dec 2025 08:51:22 -0800 Subject: [PATCH 012/109] CMakeLists.txt: Remove commented out bits --- src/state_save_restore/CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt index 7a3db83e44..8c09fc7cbc 100644 --- a/src/state_save_restore/CMakeLists.txt +++ b/src/state_save_restore/CMakeLists.txt @@ -7,10 +7,6 @@ target_link_libraries(state_save_restore PUBLIC Boost::boost # Headers-only Boost ) -#target_link_libraries(core -# PRIVATE -# ) - target_include_directories(state_save_restore PUBLIC ${PROJECT_SOURCE_DIR}/include ) From 58331e97911802a1e01258e9e7a9ee67516af546 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Mon, 15 Dec 2025 21:30:25 -0800 Subject: [PATCH 013/109] Add File_Per_Unit state saving mechanism --- include/state_save_restore/File_Per_Unit.hpp | 20 ++++ src/state_save_restore/File_Per_Unit.cpp | 101 +++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 include/state_save_restore/File_Per_Unit.hpp create mode 100644 src/state_save_restore/File_Per_Unit.cpp diff --git a/include/state_save_restore/File_Per_Unit.hpp b/include/state_save_restore/File_Per_Unit.hpp new file mode 100644 index 0000000000..cb6af8e0ac --- /dev/null +++ b/include/state_save_restore/File_Per_Unit.hpp @@ -0,0 +1,20 @@ +#ifndef NGEN_FILE_PER_UNIT_HPP +#define NGEN_FILE_PER_UNIT_HPP + +#include + +class File_Per_Unit_Saver : public State_Saver +{ +public: + File_Per_Unit_Saver(std::string base_path); + ~File_Per_Unit_Saver(); + + std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) override; + + void finalize() override; + +private: + std::string base_path_; +}; + +#endif // NGEN_FILE_PER_UNIT_HPP diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp new file mode 100644 index 0000000000..d7c7ddf2b9 --- /dev/null +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -0,0 +1,101 @@ +#include +#include + +#if __has_include() && __cpp_lib_filesystem >= 201703L + #include + using namespace std::filesystem; + #warning "Using STD Filesystem" +#elif __has_include() && defined(__cpp_lib_filesystem) + #include + using namespace std::experimental::filesystem; + #warning "Using Filesystem TS" +#elif __has_include() + #include + using namespace boost::filesystem; + #warning "Using Boost.Filesystem" +#else + #error "No Filesystem library implementation available" +#endif + +#include +#include + +// This class is only declared and defined here, in the .cpp file, +// because it is strictly an implementation detail of the top-level +// File_Per_Unit_Saver class +class File_Per_Unit_Snapshot_Saver : public State_Snapshot_Saver +{ + friend class File_Per_Unit_Saver; + + public: + File_Per_Unit_Snapshot_Saver() = delete; + File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + ~File_Per_Unit_Snapshot_Saver(); + +public: + void save_unit(std::string const& unit_name, boost::span data) override; + void finish_saving() override; + +private: + std::string format_epoch(State_Saver::snapshot_time_t epoch); + + path dir_path_; +}; + +File_Per_Unit_Saver::File_Per_Unit_Saver(std::string base_path) + : base_path_(std::move(base_path)) +{ + auto dir_path = path(base_path_); + create_directories(dir_path); +} + +File_Per_Unit_Saver::~File_Per_Unit_Saver() = default; + +std::shared_ptr File_Per_Unit_Saver::initialize_snapshot(snapshot_time_t epoch, State_Durability durability) +{ + return std::make_shared(base_path_, epoch, durability); +} + +void File_Per_Unit_Saver::finalize() +{ + // nothing to be done +} + +File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) + : State_Snapshot_Saver(epoch, durability) + , dir_path_(base_path / format_epoch(epoch)) +{ + create_directory(dir_path_); +} + +File_Per_Unit_Snapshot_Saver::~File_Per_Unit_Snapshot_Saver() = default; + +std::string File_Per_Unit_Snapshot_Saver::format_epoch(State_Saver::snapshot_time_t epoch) +{ + time_t t = std::chrono::system_clock::to_time_t(epoch); + std::tm tm = *std::gmtime(&t); + + std::stringstream tss; + tss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S"); + return tss.str(); +} + +void File_Per_Unit_Snapshot_Saver::save_unit(std::string const& unit_name, boost::span data) +{ + auto file_path = dir_path_ / unit_name; + try { + std::ofstream stream(file_path.string(), std::ios_base::out | std::ios_base::binary); + stream.write(data.data(), data.size()); + stream.close(); + } catch (std::exception &e) { + LOG("Failed to write state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } +} + +void File_Per_Unit_Snapshot_Saver::finish_saving() +{ + if (durability_ == State_Saver::State_Durability::strict) { + // fsync() or whatever + } +} From 53b3767e8f45d79f0781fc8ad9f33a37103724b3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 16 Jan 2026 14:06:13 -0500 Subject: [PATCH 014/109] State saving for multi-BMI --- data/example_state_saving_config_multi.json | 116 ++++++++++++++++++ include/core/NgenSimulation.hpp | 4 +- .../catchment/Bmi_Fortran_Formulation.hpp | 9 ++ .../catchment/Bmi_Module_Formulation.hpp | 17 +++ .../state_save_restore/State_Save_Restore.hpp | 12 +- src/NGen.cpp | 10 ++ .../catchment/Bmi_Fortran_Formulation.cpp | 10 ++ .../catchment/Bmi_Module_Formulation.cpp | 25 ++-- .../catchment/Bmi_Multi_Formulation.cpp | 69 ++++++++--- src/state_save_restore/CMakeLists.txt | 2 + src/state_save_restore/State_Save_Restore.cpp | 30 +++++ 11 files changed, 277 insertions(+), 27 deletions(-) create mode 100644 data/example_state_saving_config_multi.json diff --git a/data/example_state_saving_config_multi.json b/data/example_state_saving_config_multi.json new file mode 100644 index 0000000000..48ba4d8b58 --- /dev/null +++ b/data/example_state_saving_config_multi.json @@ -0,0 +1,116 @@ +{ + "global": { + "formulations": [ + { + "name": "bmi_multi", + "params": { + "model_type_name": "bmi_multi_noahowp_cfe", + "forcing_file": "", + "init_config": "", + "allow_exceed_end_time": true, + "main_output_variable": "Q_OUT", + "modules": [ + { + "name": "bmi_c++", + "params": { + "model_type_name": "bmi_c++_sloth", + "library_file": "./extern/sloth/cmake_build/libslothmodel.so", + "init_config": "/dev/null", + "allow_exceed_end_time": true, + "main_output_variable": "z", + "uses_forcing_file": false, + "model_params": { + "sloth_ice_fraction_schaake(1,double,m,node)": 0.0, + "sloth_ice_fraction_xinanjiang(1,double,1,node)": 0.0, + "sloth_smp(1,double,1,node)": 0.0 + } + } + }, + { + "name": "bmi_fortran", + "params": { + "model_type_name": "bmi_fortran_noahowp", + "library_file": "./extern/noah-owp-modular/cmake_build/libsurfacebmi", + "forcing_file": "", + "init_config": "./data/bmi/fortran/noah-owp-modular-init-{{id}}.namelist.input", + "allow_exceed_end_time": true, + "main_output_variable": "QINSUR", + "variables_names_map": { + "PRCPNONC": "atmosphere_water__liquid_equivalent_precipitation_rate", + "Q2": "atmosphere_air_water~vapor__relative_saturation", + "SFCTMP": "land_surface_air__temperature", + "UU": "land_surface_wind__x_component_of_velocity", + "VV": "land_surface_wind__y_component_of_velocity", + "LWDN": "land_surface_radiation~incoming~longwave__energy_flux", + "SOLDN": "land_surface_radiation~incoming~shortwave__energy_flux", + "SFCPRS": "land_surface_air__pressure" + }, + "uses_forcing_file": false + } + }, + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_pet", + "library_file": "./extern/evapotranspiration/evapotranspiration/cmake_build/libpetbmi", + "forcing_file": "", + "init_config": "./data/bmi/c/pet/{{id}}_bmi_config.ini", + "allow_exceed_end_time": true, + "main_output_variable": "water_potential_evaporation_flux", + "registration_function": "register_bmi_pet", + "variables_names_map": { + "water_potential_evaporation_flux": "potential_evapotranspiration" + }, + "uses_forcing_file": false + } + }, + { + "name": "bmi_c", + "params": { + "model_type_name": "bmi_c_cfe", + "library_file": "./extern/cfe/cmake_build/libcfebmi", + "forcing_file": "", + "init_config": "./data/bmi/c/cfe/{{id}}_bmi_config.ini", + "allow_exceed_end_time": true, + "main_output_variable": "Q_OUT", + "registration_function": "register_bmi_cfe", + "variables_names_map": { + "water_potential_evaporation_flux": "potential_evapotranspiration", + "atmosphere_air_water~vapor__relative_saturation": "SPFH_2maboveground", + "land_surface_air__temperature": "TMP_2maboveground", + "land_surface_wind__x_component_of_velocity": "UGRD_10maboveground", + "land_surface_wind__y_component_of_velocity": "VGRD_10maboveground", + "land_surface_radiation~incoming~longwave__energy_flux": "DLWRF_surface", + "land_surface_radiation~incoming~shortwave__energy_flux": "DSWRF_surface", + "land_surface_air__pressure": "PRES_surface", + "ice_fraction_schaake" : "sloth_ice_fraction_schaake", + "ice_fraction_xinanjiang" : "sloth_ice_fraction_xinanjiang", + "soil_moisture_profile" : "sloth_smp" + }, + "uses_forcing_file": false + } + } + ], + "uses_forcing_file": false + } + } + ], + "forcing": { + "file_pattern": ".*{{id}}.*..csv", + "path": "./data/forcing/", + "provider": "CsvPerFeature" + } + }, + "state_saving": { + "label": "end", + "path": "state_end", + "type": "FilePerUnit", + "when": "EndOfRun" + }, + "time": { + "start_time": "2015-12-01 00:00:00", + "end_time": "2015-12-30 23:00:00", + "output_interval": 3600 + }, + "output_root": "./output_dir/" +} diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index 0e164b664a..d60dd165b4 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -61,11 +61,11 @@ class NgenSimulation size_t get_num_output_times() const; std::string get_timestamp_for_step(int step) const; + void save_state_snapshot(std::shared_ptr snapshot_saver); + private: void advance_models_one_output_step(); - void save_state_snapshot(std::shared_ptr snapshot_saver); - int simulation_step_; std::shared_ptr sim_time_; diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 4f6863b883..71225d4885 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -23,6 +23,15 @@ namespace realization { std::string get_formulation_type() const override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface and copy the results to `size`. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + const char* create_save_state(uint64_t *size) const override; + protected: /** diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 2b09ea9f75..477c53a173 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -47,8 +47,25 @@ namespace realization { void create_formulation(boost::property_tree::ptree &config, geojson::PropertyMap *global = nullptr) override; void create_formulation(geojson::PropertyMap properties) override; + /** + * Create a save state, save it using the `State_Snapshot_Saver`, then clear the save state from memory. + * `this->get_id()` will be used as the unique ID for the saver. + */ void save_state(std::shared_ptr saver) const override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + virtual const char* create_save_state(uint64_t *size) const; + + /** + * Clears any serialized data stored by the BMI from memory. + */ + virtual void free_save_state() const; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index a7226a5df8..fea1555d21 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -12,6 +12,9 @@ #include #include +class State_Saver; +class State_Snapshot_Saver; + class State_Save_Config { public: @@ -22,6 +25,10 @@ class State_Save_Config */ State_Save_Config(boost::property_tree::ptree const& config); + bool has_end_of_run() const; + + std::shared_ptr end_of_run_saver() const; + struct instance { instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); @@ -32,11 +39,10 @@ class State_Save_Config std::string timing_; }; +private: std::vector instances_; }; -class State_Snapshot_Saver; - class State_Saver { public: @@ -50,6 +56,8 @@ class State_Saver State_Saver() = default; virtual ~State_Saver() = default; + static snapshot_time_t snapshot_time_now(); + /** * Return an object suitable for saving a simulation state as of a * particular moment in time, @param epoch diff --git a/src/NGen.cpp b/src/NGen.cpp index ce6d705fa9..b54a12769d 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -703,6 +703,16 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); + if (state_saving_config.has_end_of_run()) { + LOG("Saving end-of-run state.", LogLevel::INFO); + std::shared_ptr saver = state_saving_config.end_of_run_saver(); + std::shared_ptr snapshot = saver->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_state_snapshot(snapshot); + } + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 390a6fb4e7..ae43929ac6 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -93,4 +93,14 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const return 1.0; } +const char* Bmi_Fortran_Formulation::create_save_state(uint64_t *size) const { + auto model = get_bmi_model(); + int size_int = 1; + model->SetValue("serialization_create", &size_int); + model->GetValue("serialization_size", &size_int); + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + *size = static_cast(size_int); + return serialization_state; +} + #endif // NGEN_WITH_BMI_FORTRAN diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index af50ad92e5..bdad7105db 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -17,13 +17,8 @@ namespace realization { } void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { - auto model = get_bmi_model(); - - size_t size = 1; - model->SetValue("serialization_create", &size); - model->GetValue("serialization_size", &size); - - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + uint64_t size = 1; + const char* serialization_state = this->create_save_state(&size); boost::span data(serialization_state, size); // Rely on Formulation_Manager also using this->get_id() @@ -31,7 +26,21 @@ namespace realization { // formulations saver->save_unit(this->get_id(), data); - model->SetValue("serialization_free", &size); + this->free_save_state(); + } + + const char* Bmi_Module_Formulation::create_save_state(uint64_t *size) const { + auto model = get_bmi_model(); + model->SetValue("serialization_create", size); + model->GetValue("serialization_size", size); + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + return serialization_state; + } + + void Bmi_Module_Formulation::free_save_state() const { + auto model = get_bmi_model(); + int _; + model->SetValue("serialization_free", &_); } boost::span Bmi_Module_Formulation::get_available_variable_names() const { diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index b934166068..9de11a02d7 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -15,26 +15,65 @@ #include +#if (__cplusplus >= 202002L) +#include +#endif + using namespace realization; void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { -#if 0 - auto model = get_bmi_model(); - - size_t size = 1; - model->SetValue("serialization_create", &size); - model->GetValue("serialization_size", &size); - - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); - boost::span data(serialization_state, size); - - // Rely on Formulation_Manager also using this->get_id() - // as a unique key for the individual catchment - // formulations - saver->save_unit(this->get_id(), data); +#if (__cplusplus < 202002L) + // get system endianness + uint16_t endian_bytes = 0xFF00; + uint8_t *endian_bits = reinterpret_cast(&endian_bytes); + bool is_little_endian = endian_bits[0] == 0; +#endif - model->SetValue("serialization_free", &size); + std::vector> bmi_data; + size_t data_size = 0; + uint64_t ser_size; + // TODO: something more elegant than just skipping sloth + for (const nested_module_ptr &m : modules) { + auto bmi = static_cast(m.get()); + if (bmi->get_model_type_name() != "bmi_c++_sloth") { + ser_size = 1; + const char* serialized = bmi->create_save_state(&ser_size); + bmi_data.push_back(std::make_pair(serialized, ser_size)); + data_size += sizeof(uint64_t) + ser_size; + LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), ser_size); + } + } + char *data = new char[data_size]; + size_t index = 0; + for (const auto &bmi : bmi_data) { + // write the size of the data +#if (__cplusplus < 202002L) + if (is_little_endian) { +#else + if constexpr (std::endian::native == std::endian::little) { #endif + std::memcpy(&data[index], &bmi.second, sizeof(uint64_t)); + } else { + // store the size bytes in reverse order to ensure saved data is always little endian + const char *bytes = reinterpret_cast(&bmi.second); + size_t endian_index = index + sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + data[--endian_index] = bytes[i]; + } + } + // write the serialized data + std::memcpy(data + index + sizeof(uint64_t), &bmi.first, bmi.second); + index += sizeof(uint64_t) + bmi.second; + } + boost::span span(data, data_size); + saver->save_unit(this->get_id(), span); + + delete[] data; + for (const nested_module_ptr &m : modules) { + auto bmi = static_cast(m.get()); + bmi->free_save_state(); + } } void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap properties, bool needs_param_validation) { diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt index 8c09fc7cbc..cde58c6073 100644 --- a/src/state_save_restore/CMakeLists.txt +++ b/src/state_save_restore/CMakeLists.txt @@ -5,6 +5,8 @@ add_library(NGen::state_save_restore ALIAS state_save_restore) target_link_libraries(state_save_restore PUBLIC NGen::config_header Boost::boost # Headers-only Boost + Boost::system + Boost::filesystem ) target_include_directories(state_save_restore PUBLIC diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 1b53fb79fe..2df94b9515 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -1,4 +1,5 @@ #include +#include #include @@ -35,6 +36,26 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) LOG("State saving configured", LogLevel::INFO); } +bool State_Save_Config::has_end_of_run() const { + for (const auto& i : instances_) + if (i.timing_ == "EndOfRun") + return true; + return false; +} + +std::shared_ptr State_Save_Config::end_of_run_saver() const { + for (const auto& i : instances_) { + if (i.timing_ == "EndOfRun") { + if (i.mechanism_ == "FilePerUnit") { + return std::make_shared(i.path_); + } else { + Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_ + " is not supported for end of run saving."); + } + } + } + Logger::logMsgAndThrowError("State_Save_Config: No end of run was defined in the realization config."); +} + State_Save_Config::instance::instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) : label_(label) , path_(path) @@ -62,3 +83,12 @@ State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, S { } + +State_Saver::snapshot_time_t State_Saver::snapshot_time_now() { +#if __cplusplus < 201703L // C++ < 17 + auto now = std::chrono::system_clock::now(); + return std::chrono::time_point_cast(now); +#else + return std::chrono::floor(std::chrono::system_clock::now()); +#endif +} From ca919a33c229a87744d9d70f630f95a600d9e644 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 19 Jan 2026 13:23:42 -0500 Subject: [PATCH 015/109] Cold start config structure --- include/state_save_restore/File_Per_Unit.hpp | 14 ++ .../state_save_restore/State_Save_Restore.hpp | 35 ++++- src/state_save_restore/File_Per_Unit.cpp | 102 ++++++++++++-- src/state_save_restore/State_Save_Restore.cpp | 127 +++++++++++++----- 4 files changed, 227 insertions(+), 51 deletions(-) diff --git a/include/state_save_restore/File_Per_Unit.hpp b/include/state_save_restore/File_Per_Unit.hpp index cb6af8e0ac..3fdefe71b8 100644 --- a/include/state_save_restore/File_Per_Unit.hpp +++ b/include/state_save_restore/File_Per_Unit.hpp @@ -17,4 +17,18 @@ class File_Per_Unit_Saver : public State_Saver std::string base_path_; }; + +class File_Per_Unit_Loader : public State_Loader +{ +public: + File_Per_Unit_Loader(std::string dir_path); + ~File_Per_Unit_Loader() = default; + + void finalize() override { }; + + std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) override; +private: + std::string dir_path_; +}; + #endif // NGEN_FILE_PER_UNIT_HPP diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index fea1555d21..85c48d5d21 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -12,7 +12,26 @@ #include #include +enum class State_Save_Direction { + None = 0, + Save, + Load +}; + +enum class State_Save_Mechanism { + None = 0, + FilePerUnit +}; + +enum class State_Save_When { + None = 0, + EndOfRun, + FirstOfMonth, + StartOfRun +}; + class State_Saver; +class State_Loader; class State_Snapshot_Saver; class State_Save_Config @@ -27,20 +46,29 @@ class State_Save_Config bool has_end_of_run() const; + bool has_cold_start() const; + std::shared_ptr end_of_run_saver() const; + std::shared_ptr cold_start_saver() const; + struct instance { - instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); + instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); + State_Save_Direction direction_; + State_Save_Mechanism mechanism_; + State_Save_When timing_; std::string label_; std::string path_; - std::string mechanism_; - std::string timing_; + + std::string mechanism_string() const; }; private: std::vector instances_; + int end_of_run() const; + int cold_start() const; }; class State_Saver @@ -153,6 +181,7 @@ class State_Snapshot_Loader class State_Unit_Loader { +public: State_Unit_Loader() = default; virtual ~State_Unit_Loader() = default; virtual void load(boost::span data) = 0; diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index d7c7ddf2b9..9ab1993bb5 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -20,6 +20,18 @@ #include #include +namespace unit_saving_utils { + std::string format_epoch(State_Saver::snapshot_time_t epoch) + { + time_t t = std::chrono::system_clock::to_time_t(epoch); + std::tm tm = *std::gmtime(&t); + + std::stringstream tss; + tss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S"); + return tss.str(); + } +} + // This class is only declared and defined here, in the .cpp file, // because it is strictly an implementation detail of the top-level // File_Per_Unit_Saver class @@ -37,8 +49,6 @@ class File_Per_Unit_Snapshot_Saver : public State_Snapshot_Saver void finish_saving() override; private: - std::string format_epoch(State_Saver::snapshot_time_t epoch); - path dir_path_; }; @@ -63,23 +73,13 @@ void File_Per_Unit_Saver::finalize() File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) : State_Snapshot_Saver(epoch, durability) - , dir_path_(base_path / format_epoch(epoch)) + , dir_path_(base_path / unit_saving_utils::format_epoch(epoch)) { create_directory(dir_path_); } File_Per_Unit_Snapshot_Saver::~File_Per_Unit_Snapshot_Saver() = default; -std::string File_Per_Unit_Snapshot_Saver::format_epoch(State_Saver::snapshot_time_t epoch) -{ - time_t t = std::chrono::system_clock::to_time_t(epoch); - std::tm tm = *std::gmtime(&t); - - std::stringstream tss; - tss << std::put_time(&tm, "%Y-%m-%dT%H:%M:%S"); - return tss.str(); -} - void File_Per_Unit_Snapshot_Saver::save_unit(std::string const& unit_name, boost::span data) { auto file_path = dir_path_ / unit_name; @@ -99,3 +99,79 @@ void File_Per_Unit_Snapshot_Saver::finish_saving() // fsync() or whatever } } + + +// This class is only declared and defined here, in the .cpp file, +// because it is strictly an implementation detail of the top-level +// File_Per_Unit_Saver class +class File_Per_Unit_Snapshot_Loader : public State_Snapshot_Loader +{ + friend class State_Snapshot_Loader; +public: + File_Per_Unit_Snapshot_Loader() = default; + File_Per_Unit_Snapshot_Loader(path dir_path); + ~File_Per_Unit_Snapshot_Loader() override = default; + + /** + * Load data from whatever source, and pass it to @param unit_loader->load() + */ + void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) override; + + /** + * Execute logic to complete the saving process + * + * Data may be flushed here, and delayed errors may be detected + * and reported here. With relaxed durability, error reports may + * not come until the parent State_Saver::finalize() call is made, + * or ever. + */ + void finish_saving() override { }; + +private: + path dir_path_; + std::vector data_; +}; + +File_Per_Unit_Snapshot_Loader::File_Per_Unit_Snapshot_Loader(path dir_path) + : dir_path_(dir_path) +{ + +} + +void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) { + auto file_path = dir_path_ / unit_name; + std::uintmax_t size; + try { + size = file_size(file_path.string()); + } catch (std::exception &e) { + LOG("Failed to read state save data size for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } + std::ifstream stream(file_path.string(), std::ios_base::ate | std::ios_base::binary); + if (!stream) { + LOG("Failed to open state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } + try { + std::vector buffer(size); + stream.read(buffer.data(), size); + boost::span data(buffer.data(), size); + unit_loader->load(data); + } catch (std::exception &e) { + LOG("Failed to read state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); + throw; + } +} + +File_Per_Unit_Loader::File_Per_Unit_Loader(std::string dir_path) + : dir_path_(dir_path) +{ + +} + +std::shared_ptr File_Per_Unit_Loader::initialize_snapshot(State_Saver::snapshot_time_t epoch) +{ + path dir_path(dir_path_); + return std::make_shared(dir_path); +} + diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 2df94b9515..12ddf1e74e 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -18,62 +18,119 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) return; } - auto subtree = *maybe; - - try { - auto single_what = subtree.get("label"); - auto single_where = subtree.get("path"); - auto single_how = subtree.get("type"); - auto single_when = subtree.get("when"); - - instance i{single_what, single_where, single_how, single_when}; - instances_.push_back(i); - } catch (...) { - LOG("Bad state saving config", LogLevel::WARNING); - throw; + auto saving_config = *maybe; + size_t length = saving_config.size(); + for (size_t i = 0; i < length; ++i) { + try { + auto subtree = tree.get_child(std::to_string(i)); + auto direction = subtree.get("direction"); + auto what = subtree.get("label"); + auto where = subtree.get("path"); + auto how = subtree.get("type"); + auto when = subtree.get("when"); + + instance i{direction, what, where, how, when}; + instances_.push_back(i); + } catch (...) { + LOG("Bad state saving config", LogLevel::WARNING); + throw; + } } LOG("State saving configured", LogLevel::INFO); } +int State_Save_Config::end_of_run() const { + for (size_t i = 0; i < instances_.size(); ++i) { + auto &instance = instances_[i]; + if (instance.timing_ == State_Save_When::EndOfRun + && instance.direction_ == State_Save_Direction::Save) + return i; + } + return -1; +} + bool State_Save_Config::has_end_of_run() const { - for (const auto& i : instances_) - if (i.timing_ == "EndOfRun") - return true; - return false; + this->end_of_run() >= 0; } std::shared_ptr State_Save_Config::end_of_run_saver() const { - for (const auto& i : instances_) { - if (i.timing_ == "EndOfRun") { - if (i.mechanism_ == "FilePerUnit") { - return std::make_shared(i.path_); - } else { - Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_ + " is not supported for end of run saving."); - } + int index = this->end_of_run(); + if (index >= 0) { + const auto& i = instances_[index]; + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + return std::make_shared(i.path_); + } else { + Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } Logger::logMsgAndThrowError("State_Save_Config: No end of run was defined in the realization config."); } -State_Save_Config::instance::instance(std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) +int State_Save_Config::cold_start() const { + for (size_t i = 0; i < instances_.size(); ++i) { + const auto& instance = instances_[i]; + if (instance.timing_ == State_Save_When::StartOfRun + && instance.direction_ == State_Save_Direction::Load) { + return i; + } + } + return -1; +} + +bool State_Save_Config::has_cold_start() const { + return this->cold_start() >= 0; +} + +std::shared_ptr State_Save_Config::cold_start_saver() const { + int index = this->cold_start(); + if (index >= 0) { + const auto& i = instances_[index]; + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + return std::make_shared(i.path_); + } else { + Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); + } + } +} + +State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) : label_(label) , path_(path) - , mechanism_(mechanism) - , timing_(timing) { - if (mechanism_ == "FilePerUnit") { - // nothing to do here + if (direction == "save") { + direction_ = State_Save_Direction::Save; + } else if (direction == "load") { + direction_ = State_Save_Direction::Load; + } else { + Logger::logMsgAndThrowError("Unrecognized state saving direction '" + direction + "'"); + } + + if (mechanism == "FilePerUnit") { + mechanism_ = State_Save_Mechanism::FilePerUnit; } else { - Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism_ + "'"); + Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism + "'"); } - if (timing_ == "EndOfRun") { - // nothing to do here - } else if (timing_ == "FirstOfMonth") { - // nothing to do here + if (timing == "EndOfRun") { + timing_ = State_Save_When::EndOfRun; + } else if (timing == "FirstOfMonth") { + timing_ = State_Save_When::FirstOfMonth; + } else if (timing == "StartOfRun") { + timing_ = State_Save_When::StartOfRun; } else { - Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing_ + "'"); + Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing + "'"); + } +} + +std::string State_Save_Config::instance::instance::mechanism_string() const { + switch (mechanism_) { + case State_Save_Mechanism::None: + return "None"; + case State_Save_Mechanism::FilePerUnit: + return "FilePerUnit"; + default: + return "Other"; } } From c9b17bb6f962addee53dbe8f84ed368af18f0e8a Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 20 Jan 2026 11:22:00 -0500 Subject: [PATCH 016/109] Cold start loading --- data/example_state_saving_config_multi.json | 5 +- include/core/Layer.hpp | 2 + include/core/NgenSimulation.hpp | 2 + .../catchment/Bmi_Formulation.hpp | 8 +++ .../catchment/Bmi_Fortran_Formulation.hpp | 7 +-- .../catchment/Bmi_Module_Formulation.hpp | 27 ++++----- .../catchment/Bmi_Multi_Formulation.hpp | 2 + .../state_save_restore/State_Save_Restore.hpp | 14 +---- src/NGen.cpp | 7 +++ src/core/Layer.cpp | 11 ++++ src/core/NgenSimulation.cpp | 6 ++ .../catchment/Bmi_Fortran_Formulation.cpp | 10 ++-- .../catchment/Bmi_Module_Formulation.cpp | 34 +++++------- .../catchment/Bmi_Multi_Formulation.cpp | 55 ++++++++++++++++--- src/state_save_restore/File_Per_Unit.cpp | 14 ++--- src/state_save_restore/State_Save_Restore.cpp | 3 +- 16 files changed, 133 insertions(+), 74 deletions(-) diff --git a/data/example_state_saving_config_multi.json b/data/example_state_saving_config_multi.json index 48ba4d8b58..ecfaa272fd 100644 --- a/data/example_state_saving_config_multi.json +++ b/data/example_state_saving_config_multi.json @@ -101,12 +101,13 @@ "provider": "CsvPerFeature" } }, - "state_saving": { + "state_saving": [{ + "direction": "save", "label": "end", "path": "state_end", "type": "FilePerUnit", "when": "EndOfRun" - }, + }], "time": { "start_time": "2015-12-01 00:00:00", "end_time": "2015-12-30 23:00:00", diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 6e5fe707e8..ab4a0e4268 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -17,6 +17,7 @@ namespace hy_features } class State_Snapshot_Saver; +class State_Snapshot_Loader; namespace ngen { @@ -113,6 +114,7 @@ namespace ngen int current_step); virtual void save_state_snapshot(std::shared_ptr snapshot_saver); + virtual void load_state_snapshot(std::shared_ptr snapshot_loader); protected: diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index d60dd165b4..c29280d719 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -13,6 +13,7 @@ namespace hy_features } class State_Snapshot_Saver; +class State_Snapshot_Loader; #include #include @@ -62,6 +63,7 @@ class NgenSimulation std::string get_timestamp_for_step(int step) const; void save_state_snapshot(std::shared_ptr snapshot_saver); + void load_state_snapshot(std::shared_ptr snapshot_loader); private: void advance_models_one_output_step(); diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 40cb1df399..cc1cec0de6 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -42,6 +42,7 @@ class Bmi_C_Formulation_Test; class Bmi_C_Pet_IT; class State_Snapshot_Saver; +class State_Snapshot_Loader; namespace realization { @@ -78,6 +79,13 @@ namespace realization { */ virtual void save_state(std::shared_ptr saver) const = 0; + /** + * Passes a serialized representation of the model's state to ``loader`` + * + * Asks saver to find data for the BMI and passes that data to the BMI for loading. + */ + virtual void load_state(std::shared_ptr loader) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 71225d4885..6cc8618bc4 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -24,13 +24,12 @@ namespace realization { std::string get_formulation_type() const override; /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_serialization_state` is called. * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface and copy the results to `size`. * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. + * @return Span of the serialized data. */ - const char* create_save_state(uint64_t *size) const override; + const boost::span get_serialization_state() const override; protected: diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 477c53a173..9019904058 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -53,18 +53,7 @@ namespace realization { */ void save_state(std::shared_ptr saver) const override; - /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. - * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. - */ - virtual const char* create_save_state(uint64_t *size) const; - - /** - * Clears any serialized data stored by the BMI from memory. - */ - virtual void free_save_state() const; + void load_state(std::shared_ptr loader) const override; /** * Get the collection of forcing output property names this instance can provide. @@ -300,8 +289,20 @@ namespace realization { const std::vector get_bmi_input_variables() const override; const std::vector get_bmi_output_variables() const override; - const boost::span get_serialization_state() const; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_serialization_state` is called. + * + * @return Span of the serialized data. + */ + virtual const boost::span get_serialization_state() const; + /** + * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. + */ void load_serialization_state(const boost::span state) const; + /** + * Requests the BMI to clear a currently saved state from memory. + * Existing state pointers should not be used as the stored data may be freed depending on implementation. + */ void free_serialization_state() const; void set_realization_file_format(bool is_legacy_format); diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index b750234cff..3b2f6c7d53 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -56,6 +56,8 @@ namespace realization { } }; + void load_state(std::shared_ptr loader) const override; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 85c48d5d21..bbf54a3dbe 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -50,7 +50,7 @@ class State_Save_Config std::shared_ptr end_of_run_saver() const; - std::shared_ptr cold_start_saver() const; + std::shared_ptr cold_start_loader() const; struct instance { @@ -155,8 +155,6 @@ class State_Loader virtual void finalize() = 0; }; -class State_Unit_Loader; - class State_Snapshot_Loader { public: @@ -166,7 +164,7 @@ class State_Snapshot_Loader /** * Load data from whatever source, and pass it to @param unit_loader->load() */ - virtual void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) = 0; + virtual void load_unit(std::string const& unit_name, std::vector &data) = 0; /** * Execute logic to complete the saving process @@ -179,12 +177,4 @@ class State_Snapshot_Loader virtual void finish_saving() = 0; }; -class State_Unit_Loader -{ -public: - State_Unit_Loader() = default; - virtual ~State_Unit_Loader() = default; - virtual void load(boost::span data) = 0; -}; - #endif // NGEN_STATE_SAVE_RESTORE_HPP diff --git a/src/NGen.cpp b/src/NGen.cpp index b54a12769d..016c759dc5 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -701,6 +701,13 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); + if (state_saving_config.has_cold_start()) { + LOG(LogLevel::INFO, "Loading simulation data from cold start."); + std::shared_ptr cold_loader = state_saving_config.cold_start_loader(); + std::shared_ptr cold_snapshot_loader = cold_loader->initialize_snapshot(State_Saver::snapshot_time_now()); + simulation->load_state_snapshot(cold_snapshot_loader); + } + simulation->run_catchments(); if (state_saving_config.has_end_of_run()) { diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index cbde74e8fd..745306a885 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -104,3 +104,14 @@ void ngen::Layer::save_state_snapshot(std::shared_ptr snap r_c->save_state(snapshot_saver); } } + +void ngen::Layer::load_state_snapshot(std::shared_ptr snapshot_loader) +{ + // XXX Handle any of this class's own state as a meta-data unit + + for (auto const& id : processing_units) { + auto r = features.catchment_at(id); + auto r_c = std::dynamic_pointer_cast(r); + r_c->load_state(snapshot_loader); + } +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c2f9bdf04a..7742e3a057 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -117,6 +117,12 @@ void NgenSimulation::save_state_snapshot(std::shared_ptr s } } +void NgenSimulation::load_state_snapshot(std::shared_ptr snapshot_loader) { + for (auto& layer : layers_) { + layer->load_state_snapshot(snapshot_loader); + } +} + int NgenSimulation::get_nexus_index(std::string const& nexus_id) const { diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index ae43929ac6..b8804eab79 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -93,14 +93,14 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const return 1.0; } -const char* Bmi_Fortran_Formulation::create_save_state(uint64_t *size) const { +const boost::span Bmi_Fortran_Formulation::get_serialization_state() const { auto model = get_bmi_model(); - int size_int = 1; + int size_int = 0; model->SetValue("serialization_create", &size_int); model->GetValue("serialization_size", &size_int); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); - *size = static_cast(size_int); - return serialization_state; + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + const boost::span span(serialization_state, size_int); + return span; } #endif // NGEN_WITH_BMI_FORTRAN diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index bdad7105db..4fb2a83fa9 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -18,29 +18,21 @@ namespace realization { void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { uint64_t size = 1; - const char* serialization_state = this->create_save_state(&size); - boost::span data(serialization_state, size); + boost::span data = this->get_serialization_state(); // Rely on Formulation_Manager also using this->get_id() // as a unique key for the individual catchment // formulations saver->save_unit(this->get_id(), data); - this->free_save_state(); + this->free_serialization_state(); } - const char* Bmi_Module_Formulation::create_save_state(uint64_t *size) const { - auto model = get_bmi_model(); - model->SetValue("serialization_create", size); - model->GetValue("serialization_size", size); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); - return serialization_state; - } - - void Bmi_Module_Formulation::free_save_state() const { - auto model = get_bmi_model(); - int _; - model->SetValue("serialization_free", &_); + void Bmi_Module_Formulation::load_state(std::shared_ptr loader) const { + std::vector buffer; + loader->load_unit(this->get_id(), buffer); + boost::span data(buffer.data(), buffer.size()); + this->load_serialization_state(data); } boost::span Bmi_Module_Formulation::get_available_variable_names() const { @@ -1096,12 +1088,12 @@ namespace realization { } const boost::span Bmi_Module_Formulation::get_serialization_state() const { - auto bmi = this->bmi_model; - // create a new serialized state, getting the amount of data that was saved - uint64_t* size = (uint64_t*)bmi->GetValuePtr("serialization_create"); - // get the pointer of the new state - char* serialized = (char*)bmi->GetValuePtr("serialization_state"); - const boost::span span(serialized, *size); + auto model = get_bmi_model(); + uint64_t size = 0; + model->SetValue("serialization_create", &size); + model->GetValue("serialization_size", &size); + auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + const boost::span span(serialization_state, size); return span; } diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 9de11a02d7..b44715a093 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -31,17 +31,15 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav std::vector> bmi_data; size_t data_size = 0; - uint64_t ser_size; // TODO: something more elegant than just skipping sloth for (const nested_module_ptr &m : modules) { - auto bmi = static_cast(m.get()); + auto bmi = dynamic_cast(m.get()); if (bmi->get_model_type_name() != "bmi_c++_sloth") { - ser_size = 1; - const char* serialized = bmi->create_save_state(&ser_size); - bmi_data.push_back(std::make_pair(serialized, ser_size)); - data_size += sizeof(uint64_t) + ser_size; - LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), ser_size); + boost::span span = bmi->get_serialization_state(); + bmi_data.push_back(std::make_pair(span.data(), span.size())); + data_size += sizeof(uint64_t) + span.size(); + LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } } char *data = new char[data_size]; @@ -72,7 +70,46 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav delete[] data; for (const nested_module_ptr &m : modules) { auto bmi = static_cast(m.get()); - bmi->free_save_state(); + if (bmi->get_model_type_name() != "bmi_c++_sloth") { + bmi->free_serialization_state(); + } + } +} + +void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) const { +#if (__cplusplus < 202002L) + // get system endianness + uint16_t endian_bytes = 0xFF00; + uint8_t *endian_bits = reinterpret_cast(&endian_bytes); + bool is_little_endian = endian_bits[0] == 0; +#endif + std::vector data; + loader->load_unit(this->get_id(), data); + size_t index = 0; + for (const nested_module_ptr &m : modules) { + auto bmi = dynamic_cast(m.get()); + if (bmi->get_model_type_name() != "bmi_c++_sloth") { + uint64_t size; +#if (__cplusplus < 202002L) + if (is_little_endian) { +#else + if constexpr (std::endian::native == std::endian::little) { +#endif + memcpy(&size, data.data() + index, sizeof(uint64_t)); + } else { + // read size bytes in reverse order to interpret from little endian + char *size_bytes = reinterpret_cast(&size); + size_t endian_index = sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + size_bytes[--endian_index] = data[index + i]; + } + } + boost::span span(data.data() + index + sizeof(uint64_t), size); + bmi->load_serialization_state(span); + index += sizeof(uint64_t) + size; + LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); + } } } diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index 9ab1993bb5..103b7d6a7a 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -113,9 +113,11 @@ class File_Per_Unit_Snapshot_Loader : public State_Snapshot_Loader ~File_Per_Unit_Snapshot_Loader() override = default; /** - * Load data from whatever source, and pass it to @param unit_loader->load() + * Load data from whatever source and store it in the `data` vector. + * + * @param data The location where the loaded data will be stored. This will be resized to the amount of data loaded. */ - void load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) override; + void load_unit(std::string const& unit_name, std::vector &data) override; /** * Execute logic to complete the saving process @@ -138,7 +140,7 @@ File_Per_Unit_Snapshot_Loader::File_Per_Unit_Snapshot_Loader(path dir_path) } -void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, State_Unit_Loader *unit_loader) { +void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, std::vector &data) { auto file_path = dir_path_ / unit_name; std::uintmax_t size; try { @@ -153,10 +155,8 @@ void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, Stat throw; } try { - std::vector buffer(size); - stream.read(buffer.data(), size); - boost::span data(buffer.data(), size); - unit_loader->load(data); + data.resize(size); + stream.read(data.data(), size); } catch (std::exception &e) { LOG("Failed to read state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); throw; diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 12ddf1e74e..ebcf273d60 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -82,7 +82,7 @@ bool State_Save_Config::has_cold_start() const { return this->cold_start() >= 0; } -std::shared_ptr State_Save_Config::cold_start_saver() const { +std::shared_ptr State_Save_Config::cold_start_loader() const { int index = this->cold_start(); if (index >= 0) { const auto& i = instances_[index]; @@ -92,6 +92,7 @@ std::shared_ptr State_Save_Config::cold_start_saver() const { Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } + Logger::logMsgAndThrowError("State_Save_Config: No configuration was found for loading a cold start."); } State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) From 35f906b59ad0df1a0d66d3623ad75a3c372b40a7 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 20 Jan 2026 11:45:04 -0500 Subject: [PATCH 017/109] Fix config JSON array parsing --- src/state_save_restore/State_Save_Restore.cpp | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index ebcf273d60..f082d729a1 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -18,11 +18,10 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) return; } - auto saving_config = *maybe; - size_t length = saving_config.size(); - for (size_t i = 0; i < length; ++i) { + //auto saving_config = *maybe; + for (const auto& saving_config : *maybe) { try { - auto subtree = tree.get_child(std::to_string(i)); + auto& subtree = saving_config.second; auto direction = subtree.get("direction"); auto what = subtree.get("label"); auto where = subtree.get("path"); @@ -43,15 +42,15 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) int State_Save_Config::end_of_run() const { for (size_t i = 0; i < instances_.size(); ++i) { auto &instance = instances_[i]; - if (instance.timing_ == State_Save_When::EndOfRun - && instance.direction_ == State_Save_Direction::Save) + if (instance.timing_ == State_Save_When::EndOfRun && instance.direction_ == State_Save_Direction::Save) { return i; + } } return -1; } bool State_Save_Config::has_end_of_run() const { - this->end_of_run() >= 0; + return this->end_of_run() >= 0; } std::shared_ptr State_Save_Config::end_of_run_saver() const { @@ -64,14 +63,15 @@ std::shared_ptr State_Save_Config::end_of_run_saver() const { Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } - Logger::logMsgAndThrowError("State_Save_Config: No end of run was defined in the realization config."); + auto error = "State_Save_Config: No end of run was defined in the realization config."; + LOG(LogLevel::SEVERE, error); + throw std::runtime_error(error); } int State_Save_Config::cold_start() const { for (size_t i = 0; i < instances_.size(); ++i) { const auto& instance = instances_[i]; - if (instance.timing_ == State_Save_When::StartOfRun - && instance.direction_ == State_Save_Direction::Load) { + if (instance.timing_ == State_Save_When::StartOfRun && instance.direction_ == State_Save_Direction::Load) { return i; } } @@ -92,7 +92,9 @@ std::shared_ptr State_Save_Config::cold_start_loader() const { Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); } } - Logger::logMsgAndThrowError("State_Save_Config: No configuration was found for loading a cold start."); + auto error = "State_Save_Config: No configuration was found for loading a cold start."; + LOG(LogLevel::SEVERE, error); + throw std::runtime_error(error); } State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) From cea29148060eb7161f802306e8b45ce5e479a9e3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 21 Jan 2026 09:36:35 -0500 Subject: [PATCH 018/109] Less restrictive start and end of run save state --- .../state_save_restore/State_Save_Restore.hpp | 10 +-- src/NGen.cpp | 16 ++--- src/state_save_restore/State_Save_Restore.cpp | 72 ++++++------------- 3 files changed, 31 insertions(+), 67 deletions(-) diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index bbf54a3dbe..7bff90e474 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -44,13 +44,9 @@ class State_Save_Config */ State_Save_Config(boost::property_tree::ptree const& config); - bool has_end_of_run() const; + std::unordered_map> start_of_run_loaders() const; - bool has_cold_start() const; - - std::shared_ptr end_of_run_saver() const; - - std::shared_ptr cold_start_loader() const; + std::unordered_map> end_of_run_savers() const; struct instance { @@ -67,8 +63,6 @@ class State_Save_Config private: std::vector instances_; - int end_of_run() const; - int cold_start() const; }; class State_Saver diff --git a/src/NGen.cpp b/src/NGen.cpp index 016c759dc5..329a70a634 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -701,19 +701,17 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - if (state_saving_config.has_cold_start()) { - LOG(LogLevel::INFO, "Loading simulation data from cold start."); - std::shared_ptr cold_loader = state_saving_config.cold_start_loader(); - std::shared_ptr cold_snapshot_loader = cold_loader->initialize_snapshot(State_Saver::snapshot_time_now()); - simulation->load_state_snapshot(cold_snapshot_loader); + for (const auto& start_loader : state_saving_config.start_of_run_loaders()) { + LOG(LogLevel::INFO, "Loading start of run simulation data from state saving config " + start_loader.first); + std::shared_ptr snapshot_loader = start_loader.second->initialize_snapshot(State_Saver::snapshot_time_now()); + simulation->load_state_snapshot(snapshot_loader); } simulation->run_catchments(); - if (state_saving_config.has_end_of_run()) { - LOG("Saving end-of-run state.", LogLevel::INFO); - std::shared_ptr saver = state_saving_config.end_of_run_saver(); - std::shared_ptr snapshot = saver->initialize_snapshot( + for (const auto& end_saver : state_saving_config.end_of_run_savers()) { + LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); + std::shared_ptr snapshot = end_saver.second->initialize_snapshot( State_Saver::snapshot_time_now(), State_Saver::State_Durability::strict ); diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index f082d729a1..fe0fc6a82d 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -39,62 +39,34 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) LOG("State saving configured", LogLevel::INFO); } -int State_Save_Config::end_of_run() const { - for (size_t i = 0; i < instances_.size(); ++i) { - auto &instance = instances_[i]; - if (instance.timing_ == State_Save_When::EndOfRun && instance.direction_ == State_Save_Direction::Save) { - return i; +std::unordered_map> State_Save_Config::start_of_run_loaders() const { + std::unordered_map> loaders; + for (const auto &i : this->instances_) { + if (i.timing_ == State_Save_When::StartOfRun && i.direction_ == State_Save_Direction::Load) { + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + auto loader = std::make_shared(i.path_); + loaders[i.label_] = loader; + } else { + LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); + } } } - return -1; + return loaders; } -bool State_Save_Config::has_end_of_run() const { - return this->end_of_run() >= 0; -} - -std::shared_ptr State_Save_Config::end_of_run_saver() const { - int index = this->end_of_run(); - if (index >= 0) { - const auto& i = instances_[index]; - if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { - return std::make_shared(i.path_); - } else { - Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); - } - } - auto error = "State_Save_Config: No end of run was defined in the realization config."; - LOG(LogLevel::SEVERE, error); - throw std::runtime_error(error); -} - -int State_Save_Config::cold_start() const { - for (size_t i = 0; i < instances_.size(); ++i) { - const auto& instance = instances_[i]; - if (instance.timing_ == State_Save_When::StartOfRun && instance.direction_ == State_Save_Direction::Load) { - return i; - } - } - return -1; -} - -bool State_Save_Config::has_cold_start() const { - return this->cold_start() >= 0; -} - -std::shared_ptr State_Save_Config::cold_start_loader() const { - int index = this->cold_start(); - if (index >= 0) { - const auto& i = instances_[index]; - if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { - return std::make_shared(i.path_); - } else { - Logger::logMsgAndThrowError("State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for end of run saving."); +std::unordered_map> State_Save_Config::end_of_run_savers() const { + std::unordered_map> savers; + for (const auto &i : this->instances_) { + if (i.timing_ == State_Save_When::EndOfRun && i.direction_ == State_Save_Direction::Save) { + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + auto loader = std::make_shared(i.path_); + savers[i.label_] = loader; + } else { + LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); + } } } - auto error = "State_Save_Config: No configuration was found for loading a cold start."; - LOG(LogLevel::SEVERE, error); - throw std::runtime_error(error); + return savers; } State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) From fefc8cfce971ed87e19d7deffed276d6020de353 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 21 Jan 2026 10:03:39 -0500 Subject: [PATCH 019/109] Use parent classes for start and end of run states --- .../state_save_restore/State_Save_Restore.hpp | 14 ++++++++++++-- src/state_save_restore/State_Save_Restore.cpp | 18 ++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 7bff90e474..8f4b1b77e5 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -44,9 +44,19 @@ class State_Save_Config */ State_Save_Config(boost::property_tree::ptree const& config); - std::unordered_map> start_of_run_loaders() const; + /** + * Get state loaders that perform before the catchments are run. + * + * @return `std::pair`s of the label from the config and an instance of the loader. + */ + std::vector>> start_of_run_loaders() const; - std::unordered_map> end_of_run_savers() const; + /** + * Get state savers that perform after the catchments have run to completion. + * + * @return `std::pair`s of the label from the config and an instance of the saver. + */ + std::vector>> end_of_run_savers() const; struct instance { diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index fe0fc6a82d..84e2226e95 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -39,13 +39,14 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) LOG("State saving configured", LogLevel::INFO); } -std::unordered_map> State_Save_Config::start_of_run_loaders() const { - std::unordered_map> loaders; +std::vector>> State_Save_Config::start_of_run_loaders() const { + std::vector>> loaders; for (const auto &i : this->instances_) { if (i.timing_ == State_Save_When::StartOfRun && i.direction_ == State_Save_Direction::Load) { if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { auto loader = std::make_shared(i.path_); - loaders[i.label_] = loader; + auto pair = std::make_pair(i.label_, loader); + loaders.push_back(pair); } else { LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); } @@ -54,15 +55,16 @@ std::unordered_map> State_Sav return loaders; } -std::unordered_map> State_Save_Config::end_of_run_savers() const { - std::unordered_map> savers; +std::vector>> State_Save_Config::end_of_run_savers() const { + std::vector>> savers; for (const auto &i : this->instances_) { if (i.timing_ == State_Save_When::EndOfRun && i.direction_ == State_Save_Direction::Save) { if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { - auto loader = std::make_shared(i.path_); - savers[i.label_] = loader; + auto saver = std::make_shared(i.path_); + auto pair = std::make_pair(i.label_, saver); + savers.push_back(pair); } else { - LOG(LogLevel::WARNING, "State_Save_Config: Loading mechanism " + i.mechanism_string() + " is not supported for start of run loading."); + LOG(LogLevel::WARNING, "State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for start of run saving."); } } } From bfcccf88882e92f181c1594d7dac635c79fb9982 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 27 Jan 2026 08:34:47 -0500 Subject: [PATCH 020/109] Dynamically sized set value option for python BMI adapter --- include/bmi/Bmi_Py_Adapter.hpp | 10 ++++++++++ .../realizations/catchment/Bmi_Module_Formulation.hpp | 2 +- include/realizations/catchment/Bmi_Py_Formulation.hpp | 7 +++++++ src/bmi/Bmi_Py_Adapter.cpp | 6 ++++++ src/realizations/catchment/Bmi_Py_Formulation.cpp | 6 ++++++ 5 files changed, 30 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index d66124d534..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -597,6 +597,16 @@ namespace models { } } + /** + * Set the value of a variable. This version of setting a variable will send an array with the `size` specified instead of checking the BMI for its current size of the variable. + * + * @param name The name of the BMI variable. + * @param src Pointer to the data that will be sent to the BMI. + * @param size The number of items represented by the pointer. + */ + template + void set_value_unchecked(const std::string &name, T *src, size_t size); + /** * Set values for a model's BMI variable at specified indices. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 9019904058..1fb0cd64e8 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -298,7 +298,7 @@ namespace realization { /** * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. */ - void load_serialization_state(const boost::span state) const; + virtual void load_serialization_state(const boost::span state) const; /** * Requests the BMI to clear a currently saved state from memory. * Existing state pointers should not be used as the stored data may be freed depending on implementation. diff --git a/include/realizations/catchment/Bmi_Py_Formulation.hpp b/include/realizations/catchment/Bmi_Py_Formulation.hpp index 76904165c9..19c6a0428e 100644 --- a/include/realizations/catchment/Bmi_Py_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Py_Formulation.hpp @@ -35,6 +35,13 @@ namespace realization { bool is_bmi_output_variable(const std::string &var_name) const override; + /** + * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. + * + * The python BMI requires additional messaging for pre-allocating memory for load + */ + void load_serialization_state(const boost::span state) const override; + protected: std::shared_ptr construct_model(const geojson::PropertyMap &properties) override; diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index 7436e2b48c..ee79de4e14 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -223,4 +223,10 @@ void Bmi_Py_Adapter::UpdateUntil(double time) { bmi_model->attr("update_until")(time); } +template +void Bmi_Py_Adapter::set_value_unchecked(const std::string &name, T *src, size_t size) { + py::array_t src_array(size, src); + bmi_model->attr("set_value")(name, src_array); +} + #endif //NGEN_WITH_PYTHON diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 7d266db9f8..e804aa5882 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -117,4 +117,10 @@ bool Bmi_Py_Formulation::is_model_initialized() const { return get_bmi_model()->is_model_initialized(); } +void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { + auto bmi = std::dynamic_pointer_cast(get_bmi_model()); + // load the state through the set value function that does not enforce the input size is the same as the current BMI's size + bmi->set_value_unchecked("serialization_state", state.data(), state.size()); +} + #endif //NGEN_WITH_PYTHON From ea924cc977184deae7e09ef85f56c74a87f1f201 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 30 Jan 2026 11:11:38 -0500 Subject: [PATCH 021/109] Add output suppression flag for UEB --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6ece41e410..93edd0d218 100644 --- a/Dockerfile +++ b/Dockerfile @@ -317,7 +317,8 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-soilfreezethaw \ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ueb-bmi \ set -eux && \ - cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ \ + -DUEB_SUPPRESS_OUTPUTS=ON -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \ cmake --build extern/ueb-bmi/cmake_build/ && \ find /ngen-app/ngen/extern/ueb-bmi/ -name '*.o' -exec rm -f {} + From ddce21c524e2c1cb860f0004cb74af3cdcead248 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 8 Jan 2026 09:56:18 -0800 Subject: [PATCH 022/109] Docker updates for python ewts packages --- extern/LASAM | 2 +- extern/SoilFreezeThaw/SoilFreezeThaw | 2 +- extern/SoilMoistureProfiles/SoilMoistureProfiles | 2 +- extern/lstm | 2 +- extern/noah-owp-modular/noah-owp-modular | 2 +- extern/sac-sma/sac-sma | 2 +- extern/snow17 | 2 +- extern/t-route | 2 +- extern/topoflow-glacier | 2 +- extern/ueb-bmi | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/extern/LASAM b/extern/LASAM index 764dc82e8f..e3411b3318 160000 --- a/extern/LASAM +++ b/extern/LASAM @@ -1 +1 @@ -Subproject commit 764dc82e8fb5a160e646a8f0fde35f964fd42234 +Subproject commit e3411b331854303f7344db795f3afc1e3269810a diff --git a/extern/SoilFreezeThaw/SoilFreezeThaw b/extern/SoilFreezeThaw/SoilFreezeThaw index ab641a8209..815a970af3 160000 --- a/extern/SoilFreezeThaw/SoilFreezeThaw +++ b/extern/SoilFreezeThaw/SoilFreezeThaw @@ -1 +1 @@ -Subproject commit ab641a820920acb788dc47513a1e0ccbf31483c2 +Subproject commit 815a970af30e407785bc767a4a5f5dffa2708eb5 diff --git a/extern/SoilMoistureProfiles/SoilMoistureProfiles b/extern/SoilMoistureProfiles/SoilMoistureProfiles index 41c802cb48..d29f268223 160000 --- a/extern/SoilMoistureProfiles/SoilMoistureProfiles +++ b/extern/SoilMoistureProfiles/SoilMoistureProfiles @@ -1 +1 @@ -Subproject commit 41c802cb4862e7bd01f6081816a093135bd51282 +Subproject commit d29f26822391740549642a9818065ea296688b2e diff --git a/extern/lstm b/extern/lstm index 85a3301dae..b3ccaeb458 160000 --- a/extern/lstm +++ b/extern/lstm @@ -1 +1 @@ -Subproject commit 85a3301daeff761a54b6ebda6fee7aac977a62ce +Subproject commit b3ccaeb45897693d7ebb05dc38788a5766a6921e diff --git a/extern/noah-owp-modular/noah-owp-modular b/extern/noah-owp-modular/noah-owp-modular index 25579b4948..029fc17a35 160000 --- a/extern/noah-owp-modular/noah-owp-modular +++ b/extern/noah-owp-modular/noah-owp-modular @@ -1 +1 @@ -Subproject commit 25579b4948e28e5afd0bed3e99e08a806fd9fc7c +Subproject commit 029fc17a3552356c1a62afd31bcf0a00f70948e4 diff --git a/extern/sac-sma/sac-sma b/extern/sac-sma/sac-sma index b40f61ca9e..ca16b6e705 160000 --- a/extern/sac-sma/sac-sma +++ b/extern/sac-sma/sac-sma @@ -1 +1 @@ -Subproject commit b40f61ca9e82d4c4d0fc6171b314714af0160ab3 +Subproject commit ca16b6e705129913a5bca897428faf8d9a7561ee diff --git a/extern/snow17 b/extern/snow17 index 10c2510bfa..e73f49ba8d 160000 --- a/extern/snow17 +++ b/extern/snow17 @@ -1 +1 @@ -Subproject commit 10c2510bfa45743a3828ea0fc890f79974b48390 +Subproject commit e73f49ba8d3180af48cfc0726bebc7d08a792035 diff --git a/extern/t-route b/extern/t-route index 4c100cb7b9..595d56cb6f 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit 4c100cb7b94ff61b8028b4169571057ccdcea02f +Subproject commit 595d56cb6fe79e724e9ce7045dfdd7c200fc01fc diff --git a/extern/topoflow-glacier b/extern/topoflow-glacier index 6ccd49938a..4da1b75b8f 160000 --- a/extern/topoflow-glacier +++ b/extern/topoflow-glacier @@ -1 +1 @@ -Subproject commit 6ccd49938ab79d61a36791cc25144f3e8dd1229b +Subproject commit 4da1b75b8fe0fe98a247a4a23f7054089c30a1e5 diff --git a/extern/ueb-bmi b/extern/ueb-bmi index 299976367a..cb18da2a41 160000 --- a/extern/ueb-bmi +++ b/extern/ueb-bmi @@ -1 +1 @@ -Subproject commit 299976367a5329602fc1443f932e9cbf6de4ace6 +Subproject commit cb18da2a411f6e6f43c4cc13845c1c2837a1346a From 88f864c0f775eb0252956105c9d8dc98e96ea6e6 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:15:24 -0500 Subject: [PATCH 023/109] Ensure bmi_model has not been destroyed --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From 5946073aad754c9c5dda2b84a536432cf1662de4 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 11 Dec 2025 12:30:47 -0500 Subject: [PATCH 024/109] Remove null check since that should be an error state if null --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From 903d20450055750b8ee809902b6535a861bcf84f Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 025/109] Revert "Release BMI reference" This reverts commit 7acb111f14ce6f0adeee0f4d69a0dbfbc7127055. --- include/forcing/ForcingsEngineDataProvider.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index 37df90d59b..d4ce1f957d 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -108,10 +108,7 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override - { - bmi_.reset(); - } + void finalize() override = default; boost::span get_available_variable_names() const override { From 686ded9b757ecd852cc1deb6c19f8b6302867c89 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 026/109] Revert "Finalize forcing engine providers on instances clear" This reverts commit db3ba59587837d26b6c9dde6a38c3ff2f4740ad0. --- include/forcing/ForcingsEngineDataProvider.hpp | 15 +++++++++------ .../catchment/Formulation_Manager.hpp | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index d4ce1f957d..f1477f9eb5 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -78,14 +78,11 @@ struct ForcingsEngineStorage { data_[key] = value; } - //! Clear all references to Forcings Engine instances and run the Finalize methods on each BMI instance. + //! Clear all references to Forcings Engine instances. //! @note This will not necessarily destroy the Forcings Engine instances. Since they //! are reference counted, it will only decrement their instance by one. - void finalize() + void clear() { - for (auto &provider : data_) { - provider.second->Finalize(); - } data_.clear(); } @@ -108,7 +105,13 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override = default; + void finalize() override + { + if (bmi_ != nullptr) { + bmi_->Finalize(); + bmi_ = nullptr; + } + } boost::span get_available_variable_names() const override { diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 1865eec9a0..ee54798852 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -318,7 +318,7 @@ namespace realization { data_access::NetCDFPerFeatureDataProvider::cleanup_shared_providers(); #endif #if NGEN_WITH_PYTHON - data_access::detail::ForcingsEngineStorage::instances.finalize(); + data_access::detail::ForcingsEngineStorage::instances.clear(); #endif ss.str(""); ss << "Formulation_Manager finalized" << std::endl; LOG(ss.str(), LogLevel::DEBUG); From cca2b9c35450e681b07a6d8daadecf94ca0a37f2 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 027/109] Revert "Remove null check since that should be an error state if null" This reverts commit f4b1ce8520be147a9484a3dfe9c2276effd109a7. --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From 576a816477e99f12e7c3b7cf266edcad1421a4f7 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 028/109] Revert "Ensure bmi_model has not been destroyed" This reverts commit c77b3e38869304a51a813f7e06e1f07011423ed0. --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From b8d830e8c3866f9df5eeaf818c1fea81e196f5cb Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 029/109] Revert "Remove explicit interpreter release" This reverts commit 82ce9bcd746465b92dae87f70cb4436d9dcc803b. --- src/NGen.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/NGen.cpp b/src/NGen.cpp index 329a70a634..e0246142e8 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -781,6 +781,8 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif + _interp.reset(); + auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; From 9663faae21be3dc90281479811da5825d9ce150d Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 21 Jan 2026 07:08:02 -0800 Subject: [PATCH 030/109] Revert "Fix use-after-free error" This reverts commit 8869534d3820e1e962266f59fe0f20962966fcf8. --- include/realizations/catchment/Formulation_Manager.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index ee54798852..39841c2dc6 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -726,14 +726,12 @@ namespace realization { // Iterate over directory entries if (directory != nullptr) { - // handle closing the directory regardless of how the function returns - auto closer = [](DIR *dir){ closedir(dir); }; - std::unique_ptr _(directory, closer); while ((entry = readdir(directory))) { if (std::regex_match(entry->d_name, pattern)) { // Check for regular files and symlinks #ifdef _DIRENT_HAVE_D_TYPE if (entry->d_type == DT_REG || entry->d_type == DT_LNK) { + closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -753,6 +751,7 @@ namespace realization { } if (S_ISREG(st.st_mode)) { + closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -771,6 +770,7 @@ namespace realization { #endif } } + closedir(directory); } else { // The directory wasn't found or otherwise couldn't be opened; forcing data cannot be retrieved std::string throw_msg = "Error opening forcing data dir '" + path + "' after " + std::to_string(attemptCount) + " attempts: " + errMsg; From f61741ec20641109341c92cadaf6fd0d32f08442 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 09:59:17 -0500 Subject: [PATCH 031/109] Fix use-after-free error --- include/realizations/catchment/Formulation_Manager.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 39841c2dc6..ee54798852 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -726,12 +726,14 @@ namespace realization { // Iterate over directory entries if (directory != nullptr) { + // handle closing the directory regardless of how the function returns + auto closer = [](DIR *dir){ closedir(dir); }; + std::unique_ptr _(directory, closer); while ((entry = readdir(directory))) { if (std::regex_match(entry->d_name, pattern)) { // Check for regular files and symlinks #ifdef _DIRENT_HAVE_D_TYPE if (entry->d_type == DT_REG || entry->d_type == DT_LNK) { - closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -751,7 +753,6 @@ namespace realization { } if (S_ISREG(st.st_mode)) { - closedir(directory); return forcing_params( path + entry->d_name, provider, @@ -770,7 +771,6 @@ namespace realization { #endif } } - closedir(directory); } else { // The directory wasn't found or otherwise couldn't be opened; forcing data cannot be retrieved std::string throw_msg = "Error opening forcing data dir '" + path + "' after " + std::to_string(attemptCount) + " attempts: " + errMsg; From 5db69e2fda6c35c9b32bfa66c71e2f30cdac6d76 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:14:29 -0500 Subject: [PATCH 032/109] Remove explicit interpreter release --- src/NGen.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index e0246142e8..329a70a634 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -781,8 +781,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif - _interp.reset(); - auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; From 5fe62832bde20503b9e7b58d81b69977856ddb13 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:15:24 -0500 Subject: [PATCH 033/109] Ensure bmi_model has not been destroyed --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From fcbb90406b0df77d3170cf66cba9783c3ce73ee6 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 11 Dec 2025 12:30:47 -0500 Subject: [PATCH 034/109] Remove null check since that should be an error state if null --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From b9a4411f08aa624aba00cb414844c6a421800097 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 19 Dec 2025 09:18:15 -0500 Subject: [PATCH 035/109] Finalize forcing engine providers on instances clear --- include/forcing/ForcingsEngineDataProvider.hpp | 15 ++++++--------- .../catchment/Formulation_Manager.hpp | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index f1477f9eb5..d4ce1f957d 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -78,11 +78,14 @@ struct ForcingsEngineStorage { data_[key] = value; } - //! Clear all references to Forcings Engine instances. + //! Clear all references to Forcings Engine instances and run the Finalize methods on each BMI instance. //! @note This will not necessarily destroy the Forcings Engine instances. Since they //! are reference counted, it will only decrement their instance by one. - void clear() + void finalize() { + for (auto &provider : data_) { + provider.second->Finalize(); + } data_.clear(); } @@ -105,13 +108,7 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override - { - if (bmi_ != nullptr) { - bmi_->Finalize(); - bmi_ = nullptr; - } - } + void finalize() override = default; boost::span get_available_variable_names() const override { diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index ee54798852..1865eec9a0 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -318,7 +318,7 @@ namespace realization { data_access::NetCDFPerFeatureDataProvider::cleanup_shared_providers(); #endif #if NGEN_WITH_PYTHON - data_access::detail::ForcingsEngineStorage::instances.clear(); + data_access::detail::ForcingsEngineStorage::instances.finalize(); #endif ss.str(""); ss << "Formulation_Manager finalized" << std::endl; LOG(ss.str(), LogLevel::DEBUG); From f39f849b13d2dcdaf4ed12217dfd8edb9761c8c2 Mon Sep 17 00:00:00 2001 From: Ian Todd <48330440+idtodd@users.noreply.github.com> Date: Mon, 22 Dec 2025 08:22:39 -0500 Subject: [PATCH 036/109] Release BMI reference Co-authored-by: Phil Miller - NOAA --- include/forcing/ForcingsEngineDataProvider.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index d4ce1f957d..37df90d59b 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -108,7 +108,10 @@ struct ForcingsEngineDataProvider : public DataProvider using clock_type = std::chrono::system_clock; ~ForcingsEngineDataProvider() override = default; - void finalize() override = default; + void finalize() override + { + bmi_.reset(); + } boost::span get_available_variable_names() const override { From 572d770474855a3eaa79108c8b283315692bed9b Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 3 Dec 2025 17:39:36 -0800 Subject: [PATCH 037/109] Catch csv file errors and log exceptions initializing formulation --- include/forcing/CsvPerFeatureForcingProvider.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/forcing/CsvPerFeatureForcingProvider.hpp b/include/forcing/CsvPerFeatureForcingProvider.hpp index 1570c67b2e..25368b8d9e 100644 --- a/include/forcing/CsvPerFeatureForcingProvider.hpp +++ b/include/forcing/CsvPerFeatureForcingProvider.hpp @@ -423,7 +423,7 @@ class CsvPerFeatureForcingProvider : public data_access::GenericDataProvider available_forcings_units[var_name] = units; } else { - std::string msg = "Forcing file " + file_name + " is missing a column header name"; + std::string msg = "Forcing file " + file_name + " is missing column header names"; LOG(msg, LogLevel::FATAL); throw std::runtime_error(msg); } From 585620fbad20dfb96be9c6be6090f2c914258d81 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Thu, 4 Dec 2025 09:32:55 -0800 Subject: [PATCH 038/109] Add check for empty CSV forcing file. --- include/forcing/CsvPerFeatureForcingProvider.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/forcing/CsvPerFeatureForcingProvider.hpp b/include/forcing/CsvPerFeatureForcingProvider.hpp index 25368b8d9e..1570c67b2e 100644 --- a/include/forcing/CsvPerFeatureForcingProvider.hpp +++ b/include/forcing/CsvPerFeatureForcingProvider.hpp @@ -423,7 +423,7 @@ class CsvPerFeatureForcingProvider : public data_access::GenericDataProvider available_forcings_units[var_name] = units; } else { - std::string msg = "Forcing file " + file_name + " is missing column header names"; + std::string msg = "Forcing file " + file_name + " is missing a column header name"; LOG(msg, LogLevel::FATAL); throw std::runtime_error(msg); } From f96e983142573754f5c244a288ed2d0e62d6d3ad Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 8 Dec 2025 13:15:24 -0500 Subject: [PATCH 039/109] Ensure bmi_model has not been destroyed --- include/bmi/Bmi_Py_Adapter.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..8ee9014c64 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,7 +47,8 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - Finalize(); + if (bmi_model != NULL) + Finalize(); } /** From 3ede56956c8f0308e41eec809ae03e0c68dc6c4a Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 11 Dec 2025 12:30:47 -0500 Subject: [PATCH 040/109] Remove null check since that should be an error state if null --- include/bmi/Bmi_Py_Adapter.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 8ee9014c64..e27ecbc4f6 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -47,8 +47,7 @@ namespace models { Bmi_Py_Adapter(Bmi_Py_Adapter&&) = delete; ~Bmi_Py_Adapter() override { - if (bmi_model != NULL) - Finalize(); + Finalize(); } /** From 61f96d32a105a2dae3e2d12fe7410cdedab659f0 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 30 Jan 2026 15:46:29 -0500 Subject: [PATCH 041/109] Ensure data ownership not passed to python --- include/bmi/Bmi_Py_Adapter.hpp | 10 +++++++++- src/bmi/Bmi_Py_Adapter.cpp | 6 ------ src/realizations/catchment/Bmi_Py_Formulation.cpp | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index e27ecbc4f6..3e956a41d8 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -599,13 +599,21 @@ namespace models { /** * Set the value of a variable. This version of setting a variable will send an array with the `size` specified instead of checking the BMI for its current size of the variable. + * Ownership of the pointer will remain in C++, so the consuming BMI should not maintain a reference to the values beyond the scope of its `set_value` method. * * @param name The name of the BMI variable. * @param src Pointer to the data that will be sent to the BMI. * @param size The number of items represented by the pointer. */ template - void set_value_unchecked(const std::string &name, T *src, size_t size); + void set_value_unchecked(const std::string &name, T *src, size_t size) { + // declare readonly array info with the pointer and size + py::buffer_info info(src, static_cast(size), true); + // create the array with the info and NULL handler so python doesn't take ownership + py::array_t src_array(info, nullptr); + // pass the array to python to read; the BMI should not attempt to maintain a reference beyond the scope of this function to prevent trying to use freed memory + bmi_model->attr("set_value")(name, src_array); + } /** * Set values for a model's BMI variable at specified indices. diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index ee79de4e14..7436e2b48c 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -223,10 +223,4 @@ void Bmi_Py_Adapter::UpdateUntil(double time) { bmi_model->attr("update_until")(time); } -template -void Bmi_Py_Adapter::set_value_unchecked(const std::string &name, T *src, size_t size) { - py::array_t src_array(size, src); - bmi_model->attr("set_value")(name, src_array); -} - #endif //NGEN_WITH_PYTHON diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index e804aa5882..a3dc65df1e 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -120,7 +120,7 @@ bool Bmi_Py_Formulation::is_model_initialized() const { void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { auto bmi = std::dynamic_pointer_cast(get_bmi_model()); // load the state through the set value function that does not enforce the input size is the same as the current BMI's size - bmi->set_value_unchecked("serialization_state", state.data(), state.size()); + bmi->set_value_unchecked("serialization_state", state.data(), state.size()); } #endif //NGEN_WITH_PYTHON From 1663411561f2b31b7aa929f5cfacfff6fe404699 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 13:02:51 -0500 Subject: [PATCH 042/109] T-Route save stating for hot start --- include/bmi/Bmi_Py_Adapter.hpp | 33 +++-- include/core/Layer.hpp | 1 + include/core/NgenSimulation.hpp | 21 ++++ .../catchment/Bmi_Formulation.hpp | 9 ++ .../catchment/Bmi_Module_Formulation.hpp | 2 + .../catchment/Bmi_Multi_Formulation.hpp | 2 + .../state_save_restore/State_Save_Restore.hpp | 14 ++- src/NGen.cpp | 35 +++--- src/bmi/Bmi_Py_Adapter.cpp | 30 +++-- src/core/Layer.cpp | 11 ++ src/core/NgenSimulation.cpp | 115 +++++++++++++++--- .../catchment/Bmi_Module_Formulation.cpp | 6 + .../catchment/Bmi_Multi_Formulation.cpp | 91 +++++++------- src/state_save_restore/File_Per_Unit.cpp | 9 +- src/state_save_restore/State_Save_Restore.cpp | 13 ++ 15 files changed, 286 insertions(+), 106 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 3e956a41d8..60d97be394 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -542,24 +542,32 @@ namespace models { std::string py_type = GetVarType(name); std::string cxx_type = get_analogous_cxx_type(py_type, (size_t) itemSize); - if (cxx_type == "short") { - set_value(name, (short *) src); + if (cxx_type == "signed char") { + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned char") { + this->set_value(name, static_cast(src)); + } else if (cxx_type == "short") { + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned short") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "int") { - set_value(name, (int *) src); + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned int") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "long") { - set_value(name, (long *) src); + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned long") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "long long") { - //FIXME this gets dicey -- if a python numpy array is of type np.int64 (long long), - //but a c++ int* is passed to this function as src, it will fail in undefined ways... - //the template type overload may be perferred for doing SetValue from framework components - //such as forcing providers... - set_value(name, (long long *) src); + this->set_value(name, static_cast(src)); + } else if (cxx_type == "unsigned long long") { + this->set_value(name, static_cast(src)); } else if (cxx_type == "float") { - set_value(name, (float *) src); + this->set_value(name, static_cast(src)); } else if (cxx_type == "double") { - set_value(name, (double *) src); + this->set_value(name, static_cast(src)); } else if (cxx_type == "long double") { - set_value(name, (long double *) src); + this->set_value(name, static_cast(src)); } else { std::string throw_msg; throw_msg.assign("Bmi_Py_Adapter cannot set values for variable '" + name + "' that has unrecognized C++ type '" + cxx_type + "'"); @@ -567,7 +575,6 @@ namespace models { throw std::runtime_error(throw_msg); } } - /** * Set the values of the given BMI variable for the model, sourcing new data from the provided vector. * diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index ab4a0e4268..9a21401fb2 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -115,6 +115,7 @@ namespace ngen virtual void save_state_snapshot(std::shared_ptr snapshot_saver); virtual void load_state_snapshot(std::shared_ptr snapshot_loader); + virtual void load_hot_start(std::shared_ptr snapshot_loader); protected: diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index c29280d719..3189045edc 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -15,6 +15,10 @@ namespace hy_features class State_Snapshot_Saver; class State_Snapshot_Loader; +#if NGEN_WITH_ROUTING +#include "bmi/Bmi_Py_Adapter.hpp" +#endif // NGEN_WITH_ROUTING + #include #include #include @@ -51,6 +55,9 @@ class NgenSimulation */ void run_catchments(); + // Tear down of any items stored on the NgenSimulation object that could throw errors and, thus, should be kept separate from the deconstructor. + void finalize(); + /** * Run t-route on the stored nexus outflow values for the full configured duration of the simulation */ @@ -64,6 +71,14 @@ class NgenSimulation void save_state_snapshot(std::shared_ptr snapshot_saver); void load_state_snapshot(std::shared_ptr snapshot_loader); + /** + * Saves a snapshot state that's intended to be run at the end of a simulation. + * + * This version of saving will include T-Route BMI data and exclude the nexus outflow data stored during the catchment processing. + */ + void save_end_of_run(std::shared_ptr snapshot_saver); + // Load a snapshot of the end of a previous run. This will create a T-Route python adapter if the loader finds a unit for it and the config path is not empty. + void load_hot_start(std::shared_ptr snapshot_loader, const std::string &t_route_config_file_with_path); private: void advance_models_one_output_step(); @@ -80,6 +95,12 @@ class NgenSimulation std::vector catchment_outflows_; std::unordered_map nexus_indexes_; std::vector nexus_downstream_flows_; +#if NGEN_WITH_ROUTING + std::unique_ptr py_troute_; +#endif // NGEN_WITH_ROUTING + void make_troute(const std::string &t_route_config_file_with_path); + + std::string unit_name() const; int mpi_rank_; int mpi_num_procs_; diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index cc1cec0de6..ad2009f150 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -86,6 +86,15 @@ namespace realization { */ virtual void load_state(std::shared_ptr loader) const = 0; + /** + * Passes a serialized representation of the model's state to ``loader`` + * + * Asks saver to find data for the BMI and passes that data to the BMI for loading. + * + * Differes from `load_state` by also restting the internal time value to its initial state. + */ + virtual void load_hot_start(std::shared_ptr loader) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 1fb0cd64e8..eae78a4efa 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -55,6 +55,8 @@ namespace realization { void load_state(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) const override; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 3b2f6c7d53..492fc60ac8 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -58,6 +58,8 @@ namespace realization { void load_state(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) const override; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 8f4b1b77e5..f311a5cfd4 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -58,6 +58,13 @@ class State_Save_Config */ std::vector>> end_of_run_savers() const; + /** + * Get state loader that is intended to be performed before catchment processing starts. + * + * The returned pointer may be NULL if no configuration was made for existing data. + */ + std::unique_ptr hot_start() const; + struct instance { instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing); @@ -165,10 +172,15 @@ class State_Snapshot_Loader State_Snapshot_Loader() = default; virtual ~State_Snapshot_Loader() = default; + /** + * Check if data of a unit name exists. + */ + virtual bool has_unit(const std::string &unit_name) = 0; + /** * Load data from whatever source, and pass it to @param unit_loader->load() */ - virtual void load_unit(std::string const& unit_name, std::vector &data) = 0; + virtual void load_unit(const std::string &unit_name, std::vector &data) = 0; /** * Execute logic to complete the saving process diff --git a/src/NGen.cpp b/src/NGen.cpp index 329a70a634..79cf23c555 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -701,23 +701,17 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); - for (const auto& start_loader : state_saving_config.start_of_run_loaders()) { - LOG(LogLevel::INFO, "Loading start of run simulation data from state saving config " + start_loader.first); - std::shared_ptr snapshot_loader = start_loader.second->initialize_snapshot(State_Saver::snapshot_time_now()); - simulation->load_state_snapshot(snapshot_loader); + { // optionally run hot start loader if set in state saving config + auto hot_start_loader = state_saving_config.hot_start(); + if (hot_start_loader) { + LOG(LogLevel::INFO, "Loading hot start data from prior snapshot."); + std::shared_ptr snapshot_loader = hot_start_loader->initialize_snapshot(State_Saver::snapshot_time_now()); + simulation->load_hot_start(snapshot_loader, manager->get_t_route_config_file_with_path()); + } } simulation->run_catchments(); - for (const auto& end_saver : state_saving_config.end_of_run_savers()) { - LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); - std::shared_ptr snapshot = end_saver.second->initialize_snapshot( - State_Saver::snapshot_time_now(), - State_Saver::State_Durability::strict - ); - simulation->save_state_snapshot(snapshot); - } - #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif @@ -753,8 +747,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::chrono::duration time_elapsed_nexus_output = time_done_nexus_output - time_done_simulation; LOG("[TIMING]: Nexus outflow file writing: " + std::to_string(time_elapsed_nexus_output.count()), LogLevel::INFO); - manager->finalize(); - #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif @@ -781,6 +773,19 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { LOG("[TIMING]: Coastal: " + std::to_string(time_elapsed_coastal.count()), LogLevel::INFO); #endif + // run any end-of-run state saving after T-Route has finished but before starting to tear down data structures + for (const auto& end_saver : state_saving_config.end_of_run_savers()) { + LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); + std::shared_ptr snapshot = end_saver.second->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_end_of_run(snapshot); + } + + simulation->finalize(); + manager->finalize(); + auto time_done_total = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_total = time_done_total - time_start; diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index 7436e2b48c..d1a1c6f681 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -104,25 +104,35 @@ void Bmi_Py_Adapter::GetValue(std::string name, void *dest) { msg += e.what(); Logger::logMsgAndThrowError(msg); } - - if (cxx_type == "short") { - copy_to_array(name, (short *) dest); + if (cxx_type == "signed char") { + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned char") { + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "short") { + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned short") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "int") { - copy_to_array(name, (int *) dest); + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned int") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "long") { - copy_to_array(name, (long *) dest); + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned long") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "long long") { - copy_to_array(name, (long long *) dest); + this->copy_to_array(name, static_cast(dest)); + } else if (cxx_type == "unsigned long long") { + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "float") { - copy_to_array(name, (float *) dest); + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "double") { - copy_to_array(name, (double *) dest); + this->copy_to_array(name, static_cast(dest)); } else if (cxx_type == "long double") { - copy_to_array(name, (long double *) dest); + this->copy_to_array(name, static_cast(dest)); } else { Logger::logMsgAndThrowError("Bmi_Py_Adapter can't get value of unsupported type: " + cxx_type); } - } void Bmi_Py_Adapter::GetValueAtIndices(std::string name, void *dest, int *inds, int count) { diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 745306a885..432b918aa3 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -115,3 +115,14 @@ void ngen::Layer::load_state_snapshot(std::shared_ptr sna r_c->load_state(snapshot_loader); } } + +void ngen::Layer::load_hot_start(std::shared_ptr snapshot_loader) +{ + // XXX Handle any of this class's own state as a meta-data unit + + for (auto const& id : processing_units) { + auto r = features.catchment_at(id); + auto r_c = std::dynamic_pointer_cast(r); + r_c->load_hot_start(snapshot_loader); + } +} diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 7742e3a057..3000599634 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -8,12 +8,14 @@ #include "HY_Features.hpp" #endif -#if NGEN_WITH_ROUTING -#include "bmi/Bmi_Py_Adapter.hpp" -#endif // NGEN_WITH_ROUTING - +#include "state_save_restore/State_Save_Restore.hpp" #include "parallel_utils.h" +namespace { + const auto NGEN_UNIT_NAME = "ngen"; + const auto TROUTE_UNIT_NAME = "troute"; +} + NgenSimulation::NgenSimulation( Simulation_Time const& sim_time, std::vector> layers, @@ -54,6 +56,15 @@ void NgenSimulation::run_catchments() } } +void NgenSimulation::finalize() { +#if NGEN_WITH_ROUTING + if (this->py_troute_) { + this->py_troute_->Finalize(); + this->py_troute_.reset(); + } +#endif // NGEN_WITH_ROUTING +} + void NgenSimulation::advance_models_one_output_step() { // The Inner loop will advance all layers unless doing so will break one of two constraints @@ -110,19 +121,90 @@ void NgenSimulation::advance_models_one_output_step() void NgenSimulation::save_state_snapshot(std::shared_ptr snapshot_saver) { - + // TODO: save the current nexus data + auto unit_name = this->unit_name(); // XXX Handle self, then recursively pass responsibility to Layers for (auto& layer : layers_) { layer->save_state_snapshot(snapshot_saver); } } +void NgenSimulation::save_end_of_run(std::shared_ptr snapshot_saver) +{ + for (auto& layer : layers_) { + layer->save_state_snapshot(snapshot_saver); + } +#if NGEN_WITH_ROUTING + if (this->mpi_rank_ == 0 && this->py_troute_) { + uint64_t serialization_size; + this->py_troute_->SetValue("serialization_create", &serialization_size); + this->py_troute_->GetValue("serialization_size", &serialization_size); + void *troute_state = this->py_troute_->GetValuePtr("serialization_state"); + boost::span span(static_cast(troute_state), serialization_size); + snapshot_saver->save_unit(TROUTE_UNIT_NAME, span); + this->py_troute_->SetValue("serialization_free", &serialization_size); + } +#endif // NGEN_WITH_ROUTING +} + void NgenSimulation::load_state_snapshot(std::shared_ptr snapshot_loader) { + // TODO: load the state data related to nexus outflows + auto unit_name = this->unit_name(); for (auto& layer : layers_) { layer->load_state_snapshot(snapshot_loader); } } +void NgenSimulation::load_hot_start(std::shared_ptr snapshot_loader, const std::string &t_route_config_file_with_path) { + for (auto& layer : layers_) { + layer->load_hot_start(snapshot_loader); + } +#if NGEN_WITH_ROUTING + if (this->mpi_rank_ == 0) { + bool config_file_set = !t_route_config_file_with_path.empty(); + bool snapshot_exists = snapshot_loader->has_unit(TROUTE_UNIT_NAME); + if (config_file_set && snapshot_exists) { + LOG(LogLevel::DEBUG, "Loading T-Route data from snapshot."); + std::vector troute_data; + snapshot_loader->load_unit(TROUTE_UNIT_NAME, troute_data); + if (py_troute_ == NULL) { + this->make_troute(t_route_config_file_with_path); + } + py_troute_->set_value_unchecked("serialization_state", troute_data.data(), troute_data.size()); + double rt; // unused by the BMI but needed for messaging + py_troute_->SetValue("reset_time", &rt); + } else if (!config_file_set && !snapshot_exists) { + LOG(LogLevel::DEBUG, "No data set for loading T-Route."); + } else if (config_file_set && !snapshot_exists) { + LOG(LogLevel::WARNING, "A T-Route config file was provided but the load data does not contain T-Route data. T-Route will be run as a cold start."); + } else if (!config_file_set && snapshot_exists) { + LOG(LogLevel::WARNING, "A T-Route hot start snapshot exists but no config file was provided. T-Route will not be loaded or run,"); + } + } +#endif // NGEN_WITH_ROUTING +} + + +void NgenSimulation::make_troute(const std::string &t_route_config_file_with_path) { +#if NGEN_WITH_ROUTING + this->py_troute_ = std::make_unique( + "T-Route", + t_route_config_file_with_path, + "troute_nwm_bmi.troute_bmi.BmiTroute", + true + ); +#endif // NGEN_WITH_ROUTING +} + + +std::string NgenSimulation::unit_name() const { +#if NGEN_WITH_MPI + return "ngen_" + std::to_string(this->mpi_rank_); +#else + return "ngen_0"; +#endif // NGEN_WITH_MPI +} + int NgenSimulation::get_nexus_index(std::string const& nexus_id) const { @@ -219,14 +301,12 @@ void NgenSimulation::run_routing(NgenSimulation::hy_features_t &features, std::s int delta_time = sim_time_->get_output_interval_seconds(); // model for routing - models::bmi::Bmi_Py_Adapter py_troute("T-Route", t_route_config_file_with_path, "troute_nwm_bmi.troute_bmi.BmiTroute", true); + if (this->py_troute_ == NULL) { + this->make_troute(t_route_config_file_with_path); + } - // tell BMI to resize nexus containers - int64_t nexus_count = routing_nexus_indexes->size(); - py_troute.SetValue("land_surface_water_source__volume_flow_rate__count", &nexus_count); - py_troute.SetValue("land_surface_water_source__id__count", &nexus_count); // set up nexus id indexes - std::vector nexus_df_index(nexus_count); + std::vector nexus_df_index(routing_nexus_indexes->size()); for (const auto& key_value : *routing_nexus_indexes) { int id_index = key_value.second; @@ -244,14 +324,11 @@ void NgenSimulation::run_routing(NgenSimulation::hy_features_t &features, std::s } nexus_df_index[id_index] = id_as_int; } - py_troute.SetValue("land_surface_water_source__id", nexus_df_index.data()); - for (int i = 0; i < number_of_timesteps; ++i) { - py_troute.SetValue("land_surface_water_source__volume_flow_rate", - routing_nexus_downflows->data() + (i * nexus_count)); - py_troute.Update(); - } - // Finalize will write the output file - py_troute.Finalize(); + // use unchecked messaging to allow the BMI to change its container size + py_troute_->set_value_unchecked("land_surface_water_source__id", nexus_df_index.data(), nexus_df_index.size()); + py_troute_->set_value_unchecked("land_surface_water_source__volume_flow_rate", routing_nexus_downflows->data(), routing_nexus_downflows->size()); + // run the T-Route model and create outputs through Update + py_troute_->Update(); } #endif // NGEN_WITH_ROUTING } diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 4fb2a83fa9..d671d62776 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -35,6 +35,12 @@ namespace realization { this->load_serialization_state(data); } + void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) const { + this->load_state(loader); + double rt; + this->get_bmi_model()->SetValue("reset_time", &rt); + } + boost::span Bmi_Module_Formulation::get_available_variable_names() const { return available_forcings; } diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index b44715a093..b6f91b8b09 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -21,36 +21,38 @@ using namespace realization; -void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { +namespace { + // Check if the system's byte order is little endianness + constexpr bool is_little_endian() { #if (__cplusplus < 202002L) - // get system endianness - uint16_t endian_bytes = 0xFF00; - uint8_t *endian_bits = reinterpret_cast(&endian_bytes); - bool is_little_endian = endian_bits[0] == 0; + // C++ less than 2020 requires making a two byte object and checking which byte is 0 + uint16_t endian_bytes = 0xFF00; + uint8_t *endian_bits = reinterpret_cast(&endian_bytes); + return endian_bits[0] == 0; +#else + // C++ 2020+ has as simpler method + return std::endian::native == std::endian::little; #endif + } +} + +void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { std::vector> bmi_data; size_t data_size = 0; - // TODO: something more elegant than just skipping sloth for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); - if (bmi->get_model_type_name() != "bmi_c++_sloth") { - boost::span span = bmi->get_serialization_state(); - bmi_data.push_back(std::make_pair(span.data(), span.size())); - data_size += sizeof(uint64_t) + span.size(); - LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); - } + boost::span span = bmi->get_serialization_state(); + bmi_data.push_back(std::make_pair(span.data(), span.size())); + data_size += sizeof(uint64_t) + span.size(); + LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } char *data = new char[data_size]; size_t index = 0; for (const auto &bmi : bmi_data) { // write the size of the data -#if (__cplusplus < 202002L) - if (is_little_endian) { -#else - if constexpr (std::endian::native == std::endian::little) { -#endif + if (is_little_endian()) { std::memcpy(&data[index], &bmi.second, sizeof(uint64_t)); } else { // store the size bytes in reverse order to ensure saved data is always little endian @@ -70,46 +72,41 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav delete[] data; for (const nested_module_ptr &m : modules) { auto bmi = static_cast(m.get()); - if (bmi->get_model_type_name() != "bmi_c++_sloth") { - bmi->free_serialization_state(); - } + bmi->free_serialization_state(); } } void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) const { -#if (__cplusplus < 202002L) - // get system endianness - uint16_t endian_bytes = 0xFF00; - uint8_t *endian_bits = reinterpret_cast(&endian_bytes); - bool is_little_endian = endian_bits[0] == 0; -#endif std::vector data; loader->load_unit(this->get_id(), data); size_t index = 0; for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); - if (bmi->get_model_type_name() != "bmi_c++_sloth") { - uint64_t size; -#if (__cplusplus < 202002L) - if (is_little_endian) { -#else - if constexpr (std::endian::native == std::endian::little) { -#endif - memcpy(&size, data.data() + index, sizeof(uint64_t)); - } else { - // read size bytes in reverse order to interpret from little endian - char *size_bytes = reinterpret_cast(&size); - size_t endian_index = sizeof(uint64_t); - for (size_t i = 0; i < sizeof(uint64_t); ++i) { - size_bytes[--endian_index] = data[index + i]; - } + uint64_t size; + if (is_little_endian()) { + memcpy(&size, data.data() + index, sizeof(uint64_t)); + } else { + // read size bytes in reverse order to interpret from little endian + char *size_bytes = reinterpret_cast(&size); + size_t endian_index = sizeof(uint64_t); + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + size_bytes[--endian_index] = data[index + i]; } - boost::span span(data.data() + index + sizeof(uint64_t), size); - bmi->load_serialization_state(span); - index += sizeof(uint64_t) + size; - LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } + boost::span span(data.data() + index + sizeof(uint64_t), size); + bmi->load_serialization_state(span); + index += sizeof(uint64_t) + size; + LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", + bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); + } +} + +void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr loader) const { + this->load_state(loader); + double rt; + for (const nested_module_ptr &m : modules) { + auto bmi = dynamic_cast(m.get()); + bmi->get_bmi_model()->SetValue("reset_time", &rt); } } diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index 103b7d6a7a..cfa152a92e 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -112,12 +112,14 @@ class File_Per_Unit_Snapshot_Loader : public State_Snapshot_Loader File_Per_Unit_Snapshot_Loader(path dir_path); ~File_Per_Unit_Snapshot_Loader() override = default; + bool has_unit(const std::string &unit_name) override; + /** * Load data from whatever source and store it in the `data` vector. * * @param data The location where the loaded data will be stored. This will be resized to the amount of data loaded. */ - void load_unit(std::string const& unit_name, std::vector &data) override; + void load_unit(const std::string &unit_name, std::vector &data) override; /** * Execute logic to complete the saving process @@ -140,6 +142,11 @@ File_Per_Unit_Snapshot_Loader::File_Per_Unit_Snapshot_Loader(path dir_path) } +bool File_Per_Unit_Snapshot_Loader::has_unit(const std::string &unit_name) { + auto file_path = dir_path_ / unit_name; + return exists(file_path.string()); +} + void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, std::vector &data) { auto file_path = dir_path_ / unit_name; std::uintmax_t size; diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 84e2226e95..3f5838e195 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -71,6 +71,19 @@ std::vector>> State_Save_Con return savers; } +std::unique_ptr State_Save_Config::hot_start() const { + for (const auto &i : this->instances_) { + if (i.direction_ == State_Save_Direction::Load && i.timing_ == State_Save_When::StartOfRun) { + if (i.mechanism_ == State_Save_Mechanism::FilePerUnit) { + return std::make_unique(i.path_); + } else { + LOG(LogLevel::WARNING, "State_Save_Config: Saving mechanism " + i.mechanism_string() + " is not supported for start of run saving."); + } + } + } + return std::unique_ptr(); +} + State_Save_Config::instance::instance(std::string const& direction, std::string const& label, std::string const& path, std::string const& mechanism, std::string const& timing) : label_(label) , path_(path) From 8c50295a23a27cdebe420dd916168afcf2096e90 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 13:53:05 -0500 Subject: [PATCH 043/109] Global source for serialization message names --- .../catchment/Bmi_Fortran_Formulation.hpp | 2 +- include/state_save_restore/State_Save_Utils.hpp | 12 ++++++++++++ src/core/NgenSimulation.cpp | 13 +++++++------ .../catchment/Bmi_Fortran_Formulation.cpp | 7 ++++--- .../catchment/Bmi_Module_Formulation.cpp | 13 +++++++------ .../catchment/Bmi_Multi_Formulation.cpp | 13 +++++++------ src/realizations/catchment/Bmi_Py_Formulation.cpp | 3 ++- 7 files changed, 40 insertions(+), 23 deletions(-) create mode 100644 include/state_save_restore/State_Save_Utils.hpp diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 6cc8618bc4..4da087be52 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -25,7 +25,7 @@ namespace realization { /** * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_serialization_state` is called. - * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface and copy the results to `size`. + * Because the Fortran BMI has no messaging for 64-bit integers, this overload will use the 32-bit integer interface. * * @return Span of the serialized data. */ diff --git a/include/state_save_restore/State_Save_Utils.hpp b/include/state_save_restore/State_Save_Utils.hpp new file mode 100644 index 0000000000..8c0504f7a8 --- /dev/null +++ b/include/state_save_restore/State_Save_Utils.hpp @@ -0,0 +1,12 @@ +#ifndef NGEN_STATE_SAVE_UTILS_HPP +#define NGEN_STATE_SAVE_UTILS_HPP + +namespace StateSaveNames { + inline constexpr auto CREATE = "serialization_create"; + inline constexpr auto STATE = "serialization_state"; + inline constexpr auto FREE = "serialization_free"; + inline constexpr auto SIZE = "serialization_size"; + inline constexpr auto RESET = "reset_time"; +} + +#endif diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 3000599634..86e0112f86 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -8,6 +8,7 @@ #include "HY_Features.hpp" #endif +#include "state_save_restore/State_Save_Utils.hpp" #include "state_save_restore/State_Save_Restore.hpp" #include "parallel_utils.h" @@ -137,12 +138,12 @@ void NgenSimulation::save_end_of_run(std::shared_ptr snaps #if NGEN_WITH_ROUTING if (this->mpi_rank_ == 0 && this->py_troute_) { uint64_t serialization_size; - this->py_troute_->SetValue("serialization_create", &serialization_size); - this->py_troute_->GetValue("serialization_size", &serialization_size); - void *troute_state = this->py_troute_->GetValuePtr("serialization_state"); + this->py_troute_->SetValue(StateSaveNames::CREATE, &serialization_size); + this->py_troute_->GetValue(StateSaveNames::SIZE, &serialization_size); + void *troute_state = this->py_troute_->GetValuePtr(StateSaveNames::STATE); boost::span span(static_cast(troute_state), serialization_size); snapshot_saver->save_unit(TROUTE_UNIT_NAME, span); - this->py_troute_->SetValue("serialization_free", &serialization_size); + this->py_troute_->SetValue(StateSaveNames::FREE, &serialization_size); } #endif // NGEN_WITH_ROUTING } @@ -170,9 +171,9 @@ void NgenSimulation::load_hot_start(std::shared_ptr snaps if (py_troute_ == NULL) { this->make_troute(t_route_config_file_with_path); } - py_troute_->set_value_unchecked("serialization_state", troute_data.data(), troute_data.size()); + py_troute_->set_value_unchecked(StateSaveNames::STATE, troute_data.data(), troute_data.size()); double rt; // unused by the BMI but needed for messaging - py_troute_->SetValue("reset_time", &rt); + py_troute_->SetValue(StateSaveNames::RESET, &rt); } else if (!config_file_set && !snapshot_exists) { LOG(LogLevel::DEBUG, "No data set for loading T-Route."); } else if (config_file_set && !snapshot_exists) { diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index b8804eab79..ee9d3179ef 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -6,6 +6,7 @@ #include "Bmi_Fortran_Formulation.hpp" #include "Bmi_Fortran_Adapter.hpp" #include "Constants.h" +#include "state_save_restore/State_Save_Utils.hpp" using namespace realization; using namespace models::bmi; @@ -96,9 +97,9 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const const boost::span Bmi_Fortran_Formulation::get_serialization_state() const { auto model = get_bmi_model(); int size_int = 0; - model->SetValue("serialization_create", &size_int); - model->GetValue("serialization_size", &size_int); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + model->SetValue(StateSaveNames::CREATE, &size_int); + model->GetValue(StateSaveNames::SIZE, &size_int); + auto serialization_state = static_cast(model->GetValuePtr(StateSaveNames::STATE)); const boost::span span(serialization_state, size_int); return span; } diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index d671d62776..2ede682b3e 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -2,6 +2,7 @@ #include "utilities/logging_utils.h" #include #include "Logger.hpp" +#include "state_save_restore/State_Save_Utils.hpp" #include std::stringstream bmiform_ss; @@ -38,7 +39,7 @@ namespace realization { void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) const { this->load_state(loader); double rt; - this->get_bmi_model()->SetValue("reset_time", &rt); + this->get_bmi_model()->SetValue(StateSaveNames::FREE, &rt); } boost::span Bmi_Module_Formulation::get_available_variable_names() const { @@ -1096,9 +1097,9 @@ namespace realization { const boost::span Bmi_Module_Formulation::get_serialization_state() const { auto model = get_bmi_model(); uint64_t size = 0; - model->SetValue("serialization_create", &size); - model->GetValue("serialization_size", &size); - auto serialization_state = static_cast(model->GetValuePtr("serialization_state")); + model->SetValue(StateSaveNames::CREATE, &size); + model->GetValue(StateSaveNames::SIZE, &size); + auto serialization_state = static_cast(model->GetValuePtr(StateSaveNames::STATE)); const boost::span span(serialization_state, size); return span; } @@ -1108,13 +1109,13 @@ namespace realization { // grab the pointer to the underlying state data void* data = (void*)state.data(); // load the state through SetValue - bmi->SetValue("serialization_state", data); + bmi->SetValue(StateSaveNames::STATE, data); } void Bmi_Module_Formulation::free_serialization_state() const { auto bmi = this->bmi_model; // send message to clear memory associated with serialized data void* _; // this pointer will be unused by SetValue - bmi->SetValue("serialization_free", _); + bmi->SetValue(StateSaveNames::FREE, _); } } diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index b6f91b8b09..e2b0ea8106 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -13,6 +13,7 @@ #include "Bmi_Py_Formulation.hpp" #include "Logger.hpp" +#include "state_save_restore/State_Save_Utils.hpp" #include #if (__cplusplus >= 202002L) @@ -48,12 +49,13 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); } - char *data = new char[data_size]; + std::vector data; + data.reserve(data_size); size_t index = 0; for (const auto &bmi : bmi_data) { // write the size of the data if (is_little_endian()) { - std::memcpy(&data[index], &bmi.second, sizeof(uint64_t)); + std::memcpy(data.data() + index, &bmi.second, sizeof(uint64_t)); } else { // store the size bytes in reverse order to ensure saved data is always little endian const char *bytes = reinterpret_cast(&bmi.second); @@ -63,13 +65,12 @@ void Bmi_Multi_Formulation::save_state(std::shared_ptr sav } } // write the serialized data - std::memcpy(data + index + sizeof(uint64_t), &bmi.first, bmi.second); + std::memcpy(data.data() + index + sizeof(uint64_t), &bmi.first, bmi.second); index += sizeof(uint64_t) + bmi.second; } - boost::span span(data, data_size); + boost::span span(data.data(), data_size); saver->save_unit(this->get_id(), span); - delete[] data; for (const nested_module_ptr &m : modules) { auto bmi = static_cast(m.get()); bmi->free_serialization_state(); @@ -106,7 +107,7 @@ void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr(m.get()); - bmi->get_bmi_model()->SetValue("reset_time", &rt); + bmi->get_bmi_model()->SetValue(StateSaveNames::RESET, &rt); } } diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index a3dc65df1e..15e66677d0 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -1,5 +1,6 @@ #include #include "Logger.hpp" +#include "state_save_restore/State_Save_Utils.hpp" #if NGEN_WITH_PYTHON @@ -120,7 +121,7 @@ bool Bmi_Py_Formulation::is_model_initialized() const { void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { auto bmi = std::dynamic_pointer_cast(get_bmi_model()); // load the state through the set value function that does not enforce the input size is the same as the current BMI's size - bmi->set_value_unchecked("serialization_state", state.data(), state.size()); + bmi->set_value_unchecked(StateSaveNames::STATE, state.data(), state.size()); } #endif //NGEN_WITH_PYTHON From 79d45996195b0e0ea147796d38e76363874b757b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 14:20:23 -0500 Subject: [PATCH 044/109] Expand applicable types for get value as double --- .../catchment/Bmi_Py_Formulation.cpp | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 15e66677d0..f0c912fa44 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -54,50 +54,30 @@ double Bmi_Py_Formulation::get_var_value_as_double(const int &index, const std:: std::string val_type = model->GetVarType(var_name); size_t val_item_size = (size_t)model->GetVarItemsize(var_name); + std::string cxx_type = model->get_analogous_cxx_type(val_type, val_item_size); //void *dest; int indices[1]; indices[0] = index; - - // The available types and how they are handled here should match what is in SetValueAtIndices - if (val_type == "int" && val_item_size == sizeof(short)) { - short dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "int" && val_item_size == sizeof(int)) { - int dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "int" && val_item_size == sizeof(long)) { - long dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "int" && val_item_size == sizeof(long long)) { - long long dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double)dest; - } - if (val_type == "float" || val_type == "float16" || val_type == "float32" || val_type == "float64") { - if (val_item_size == sizeof(float)) { - float dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double) dest; - } - if (val_item_size == sizeof(double)) { - double dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return dest; - } - if (val_item_size == sizeof(long double)) { - long double dest; - model->get_value_at_indices(var_name, &dest, indices, 1, false); - return (double) dest; - } - } - + // macro for both checking and converting based on type from get_analogous_cxx_type +#define GET_DOUBLE(type) if (cxx_type == #type) {\ + type dest;\ + model->get_value_at_indices(var_name, &dest, indices, 1, false);\ + return static_cast(dest);} + GET_DOUBLE(signed char) + else GET_DOUBLE(unsigned char) + else GET_DOUBLE(short) + else GET_DOUBLE(unsigned short) + else GET_DOUBLE(int) + else GET_DOUBLE(unsigned int) + else GET_DOUBLE(long) + else GET_DOUBLE(unsigned long) + else GET_DOUBLE(long long) + else GET_DOUBLE(unsigned long long) + else GET_DOUBLE(float) + else GET_DOUBLE(double) + else GET_DOUBLE(long double) +#undef GET_DOUBLE Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + " as double: no logic for converting variable type " + val_type); From 0d5f27f03cb01e7c3d603f8939de876a7cd8bb98 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 4 Feb 2026 14:29:51 -0500 Subject: [PATCH 045/109] C++14 consts --- .../state_save_restore/State_Save_Utils.hpp | 10 +++--- .../catchment/Bmi_Py_Formulation.cpp | 36 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/include/state_save_restore/State_Save_Utils.hpp b/include/state_save_restore/State_Save_Utils.hpp index 8c0504f7a8..8e6c127f6a 100644 --- a/include/state_save_restore/State_Save_Utils.hpp +++ b/include/state_save_restore/State_Save_Utils.hpp @@ -2,11 +2,11 @@ #define NGEN_STATE_SAVE_UTILS_HPP namespace StateSaveNames { - inline constexpr auto CREATE = "serialization_create"; - inline constexpr auto STATE = "serialization_state"; - inline constexpr auto FREE = "serialization_free"; - inline constexpr auto SIZE = "serialization_size"; - inline constexpr auto RESET = "reset_time"; + const auto CREATE = "serialization_create"; + const auto STATE = "serialization_state"; + const auto FREE = "serialization_free"; + const auto SIZE = "serialization_size"; + const auto RESET = "reset_time"; } #endif diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index f0c912fa44..046bd68207 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -60,24 +60,24 @@ double Bmi_Py_Formulation::get_var_value_as_double(const int &index, const std:: int indices[1]; indices[0] = index; // macro for both checking and converting based on type from get_analogous_cxx_type -#define GET_DOUBLE(type) if (cxx_type == #type) {\ - type dest;\ - model->get_value_at_indices(var_name, &dest, indices, 1, false);\ - return static_cast(dest);} - GET_DOUBLE(signed char) - else GET_DOUBLE(unsigned char) - else GET_DOUBLE(short) - else GET_DOUBLE(unsigned short) - else GET_DOUBLE(int) - else GET_DOUBLE(unsigned int) - else GET_DOUBLE(long) - else GET_DOUBLE(unsigned long) - else GET_DOUBLE(long long) - else GET_DOUBLE(unsigned long long) - else GET_DOUBLE(float) - else GET_DOUBLE(double) - else GET_DOUBLE(long double) -#undef GET_DOUBLE +#define PY_BMI_DOUBLE_AT_INDEX(type) if (cxx_type == #type) {\ + type dest;\ + model->get_value_at_indices(var_name, &dest, indices, 1, false);\ + return static_cast(dest);} + PY_BMI_DOUBLE_AT_INDEX(signed char) + else PY_BMI_DOUBLE_AT_INDEX(unsigned char) + else PY_BMI_DOUBLE_AT_INDEX(short) + else PY_BMI_DOUBLE_AT_INDEX(unsigned short) + else PY_BMI_DOUBLE_AT_INDEX(int) + else PY_BMI_DOUBLE_AT_INDEX(unsigned int) + else PY_BMI_DOUBLE_AT_INDEX(long) + else PY_BMI_DOUBLE_AT_INDEX(unsigned long) + else PY_BMI_DOUBLE_AT_INDEX(long long) + else PY_BMI_DOUBLE_AT_INDEX(unsigned long long) + else PY_BMI_DOUBLE_AT_INDEX(float) + else PY_BMI_DOUBLE_AT_INDEX(double) + else PY_BMI_DOUBLE_AT_INDEX(long double) +#undef PY_BMI_DOUBLE_AT_INDEX Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + " as double: no logic for converting variable type " + val_type); From d862fa42fc2b0f3066c3e05ec772375c27f9d2b2 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 5 Feb 2026 07:53:02 -0500 Subject: [PATCH 046/109] Expand python typing interface --- include/bmi/Bmi_Py_Adapter.hpp | 46 +++++++++++++--------------------- src/bmi/Bmi_Py_Adapter.cpp | 38 ++++++++++++++-------------- 2 files changed, 37 insertions(+), 47 deletions(-) diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 60d97be394..2e48dc7845 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -541,39 +541,29 @@ namespace models { int itemSize = GetVarItemsize(name); std::string py_type = GetVarType(name); std::string cxx_type = get_analogous_cxx_type(py_type, (size_t) itemSize); - - if (cxx_type == "signed char") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned char") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "short") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned short") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "int") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned int") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "long long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "unsigned long long") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "float") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "double") { - this->set_value(name, static_cast(src)); - } else if (cxx_type == "long double") { - this->set_value(name, static_cast(src)); - } else { + // macro for checking type and setting value + #define BMI_PY_SET_VALUE(type) if (cxx_type == #type) {\ + this->set_value(name, static_cast(src)); } + BMI_PY_SET_VALUE(signed char) + else BMI_PY_SET_VALUE(unsigned char) + else BMI_PY_SET_VALUE(short) + else BMI_PY_SET_VALUE(unsigned short) + else BMI_PY_SET_VALUE(int) + else BMI_PY_SET_VALUE(unsigned int) + else BMI_PY_SET_VALUE(long) + else BMI_PY_SET_VALUE(unsigned long) + else BMI_PY_SET_VALUE(long long) + else BMI_PY_SET_VALUE(unsigned long long) + else BMI_PY_SET_VALUE(float) + else BMI_PY_SET_VALUE(double) + else BMI_PY_SET_VALUE(long double) + else { std::string throw_msg; throw_msg.assign("Bmi_Py_Adapter cannot set values for variable '" + name + "' that has unrecognized C++ type '" + cxx_type + "'"); LOG(throw_msg, LogLevel::WARNING); throw std::runtime_error(throw_msg); } + #undef BMI_PY_SET_VALUE } /** * Set the values of the given BMI variable for the model, sourcing new data from the provided vector. diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index d1a1c6f681..5fdc3eb8ec 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -199,30 +199,30 @@ std::string Bmi_Py_Adapter::get_bmi_type_simple_name() const { void Bmi_Py_Adapter::SetValueAtIndices(std::string name, int *inds, int count, void *src) { std::string val_type = GetVarType(name); size_t val_item_size = (size_t)GetVarItemsize(name); - - // The available types and how they are handled here should match what is in get_value_at_indices - if (val_type == "int" && val_item_size == sizeof(short)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "int" && val_item_size == sizeof(int)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "int" && val_item_size == sizeof(long)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "int" && val_item_size == sizeof(long long)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float" && val_item_size == sizeof(float)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float" && val_item_size == sizeof(double)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float64" && val_item_size == sizeof(double)) { - set_value_at_indices(name, inds, count, src, val_type); - } else if (val_type == "float" && val_item_size == sizeof(long double)) { - set_value_at_indices(name, inds, count, src, val_type); - } else { + std::string cxx_type = this->get_analogous_cxx_type(val_type, val_item_size); + + // macro for checking type and calling `set_value_at_indices` with that type + #define BMI_PY_SET_VALUE_INDEX(type) if (cxx_type == #type) { this->set_value_at_indices(name, inds, count, src, val_type); } + BMI_PY_SET_VALUE_INDEX(signed char) + else BMI_PY_SET_VALUE_INDEX(unsigned char) + else BMI_PY_SET_VALUE_INDEX(short) + else BMI_PY_SET_VALUE_INDEX(unsigned short) + else BMI_PY_SET_VALUE_INDEX(int) + else BMI_PY_SET_VALUE_INDEX(unsigned int) + else BMI_PY_SET_VALUE_INDEX(long) + else BMI_PY_SET_VALUE_INDEX(unsigned long) + else BMI_PY_SET_VALUE_INDEX(long long) + else BMI_PY_SET_VALUE_INDEX(unsigned long long) + else BMI_PY_SET_VALUE_INDEX(float) + else BMI_PY_SET_VALUE_INDEX(double) + else BMI_PY_SET_VALUE_INDEX(long double) + else { Logger::logMsgAndThrowError( "(Bmi_Py_Adapter) Failed attempt to SET values of BMI variable '" + name + "' from '" + model_name + "' model: model advertises unsupported combination of type (" + val_type + ") and size (" + std::to_string(val_item_size) + ")."); } + #undef BMI_PY_SET_VALUE_INDEX } void Bmi_Py_Adapter::Update() { From c86130b32128262952ce01f0e3da6fce96d154c8 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 5 Feb 2026 07:53:26 -0500 Subject: [PATCH 047/109] Prevent multiple hot start config definitions --- src/state_save_restore/State_Save_Restore.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 3f5838e195..58fe75fad1 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -18,7 +18,7 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) return; } - //auto saving_config = *maybe; + bool hot_start = false; for (const auto& saving_config : *maybe) { try { auto& subtree = saving_config.second; @@ -29,9 +29,14 @@ State_Save_Config::State_Save_Config(boost::property_tree::ptree const& tree) auto when = subtree.get("when"); instance i{direction, what, where, how, when}; + if (i.timing_ == State_Save_When::StartOfRun && i.direction_ == State_Save_Direction::Load) { + if (hot_start) + throw std::runtime_error("Only one hot start state saving configuration is allowed."); + hot_start = true; + } instances_.push_back(i); - } catch (...) { - LOG("Bad state saving config", LogLevel::WARNING); + } catch (std::exception &e) { + LOG("Bad state saving config: " + std::string(e.what()), LogLevel::WARNING); throw; } } From 4314da85db8d55e171638e4f9fd45c644e55a033 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 6 Feb 2026 10:26:57 -0500 Subject: [PATCH 048/109] Use Boost for serializing Multi-BMI --- .../catchment/Bmi_Formulation.hpp | 6 +- .../catchment/Bmi_Fortran_Formulation.hpp | 2 +- .../catchment/Bmi_Module_Formulation.hpp | 12 +- .../catchment/Bmi_Multi_Formulation.hpp | 8 +- .../catchment/Bmi_Py_Formulation.hpp | 2 +- include/state_save_restore/File_Per_Unit.hpp | 8 +- .../state_save_restore/State_Save_Restore.hpp | 11 +- include/state_save_restore/vecbuf.hpp | 125 ++++++++++++++++++ src/NGen.cpp | 7 +- .../catchment/Bmi_Fortran_Formulation.cpp | 2 +- .../catchment/Bmi_Module_Formulation.cpp | 12 +- .../catchment/Bmi_Multi_Formulation.cpp | 121 +++++++---------- .../catchment/Bmi_Py_Formulation.cpp | 2 +- src/realizations/catchment/CMakeLists.txt | 9 ++ src/state_save_restore/File_Per_Unit.cpp | 32 +++-- src/state_save_restore/State_Save_Restore.cpp | 5 +- 16 files changed, 249 insertions(+), 115 deletions(-) create mode 100644 include/state_save_restore/vecbuf.hpp diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index ad2009f150..91236fea42 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -77,14 +77,14 @@ namespace realization { * Asks the model to serialize its state, queries the pointer * and length, passes that to saver, and then releases it */ - virtual void save_state(std::shared_ptr saver) const = 0; + virtual void save_state(std::shared_ptr saver) = 0; /** * Passes a serialized representation of the model's state to ``loader`` * * Asks saver to find data for the BMI and passes that data to the BMI for loading. */ - virtual void load_state(std::shared_ptr loader) const = 0; + virtual void load_state(std::shared_ptr loader) = 0; /** * Passes a serialized representation of the model's state to ``loader`` @@ -93,7 +93,7 @@ namespace realization { * * Differes from `load_state` by also restting the internal time value to its initial state. */ - virtual void load_hot_start(std::shared_ptr loader) const = 0; + virtual void load_hot_start(std::shared_ptr loader) = 0; /** * Convert a time value from the model to an epoch time in seconds. diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 4da087be52..205303f9d5 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -29,7 +29,7 @@ namespace realization { * * @return Span of the serialized data. */ - const boost::span get_serialization_state() const override; + const boost::span get_serialization_state() override; protected: diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index eae78a4efa..10fadd5327 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -51,11 +51,11 @@ namespace realization { * Create a save state, save it using the `State_Snapshot_Saver`, then clear the save state from memory. * `this->get_id()` will be used as the unique ID for the saver. */ - void save_state(std::shared_ptr saver) const override; + void save_state(std::shared_ptr saver) override; - void load_state(std::shared_ptr loader) const override; + void load_state(std::shared_ptr loader) override; - void load_hot_start(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) override; /** * Get the collection of forcing output property names this instance can provide. @@ -296,16 +296,16 @@ namespace realization { * * @return Span of the serialized data. */ - virtual const boost::span get_serialization_state() const; + virtual const boost::span get_serialization_state(); /** * Requests the BMI to load data from a previously saved state. This has a side effect of freeing a current state if it currently exists. */ - virtual void load_serialization_state(const boost::span state) const; + virtual void load_serialization_state(const boost::span state); /** * Requests the BMI to clear a currently saved state from memory. * Existing state pointers should not be used as the stored data may be freed depending on implementation. */ - void free_serialization_state() const; + void free_serialization_state(); void set_realization_file_format(bool is_legacy_format); virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const override { diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 492fc60ac8..7ebc7dc0b1 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -15,6 +15,7 @@ #include "ConfigurationException.hpp" #include "ExternalIntegrationException.hpp" +#include #define BMI_REALIZATION_CFG_PARAM_REQ__MODULES "modules" #define BMI_REALIZATION_CFG_PARAM_OPT__DEFAULT_OUT_VALS "default_output_values" @@ -56,9 +57,9 @@ namespace realization { } }; - void load_state(std::shared_ptr loader) const override; + void load_state(std::shared_ptr loader) override; - void load_hot_start(std::shared_ptr loader) const override; + void load_hot_start(std::shared_ptr loader) override; /** * Convert a time value from the model to an epoch time in seconds. @@ -678,6 +679,7 @@ namespace realization { bool is_realization_legacy_format() const; private: + friend class boost::serialization::access; /** * Setup a deferred provider for a nested module, tracking the class as needed. @@ -777,6 +779,8 @@ namespace realization { friend Bmi_Multi_Formulation_Test; friend class ::Bmi_Cpp_Multi_Array_Test; + template + void serialize(Archive& ar, const unsigned int version); }; } diff --git a/include/realizations/catchment/Bmi_Py_Formulation.hpp b/include/realizations/catchment/Bmi_Py_Formulation.hpp index 19c6a0428e..d3830d6282 100644 --- a/include/realizations/catchment/Bmi_Py_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Py_Formulation.hpp @@ -40,7 +40,7 @@ namespace realization { * * The python BMI requires additional messaging for pre-allocating memory for load */ - void load_serialization_state(const boost::span state) const override; + void load_serialization_state(const boost::span state) override; protected: diff --git a/include/state_save_restore/File_Per_Unit.hpp b/include/state_save_restore/File_Per_Unit.hpp index 3fdefe71b8..faec8d966a 100644 --- a/include/state_save_restore/File_Per_Unit.hpp +++ b/include/state_save_restore/File_Per_Unit.hpp @@ -9,7 +9,9 @@ class File_Per_Unit_Saver : public State_Saver File_Per_Unit_Saver(std::string base_path); ~File_Per_Unit_Saver(); - std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) override; + std::shared_ptr initialize_snapshot(State_Durability durability) override; + + std::shared_ptr initialize_checkpoint_snapshot(snapshot_time_t epoch, State_Durability durability) override; void finalize() override; @@ -26,7 +28,9 @@ class File_Per_Unit_Loader : public State_Loader void finalize() override { }; - std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) override; + std::shared_ptr initialize_snapshot() override; + + std::shared_ptr initialize_checkpoint_snapshot(State_Saver::snapshot_time_t epoch) override; private: std::string dir_path_; }; diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index f311a5cfd4..3d3a3bf692 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -105,7 +105,9 @@ class State_Saver * potential errors to be checked and reported before finalize() * and/or State_Snapshot_Saver::finish_saving() return */ - virtual std::shared_ptr initialize_snapshot(snapshot_time_t epoch, State_Durability durability) = 0; + virtual std::shared_ptr initialize_snapshot(State_Durability durability) = 0; + + virtual std::shared_ptr initialize_checkpoint_snapshot(snapshot_time_t epoch, State_Durability durability) = 0; /** * Execute any logic necessary to cleanly finish usage, and @@ -120,7 +122,7 @@ class State_Snapshot_Saver { public: State_Snapshot_Saver() = delete; - State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + State_Snapshot_Saver(State_Saver::State_Durability durability); virtual ~State_Snapshot_Saver() = default; /** @@ -139,7 +141,6 @@ class State_Snapshot_Saver virtual void finish_saving() = 0; protected: - State_Saver::snapshot_time_t epoch_; State_Saver::State_Durability durability_; }; @@ -155,7 +156,9 @@ class State_Loader * Return an object suitable for loading a simulation state as of * a particular moment in time, @param epoch */ - virtual std::shared_ptr initialize_snapshot(State_Saver::snapshot_time_t epoch) = 0; + virtual std::shared_ptr initialize_snapshot() = 0; + + virtual std::shared_ptr initialize_checkpoint_snapshot(State_Saver::snapshot_time_t epoch) = 0; /** * Execute any logic necessary to cleanly finish usage, and diff --git a/include/state_save_restore/vecbuf.hpp b/include/state_save_restore/vecbuf.hpp new file mode 100644 index 0000000000..a20bc70e73 --- /dev/null +++ b/include/state_save_restore/vecbuf.hpp @@ -0,0 +1,125 @@ +#ifndef HPP_STRING_VECBUF +#define HPP_STRING_VECBUF +// https://gist.github.com/stephanlachnit/4a06f8475afd144e73235e2a2584b000 +// SPDX-FileCopyrightText: 2023 Stephan Lachnit +// SPDX-License-Identifier: MIT + +#include +#include +#include + +template> +class vecbuf : public std::basic_streambuf { +public: + using streambuf = std::basic_streambuf; + using char_type = typename streambuf::char_type; + using int_type = typename streambuf::int_type; + using traits_type = typename streambuf::traits_type; + using vector = std::vector; + using value_type = typename vector::value_type; + using size_type = typename vector::size_type; + + // Constructor for vecbuf with optional initial capacity + vecbuf(size_type capacity = 0) : vector_() { reserve(capacity); } + + // Forwarder for std::vector::shrink_to_fit() + constexpr void shrink_to_fit() { vector_.shrink_to_fit(); } + + // Forwarder for std::vector::clear() + constexpr void clear() { vector_.clear(); } + + // Forwarder for std::vector::resize(size) + constexpr void resize(size_type size) { vector_.resize(size); } + + // Forwarder for std::vector::reserve + constexpr void reserve(size_type capacity) { vector_.reserve(capacity); setp_from_vector(); } + + // Increase the capacity of the buffer by reserving the current_size + additional_capacity + constexpr void reserve_additional(size_type additional_capacity) { reserve(size() + additional_capacity); } + + // Forwarder for std::vector::data + constexpr value_type* data() { return vector_.data(); } + + // Forwarder for std::vector::size + constexpr size_type size() const { return vector_.size(); } + + // Forwarder for std::vector::capacity + constexpr size_type capacity() const { return vector_.capacity(); } + + // Implements std::basic_streambuf::xsputn + std::streamsize xsputn(const char_type* s, std::streamsize count) override { + try { + reserve_additional(count); + } + catch (const std::bad_alloc& error) { + // reserve did not work, use slow algorithm + return xsputn_slow(s, count); + } + // reserve worked, use fast algorithm + return xsputn_fast(s, count); + } + +protected: + // Calculates value to std::basic_streambuf::pbase from vector + constexpr value_type* pbase_from_vector() const { return const_cast(vector_.data()); } + + // Calculates value to std::basic_streambuf::pptr from vector + constexpr value_type* pptr_from_vector() const { return const_cast(vector_.data() + vector_.size()); } + + // Calculates value to std::basic_streambuf::epptr from vector + constexpr value_type* epptr_from_vector() const { return const_cast(vector_.data()) + vector_.capacity(); } + + // Sets the values for std::basic_streambuf::pbase, std::basic_streambuf::pptr and std::basic_streambuf::epptr from vector + constexpr void setp_from_vector() { streambuf::setp(pbase_from_vector(), epptr_from_vector()); streambuf::pbump(size()); } + +private: + // std::vector containing the data + vector vector_; + + // Fast implementation of std::basic_streambuf::xsputn if reserve_additional(count) succeeded + std::streamsize xsputn_fast(const char_type* s, std::streamsize count) { + // store current pptr (end of vector location) + auto* old_pptr = pptr_from_vector(); + // resize the vector, does not move since space already reserved + vector_.resize(vector_.size() + count); + // directly memcpy new content to old pptr (end of vector before it was resized) + traits_type::copy(old_pptr, s, count); + // reserve() already calls setp_from_vector(), only adjust pptr to new epptr + streambuf::pbump(count); + + return count; + } + + // Slow implementation of std::basic_streambuf::xsputn if reserve_additional(count) did not succeed, might calls std::basic_streambuf::overflow() + std::streamsize xsputn_slow(const char_type* s, std::streamsize count) { + // reserving entire vector failed, emplace char for char + std::streamsize written = 0; + while (written < count) { + try { + // copy one char, should throw eventually std::bad_alloc + vector_.emplace_back(s[written]); + } + catch (const std::bad_alloc& error) { + // try overflow(), if eof return, else continue + int_type c = this->overflow(traits_type::to_int_type(s[written])); + if (traits_type::eq_int_type(c, traits_type::eof())) { + return written; + } + } + // update pbase, pptr and epptr + setp_from_vector(); + written++; + } + return written; + } + +}; + +class membuf : public std::streambuf { +public: + membuf(char *begin, size_t size) { + this->setg(begin, begin, begin + size); + } +}; + +#endif diff --git a/src/NGen.cpp b/src/NGen.cpp index 79cf23c555..ed713151a9 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -705,7 +705,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { auto hot_start_loader = state_saving_config.hot_start(); if (hot_start_loader) { LOG(LogLevel::INFO, "Loading hot start data from prior snapshot."); - std::shared_ptr snapshot_loader = hot_start_loader->initialize_snapshot(State_Saver::snapshot_time_now()); + std::shared_ptr snapshot_loader = hot_start_loader->initialize_snapshot(); simulation->load_hot_start(snapshot_loader, manager->get_t_route_config_file_with_path()); } } @@ -776,10 +776,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { // run any end-of-run state saving after T-Route has finished but before starting to tear down data structures for (const auto& end_saver : state_saving_config.end_of_run_savers()) { LOG(LogLevel::INFO, "Saving end of run simulation data for state saving config " + end_saver.first); - std::shared_ptr snapshot = end_saver.second->initialize_snapshot( - State_Saver::snapshot_time_now(), - State_Saver::State_Durability::strict - ); + std::shared_ptr snapshot = end_saver.second->initialize_snapshot(State_Saver::State_Durability::strict); simulation->save_end_of_run(snapshot); } diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index ee9d3179ef..3f1e338813 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -94,7 +94,7 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const return 1.0; } -const boost::span Bmi_Fortran_Formulation::get_serialization_state() const { +const boost::span Bmi_Fortran_Formulation::get_serialization_state() { auto model = get_bmi_model(); int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 2ede682b3e..3fd0b3dcd7 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -17,7 +17,7 @@ namespace realization { inner_create_formulation(properties, true); } - void Bmi_Module_Formulation::save_state(std::shared_ptr saver) const { + void Bmi_Module_Formulation::save_state(std::shared_ptr saver) { uint64_t size = 1; boost::span data = this->get_serialization_state(); @@ -29,14 +29,14 @@ namespace realization { this->free_serialization_state(); } - void Bmi_Module_Formulation::load_state(std::shared_ptr loader) const { + void Bmi_Module_Formulation::load_state(std::shared_ptr loader) { std::vector buffer; loader->load_unit(this->get_id(), buffer); boost::span data(buffer.data(), buffer.size()); this->load_serialization_state(data); } - void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) const { + void Bmi_Module_Formulation::load_hot_start(std::shared_ptr loader) { this->load_state(loader); double rt; this->get_bmi_model()->SetValue(StateSaveNames::FREE, &rt); @@ -1094,7 +1094,7 @@ namespace realization { } - const boost::span Bmi_Module_Formulation::get_serialization_state() const { + const boost::span Bmi_Module_Formulation::get_serialization_state() { auto model = get_bmi_model(); uint64_t size = 0; model->SetValue(StateSaveNames::CREATE, &size); @@ -1104,7 +1104,7 @@ namespace realization { return span; } - void Bmi_Module_Formulation::load_serialization_state(const boost::span state) const { + void Bmi_Module_Formulation::load_serialization_state(const boost::span state) { auto bmi = this->bmi_model; // grab the pointer to the underlying state data void* data = (void*)state.data(); @@ -1112,7 +1112,7 @@ namespace realization { bmi->SetValue(StateSaveNames::STATE, data); } - void Bmi_Module_Formulation::free_serialization_state() const { + void Bmi_Module_Formulation::free_serialization_state() { auto bmi = this->bmi_model; // send message to clear memory associated with serialized data void* _; // this pointer will be unused by SetValue diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index e2b0ea8106..290c475cba 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -13,98 +13,51 @@ #include "Bmi_Py_Formulation.hpp" #include "Logger.hpp" +#include "state_save_restore/vecbuf.hpp" #include "state_save_restore/State_Save_Utils.hpp" #include +#include +#include +#include + #if (__cplusplus >= 202002L) #include #endif using namespace realization; -namespace { - // Check if the system's byte order is little endianness - constexpr bool is_little_endian() { -#if (__cplusplus < 202002L) - // C++ less than 2020 requires making a two byte object and checking which byte is 0 - uint16_t endian_bytes = 0xFF00; - uint8_t *endian_bits = reinterpret_cast(&endian_bytes); - return endian_bits[0] == 0; -#else - // C++ 2020+ has as simpler method - return std::endian::native == std::endian::little; -#endif - } -} - -void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) const { - std::vector> bmi_data; - size_t data_size = 0; +void Bmi_Multi_Formulation::save_state(std::shared_ptr saver) { + LOG(LogLevel::DEBUG, "Saving state for Multi-BMI %s", this->get_id()); + vecbuf data; + boost::archive::binary_oarchive archive(data); + // serialization function handles freeing the sub-BMI states after archiving them + archive << (*this); + // it's recommended to keep data pointers around until serialization completes, + // so freeing the BMI states is done after the data buffer has been completely written to for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); - boost::span span = bmi->get_serialization_state(); - bmi_data.push_back(std::make_pair(span.data(), span.size())); - data_size += sizeof(uint64_t) + span.size(); - LOG(LogLevel::DEBUG, "Serialization of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); - } - std::vector data; - data.reserve(data_size); - size_t index = 0; - for (const auto &bmi : bmi_data) { - // write the size of the data - if (is_little_endian()) { - std::memcpy(data.data() + index, &bmi.second, sizeof(uint64_t)); - } else { - // store the size bytes in reverse order to ensure saved data is always little endian - const char *bytes = reinterpret_cast(&bmi.second); - size_t endian_index = index + sizeof(uint64_t); - for (size_t i = 0; i < sizeof(uint64_t); ++i) { - data[--endian_index] = bytes[i]; - } - } - // write the serialized data - std::memcpy(data.data() + index + sizeof(uint64_t), &bmi.first, bmi.second); - index += sizeof(uint64_t) + bmi.second; - } - boost::span span(data.data(), data_size); - saver->save_unit(this->get_id(), span); - - for (const nested_module_ptr &m : modules) { - auto bmi = static_cast(m.get()); bmi->free_serialization_state(); } + boost::span span(data.data(), data.size()); + saver->save_unit(this->get_id(), span); } -void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) const { +void Bmi_Multi_Formulation::load_state(std::shared_ptr loader) { + LOG(LogLevel::DEBUG, "Loading save state for Multi-BMI %s", this->get_id()); std::vector data; loader->load_unit(this->get_id(), data); - size_t index = 0; - for (const nested_module_ptr &m : modules) { - auto bmi = dynamic_cast(m.get()); - uint64_t size; - if (is_little_endian()) { - memcpy(&size, data.data() + index, sizeof(uint64_t)); - } else { - // read size bytes in reverse order to interpret from little endian - char *size_bytes = reinterpret_cast(&size); - size_t endian_index = sizeof(uint64_t); - for (size_t i = 0; i < sizeof(uint64_t); ++i) { - size_bytes[--endian_index] = data[index + i]; - } - } - boost::span span(data.data() + index + sizeof(uint64_t), size); - bmi->load_serialization_state(span); - index += sizeof(uint64_t) + size; - LOG(LogLevel::DEBUG, "Loading of multi-BMI %s %s completed with a size of %d bytes.", - bmi->get_id().c_str(), bmi->get_model_type_name().c_str(), span.size()); - } + membuf stream(data.data(), data.size()); + boost::archive::binary_iarchive archive(stream); + archive >> (*this); } -void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr loader) const { +void Bmi_Multi_Formulation::load_hot_start(std::shared_ptr loader) { this->load_state(loader); double rt; + LOG(LogLevel::DEBUG, "Resetting time for sub-BMIs"); + // Multi-BMI's current forwards its primary BMI's current time, so no additional action needed for the formulation's reset time for (const nested_module_ptr &m : modules) { auto bmi = dynamic_cast(m.get()); bmi->get_bmi_model()->SetValue(StateSaveNames::RESET, &rt); @@ -705,6 +658,34 @@ void Bmi_Multi_Formulation::set_realization_file_format(bool is_legacy_format){ legacy_json_format = is_legacy_format; } +template +void Bmi_Multi_Formulation::serialize(Archive &ar, const unsigned int version) { + uint64_t data_size; + std::vector buffer; + for (const nested_module_ptr &m : modules) { + auto bmi = dynamic_cast(m.get()); + // if saving, make the BMI's state and record its size and data + if (Archive::is_saving::value) { + LOG(LogLevel::DEBUG, "Saving state from sub-BMI " + bmi->get_model_type_name()); + boost::span span = bmi->get_serialization_state(); + data_size = span.size(); + ar & data_size; + ar & boost::serialization::make_array(span.data(), data_size); + // it's recommended to keep raw pointers alive throughout the entire seiralization process, + // so responsibility for freeing the BMIs' state is left to the caller of this function + } + // if loading, get the current data size stored at the front, then load that much data as a char blob passed to the BMI + else { + LOG(LogLevel::DEBUG, "Loading state from sub-BMI " + bmi->get_model_type_name()); + ar & data_size; + buffer.resize(data_size); + ar & boost::serialization::make_array(buffer.data(), data_size); + boost::span span(buffer.data(), data_size); + bmi->load_serialization_state(span); + } + } +} + //Function to find whether any item in the string vector is empty or blank int find_empty_string_index(const std::vector& str_vector) { for (int i = 0; i < str_vector.size(); ++i) { diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 046bd68207..1712ec9cc7 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -98,7 +98,7 @@ bool Bmi_Py_Formulation::is_model_initialized() const { return get_bmi_model()->is_model_initialized(); } -void Bmi_Py_Formulation::load_serialization_state(const boost::span state) const { +void Bmi_Py_Formulation::load_serialization_state(const boost::span state) { auto bmi = std::dynamic_pointer_cast(get_bmi_model()); // load the state through the set value function that does not enforce the input size is the same as the current BMI's size bmi->set_value_unchecked(StateSaveNames::STATE, state.data(), state.size()); diff --git a/src/realizations/catchment/CMakeLists.txt b/src/realizations/catchment/CMakeLists.txt index 48771b18bf..6df221d6af 100644 --- a/src/realizations/catchment/CMakeLists.txt +++ b/src/realizations/catchment/CMakeLists.txt @@ -3,6 +3,15 @@ dynamic_sourced_cxx_library(realizations_catchment "${CMAKE_CURRENT_SOURCE_DIR}" add_library(NGen::realizations_catchment ALIAS realizations_catchment) +# ----------------------------------------------------------------------------- +# Find the Boost library and configure usage +set(Boost_USE_STATIC_LIBS OFF) +set(Boost_USE_MULTITHREADED ON) +set(Boost_USE_STATIC_RUNTIME OFF) +find_package(Boost 1.79.0 REQUIRED COMPONENTS serialization) + +target_link_libraries(realizations_catchment PRIVATE Boost::serialization) + target_include_directories(realizations_catchment PUBLIC ${PROJECT_SOURCE_DIR}/include/core ${PROJECT_SOURCE_DIR}/include/core/catchment diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index cfa152a92e..80bce91418 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -41,7 +41,7 @@ class File_Per_Unit_Snapshot_Saver : public State_Snapshot_Saver public: File_Per_Unit_Snapshot_Saver() = delete; - File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability); + File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::State_Durability durability); ~File_Per_Unit_Snapshot_Saver(); public: @@ -61,9 +61,16 @@ File_Per_Unit_Saver::File_Per_Unit_Saver(std::string base_path) File_Per_Unit_Saver::~File_Per_Unit_Saver() = default; -std::shared_ptr File_Per_Unit_Saver::initialize_snapshot(snapshot_time_t epoch, State_Durability durability) +std::shared_ptr File_Per_Unit_Saver::initialize_snapshot(State_Durability durability) { + // TODO + return std::make_shared(path(this->base_path_), durability); +} + +std::shared_ptr File_Per_Unit_Saver::initialize_checkpoint_snapshot(snapshot_time_t epoch, State_Durability durability) { - return std::make_shared(base_path_, epoch, durability); + path checkpoint_path = path(this->base_path_) / unit_saving_utils::format_epoch(epoch); + create_directory(checkpoint_path); + return std::make_shared(checkpoint_path, durability); } void File_Per_Unit_Saver::finalize() @@ -71,9 +78,9 @@ void File_Per_Unit_Saver::finalize() // nothing to be done } -File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) - : State_Snapshot_Saver(epoch, durability) - , dir_path_(base_path / unit_saving_utils::format_epoch(epoch)) +File_Per_Unit_Snapshot_Saver::File_Per_Unit_Snapshot_Saver(path base_path, State_Saver::State_Durability durability) + : State_Snapshot_Saver(durability) + , dir_path_(base_path) { create_directory(dir_path_); } @@ -156,7 +163,7 @@ void File_Per_Unit_Snapshot_Loader::load_unit(std::string const& unit_name, std: LOG("Failed to read state save data size for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); throw; } - std::ifstream stream(file_path.string(), std::ios_base::ate | std::ios_base::binary); + std::ifstream stream(file_path.string(), std::ios_base::binary); if (!stream) { LOG("Failed to open state save data for unit '" + unit_name + "' in file '" + file_path.string() + "'", LogLevel::WARNING); throw; @@ -176,9 +183,14 @@ File_Per_Unit_Loader::File_Per_Unit_Loader(std::string dir_path) } -std::shared_ptr File_Per_Unit_Loader::initialize_snapshot(State_Saver::snapshot_time_t epoch) +std::shared_ptr File_Per_Unit_Loader::initialize_snapshot() +{ + return std::make_shared(path(dir_path_)); +} + +std::shared_ptr File_Per_Unit_Loader::initialize_checkpoint_snapshot(State_Saver::snapshot_time_t epoch) { - path dir_path(dir_path_); - return std::make_shared(dir_path); + path checkpoint_path = path(dir_path_) / unit_saving_utils::format_epoch(epoch);; + return std::make_shared(checkpoint_path); } diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index 58fe75fad1..ee3f5ae3c9 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -129,9 +129,8 @@ std::string State_Save_Config::instance::instance::mechanism_string() const { } } -State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::snapshot_time_t epoch, State_Saver::State_Durability durability) - : epoch_(epoch) - , durability_(durability) +State_Snapshot_Saver::State_Snapshot_Saver(State_Saver::State_Durability durability) + : durability_(durability) { } From 1f67c180fb6c9b1b3677f415832d54c262f0709c Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 10:51:35 -0500 Subject: [PATCH 049/109] Add GetValuePtrInt for Fortran Adapter --- include/bmi/Bmi_Fortran_Adapter.hpp | 16 ++++++++++++++++ .../catchment/Bmi_Fortran_Formulation.cpp | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/bmi/Bmi_Fortran_Adapter.hpp b/include/bmi/Bmi_Fortran_Adapter.hpp index f3818c96e8..21f416bacc 100644 --- a/include/bmi/Bmi_Fortran_Adapter.hpp +++ b/include/bmi/Bmi_Fortran_Adapter.hpp @@ -231,6 +231,22 @@ namespace models { return ptr; } + int* GetValuePtrInt(const std::string &name) { + int nbytes; + if (get_var_nbytes(&bmi_model->handle, name.c_str(), &nbytes) != BMI_SUCCESS) { + std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + int *dest; + if (get_value_ptr_int(&bmi_model->handle, name.c_str(), dest) != BMI_SUCCESS) { + std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); + } + return dest; + } + /** * Get the size (in bytes) of one item of a variable. * diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 3f1e338813..0ceb36bc93 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -95,11 +95,12 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const } const boost::span Bmi_Fortran_Formulation::get_serialization_state() { - auto model = get_bmi_model(); + auto model = dynamic_cast(get_bmi_model().get()); int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); model->GetValue(StateSaveNames::SIZE, &size_int); - auto serialization_state = static_cast(model->GetValuePtr(StateSaveNames::STATE)); + int *serialization_ptr = model->GetValuePtrInt(StateSaveNames::STATE); + char *serialization_state = reinterpret_cast(serialization_ptr); const boost::span span(serialization_state, size_int); return span; } From 88fd9a29269dc15ffe9a87b5dbb7eb3bc3bfa116 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 17 Feb 2026 09:21:39 -0500 Subject: [PATCH 050/109] Fortran state hackery --- include/bmi/Bmi_Fortran_Adapter.hpp | 16 ---------- .../catchment/Bmi_Fortran_Formulation.hpp | 9 ++++++ .../catchment/Bmi_Module_Formulation.hpp | 2 +- .../state_save_restore/State_Save_Restore.hpp | 18 +---------- .../state_save_restore/State_Save_Utils.hpp | 18 +++++++++++ .../catchment/Bmi_Fortran_Formulation.cpp | 30 ++++++++++++++++--- 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/include/bmi/Bmi_Fortran_Adapter.hpp b/include/bmi/Bmi_Fortran_Adapter.hpp index 21f416bacc..f3818c96e8 100644 --- a/include/bmi/Bmi_Fortran_Adapter.hpp +++ b/include/bmi/Bmi_Fortran_Adapter.hpp @@ -231,22 +231,6 @@ namespace models { return ptr; } - int* GetValuePtrInt(const std::string &name) { - int nbytes; - if (get_var_nbytes(&bmi_model->handle, name.c_str(), &nbytes) != BMI_SUCCESS) { - std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); - LOG(throw_msg, LogLevel::WARNING); - throw std::runtime_error(throw_msg); - } - int *dest; - if (get_value_ptr_int(&bmi_model->handle, name.c_str(), dest) != BMI_SUCCESS) { - std::string throw_msg; throw_msg.assign(model_name + " failed to get int pointer for BMI variable " + name + "."); - LOG(throw_msg, LogLevel::WARNING); - throw std::runtime_error(throw_msg); - } - return dest; - } - /** * Get the size (in bytes) of one item of a variable. * diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index 205303f9d5..e7736b04a1 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -5,6 +5,7 @@ #if NGEN_WITH_BMI_FORTRAN +#include #include "Bmi_Module_Formulation.hpp" #include @@ -31,6 +32,10 @@ namespace realization { */ const boost::span get_serialization_state() override; + void load_serialization_state(boost::span state) override; + + void free_serialization_state() override; + protected: /** @@ -57,6 +62,10 @@ namespace realization { friend class ::Bmi_Multi_Formulation_Test; friend class ::Bmi_Formulation_Test; friend class ::Bmi_Fortran_Formulation_Test; + + private: + // location to store serialized state from the BMI because pointer interfaces are not available for Fotran + std::vector serialized_state; }; } diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 10fadd5327..150bd2ac38 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -305,7 +305,7 @@ namespace realization { * Requests the BMI to clear a currently saved state from memory. * Existing state pointers should not be used as the stored data may be freed depending on implementation. */ - void free_serialization_state(); + virtual void free_serialization_state(); void set_realization_file_format(bool is_legacy_format); virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const override { diff --git a/include/state_save_restore/State_Save_Restore.hpp b/include/state_save_restore/State_Save_Restore.hpp index 3d3a3bf692..57a88b5f3e 100644 --- a/include/state_save_restore/State_Save_Restore.hpp +++ b/include/state_save_restore/State_Save_Restore.hpp @@ -12,23 +12,7 @@ #include #include -enum class State_Save_Direction { - None = 0, - Save, - Load -}; - -enum class State_Save_Mechanism { - None = 0, - FilePerUnit -}; - -enum class State_Save_When { - None = 0, - EndOfRun, - FirstOfMonth, - StartOfRun -}; +#include "State_Save_Utils.hpp" class State_Saver; class State_Loader; diff --git a/include/state_save_restore/State_Save_Utils.hpp b/include/state_save_restore/State_Save_Utils.hpp index 8e6c127f6a..9713b660af 100644 --- a/include/state_save_restore/State_Save_Utils.hpp +++ b/include/state_save_restore/State_Save_Utils.hpp @@ -9,4 +9,22 @@ namespace StateSaveNames { const auto RESET = "reset_time"; } +enum class State_Save_Direction { + None = 0, + Save, + Load +}; + +enum class State_Save_Mechanism { + None = 0, + FilePerUnit +}; + +enum class State_Save_When { + None = 0, + EndOfRun, + FirstOfMonth, + StartOfRun +}; + #endif diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 0ceb36bc93..62b327ac32 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -95,14 +95,36 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const } const boost::span Bmi_Fortran_Formulation::get_serialization_state() { - auto model = dynamic_cast(get_bmi_model().get()); + auto model = this->get_bmi_model(); + // create the serialized state on the Fortran BMI int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); model->GetValue(StateSaveNames::SIZE, &size_int); - int *serialization_ptr = model->GetValuePtrInt(StateSaveNames::STATE); - char *serialization_state = reinterpret_cast(serialization_ptr); - const boost::span span(serialization_state, size_int); + // since GetValuePtr on the Fortran BMI does not work currently, store the data on the formulation + this->serialized_state.resize(size_int); + model->GetValue(StateSaveNames::STATE, this->serialized_state.data()); + // the BMI can have its state freed immediately since the data is now stored on the formulation + model->SetValue(StateSaveNames::FREE, &size_int); + // return a span of the data stored on the formulation + const boost::span(this->serialized_state.data(), this->serialized_state.size()); return span; } +void Bmi_Fortran_Formulation::load_serialization_state(boost::span state) { + auto model = this->get_bmi_model(); + int int_array_size = std::ceil(state.size() / static_cast(sizeof(int))); + // setting size is a workaround for loading the state. + // The BMI Fortran interface shapes the incoming pointer to the same size as the data currently backing the BMI's variable. + // By setting the size, the BMI can lie about the size of its state variable to that interface. + model->SetValue(StateSaveNames::SIZE, &int_array_size); + model->SetValue(StateSaveNames::STATE, state.data()); +} + +void Bmi_Fortran_Formulation::free_serialization_state() { + // The serialized data needs to be stored on the formluation since GetValuePtr is not available on Fortran BMIs. + // The backing BMI's serialization data should already be freed during `get_serialization_state`, so clearing the formulation's data is all that is needed. + this->serialized_state.clear(); + this->serialized_state.shrink_to_fit(); +} + #endif // NGEN_WITH_BMI_FORTRAN From 34fa2665eb6ba083c3fc7b8de76d9012b7af5275 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 17 Feb 2026 09:25:14 -0500 Subject: [PATCH 051/109] Add variable name --- src/realizations/catchment/Bmi_Fortran_Formulation.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 62b327ac32..7e7407e2b3 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -106,7 +106,7 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { // the BMI can have its state freed immediately since the data is now stored on the formulation model->SetValue(StateSaveNames::FREE, &size_int); // return a span of the data stored on the formulation - const boost::span(this->serialized_state.data(), this->serialized_state.size()); + const boost::span span(this->serialized_state.data(), this->serialized_state.size()); return span; } From 1d7f39c5b3e06225e63cb1be2f813291d08c5f34 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 09:51:18 -0500 Subject: [PATCH 052/109] Update submodules for testing --- extern/evapotranspiration/evapotranspiration | 2 +- extern/sloth | 2 +- extern/topmodel/topmodel | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extern/evapotranspiration/evapotranspiration b/extern/evapotranspiration/evapotranspiration index 096208ad62..836c146cbe 160000 --- a/extern/evapotranspiration/evapotranspiration +++ b/extern/evapotranspiration/evapotranspiration @@ -1 +1 @@ -Subproject commit 096208ad624e07216617f770a3447eb829266112 +Subproject commit 836c146cbeef10740af0dd2e570a7764bf4dadd2 diff --git a/extern/sloth b/extern/sloth index ee0d982ccc..2745e1b0f9 160000 --- a/extern/sloth +++ b/extern/sloth @@ -1 +1 @@ -Subproject commit ee0d982ccc07663cfea7bf0ac4d645841e19ccc1 +Subproject commit 2745e1b0f954f5a98afa00f844e96bb436827996 diff --git a/extern/topmodel/topmodel b/extern/topmodel/topmodel index fa4f7e56db..b35249c195 160000 --- a/extern/topmodel/topmodel +++ b/extern/topmodel/topmodel @@ -1 +1 @@ -Subproject commit fa4f7e56dbe46df8cc0d7ca9095102290170b866 +Subproject commit b35249c1954abee61885bbd75d8b1765b5a311a5 From 761622f5a853a6ccaa3e1c69912fc3b074e06cce Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Wed, 4 Feb 2026 20:25:43 -0800 Subject: [PATCH 053/109] updates to cicd and dockerfile --- .github/workflows/ngwpc-cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 2e293bef98..b7160a614e 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -233,6 +233,8 @@ jobs: - name: Build & push image id: build_image uses: docker/build-push-action@v6 + env: + BUILD_DATE: ${{ env.BUILD_DATE }} with: context: . # file: Dockerfile.test # comment out when done testing From 695d05fc509c6a5cb2bd9da1e01b7de6dfe69a62 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Wed, 4 Feb 2026 21:56:39 -0800 Subject: [PATCH 054/109] updated cicd file --- extern/evapotranspiration/evapotranspiration | 2 +- extern/sloth | 2 +- extern/topmodel/topmodel | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extern/evapotranspiration/evapotranspiration b/extern/evapotranspiration/evapotranspiration index 836c146cbe..096208ad62 160000 --- a/extern/evapotranspiration/evapotranspiration +++ b/extern/evapotranspiration/evapotranspiration @@ -1 +1 @@ -Subproject commit 836c146cbeef10740af0dd2e570a7764bf4dadd2 +Subproject commit 096208ad624e07216617f770a3447eb829266112 diff --git a/extern/sloth b/extern/sloth index 2745e1b0f9..ee0d982ccc 160000 --- a/extern/sloth +++ b/extern/sloth @@ -1 +1 @@ -Subproject commit 2745e1b0f954f5a98afa00f844e96bb436827996 +Subproject commit ee0d982ccc07663cfea7bf0ac4d645841e19ccc1 diff --git a/extern/topmodel/topmodel b/extern/topmodel/topmodel index b35249c195..fa4f7e56db 160000 --- a/extern/topmodel/topmodel +++ b/extern/topmodel/topmodel @@ -1 +1 @@ -Subproject commit b35249c1954abee61885bbd75d8b1765b5a311a5 +Subproject commit fa4f7e56dbe46df8cc0d7ca9095102290170b866 From 5555cf3f15768e57238ef9242378611e16c6032c Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 10 Feb 2026 20:56:44 -0800 Subject: [PATCH 055/109] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index b7160a614e..2e293bef98 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -233,8 +233,6 @@ jobs: - name: Build & push image id: build_image uses: docker/build-push-action@v6 - env: - BUILD_DATE: ${{ env.BUILD_DATE }} with: context: . # file: Dockerfile.test # comment out when done testing From b76ddfac24955263704715dfb66213ebd8665394 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 09:47:23 -0500 Subject: [PATCH 056/109] Merge remote-tracking branch 'NOAA-OWP/master' into development --- .../core/nexus/HY_PointHydroNexusRemote.hpp | 1 + .../catchment/Bmi_Multi_Formulation.hpp | 9 ++ src/core/nexus/HY_PointHydroNexusRemote.cpp | 126 ++++++++++++------ 3 files changed, 93 insertions(+), 43 deletions(-) diff --git a/include/core/nexus/HY_PointHydroNexusRemote.hpp b/include/core/nexus/HY_PointHydroNexusRemote.hpp index 34d98f9a96..ebac2e9ace 100644 --- a/include/core/nexus/HY_PointHydroNexusRemote.hpp +++ b/include/core/nexus/HY_PointHydroNexusRemote.hpp @@ -72,6 +72,7 @@ class HY_PointHydroNexusRemote : public HY_PointHydroNexus communication_type get_communicator_type() { return type; } private: + void post_receives(); void process_communications(); int world_rank; diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 7ebc7dc0b1..ec520e112f 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -60,6 +60,15 @@ namespace realization { void load_state(std::shared_ptr loader) override; void load_hot_start(std::shared_ptr loader) override; + + virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const final { + for( const auto &module : modules ) { + // TODO may need to check on outputs form each module indepdently??? + // Right now, the assumption is that if each component is mass balanced + // then the entire formulation is mass balanced + module->check_mass_balance(iteration, total_steps, timestamp); + } + }; /** * Convert a time value from the model to an epoch time in seconds. diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 1f1e0ab544..0dc571d2fc 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -18,7 +18,7 @@ void MPI_Handle_Error(int status) } else { - MPI_Abort(MPI_COMM_WORLD,1); + MPI_Abort(MPI_COMM_WORLD, status); } } @@ -92,9 +92,9 @@ HY_PointHydroNexusRemote::HY_PointHydroNexusRemote(std::string nexus_id, Catchme HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() { - long wait_time = 0; - - // This destructore might be called after MPI_Finalize so do not attempt communication if + const unsigned int timeout = 120000; // timeout threshold in milliseconds + unsigned int wait_time = 0; + // This destructor might be called after MPI_Finalize so do not attempt communication if // this has occured int mpi_finalized; MPI_Finalized(&mpi_finalized); @@ -105,14 +105,46 @@ HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - + if( wait_time < timeout && wait_time > 0 ){ // don't sleep if first call clears comms! + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + else + { + std::cerr << "HY_PointHydroNexusRemote: "<< id + << " destructor timed out after " << timeout/1000 + << " seconds waiting on pending MPI communications\n"; + // The return is is probably best, logging the error. + // There is no good way to recover from this. + // Throwing an exception from destructors is generally not a good idea + // as it can lead to undefined behavior. + // and using std::exit forces the program to terminate immediately, + // even if this situation is recoverable/acceptable in some cases. + return; + } wait_time += 1; + MPI_Finalized(&mpi_finalized); + } +} - if ( wait_time > 120000 ) +void HY_PointHydroNexusRemote::post_receives() +{ + // Post receives if not already posted (for pure receiver nexuses) + if (stored_receives.empty()) + { + for (int rank : upstream_ranks) { - // TODO log warning message that some comunications could not complete - + stored_receives.push_back({}); + stored_receives.back().buffer = std::make_shared(); + int tag = extract(id); + + MPI_Handle_Error(MPI_Irecv( + stored_receives.back().buffer.get(), + 1, + time_step_and_flow_type, + rank, + tag, + MPI_COMM_WORLD, + &stored_receives.back().mpi_request)); } } } @@ -130,31 +162,15 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t } else if ( type == receiver || type == sender_receiver ) { - for ( int rank : upstream_ranks ) - { - int status; - - stored_receives.resize(stored_receives.size() + 1); - stored_receives.back().buffer = std::make_shared(); - - int tag = extract(id); - - //Receive downstream_flow from Upstream Remote Nexus to this Downstream Remote Nexus - status = MPI_Irecv( - stored_receives.back().buffer.get(), - 1, - time_step_and_flow_type, - rank, - tag, - MPI_COMM_WORLD, - &stored_receives.back().mpi_request); - - MPI_Handle_Error(status); - - //std::cerr << "Creating receive with target_rank=" << rank << " on tag=" << tag << "\n"; - } - - //std::cerr << "Waiting on receives\n"; + post_receives(); + // Wait for receives to complete + // This ensures all upstream flows are received before returning + // and that we have matched all sends with receives for a given time step. + // As long as the functions are called appropriately, e.g. one call to + // `add_upstream_flow` per upstream catchment per time step, followed + // by a call to `get_downstream_flow` for each downstream catchment per time step, + // this loop will terminate and ensures the synchronization of flows between + // ranks. while ( stored_receives.size() > 0 ) { process_communications(); @@ -167,6 +183,28 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchment_id, time_step_t t) { + // Process any completed communications to free resources + // If no communications are pending, this call will do nothing. + process_communications(); + // NOTE: It is possible for a partition to get "too far" ahead since the sends are now + // truely asynchronous. For pure receivers and sender_receivers, this isn't a problem + // because the get_downstream_flow function will block until all receives are processed. + // However, for pure senders, this could be a problem. + // We can use this spinlock here to limit how far ahead a partition can get. + // in this case, approximately 100 time steps per downstream catchment... + while( stored_sends.size() > downstream_ranks.size()*100 ) + { + process_communications(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + // Post receives before sending to prevent deadlock + // When stored_receives is empty, we need to post for incoming messages + if ((type == receiver || type == sender_receiver) && stored_receives.empty()) + { + post_receives(); + } + // first add flow to local copy HY_PointHydroNexus::add_upstream_flow(val, catchment_id, t); @@ -205,23 +243,25 @@ void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchme int tag = extract(id); //Send downstream_flow from this Upstream Remote Nexus to the Downstream Remote Nexus - MPI_Isend( + MPI_Handle_Error( + MPI_Isend( stored_sends.back().buffer.get(), 1, time_step_and_flow_type, *downstream_ranks.begin(), //TODO currently only support a SINGLE downstream message pairing tag, MPI_COMM_WORLD, - &stored_sends.back().mpi_request); - - //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; + &stored_sends.back().mpi_request) + ); + //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - while ( stored_sends.size() > 0 ) - { - process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } + // Send is async, the next call to add_upstream_flow will test and ensure the send has completed + // and free the memory associated with the send. + // This prevents a potential deadlock situation where a send isn't able to complete + // because the remote receiver is also trying to send and the underlying mpi buffers/protocol + // are forced into a rendevous protocol. So we ensure that we always post receives before sends. + // and that we always test for completed sends before freeing the memory associated with the send. } } } From b60812150e242bc4abcec7678fe85348c3f99558 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 15:55:21 -0800 Subject: [PATCH 057/109] Flesh out API for state saving and restoring, with adjusted use in Bmi_Module_Formulation --- include/realizations/catchment/Bmi_Multi_Formulation.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index ec520e112f..565d7cf53e 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -70,6 +70,8 @@ namespace realization { } }; + void save_state(std::shared_ptr saver) const override; + /** * Convert a time value from the model to an epoch time in seconds. * From 87ae5136f3e7eff56fd51badd865659f759b0055 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Mon, 8 Dec 2025 17:53:19 -0800 Subject: [PATCH 058/109] Add logic and structures for parsing state saving configuration from realization config --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index a4ebf877c8..aa51edd8d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -339,6 +339,7 @@ target_link_libraries(ngen NGen::logging NGen::parallel NGen::bmi_protocols + NGen::state_save_restore ) if(NGEN_WITH_SQLITE) From 97be208de304d3bb541a799aa569174ede5a278c Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 16 Jan 2026 14:06:13 -0500 Subject: [PATCH 059/109] State saving for multi-BMI --- .../catchment/Bmi_Module_Formulation.hpp | 13 +++++++++++++ src/NGen.cpp | 10 ++++++++++ .../catchment/Bmi_Multi_Formulation.cpp | 4 ---- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 150bd2ac38..bcc1f251e1 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -57,6 +57,19 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + virtual const char* create_save_state(uint64_t *size) const; + + /** + * Clears any serialized data stored by the BMI from memory. + */ + virtual void free_save_state() const; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/src/NGen.cpp b/src/NGen.cpp index ed713151a9..e8944dc6e2 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -712,6 +712,16 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); + if (state_saving_config.has_end_of_run()) { + LOG("Saving end-of-run state.", LogLevel::INFO); + std::shared_ptr saver = state_saving_config.end_of_run_saver(); + std::shared_ptr snapshot = saver->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_state_snapshot(snapshot); + } + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 290c475cba..1f4fbbf245 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -21,10 +21,6 @@ #include #include -#if (__cplusplus >= 202002L) -#include -#endif - using namespace realization; From 0bd330cb3fa8109b4d6b635d46b1ece7f36e714e Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 6 Feb 2026 10:26:57 -0500 Subject: [PATCH 060/109] Use Boost for serializing Multi-BMI --- include/realizations/catchment/Bmi_Multi_Formulation.hpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 565d7cf53e..6126abca39 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -70,7 +70,11 @@ namespace realization { } }; - void save_state(std::shared_ptr saver) const override; + void save_state(std::shared_ptr saver) override; + + void load_state(std::shared_ptr loader) override; + + void load_hot_start(std::shared_ptr loader) override; /** * Convert a time value from the model to an epoch time in seconds. From 373ea108d343307c78f1b72cd7da43c32b2d435b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 14:39:05 -0500 Subject: [PATCH 061/109] Remove old state calls --- src/NGen.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index e8944dc6e2..ed713151a9 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -712,16 +712,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); - if (state_saving_config.has_end_of_run()) { - LOG("Saving end-of-run state.", LogLevel::INFO); - std::shared_ptr saver = state_saving_config.end_of_run_saver(); - std::shared_ptr snapshot = saver->initialize_snapshot( - State_Saver::snapshot_time_now(), - State_Saver::State_Durability::strict - ); - simulation->save_state_snapshot(snapshot); - } - #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif From d90e9aba18c76fe11705169236782b3615865772 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 18 Feb 2026 14:44:03 -0500 Subject: [PATCH 062/109] Remove old state saving definitions --- .../catchment/Bmi_Module_Formulation.hpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index bcc1f251e1..150bd2ac38 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -57,19 +57,6 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; - /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. - * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. - */ - virtual const char* create_save_state(uint64_t *size) const; - - /** - * Clears any serialized data stored by the BMI from memory. - */ - virtual void free_save_state() const; - /** * Get the collection of forcing output property names this instance can provide. * From f12fde0d9055a19e406b0a3b7b65599069a7f90f Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 24 Feb 2026 14:10:00 -0500 Subject: [PATCH 063/109] Align Fortran state size with sizeof int --- .../catchment/Bmi_Fortran_Formulation.hpp | 2 +- .../catchment/Bmi_Fortran_Formulation.cpp | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp index e7736b04a1..d37ca469e7 100644 --- a/include/realizations/catchment/Bmi_Fortran_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Fortran_Formulation.hpp @@ -32,7 +32,7 @@ namespace realization { */ const boost::span get_serialization_state() override; - void load_serialization_state(boost::span state) override; + void load_serialization_state(const boost::span state) override; void free_serialization_state() override; diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 7e7407e2b3..904dfb5f7f 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -99,9 +99,11 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { // create the serialized state on the Fortran BMI int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); + // the size coming in should be the number of int elements in the Fortran backing array, not the byte size of the array model->GetValue(StateSaveNames::SIZE, &size_int); + // resize the state to the array to the size of the Fortran's backing array + this->serialized_state.resize(size_int * sizeof(int)); // since GetValuePtr on the Fortran BMI does not work currently, store the data on the formulation - this->serialized_state.resize(size_int); model->GetValue(StateSaveNames::STATE, this->serialized_state.data()); // the BMI can have its state freed immediately since the data is now stored on the formulation model->SetValue(StateSaveNames::FREE, &size_int); @@ -110,9 +112,18 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { return span; } -void Bmi_Fortran_Formulation::load_serialization_state(boost::span state) { +void Bmi_Fortran_Formulation::load_serialization_state(const boost::span state) { auto model = this->get_bmi_model(); - int int_array_size = std::ceil(state.size() / static_cast(sizeof(int))); + // get number of ints needed to store chars + double num_ints = state.size() / static_cast(sizeof(int)); + int int_array_size = std::ceil(num_ints); + // assert the number of chars aligns with an integer array to prevent reading out of bounds + if (int_array_size != std::floor(num_ints)) { + std::string error = "Fortran Deserialization: The number of bytes in the state (" + std::to_string(state.size()) + + ") must be a multiple of the size of an int (" + std::to_string(sizeof(int)) + ")"; + LOG(LogLevel::SEVERE, error); + throw std::runtime_error(error); + } // setting size is a workaround for loading the state. // The BMI Fortran interface shapes the incoming pointer to the same size as the data currently backing the BMI's variable. // By setting the size, the BMI can lie about the size of its state variable to that interface. From f70d1f1e04d6aaf43f542ea9ad010bbbec05070c Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 26 Feb 2026 11:57:31 -0500 Subject: [PATCH 064/109] Dynamically check Fortran state size for load alignment --- .../catchment/Bmi_Fortran_Formulation.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 904dfb5f7f..4fc9e611f0 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -114,20 +114,19 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { void Bmi_Fortran_Formulation::load_serialization_state(const boost::span state) { auto model = this->get_bmi_model(); - // get number of ints needed to store chars - double num_ints = state.size() / static_cast(sizeof(int)); - int int_array_size = std::ceil(num_ints); - // assert the number of chars aligns with an integer array to prevent reading out of bounds - if (int_array_size != std::floor(num_ints)) { + int item_size = model->GetVarItemsize(StateSaveNames::STATE); + // assert the number of chars aligns with the storage array to prevent reading out of bounds + if (state.size() % item_size != 0) { std::string error = "Fortran Deserialization: The number of bytes in the state (" + std::to_string(state.size()) - + ") must be a multiple of the size of an int (" + std::to_string(sizeof(int)) + ")"; + + ") must be a multiple of the size of the storage unit (" + std::to_string(item_size) + ")"; LOG(LogLevel::SEVERE, error); throw std::runtime_error(error); } // setting size is a workaround for loading the state. // The BMI Fortran interface shapes the incoming pointer to the same size as the data currently backing the BMI's variable. // By setting the size, the BMI can lie about the size of its state variable to that interface. - model->SetValue(StateSaveNames::SIZE, &int_array_size); + int false_nbytes = state.size(); + model->SetValue(StateSaveNames::SIZE, &false_nbytes); model->SetValue(StateSaveNames::STATE, state.data()); } From 403c39a8d6cb85ebacc122f3d9dbc73a2adc5d15 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 2 Mar 2026 10:27:53 -0500 Subject: [PATCH 065/109] Fortran reports size in bytes --- src/realizations/catchment/Bmi_Fortran_Formulation.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index 4fc9e611f0..bfd59b0795 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -99,10 +99,9 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { // create the serialized state on the Fortran BMI int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); - // the size coming in should be the number of int elements in the Fortran backing array, not the byte size of the array model->GetValue(StateSaveNames::SIZE, &size_int); // resize the state to the array to the size of the Fortran's backing array - this->serialized_state.resize(size_int * sizeof(int)); + this->serialized_state.resize(size_int); // since GetValuePtr on the Fortran BMI does not work currently, store the data on the formulation model->GetValue(StateSaveNames::STATE, this->serialized_state.data()); // the BMI can have its state freed immediately since the data is now stored on the formulation From 606f4cc8ea0f75347b1d16f5d08d33540a2e18ee Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 09:47:23 -0500 Subject: [PATCH 066/109] Merge remote-tracking branch 'NOAA-OWP/master' into development --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index aa51edd8d9..c90788dd9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -338,6 +338,7 @@ target_link_libraries(ngen NGen::core_mediator NGen::logging NGen::parallel + NGen::state_save_restore NGen::bmi_protocols NGen::state_save_restore ) From 01db1cd8d8a9500ee81136eb7a714a10dbd1fb5e Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Tue, 16 Sep 2025 17:15:02 -0600 Subject: [PATCH 067/109] feat(mass_balance): define the bmi mass balance protocol --- src/utilities/bmi/mass_balance.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/utilities/bmi/mass_balance.cpp b/src/utilities/bmi/mass_balance.cpp index f542a682a0..4c81456591 100644 --- a/src/utilities/bmi/mass_balance.cpp +++ b/src/utilities/bmi/mass_balance.cpp @@ -112,12 +112,8 @@ auto NgenMassBalance::check_support(const ModelPtr& model) -> expected( ProtocolError( Error::INTEGRATION_ERROR, - "mass_balance: variables have incosistent units, cannot perform mass balance." ) ); - } - } catch (const std::exception &e) { - std::stringstream ss; ss << "mass_balance: Error getting mass balance values for module '" << model->GetComponentName() << "': " << e.what() << std::endl; return make_unexpected( ProtocolError( Error::INTEGRATION_ERROR, From faa2a6e58e1ea9bc3ddb32117021067569d5814a Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Tue, 16 Sep 2025 17:50:27 -0600 Subject: [PATCH 068/109] test(bmi_protocols): add mock for BMI protocol testing --- .../realizations/catchments/Bmi_Protocols.hpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/realizations/catchments/Bmi_Protocols.hpp diff --git a/test/realizations/catchments/Bmi_Protocols.hpp b/test/realizations/catchments/Bmi_Protocols.hpp new file mode 100644 index 0000000000..30b5570e9f --- /dev/null +++ b/test/realizations/catchments/Bmi_Protocols.hpp @@ -0,0 +1,19 @@ +#include + +class MassBalanceMock { + public: + + MassBalanceMock( bool fatal = false, double tolerance = 1e-12, int frequency = 1) + : properties() { + properties.put("check", true); + properties.put("tolerance", tolerance); + properties.put("fatal", fatal); + properties.put("frequency", frequency); + } + + const boost::property_tree::ptree& get() const { + return properties; + } + private: + boost::property_tree::ptree properties; +}; \ No newline at end of file From 76cf05d7cac00c2695608d19769102b8b887a569 Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Tue, 16 Sep 2025 18:07:30 -0600 Subject: [PATCH 069/109] feat(ngen): add mass balance check for all bmi modules during runtime --- include/realizations/catchment/Bmi_Module_Formulation.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 150bd2ac38..ee28db4f70 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -7,6 +7,7 @@ #include "Bmi_Adapter.hpp" #include #include "bmi_utilities.hpp" +#include "bmi/protocols.hpp" #include #include "bmi/protocols.hpp" From 54b2fc91f58359535232aa2905a05c77d14e1874 Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Sat, 27 Sep 2025 00:05:22 -0600 Subject: [PATCH 070/109] feat(bmi-protocols)!: v0.2 of the protocols lib using expected semantics for error handling --- include/utilities/bmi/mass_balance.hpp | 4 ++++ include/utilities/bmi/protocol.hpp | 6 ++++++ include/utilities/bmi/protocols.hpp | 13 ++++++++++++- src/utilities/bmi/mass_balance.cpp | 3 +++ src/utilities/bmi/protocols.cpp | 3 +++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/include/utilities/bmi/mass_balance.hpp b/include/utilities/bmi/mass_balance.hpp index e8883232ff..a58c5a9bf4 100644 --- a/include/utilities/bmi/mass_balance.hpp +++ b/include/utilities/bmi/mass_balance.hpp @@ -23,6 +23,10 @@ Version 0.2 Conform to updated protocol interface Removed integration and error exceptions in favor of ProtocolError +Version 0.2 +Conform to updated protocol interface +Removed integration and error exceptions in favor of ProtocolError + Version 0.1 Interface of the BMI mass balance protocol */ diff --git a/include/utilities/bmi/protocol.hpp b/include/utilities/bmi/protocol.hpp index d08dd7681c..1b8751abc9 100644 --- a/include/utilities/bmi/protocol.hpp +++ b/include/utilities/bmi/protocol.hpp @@ -23,6 +23,12 @@ Implement error handling via expected and error_or_warning Removed model member and required model reference in run(), check_support(), and initialize() Minor refactoring and style changes +Version 0.2 +Enumerate protocol error types and add ProtocolError exception class +Implement error handling via expected and error_or_warning +Removed model member and required model reference in run(), check_support(), and initialize() +Minor refactoring and style changes + Version 0.1 Virtual interface for BMI protocols */ diff --git a/include/utilities/bmi/protocols.hpp b/include/utilities/bmi/protocols.hpp index 723fcf1364..a310580515 100644 --- a/include/utilities/bmi/protocols.hpp +++ b/include/utilities/bmi/protocols.hpp @@ -25,6 +25,13 @@ per the updated (v0.2) protocol interface Keep protocols in a map for dynamic access by enumeration name add operator<< for Protocol enum +Version 0.2 +Enumerate protocol types/names +The container now holds a single model pointer and passes it to each protocol +per the updated (v0.2) protocol interface +Keep protocols in a map for dynamic access by enumeration name +add operator<< for Protocol enum + Version 0.1 Container and management for abstract BMI protocols */ @@ -59,7 +66,7 @@ class NgenBmiProtocols { * @brief Construct a new Ngen Bmi Protocols object with a null model * */ - NgenBmiProtocols(); + NgenBmiProtocols() : model(nullptr){}; /** * @brief Construct a new Ngen Bmi Protocols object for use with a known model @@ -82,6 +89,10 @@ class NgenBmiProtocols { private: + auto run(const Protocol& protocol_name, const Context& ctx) const -> expected; + + private: + /** * @brief All protocols managed by this container will utilize the same model * diff --git a/src/utilities/bmi/mass_balance.cpp b/src/utilities/bmi/mass_balance.cpp index 4c81456591..4d5dd73317 100644 --- a/src/utilities/bmi/mass_balance.cpp +++ b/src/utilities/bmi/mass_balance.cpp @@ -19,6 +19,9 @@ Version 0.3 (see mass_balance.hpp for details) Version 0.2 Implement error handling via expected and error_or_warning +Version 0.2 +Implement error handling via expected and error_or_warning + Version 0.1 Implementation the BMI mass balance checking protocol */ diff --git a/src/utilities/bmi/protocols.cpp b/src/utilities/bmi/protocols.cpp index c78651a192..7f72a17a22 100644 --- a/src/utilities/bmi/protocols.cpp +++ b/src/utilities/bmi/protocols.cpp @@ -19,6 +19,9 @@ Version 0.2.1 (See bmi/protocols.hpp for details) Version 0.2 Implement error handling via expected and error_or_warning +Version 0.2 +Implement error handling via expected and error_or_warning + Version 0.1 Container and management for abstract BMI protocols */ From 7f9490bacac416b7999a9938626d45052cd18eef Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Thu, 23 Oct 2025 16:45:10 -0600 Subject: [PATCH 071/109] fix(protocols): better default handling; add missing return --- include/utilities/bmi/protocols.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/utilities/bmi/protocols.hpp b/include/utilities/bmi/protocols.hpp index a310580515..3a1929f60c 100644 --- a/include/utilities/bmi/protocols.hpp +++ b/include/utilities/bmi/protocols.hpp @@ -66,7 +66,7 @@ class NgenBmiProtocols { * @brief Construct a new Ngen Bmi Protocols object with a null model * */ - NgenBmiProtocols() : model(nullptr){}; + NgenBmiProtocols(); /** * @brief Construct a new Ngen Bmi Protocols object for use with a known model From 17c3c96deca9378700dbce294240064b3fb0da40 Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Thu, 23 Oct 2025 16:56:38 -0600 Subject: [PATCH 072/109] test: point formulation tests to use same protocol mock --- .../realizations/catchments/Bmi_Protocols.hpp | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 test/realizations/catchments/Bmi_Protocols.hpp diff --git a/test/realizations/catchments/Bmi_Protocols.hpp b/test/realizations/catchments/Bmi_Protocols.hpp deleted file mode 100644 index 30b5570e9f..0000000000 --- a/test/realizations/catchments/Bmi_Protocols.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#include - -class MassBalanceMock { - public: - - MassBalanceMock( bool fatal = false, double tolerance = 1e-12, int frequency = 1) - : properties() { - properties.put("check", true); - properties.put("tolerance", tolerance); - properties.put("fatal", fatal); - properties.put("frequency", frequency); - } - - const boost::property_tree::ptree& get() const { - return properties; - } - private: - boost::property_tree::ptree properties; -}; \ No newline at end of file From 8a70e0e495254d1ac45b519bbbbc987145be9b43 Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Wed, 4 Feb 2026 08:45:20 -0700 Subject: [PATCH 073/109] fix(mpi): prevent potential deadlock scenario --- src/core/nexus/HY_PointHydroNexusRemote.cpp | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 0dc571d2fc..035d38f858 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -149,6 +149,29 @@ void HY_PointHydroNexusRemote::post_receives() } } +void HY_PointHydroNexusRemote::post_receives() +{ + // Post receives if not already posted (for pure receiver nexuses) + if (stored_receives.empty()) + { + for (int rank : upstream_ranks) + { + stored_receives.push_back({}); + stored_receives.back().buffer = std::make_shared(); + int tag = extract(id); + + MPI_Handle_Error(MPI_Irecv( + stored_receives.back().buffer.get(), + 1, + time_step_and_flow_type, + rank, + tag, + MPI_COMM_WORLD, + &stored_receives.back().mpi_request)); + } + } +} + double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, time_step_t t, double percent_flow) { double remote_flow = 0.0; From 3f70965bf1c321c75ebdd116e141b82aff963bf0 Mon Sep 17 00:00:00 2001 From: hellkite500 Date: Wed, 4 Feb 2026 11:54:33 -0700 Subject: [PATCH 074/109] fix(mpi): better handling of pending communication in remote nexus destructor --- src/core/nexus/HY_PointHydroNexusRemote.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 035d38f858..4e94dfde64 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -146,6 +146,21 @@ void HY_PointHydroNexusRemote::post_receives() MPI_COMM_WORLD, &stored_receives.back().mpi_request)); } + else + { + std::cerr << "HY_PointHydroNexusRemote: "<< id + << " destructor timed out after " << timeout/1000 + << " seconds waiting on pending MPI communications\n"; + // The return is is probably best, logging the error. + // There is no good way to recover from this. + // Throwing an exception from destructors is generally not a good idea + // as it can lead to undefined behavior. + // and using std::exit forces the program to terminate immediately, + // even if this situation is recoverable/acceptable in some cases. + return; + } + wait_time += 1; + MPI_Finalized(&mpi_finalized); } } From 31bdae9618fb8ab0bb9a86203283f3d6b7614599 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 20 Feb 2026 13:55:10 -0500 Subject: [PATCH 075/109] Merge resolution from OWP Master --- include/realizations/catchment/Bmi_Module_Formulation.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index ee28db4f70..c5a3cdaba5 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -7,6 +7,9 @@ #include "Bmi_Adapter.hpp" #include #include "bmi_utilities.hpp" +#include + +#include #include "bmi/protocols.hpp" #include From 4beddb453bbfddbb774a00735c0e9305ba03aedd Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Wed, 25 Feb 2026 10:12:57 -0500 Subject: [PATCH 076/109] Revert changes from OWP --- .../core/nexus/HY_PointHydroNexusRemote.hpp | 1 - src/core/nexus/HY_PointHydroNexusRemote.cpp | 126 ++++++------------ 2 files changed, 43 insertions(+), 84 deletions(-) diff --git a/include/core/nexus/HY_PointHydroNexusRemote.hpp b/include/core/nexus/HY_PointHydroNexusRemote.hpp index ebac2e9ace..34d98f9a96 100644 --- a/include/core/nexus/HY_PointHydroNexusRemote.hpp +++ b/include/core/nexus/HY_PointHydroNexusRemote.hpp @@ -72,7 +72,6 @@ class HY_PointHydroNexusRemote : public HY_PointHydroNexus communication_type get_communicator_type() { return type; } private: - void post_receives(); void process_communications(); int world_rank; diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 4e94dfde64..552f22c48b 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -18,7 +18,7 @@ void MPI_Handle_Error(int status) } else { - MPI_Abort(MPI_COMM_WORLD, status); + MPI_Abort(MPI_COMM_WORLD,1); } } @@ -92,9 +92,9 @@ HY_PointHydroNexusRemote::HY_PointHydroNexusRemote(std::string nexus_id, Catchme HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() { - const unsigned int timeout = 120000; // timeout threshold in milliseconds - unsigned int wait_time = 0; - // This destructor might be called after MPI_Finalize so do not attempt communication if + long wait_time = 0; + + // This destructore might be called after MPI_Finalize so do not attempt communication if // this has occured int mpi_finalized; MPI_Finalized(&mpi_finalized); @@ -105,46 +105,14 @@ HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() process_communications(); - if( wait_time < timeout && wait_time > 0 ){ // don't sleep if first call clears comms! - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - else - { - std::cerr << "HY_PointHydroNexusRemote: "<< id - << " destructor timed out after " << timeout/1000 - << " seconds waiting on pending MPI communications\n"; - // The return is is probably best, logging the error. - // There is no good way to recover from this. - // Throwing an exception from destructors is generally not a good idea - // as it can lead to undefined behavior. - // and using std::exit forces the program to terminate immediately, - // even if this situation is recoverable/acceptable in some cases. - return; - } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + wait_time += 1; - MPI_Finalized(&mpi_finalized); - } -} -void HY_PointHydroNexusRemote::post_receives() -{ - // Post receives if not already posted (for pure receiver nexuses) - if (stored_receives.empty()) - { - for (int rank : upstream_ranks) + if ( wait_time > 120000 ) { - stored_receives.push_back({}); - stored_receives.back().buffer = std::make_shared(); - int tag = extract(id); - - MPI_Handle_Error(MPI_Irecv( - stored_receives.back().buffer.get(), - 1, - time_step_and_flow_type, - rank, - tag, - MPI_COMM_WORLD, - &stored_receives.back().mpi_request)); + // TODO log warning message that some comunications could not complete + } else { @@ -200,15 +168,31 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t } else if ( type == receiver || type == sender_receiver ) { - post_receives(); - // Wait for receives to complete - // This ensures all upstream flows are received before returning - // and that we have matched all sends with receives for a given time step. - // As long as the functions are called appropriately, e.g. one call to - // `add_upstream_flow` per upstream catchment per time step, followed - // by a call to `get_downstream_flow` for each downstream catchment per time step, - // this loop will terminate and ensures the synchronization of flows between - // ranks. + for ( int rank : upstream_ranks ) + { + int status; + + stored_receives.resize(stored_receives.size() + 1); + stored_receives.back().buffer = std::make_shared(); + + int tag = extract(id); + + //Receive downstream_flow from Upstream Remote Nexus to this Downstream Remote Nexus + status = MPI_Irecv( + stored_receives.back().buffer.get(), + 1, + time_step_and_flow_type, + rank, + tag, + MPI_COMM_WORLD, + &stored_receives.back().mpi_request); + + MPI_Handle_Error(status); + + //std::cerr << "Creating receive with target_rank=" << rank << " on tag=" << tag << "\n"; + } + + //std::cerr << "Waiting on receives\n"; while ( stored_receives.size() > 0 ) { process_communications(); @@ -221,28 +205,6 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchment_id, time_step_t t) { - // Process any completed communications to free resources - // If no communications are pending, this call will do nothing. - process_communications(); - // NOTE: It is possible for a partition to get "too far" ahead since the sends are now - // truely asynchronous. For pure receivers and sender_receivers, this isn't a problem - // because the get_downstream_flow function will block until all receives are processed. - // However, for pure senders, this could be a problem. - // We can use this spinlock here to limit how far ahead a partition can get. - // in this case, approximately 100 time steps per downstream catchment... - while( stored_sends.size() > downstream_ranks.size()*100 ) - { - process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - // Post receives before sending to prevent deadlock - // When stored_receives is empty, we need to post for incoming messages - if ((type == receiver || type == sender_receiver) && stored_receives.empty()) - { - post_receives(); - } - // first add flow to local copy HY_PointHydroNexus::add_upstream_flow(val, catchment_id, t); @@ -281,25 +243,23 @@ void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchme int tag = extract(id); //Send downstream_flow from this Upstream Remote Nexus to the Downstream Remote Nexus - MPI_Handle_Error( - MPI_Isend( + MPI_Isend( stored_sends.back().buffer.get(), 1, time_step_and_flow_type, *downstream_ranks.begin(), //TODO currently only support a SINGLE downstream message pairing tag, MPI_COMM_WORLD, - &stored_sends.back().mpi_request) - ); + &stored_sends.back().mpi_request); + + //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - // Send is async, the next call to add_upstream_flow will test and ensure the send has completed - // and free the memory associated with the send. - // This prevents a potential deadlock situation where a send isn't able to complete - // because the remote receiver is also trying to send and the underlying mpi buffers/protocol - // are forced into a rendevous protocol. So we ensure that we always post receives before sends. - // and that we always test for completed sends before freeing the memory associated with the send. + while ( stored_sends.size() > 0 ) + { + process_communications(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } } } } From 5c52f20b984e18be48bc7ddc872d8697abfeed6d Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Wed, 4 Mar 2026 13:46:25 -0800 Subject: [PATCH 077/109] updated cicd file --- .github/workflows/ngwpc-cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 2e293bef98..b542fa190d 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -286,10 +286,10 @@ jobs: - build steps: - name: Install Trivy - uses: aquasecurity/setup-trivy@v0.2.2 + uses: aquasecurity/setup-trivy@v0.2.5 with: cache: true - version: v0.68.2 + version: v0.69.3 - name: Trivy scan env: From 8723b6f250d45ff36ff103d04d0b8c1fc29ed1e7 Mon Sep 17 00:00:00 2001 From: Phil Miller Date: Tue, 25 Nov 2025 15:55:21 -0800 Subject: [PATCH 078/109] Flesh out API for state saving and restoring, with adjusted use in Bmi_Module_Formulation --- include/realizations/catchment/Bmi_Module_Formulation.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index c5a3cdaba5..6a0abadd1d 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -7,7 +7,6 @@ #include "Bmi_Adapter.hpp" #include #include "bmi_utilities.hpp" -#include #include #include "bmi/protocols.hpp" From 0092858cabbe4c6b7bc7b259455dfb563cd96535 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 16 Jan 2026 14:06:13 -0500 Subject: [PATCH 079/109] State saving for multi-BMI --- .../catchment/Bmi_Module_Formulation.hpp | 13 +++++++++++++ src/NGen.cpp | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 6a0abadd1d..2b35ae79f6 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -60,6 +60,19 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; + /** + * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. + * + * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. + * @return Pointer to the beginning of the serialized data. + */ + virtual const char* create_save_state(uint64_t *size) const; + + /** + * Clears any serialized data stored by the BMI from memory. + */ + virtual void free_save_state() const; + /** * Get the collection of forcing output property names this instance can provide. * diff --git a/src/NGen.cpp b/src/NGen.cpp index ed713151a9..e8944dc6e2 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -712,6 +712,16 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); + if (state_saving_config.has_end_of_run()) { + LOG("Saving end-of-run state.", LogLevel::INFO); + std::shared_ptr saver = state_saving_config.end_of_run_saver(); + std::shared_ptr snapshot = saver->initialize_snapshot( + State_Saver::snapshot_time_now(), + State_Saver::State_Durability::strict + ); + simulation->save_state_snapshot(snapshot); + } + #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); #endif From a91227d98e821686f1b69a88c25a9c511c617a5b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 20 Jan 2026 11:22:00 -0500 Subject: [PATCH 080/109] Cold start loading --- include/realizations/catchment/Bmi_Formulation.hpp | 7 +++++++ .../catchment/Bmi_Module_Formulation.hpp | 13 +------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 91236fea42..02aab75fb7 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -95,6 +95,13 @@ namespace realization { */ virtual void load_hot_start(std::shared_ptr loader) = 0; + /** + * Passes a serialized representation of the model's state to ``loader`` + * + * Asks saver to find data for the BMI and passes that data to the BMI for loading. + */ + virtual void load_state(std::shared_ptr loader) const = 0; + /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 2b35ae79f6..5c84c7f9bd 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -60,18 +60,7 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; - /** - * Requests the BMI to copy its current state into memory. The state will remain in memory until either a new state is made or `free_save_state` is called. - * - * @param size A `uint64_t` pointer that will have its value set to the size of the serialized data. - * @return Pointer to the beginning of the serialized data. - */ - virtual const char* create_save_state(uint64_t *size) const; - - /** - * Clears any serialized data stored by the BMI from memory. - */ - virtual void free_save_state() const; + void load_state(std::shared_ptr loader) const override; /** * Get the collection of forcing output property names this instance can provide. From 2b8404bb004b39afae2ffc6579074893de021029 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 6 Feb 2026 10:26:57 -0500 Subject: [PATCH 081/109] Use Boost for serializing Multi-BMI --- include/realizations/catchment/Bmi_Module_Formulation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 5c84c7f9bd..3466ab6262 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -60,7 +60,7 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; - void load_state(std::shared_ptr loader) const override; + void load_state(std::shared_ptr loader) override; /** * Get the collection of forcing output property names this instance can provide. From c1f56ad2bdd8629643119d79a294fdd780d45cc3 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 12 Feb 2026 09:47:23 -0500 Subject: [PATCH 082/109] Merge remote-tracking branch 'NOAA-OWP/master' into development --- .../core/nexus/HY_PointHydroNexusRemote.hpp | 1 + .../catchment/Bmi_Multi_Formulation.hpp | 9 ++ src/core/nexus/HY_PointHydroNexusRemote.cpp | 126 ++++++++++++------ 3 files changed, 93 insertions(+), 43 deletions(-) diff --git a/include/core/nexus/HY_PointHydroNexusRemote.hpp b/include/core/nexus/HY_PointHydroNexusRemote.hpp index 34d98f9a96..ebac2e9ace 100644 --- a/include/core/nexus/HY_PointHydroNexusRemote.hpp +++ b/include/core/nexus/HY_PointHydroNexusRemote.hpp @@ -72,6 +72,7 @@ class HY_PointHydroNexusRemote : public HY_PointHydroNexus communication_type get_communicator_type() { return type; } private: + void post_receives(); void process_communications(); int world_rank; diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 6126abca39..556bd7b477 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -75,6 +75,15 @@ namespace realization { void load_state(std::shared_ptr loader) override; void load_hot_start(std::shared_ptr loader) override; + + virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const final { + for( const auto &module : modules ) { + // TODO may need to check on outputs form each module indepdently??? + // Right now, the assumption is that if each component is mass balanced + // then the entire formulation is mass balanced + module->check_mass_balance(iteration, total_steps, timestamp); + } + }; /** * Convert a time value from the model to an epoch time in seconds. diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 552f22c48b..4e94dfde64 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -18,7 +18,7 @@ void MPI_Handle_Error(int status) } else { - MPI_Abort(MPI_COMM_WORLD,1); + MPI_Abort(MPI_COMM_WORLD, status); } } @@ -92,9 +92,9 @@ HY_PointHydroNexusRemote::HY_PointHydroNexusRemote(std::string nexus_id, Catchme HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() { - long wait_time = 0; - - // This destructore might be called after MPI_Finalize so do not attempt communication if + const unsigned int timeout = 120000; // timeout threshold in milliseconds + unsigned int wait_time = 0; + // This destructor might be called after MPI_Finalize so do not attempt communication if // this has occured int mpi_finalized; MPI_Finalized(&mpi_finalized); @@ -105,14 +105,46 @@ HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - + if( wait_time < timeout && wait_time > 0 ){ // don't sleep if first call clears comms! + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + else + { + std::cerr << "HY_PointHydroNexusRemote: "<< id + << " destructor timed out after " << timeout/1000 + << " seconds waiting on pending MPI communications\n"; + // The return is is probably best, logging the error. + // There is no good way to recover from this. + // Throwing an exception from destructors is generally not a good idea + // as it can lead to undefined behavior. + // and using std::exit forces the program to terminate immediately, + // even if this situation is recoverable/acceptable in some cases. + return; + } wait_time += 1; + MPI_Finalized(&mpi_finalized); + } +} - if ( wait_time > 120000 ) +void HY_PointHydroNexusRemote::post_receives() +{ + // Post receives if not already posted (for pure receiver nexuses) + if (stored_receives.empty()) + { + for (int rank : upstream_ranks) { - // TODO log warning message that some comunications could not complete - + stored_receives.push_back({}); + stored_receives.back().buffer = std::make_shared(); + int tag = extract(id); + + MPI_Handle_Error(MPI_Irecv( + stored_receives.back().buffer.get(), + 1, + time_step_and_flow_type, + rank, + tag, + MPI_COMM_WORLD, + &stored_receives.back().mpi_request)); } else { @@ -168,31 +200,15 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t } else if ( type == receiver || type == sender_receiver ) { - for ( int rank : upstream_ranks ) - { - int status; - - stored_receives.resize(stored_receives.size() + 1); - stored_receives.back().buffer = std::make_shared(); - - int tag = extract(id); - - //Receive downstream_flow from Upstream Remote Nexus to this Downstream Remote Nexus - status = MPI_Irecv( - stored_receives.back().buffer.get(), - 1, - time_step_and_flow_type, - rank, - tag, - MPI_COMM_WORLD, - &stored_receives.back().mpi_request); - - MPI_Handle_Error(status); - - //std::cerr << "Creating receive with target_rank=" << rank << " on tag=" << tag << "\n"; - } - - //std::cerr << "Waiting on receives\n"; + post_receives(); + // Wait for receives to complete + // This ensures all upstream flows are received before returning + // and that we have matched all sends with receives for a given time step. + // As long as the functions are called appropriately, e.g. one call to + // `add_upstream_flow` per upstream catchment per time step, followed + // by a call to `get_downstream_flow` for each downstream catchment per time step, + // this loop will terminate and ensures the synchronization of flows between + // ranks. while ( stored_receives.size() > 0 ) { process_communications(); @@ -205,6 +221,28 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchment_id, time_step_t t) { + // Process any completed communications to free resources + // If no communications are pending, this call will do nothing. + process_communications(); + // NOTE: It is possible for a partition to get "too far" ahead since the sends are now + // truely asynchronous. For pure receivers and sender_receivers, this isn't a problem + // because the get_downstream_flow function will block until all receives are processed. + // However, for pure senders, this could be a problem. + // We can use this spinlock here to limit how far ahead a partition can get. + // in this case, approximately 100 time steps per downstream catchment... + while( stored_sends.size() > downstream_ranks.size()*100 ) + { + process_communications(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + // Post receives before sending to prevent deadlock + // When stored_receives is empty, we need to post for incoming messages + if ((type == receiver || type == sender_receiver) && stored_receives.empty()) + { + post_receives(); + } + // first add flow to local copy HY_PointHydroNexus::add_upstream_flow(val, catchment_id, t); @@ -243,23 +281,25 @@ void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchme int tag = extract(id); //Send downstream_flow from this Upstream Remote Nexus to the Downstream Remote Nexus - MPI_Isend( + MPI_Handle_Error( + MPI_Isend( stored_sends.back().buffer.get(), 1, time_step_and_flow_type, *downstream_ranks.begin(), //TODO currently only support a SINGLE downstream message pairing tag, MPI_COMM_WORLD, - &stored_sends.back().mpi_request); - - //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; + &stored_sends.back().mpi_request) + ); + //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - while ( stored_sends.size() > 0 ) - { - process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } + // Send is async, the next call to add_upstream_flow will test and ensure the send has completed + // and free the memory associated with the send. + // This prevents a potential deadlock situation where a send isn't able to complete + // because the remote receiver is also trying to send and the underlying mpi buffers/protocol + // are forced into a rendevous protocol. So we ensure that we always post receives before sends. + // and that we always test for completed sends before freeing the memory associated with the send. } } } From fad4e55384d7c42334f859b95cf67ce227961003 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 24 Feb 2026 14:10:00 -0500 Subject: [PATCH 083/109] Align Fortran state size with sizeof int --- src/realizations/catchment/Bmi_Fortran_Formulation.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index bfd59b0795..d8d058fe9a 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -99,6 +99,7 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { // create the serialized state on the Fortran BMI int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); + // the size coming in should be the number of int elements in the Fortran backing array, not the byte size of the array model->GetValue(StateSaveNames::SIZE, &size_int); // resize the state to the array to the size of the Fortran's backing array this->serialized_state.resize(size_int); From 832ddd498dea9d16837a57d813159ffc98a3b20b Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Mon, 2 Mar 2026 10:27:53 -0500 Subject: [PATCH 084/109] Fortran reports size in bytes --- src/realizations/catchment/Bmi_Fortran_Formulation.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index d8d058fe9a..bfd59b0795 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -99,7 +99,6 @@ const boost::span Bmi_Fortran_Formulation::get_serialization_state() { // create the serialized state on the Fortran BMI int size_int = 0; model->SetValue(StateSaveNames::CREATE, &size_int); - // the size coming in should be the number of int elements in the Fortran backing array, not the byte size of the array model->GetValue(StateSaveNames::SIZE, &size_int); // resize the state to the array to the size of the Fortran's backing array this->serialized_state.resize(size_int); From 4f75a389de6dbf34e0f309d0fe41c3bfb04a240f Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 10 Mar 2026 13:47:07 -0400 Subject: [PATCH 085/109] Fix rebase problems --- .../catchment/Bmi_Formulation.hpp | 7 - .../catchment/Bmi_Module_Formulation.hpp | 2 - .../catchment/Bmi_Multi_Formulation.hpp | 9 - include/utilities/bmi/protocols.hpp | 11 -- src/NGen.cpp | 16 +- src/core/nexus/HY_PointHydroNexusRemote.cpp | 162 +++++------------- src/utilities/bmi/mass_balance.cpp | 7 +- src/utilities/bmi/protocols.cpp | 3 - 8 files changed, 57 insertions(+), 160 deletions(-) diff --git a/include/realizations/catchment/Bmi_Formulation.hpp b/include/realizations/catchment/Bmi_Formulation.hpp index 02aab75fb7..91236fea42 100644 --- a/include/realizations/catchment/Bmi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Formulation.hpp @@ -95,13 +95,6 @@ namespace realization { */ virtual void load_hot_start(std::shared_ptr loader) = 0; - /** - * Passes a serialized representation of the model's state to ``loader`` - * - * Asks saver to find data for the BMI and passes that data to the BMI for loading. - */ - virtual void load_state(std::shared_ptr loader) const = 0; - /** * Convert a time value from the model to an epoch time in seconds. * diff --git a/include/realizations/catchment/Bmi_Module_Formulation.hpp b/include/realizations/catchment/Bmi_Module_Formulation.hpp index 3466ab6262..6a0abadd1d 100644 --- a/include/realizations/catchment/Bmi_Module_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Module_Formulation.hpp @@ -60,8 +60,6 @@ namespace realization { void load_hot_start(std::shared_ptr loader) override; - void load_state(std::shared_ptr loader) override; - /** * Get the collection of forcing output property names this instance can provide. * diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 556bd7b477..6126abca39 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -75,15 +75,6 @@ namespace realization { void load_state(std::shared_ptr loader) override; void load_hot_start(std::shared_ptr loader) override; - - virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const final { - for( const auto &module : modules ) { - // TODO may need to check on outputs form each module indepdently??? - // Right now, the assumption is that if each component is mass balanced - // then the entire formulation is mass balanced - module->check_mass_balance(iteration, total_steps, timestamp); - } - }; /** * Convert a time value from the model to an epoch time in seconds. diff --git a/include/utilities/bmi/protocols.hpp b/include/utilities/bmi/protocols.hpp index 3a1929f60c..723fcf1364 100644 --- a/include/utilities/bmi/protocols.hpp +++ b/include/utilities/bmi/protocols.hpp @@ -25,13 +25,6 @@ per the updated (v0.2) protocol interface Keep protocols in a map for dynamic access by enumeration name add operator<< for Protocol enum -Version 0.2 -Enumerate protocol types/names -The container now holds a single model pointer and passes it to each protocol -per the updated (v0.2) protocol interface -Keep protocols in a map for dynamic access by enumeration name -add operator<< for Protocol enum - Version 0.1 Container and management for abstract BMI protocols */ @@ -89,10 +82,6 @@ class NgenBmiProtocols { private: - auto run(const Protocol& protocol_name, const Context& ctx) const -> expected; - - private: - /** * @brief All protocols managed by this container will utilize the same model * diff --git a/src/NGen.cpp b/src/NGen.cpp index e8944dc6e2..c31733b61a 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -712,15 +712,21 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); - if (state_saving_config.has_end_of_run()) { - LOG("Saving end-of-run state.", LogLevel::INFO); - std::shared_ptr saver = state_saving_config.end_of_run_saver(); - std::shared_ptr snapshot = saver->initialize_snapshot( - State_Saver::snapshot_time_now(), + for (const auto& saver : state_saving_config.end_of_run_savers()) { + std::shared_ptr snapshot = saver.second->initialize_snapshot( State_Saver::State_Durability::strict ); simulation->save_state_snapshot(snapshot); } + // if (state_saving_config.has_end_of_run()) { + // LOG("Saving end-of-run state.", LogLevel::INFO); + // std::shared_ptr saver = state_saving_config.end_of_run_saver(); + // std::shared_ptr snapshot = saver->initialize_snapshot( + // State_Saver::snapshot_time_now(), + // State_Saver::State_Durability::strict + // ); + // simulation->save_state_snapshot(snapshot); + // } #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 4e94dfde64..1f1e0ab544 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -18,7 +18,7 @@ void MPI_Handle_Error(int status) } else { - MPI_Abort(MPI_COMM_WORLD, status); + MPI_Abort(MPI_COMM_WORLD,1); } } @@ -92,9 +92,9 @@ HY_PointHydroNexusRemote::HY_PointHydroNexusRemote(std::string nexus_id, Catchme HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() { - const unsigned int timeout = 120000; // timeout threshold in milliseconds - unsigned int wait_time = 0; - // This destructor might be called after MPI_Finalize so do not attempt communication if + long wait_time = 0; + + // This destructore might be called after MPI_Finalize so do not attempt communication if // this has occured int mpi_finalized; MPI_Finalized(&mpi_finalized); @@ -105,84 +105,14 @@ HY_PointHydroNexusRemote::~HY_PointHydroNexusRemote() process_communications(); - if( wait_time < timeout && wait_time > 0 ){ // don't sleep if first call clears comms! - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - else - { - std::cerr << "HY_PointHydroNexusRemote: "<< id - << " destructor timed out after " << timeout/1000 - << " seconds waiting on pending MPI communications\n"; - // The return is is probably best, logging the error. - // There is no good way to recover from this. - // Throwing an exception from destructors is generally not a good idea - // as it can lead to undefined behavior. - // and using std::exit forces the program to terminate immediately, - // even if this situation is recoverable/acceptable in some cases. - return; - } - wait_time += 1; - MPI_Finalized(&mpi_finalized); - } -} + std::this_thread::sleep_for(std::chrono::milliseconds(1)); -void HY_PointHydroNexusRemote::post_receives() -{ - // Post receives if not already posted (for pure receiver nexuses) - if (stored_receives.empty()) - { - for (int rank : upstream_ranks) - { - stored_receives.push_back({}); - stored_receives.back().buffer = std::make_shared(); - int tag = extract(id); - - MPI_Handle_Error(MPI_Irecv( - stored_receives.back().buffer.get(), - 1, - time_step_and_flow_type, - rank, - tag, - MPI_COMM_WORLD, - &stored_receives.back().mpi_request)); - } - else - { - std::cerr << "HY_PointHydroNexusRemote: "<< id - << " destructor timed out after " << timeout/1000 - << " seconds waiting on pending MPI communications\n"; - // The return is is probably best, logging the error. - // There is no good way to recover from this. - // Throwing an exception from destructors is generally not a good idea - // as it can lead to undefined behavior. - // and using std::exit forces the program to terminate immediately, - // even if this situation is recoverable/acceptable in some cases. - return; - } wait_time += 1; - MPI_Finalized(&mpi_finalized); - } -} -void HY_PointHydroNexusRemote::post_receives() -{ - // Post receives if not already posted (for pure receiver nexuses) - if (stored_receives.empty()) - { - for (int rank : upstream_ranks) + if ( wait_time > 120000 ) { - stored_receives.push_back({}); - stored_receives.back().buffer = std::make_shared(); - int tag = extract(id); - - MPI_Handle_Error(MPI_Irecv( - stored_receives.back().buffer.get(), - 1, - time_step_and_flow_type, - rank, - tag, - MPI_COMM_WORLD, - &stored_receives.back().mpi_request)); + // TODO log warning message that some comunications could not complete + } } } @@ -200,15 +130,31 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t } else if ( type == receiver || type == sender_receiver ) { - post_receives(); - // Wait for receives to complete - // This ensures all upstream flows are received before returning - // and that we have matched all sends with receives for a given time step. - // As long as the functions are called appropriately, e.g. one call to - // `add_upstream_flow` per upstream catchment per time step, followed - // by a call to `get_downstream_flow` for each downstream catchment per time step, - // this loop will terminate and ensures the synchronization of flows between - // ranks. + for ( int rank : upstream_ranks ) + { + int status; + + stored_receives.resize(stored_receives.size() + 1); + stored_receives.back().buffer = std::make_shared(); + + int tag = extract(id); + + //Receive downstream_flow from Upstream Remote Nexus to this Downstream Remote Nexus + status = MPI_Irecv( + stored_receives.back().buffer.get(), + 1, + time_step_and_flow_type, + rank, + tag, + MPI_COMM_WORLD, + &stored_receives.back().mpi_request); + + MPI_Handle_Error(status); + + //std::cerr << "Creating receive with target_rank=" << rank << " on tag=" << tag << "\n"; + } + + //std::cerr << "Waiting on receives\n"; while ( stored_receives.size() > 0 ) { process_communications(); @@ -221,28 +167,6 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchment_id, time_step_t t) { - // Process any completed communications to free resources - // If no communications are pending, this call will do nothing. - process_communications(); - // NOTE: It is possible for a partition to get "too far" ahead since the sends are now - // truely asynchronous. For pure receivers and sender_receivers, this isn't a problem - // because the get_downstream_flow function will block until all receives are processed. - // However, for pure senders, this could be a problem. - // We can use this spinlock here to limit how far ahead a partition can get. - // in this case, approximately 100 time steps per downstream catchment... - while( stored_sends.size() > downstream_ranks.size()*100 ) - { - process_communications(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - // Post receives before sending to prevent deadlock - // When stored_receives is empty, we need to post for incoming messages - if ((type == receiver || type == sender_receiver) && stored_receives.empty()) - { - post_receives(); - } - // first add flow to local copy HY_PointHydroNexus::add_upstream_flow(val, catchment_id, t); @@ -281,25 +205,23 @@ void HY_PointHydroNexusRemote::add_upstream_flow(double val, std::string catchme int tag = extract(id); //Send downstream_flow from this Upstream Remote Nexus to the Downstream Remote Nexus - MPI_Handle_Error( - MPI_Isend( + MPI_Isend( stored_sends.back().buffer.get(), 1, time_step_and_flow_type, *downstream_ranks.begin(), //TODO currently only support a SINGLE downstream message pairing tag, MPI_COMM_WORLD, - &stored_sends.back().mpi_request) - ); + &stored_sends.back().mpi_request); + + //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - //std::cerr << "Creating send with target_rank=" << *downstream_ranks.begin() << " on tag=" << tag << "\n"; - // Send is async, the next call to add_upstream_flow will test and ensure the send has completed - // and free the memory associated with the send. - // This prevents a potential deadlock situation where a send isn't able to complete - // because the remote receiver is also trying to send and the underlying mpi buffers/protocol - // are forced into a rendevous protocol. So we ensure that we always post receives before sends. - // and that we always test for completed sends before freeing the memory associated with the send. + while ( stored_sends.size() > 0 ) + { + process_communications(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } } } } diff --git a/src/utilities/bmi/mass_balance.cpp b/src/utilities/bmi/mass_balance.cpp index 4d5dd73317..f542a682a0 100644 --- a/src/utilities/bmi/mass_balance.cpp +++ b/src/utilities/bmi/mass_balance.cpp @@ -19,9 +19,6 @@ Version 0.3 (see mass_balance.hpp for details) Version 0.2 Implement error handling via expected and error_or_warning -Version 0.2 -Implement error handling via expected and error_or_warning - Version 0.1 Implementation the BMI mass balance checking protocol */ @@ -115,8 +112,12 @@ auto NgenMassBalance::check_support(const ModelPtr& model) -> expected( ProtocolError( Error::INTEGRATION_ERROR, + "mass_balance: variables have incosistent units, cannot perform mass balance." ) ); + } + } catch (const std::exception &e) { + std::stringstream ss; ss << "mass_balance: Error getting mass balance values for module '" << model->GetComponentName() << "': " << e.what() << std::endl; return make_unexpected( ProtocolError( Error::INTEGRATION_ERROR, diff --git a/src/utilities/bmi/protocols.cpp b/src/utilities/bmi/protocols.cpp index 7f72a17a22..c78651a192 100644 --- a/src/utilities/bmi/protocols.cpp +++ b/src/utilities/bmi/protocols.cpp @@ -19,9 +19,6 @@ Version 0.2.1 (See bmi/protocols.hpp for details) Version 0.2 Implement error handling via expected and error_or_warning -Version 0.2 -Implement error handling via expected and error_or_warning - Version 0.1 Container and management for abstract BMI protocols */ From c7061b32a0d8f60ae12d64f0e504ed1e3a559c5d Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 10 Mar 2026 14:01:36 -0400 Subject: [PATCH 086/109] Remove commented old code --- src/NGen.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index c31733b61a..ed96b60c2b 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -718,15 +718,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { ); simulation->save_state_snapshot(snapshot); } - // if (state_saving_config.has_end_of_run()) { - // LOG("Saving end-of-run state.", LogLevel::INFO); - // std::shared_ptr saver = state_saving_config.end_of_run_saver(); - // std::shared_ptr snapshot = saver->initialize_snapshot( - // State_Saver::snapshot_time_now(), - // State_Saver::State_Durability::strict - // ); - // simulation->save_state_snapshot(snapshot); - // } #if NGEN_WITH_MPI MPI_Barrier(MPI_COMM_WORLD); From bbe2731ab15eed6d527685cbbb98fd96ad2ca434 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 10 Mar 2026 14:08:33 -0400 Subject: [PATCH 087/109] Log end of run saving --- src/NGen.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index ed96b60c2b..67177864ef 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -713,10 +713,11 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { simulation->run_catchments(); for (const auto& saver : state_saving_config.end_of_run_savers()) { + LOG(LogLevel::INFO, "Saving end of run data for state " + saver.first); std::shared_ptr snapshot = saver.second->initialize_snapshot( State_Saver::State_Durability::strict ); - simulation->save_state_snapshot(snapshot); + simulation->save_end_of_run(snapshot); } #if NGEN_WITH_MPI From caef072e272127a9f89dfa7e54c7685c799491c2 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 10 Mar 2026 14:18:16 -0400 Subject: [PATCH 088/109] Fix documentation --- include/utilities/bmi/mass_balance.hpp | 5 ----- include/utilities/bmi/protocol.hpp | 6 ------ 2 files changed, 11 deletions(-) diff --git a/include/utilities/bmi/mass_balance.hpp b/include/utilities/bmi/mass_balance.hpp index a58c5a9bf4..046b1002e8 100644 --- a/include/utilities/bmi/mass_balance.hpp +++ b/include/utilities/bmi/mass_balance.hpp @@ -23,10 +23,6 @@ Version 0.2 Conform to updated protocol interface Removed integration and error exceptions in favor of ProtocolError -Version 0.2 -Conform to updated protocol interface -Removed integration and error exceptions in favor of ProtocolError - Version 0.1 Interface of the BMI mass balance protocol */ @@ -156,4 +152,3 @@ namespace models{ namespace bmi{ namespace protocols{ }; }}} - diff --git a/include/utilities/bmi/protocol.hpp b/include/utilities/bmi/protocol.hpp index 1b8751abc9..d08dd7681c 100644 --- a/include/utilities/bmi/protocol.hpp +++ b/include/utilities/bmi/protocol.hpp @@ -23,12 +23,6 @@ Implement error handling via expected and error_or_warning Removed model member and required model reference in run(), check_support(), and initialize() Minor refactoring and style changes -Version 0.2 -Enumerate protocol error types and add ProtocolError exception class -Implement error handling via expected and error_or_warning -Removed model member and required model reference in run(), check_support(), and initialize() -Minor refactoring and style changes - Version 0.1 Virtual interface for BMI protocols */ From 0830c2cf7c0c0f393df87b7cae153fe56953aab1 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 13 Mar 2026 09:22:02 -0400 Subject: [PATCH 089/109] Send NGEN's delta time to T-route --- src/core/NgenSimulation.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 86e0112f86..05d4a8076a 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -305,6 +305,7 @@ void NgenSimulation::run_routing(NgenSimulation::hy_features_t &features, std::s if (this->py_troute_ == NULL) { this->make_troute(t_route_config_file_with_path); } + this->py_troute_->set_value_unchecked("ngen_dt", &delta_time, 1); // set up nexus id indexes std::vector nexus_df_index(routing_nexus_indexes->size()); From a082fd83ea0e99c50c59f1f589595106dbfb2367 Mon Sep 17 00:00:00 2001 From: Chris Nealen Date: Mon, 23 Mar 2026 11:29:26 -0400 Subject: [PATCH 090/109] Update setup-trivy action to v0.2.6 --- .github/workflows/ngwpc-cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index b542fa190d..87ed350453 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -286,7 +286,7 @@ jobs: - build steps: - name: Install Trivy - uses: aquasecurity/setup-trivy@v0.2.5 + uses: aquasecurity/setup-trivy@v0.2.6 with: cache: true version: v0.69.3 From f2635d02ad33cda5f18c5ed2025b2b4f00a171cf Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Mon, 9 Mar 2026 18:11:46 -0700 Subject: [PATCH 091/109] Use new nwm-ewts libraries and python package EWTS versions to build summary output to console Updates to CMakeLists.txt for integrating the nwm-ewts libraries Initial conversion to ewts_ngen_bridge library. Updated CMakeLists. Replaced logMsgAndThrowError with separate calls to LOG and throw. --- CMakeLists.txt | 35 +- Dockerfile | 4 +- extern/cfe/CMakeLists.txt | 15 +- extern/noah-owp-modular/CMakeLists.txt | 14 + extern/sac-sma/CMakeLists.txt | 16 +- extern/topmodel/CMakeLists.txt | 14 +- include/bmi/Bmi_C_Adapter.hpp | 2 +- include/bmi/Bmi_Cpp_Adapter.hpp | 2 +- include/bmi/Bmi_Fortran_Adapter.hpp | 2 +- include/bmi/Bmi_Py_Adapter.hpp | 2 +- include/core/Layer.hpp | 2 +- include/core/Partition_Parser.hpp | 2 +- include/core/mediator/UnitsHelper.hpp | 2 +- .../core/nexus/HY_PointHydroNexusRemote.hpp | 2 +- .../forcing/CsvPerFeatureForcingProvider.hpp | 2 +- .../forcing/ForcingsEngineDataProvider.hpp | 2 +- .../forcing/OptionalWrappedDataProvider.hpp | 2 +- include/geojson/JSONProperty.hpp | 2 +- include/geojson/features/FeatureBase.hpp | 2 +- .../catchment/Bmi_Multi_Formulation.hpp | 2 +- .../catchment/Catchment_Formulation.hpp | 2 +- .../realizations/catchment/Formulation.hpp | 2 +- .../catchment/Formulation_Constructors.hpp | 2 +- .../catchment/Formulation_Manager.hpp | 2 +- include/realizations/config/layer.hpp | 2 +- include/realizations/config/time.hpp | 2 +- include/simulation_time/Simulation_Time.hpp | 2 +- include/utilities/CSV_Reader.h | 2 +- include/utilities/FileChecker.h | 5 +- include/utilities/Logger.hpp | 84 --- include/utilities/bmi_utilities.hpp | 2 +- include/utilities/mdarray/mdarray.hpp | 2 +- include/utilities/mdframe/mdframe.hpp | 2 +- .../utilities/python/HydrofabricSubsetter.hpp | 2 +- src/NGen.cpp | 8 +- src/bmi/AbstractCLibBmiAdapter.cpp | 20 +- src/bmi/Bmi_Adapter.cpp | 28 +- src/bmi/Bmi_C_Adapter.cpp | 89 ++- src/bmi/Bmi_Fortran_Adapter.cpp | 89 ++- src/bmi/Bmi_Py_Adapter.cpp | 19 +- src/bmi/CMakeLists.txt | 2 + src/core/CMakeLists.txt | 4 + src/core/HY_Features.cpp | 2 +- src/core/HY_Features_MPI.cpp | 2 +- src/core/NgenSimulation.cpp | 2 +- src/core/SurfaceLayer.cpp | 2 +- src/core/mediator/CMakeLists.txt | 2 + src/core/nexus/CMakeLists.txt | 2 + src/core/nexus/HY_PointHydroNexus.cpp | 2 +- src/core/nexus/HY_PointHydroNexusRemote.cpp | 6 +- src/forcing/CMakeLists.txt | 1 + src/forcing/ForcingsEngineDataProvider.cpp | 14 +- src/forcing/NetCDFPerFeatureDataProvider.cpp | 15 +- src/forcing/NullForcingProvider.cpp | 2 +- src/geojson/CMakeLists.txt | 4 + src/geojson/JSONProperty.cpp | 2 +- src/geopackage/CMakeLists.txt | 2 + src/geopackage/feature.cpp | 2 +- src/geopackage/geometry.cpp | 6 +- src/geopackage/ngen_sqlite.cpp | 2 +- src/geopackage/proj.cpp | 2 +- src/geopackage/read.cpp | 15 +- src/geopackage/wkb.cpp | 16 +- src/partitionGenerator.cpp | 31 +- .../catchment/Bmi_C_Formulation.cpp | 13 +- .../catchment/Bmi_Cpp_Formulation.cpp | 14 +- .../catchment/Bmi_Fortran_Formulation.cpp | 15 +- .../catchment/Bmi_Module_Formulation.cpp | 7 +- .../catchment/Bmi_Multi_Formulation.cpp | 18 +- .../catchment/Bmi_Py_Formulation.cpp | 13 +- src/realizations/catchment/CMakeLists.txt | 3 + src/utilities/CMakeLists.txt | 2 + src/utilities/bmi/CMakeLists.txt | 2 + src/utilities/bmi/mass_balance.cpp | 4 +- src/utilities/logging/CMakeLists.txt | 4 +- src/utilities/logging/Logger.cpp | 671 ------------------ src/utilities/logging/logging_utils.cpp | 2 +- src/utilities/mdframe/CMakeLists.txt | 1 + src/utilities/mdframe/handler_csv.cpp | 12 +- src/utilities/mdframe/handler_netcdf.cpp | 3 +- src/utilities/python/CMakeLists.txt | 3 + src/utilities/python/InterpreterUtil.cpp | 2 +- 82 files changed, 454 insertions(+), 967 deletions(-) delete mode 100644 include/utilities/Logger.hpp delete mode 100644 src/utilities/logging/Logger.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index c90788dd9a..dbb9adcb48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,7 @@ +# +# ngen/CMakeLists.txt +# + # Ensure CMake policies have defaults depending on the CMake version used # between the two versions specified. e.g. if 3.18 is used, then 3.18 defaults # will be used instead of 3.17 defaults. @@ -118,6 +122,30 @@ project(ngen VERSION 0.3.0) add_executable(ngen "${NGEN_SRC_DIR}/NGen.cpp") +# --- EWTS (installed from nwm-ewts) --- +find_package(ewts CONFIG REQUIRED) + +if(DEFINED ewts_VERSION AND NOT "${ewts_VERSION}" STREQUAL "") + set(NGEN_EWTS_VERSION "${ewts_VERSION}") +else() + set(NGEN_EWTS_VERSION "") +endif() + +if(DEFINED EWTS_NGWPC_VERSION AND NOT "${EWTS_NGWPC_VERSION}" STREQUAL "") + set(NGEN_EWTS_NGWPC_VERSION "${EWTS_NGWPC_VERSION}") +else() + set(NGEN_EWTS_NGWPC_VERSION "") +endif() + +get_filename_component(EWTS_PREFIX "${ewts_DIR}" DIRECTORY) +get_filename_component(EWTS_PREFIX "${EWTS_PREFIX}" DIRECTORY) +get_filename_component(EWTS_PREFIX "${EWTS_PREFIX}" DIRECTORY) + +message(STATUS "Found EWTS: ${EWTS_PREFIX} (found version ${ewts_VERSION})") + +# NGen itself uses the NGen-specific EWTS logger/bridge +target_link_libraries(ngen PRIVATE ewts::ewts_ngen_bridge) + # Dependencies ================================================================ # ----------------------------------------------------------------------------- @@ -281,7 +309,6 @@ if(UDUNITS_QUIET) add_compile_definitions(UDUNITS_QUIET) endif() - # ----------------------------------------------------------------------------- # Project Targets # ----------------------------------------------------------------------------- @@ -354,6 +381,7 @@ if(NGEN_WITH_ROUTING) endif() add_executable(partitionGenerator src/partitionGenerator.cpp) +target_link_libraries(partitionGenerator PRIVATE ewts::ewts_ngen_bridge) target_link_libraries(partitionGenerator PUBLIC NGen::logging) target_include_directories(partitionGenerator PUBLIC "${PROJECT_BINARY_DIR}/include") if(NGEN_WITH_SQLITE) @@ -444,7 +472,10 @@ ngen_multiline_message( " Boost:" " Version: ${Boost_VERSION}" " Include: ${Boost_INCLUDE_DIRS}" -" Library: ${Boost_LIBRARY_DIRS}") +" Library: ${Boost_LIBRARY_DIRS}" +" EWTS:" +" Package Version: ${NGEN_EWTS_VERSION}" +" NGWPC Version: ${NGEN_EWTS_NGWPC_VERSION}") ngen_dependent_multiline_message(INTEL_DPCPP " Intel DPC++:" " Version: ${SYCL_LANGUAGE_VERSION}" diff --git a/Dockerfile b/Dockerfile index 93edd0d218..ca6b38dbfc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,10 +7,10 @@ ARG ORG=ngwpc ARG NGEN_FORCING_IMAGE_TAG=latest ARG NGEN_FORCING_IMAGE=ghcr.io/ngwpc/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG} -FROM ${NGEN_FORCING_IMAGE} AS base +#FROM ${NGEN_FORCING_IMAGE} AS base # Uncomment when building locally -#FROM ngen-bmi-forcing AS base +FROM ngen-bmi-forcing AS base # OCI Metadata Arguments ARG NGEN_FORCING_IMAGE diff --git a/extern/cfe/CMakeLists.txt b/extern/cfe/CMakeLists.txt index 6ef23bf9d5..52bfb303a4 100644 --- a/extern/cfe/CMakeLists.txt +++ b/extern/cfe/CMakeLists.txt @@ -22,9 +22,9 @@ find_package(Boost 1.79.0 REQUIRED COMPONENTS serialization) if(WIN32) - add_library(cfebmi cfe/src/bmi_cfe.c cfe/src/cfe.c cfe/src/giuh.c cfe/src/logger.c cfe/src/conceptual_reservoir.c cfe/src/nash_cascade.c cfe/src/bmi_serialization.cxx) + add_library(cfebmi cfe/src/bmi_cfe.c cfe/src/cfe.c cfe/src/giuh.c cfe/src/conceptual_reservoir.c cfe/src/nash_cascade.c cfe/src/bmi_serialization.cxx) else() - add_library(cfebmi SHARED cfe/src/bmi_cfe.c cfe/src/cfe.c cfe/src/giuh.c cfe/src/logger.c cfe/src/conceptual_reservoir.c cfe/src/nash_cascade.c cfe/src/bmi_serialization.cxx) + add_library(cfebmi SHARED cfe/src/bmi_cfe.c cfe/src/cfe.c cfe/src/giuh.c cfe/src/conceptual_reservoir.c cfe/src/nash_cascade.c cfe/src/bmi_serialization.cxx) endif() target_include_directories(cfebmi PRIVATE cfe/include) @@ -35,6 +35,17 @@ set_target_properties(cfebmi PROPERTIES PUBLIC_HEADER cfe/include/bmi_cfe.h) target_link_libraries(cfebmi PRIVATE Boost::serialization) +# --- EWTS (installed from nwm-ewts) --- +find_package(ewts CONFIG REQUIRED) + +# Always use EWTS runtime logger for C +target_link_libraries(cfebmi PRIVATE ewts::ewts_c) + +# Built with ngen bridge +target_link_libraries(cfebmi PRIVATE ewts::ewts_ngen_bridge) +target_compile_definitions(cfebmi PRIVATE EWTS_HAVE_NGEN_BRIDGE) + + # Code requires minimum of C99 standard to compile set_target_properties(cfebmi PROPERTIES C_STANDARD 99 C_STANDARD_REQUIRED ON) diff --git a/extern/noah-owp-modular/CMakeLists.txt b/extern/noah-owp-modular/CMakeLists.txt index 526e6c9239..1aeb29f7ea 100644 --- a/extern/noah-owp-modular/CMakeLists.txt +++ b/extern/noah-owp-modular/CMakeLists.txt @@ -1,3 +1,7 @@ +# +# noah-owp-modular CMakeLists.txt +# + cmake_minimum_required(VERSION 3.12...3.26) # NGen NOAH-OWP-MODULAR listfile shim @@ -28,6 +32,12 @@ target_compile_options(surfacebmi PRIVATE -cpp -ffree-line-length-none) target_include_directories(surfacebmi INTERFACE "${_SURFACEBMI_BINARY_DIR}/mod") target_compile_definitions(surfacebmi PRIVATE BMI_ACTIVE) +# --- EWTS (installed from nwm-ewts) --- +find_package(ewts CONFIG REQUIRED) + +# Always use EWTS runtime logger for Fortran +target_link_libraries(surfacebmi PRIVATE ewts::ewts_fortran) + if(NGEN_IS_MAIN_PROJECT) # This ensures we can build NOAH-OWP-Modular with NGen support, but @@ -41,6 +51,10 @@ if(NGEN_IS_MAIN_PROJECT) target_link_libraries(surfacebmi PUBLIC iso_c_bmi) target_compile_definitions(surfacebmi PRIVATE NGEN_FORCING_ACTIVE NGEN_OUTPUT_ACTIVE NGEN_ACTIVE) + + # Built with ngen bridge + target_link_libraries(surfacebmi PRIVATE ewts::ewts_ngen_bridge) + target_compile_definitions(surfacebmi PRIVATE EWTS_HAVE_NGEN_BRIDGE) else() find_path(NETCDF_MODULE_DIR netcdf.mod PATHS "${NETCDF_ROOT}/include" diff --git a/extern/sac-sma/CMakeLists.txt b/extern/sac-sma/CMakeLists.txt index 4d06eaa19e..2646aa7ab2 100644 --- a/extern/sac-sma/CMakeLists.txt +++ b/extern/sac-sma/CMakeLists.txt @@ -1,3 +1,7 @@ +# +# ngen/extern/sac-sma/CMakeLists.txt +# + cmake_minimum_required(VERSION 3.12) enable_language( Fortran ) add_subdirectory(../iso_c_fortran_bmi ${CMAKE_BINARY_DIR}/iso_c_bmi) @@ -83,7 +87,17 @@ target_link_libraries( sacbmi PUBLIC iso_c_bmi ) set_target_properties(sacbmi PROPERTIES VERSION ${PROJECT_VERSION}) #TODO is this needed for fortran? -#set_target_properties(surfacebmi PROPERTIES PUBLIC_HEADER ${BMI_SOURCE}) +#set_target_properties(sacbmi PROPERTIES PUBLIC_HEADER ${BMI_SOURCE}) + +# --- EWTS (installed from nwm-ewts) --- +find_package(ewts CONFIG REQUIRED) + +# Always use EWTS runtime logger for Fortran +target_link_libraries(sacbmi PRIVATE ewts::ewts_fortran) + +# Built with ngen bridge +target_link_libraries(sacbmi PRIVATE ewts::ewts_ngen_bridge) +target_compile_definitions(sacbmi PRIVATE EWTS_HAVE_NGEN_BRIDGE) include(GNUInstallDirs) diff --git a/extern/topmodel/CMakeLists.txt b/extern/topmodel/CMakeLists.txt index 0bc32b327b..604a11c45e 100644 --- a/extern/topmodel/CMakeLists.txt +++ b/extern/topmodel/CMakeLists.txt @@ -19,9 +19,9 @@ set(Boost_USE_STATIC_RUNTIME OFF) find_package(Boost 1.79.0 REQUIRED COMPONENTS serialization) if(WIN32) - add_library(topmodelbmi topmodel/src/bmi_topmodel.c topmodel/src/topmodel.c topmodel/src/logger.c topmodel/src/bmi_serialization.cpp) + add_library(topmodelbmi topmodel/src/bmi_topmodel.c topmodel/src/topmodel.c topmodel/src/bmi_serialization.cpp) else() - add_library(topmodelbmi SHARED topmodel/src/bmi_topmodel.c topmodel/src/topmodel.c topmodel/src/logger.c topmodel/src/bmi_serialization.cpp) + add_library(topmodelbmi SHARED topmodel/src/bmi_topmodel.c topmodel/src/topmodel.c topmodel/src/bmi_serialization.cpp) endif() target_include_directories(topmodelbmi PRIVATE topmodel/include) @@ -32,6 +32,16 @@ set_target_properties(topmodelbmi PROPERTIES PUBLIC_HEADER topmodel/include/bmi_ target_link_libraries(topmodelbmi PRIVATE Boost::serialization) +# --- EWTS (installed from nwm-ewts) --- +find_package(ewts CONFIG REQUIRED) + +# Always use EWTS runtime logger for C +target_link_libraries(topmodelbmi PRIVATE ewts::ewts_c) + +# Built with ngen bridge +target_link_libraries(topmodelbmi PRIVATE ewts::ewts_ngen_bridge) +target_compile_definitions(topmodelbmi PRIVATE EWTS_HAVE_NGEN_BRIDGE) + # Code requires minimum of C99 standard to compile set_target_properties(topmodelbmi PROPERTIES C_STANDARD 99 C_STANDARD_REQUIRED ON) diff --git a/include/bmi/Bmi_C_Adapter.hpp b/include/bmi/Bmi_C_Adapter.hpp index 9ac7b1620f..e2068f8b59 100644 --- a/include/bmi/Bmi_C_Adapter.hpp +++ b/include/bmi/Bmi_C_Adapter.hpp @@ -3,7 +3,7 @@ #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include "bmi.h" #include "AbstractCLibBmiAdapter.hpp" diff --git a/include/bmi/Bmi_Cpp_Adapter.hpp b/include/bmi/Bmi_Cpp_Adapter.hpp index 91a67b1601..66abb1f648 100644 --- a/include/bmi/Bmi_Cpp_Adapter.hpp +++ b/include/bmi/Bmi_Cpp_Adapter.hpp @@ -3,7 +3,7 @@ #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include "bmi.hpp" #include "AbstractCLibBmiAdapter.hpp" diff --git a/include/bmi/Bmi_Fortran_Adapter.hpp b/include/bmi/Bmi_Fortran_Adapter.hpp index f3818c96e8..c168819e9d 100644 --- a/include/bmi/Bmi_Fortran_Adapter.hpp +++ b/include/bmi/Bmi_Fortran_Adapter.hpp @@ -2,7 +2,7 @@ #define NGEN_BMI_FORTRAN_ADAPTER_HPP #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_BMI_FORTRAN diff --git a/include/bmi/Bmi_Py_Adapter.hpp b/include/bmi/Bmi_Py_Adapter.hpp index 2e48dc7845..3e47fc2d16 100644 --- a/include/bmi/Bmi_Py_Adapter.hpp +++ b/include/bmi/Bmi_Py_Adapter.hpp @@ -2,7 +2,7 @@ #define NGEN_BMI_PY_ADAPTER_H #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_PYTHON diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 9a21401fb2..643f1b6970 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -2,7 +2,7 @@ #define __NGEN_LAYER__ #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include "LayerData.hpp" #include "Simulation_Time.hpp" diff --git a/include/core/Partition_Parser.hpp b/include/core/Partition_Parser.hpp index d479aec04c..c05c88021e 100644 --- a/include/core/Partition_Parser.hpp +++ b/include/core/Partition_Parser.hpp @@ -17,7 +17,7 @@ #include #include "JSONProperty.hpp" #include "Partition_Data.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" class Partitions_Parser { diff --git a/include/core/mediator/UnitsHelper.hpp b/include/core/mediator/UnitsHelper.hpp index 7d61f90d30..6887389ee5 100644 --- a/include/core/mediator/UnitsHelper.hpp +++ b/include/core/mediator/UnitsHelper.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_UNITSHELPER_H #define NGEN_UNITSHELPER_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" // FIXME: Workaround to handle UDUNITS2 includes with differing paths. // Not exactly sure why CMake can't handle this, but even with diff --git a/include/core/nexus/HY_PointHydroNexusRemote.hpp b/include/core/nexus/HY_PointHydroNexusRemote.hpp index ebac2e9ace..aa3857d67b 100644 --- a/include/core/nexus/HY_PointHydroNexusRemote.hpp +++ b/include/core/nexus/HY_PointHydroNexusRemote.hpp @@ -1,6 +1,6 @@ #ifndef HY_POINTHDRONEXUSREMOTE_H #define HY_POINTHDRONEXUSREMOTE_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #if NGEN_WITH_MPI diff --git a/include/forcing/CsvPerFeatureForcingProvider.hpp b/include/forcing/CsvPerFeatureForcingProvider.hpp index 1570c67b2e..a8b4bbd22c 100644 --- a/include/forcing/CsvPerFeatureForcingProvider.hpp +++ b/include/forcing/CsvPerFeatureForcingProvider.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_CSVPERFEATUREFORCING_H #define NGEN_CSVPERFEATUREFORCING_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/forcing/ForcingsEngineDataProvider.hpp b/include/forcing/ForcingsEngineDataProvider.hpp index 37df90d59b..5987c9bf48 100644 --- a/include/forcing/ForcingsEngineDataProvider.hpp +++ b/include/forcing/ForcingsEngineDataProvider.hpp @@ -16,7 +16,7 @@ #include "DataProvider.hpp" #include "bmi/Bmi_Py_Adapter.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" namespace data_access { diff --git a/include/forcing/OptionalWrappedDataProvider.hpp b/include/forcing/OptionalWrappedDataProvider.hpp index 3a2c2c7c94..06dab46c61 100644 --- a/include/forcing/OptionalWrappedDataProvider.hpp +++ b/include/forcing/OptionalWrappedDataProvider.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_OPTIONALWRAPPEDPROVIDER_HPP #define NGEN_OPTIONALWRAPPEDPROVIDER_HPP -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include diff --git a/include/geojson/JSONProperty.hpp b/include/geojson/JSONProperty.hpp index edcdb3a781..61e2836fd6 100644 --- a/include/geojson/JSONProperty.hpp +++ b/include/geojson/JSONProperty.hpp @@ -9,7 +9,7 @@ #include #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" namespace geojson { class JSONProperty; diff --git a/include/geojson/features/FeatureBase.hpp b/include/geojson/features/FeatureBase.hpp index a11669808f..2c89aa2474 100644 --- a/include/geojson/features/FeatureBase.hpp +++ b/include/geojson/features/FeatureBase.hpp @@ -1,6 +1,6 @@ #ifndef GEOJSON_FEATURE_H #define GEOJSON_FEATURE_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include "JSONGeometry.hpp" #include "JSONProperty.hpp" diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 6126abca39..458b63949a 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_BMI_MULTI_FORMULATION_HPP #define NGEN_BMI_MULTI_FORMULATION_HPP -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/realizations/catchment/Catchment_Formulation.hpp b/include/realizations/catchment/Catchment_Formulation.hpp index 06d4ef5f7a..b9ababa3b2 100644 --- a/include/realizations/catchment/Catchment_Formulation.hpp +++ b/include/realizations/catchment/Catchment_Formulation.hpp @@ -7,7 +7,7 @@ #include #include "GenericDataProvider.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #define DEFAULT_FORMULATION_OUTPUT_DELIMITER "," diff --git a/include/realizations/catchment/Formulation.hpp b/include/realizations/catchment/Formulation.hpp index ad8c6f097c..5b63d509ff 100644 --- a/include/realizations/catchment/Formulation.hpp +++ b/include/realizations/catchment/Formulation.hpp @@ -1,6 +1,6 @@ #ifndef FORMULATION_H #define FORMULATION_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/realizations/catchment/Formulation_Constructors.hpp b/include/realizations/catchment/Formulation_Constructors.hpp index a1afd75f46..c24b686111 100644 --- a/include/realizations/catchment/Formulation_Constructors.hpp +++ b/include/realizations/catchment/Formulation_Constructors.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_FORMULATION_CONSTRUCTORS_H #define NGEN_FORMULATION_CONSTRUCTORS_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include diff --git a/include/realizations/catchment/Formulation_Manager.hpp b/include/realizations/catchment/Formulation_Manager.hpp index 1865eec9a0..5584b03115 100644 --- a/include/realizations/catchment/Formulation_Manager.hpp +++ b/include/realizations/catchment/Formulation_Manager.hpp @@ -2,7 +2,7 @@ #define NGEN_FORMULATION_MANAGER_H #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/realizations/config/layer.hpp b/include/realizations/config/layer.hpp index fffef3c5d9..c947079306 100644 --- a/include/realizations/config/layer.hpp +++ b/include/realizations/config/layer.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_REALIZATION_CONFIG_LAYER_H #define NGEN_REALIZATION_CONFIG_LAYER_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include diff --git a/include/realizations/config/time.hpp b/include/realizations/config/time.hpp index 0825eef698..b00ed4e09f 100644 --- a/include/realizations/config/time.hpp +++ b/include/realizations/config/time.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_REALIZATION_CONFIG_TIME_H #define NGEN_REALIZATION_CONFIG_TIME_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/simulation_time/Simulation_Time.hpp b/include/simulation_time/Simulation_Time.hpp index c017e2c267..d2ea4969d4 100644 --- a/include/simulation_time/Simulation_Time.hpp +++ b/include/simulation_time/Simulation_Time.hpp @@ -1,7 +1,7 @@ #ifndef SIMULATION_TIME_H #define SIMULATION_TIME_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/utilities/CSV_Reader.h b/include/utilities/CSV_Reader.h index 21adf3d745..bcda340385 100644 --- a/include/utilities/CSV_Reader.h +++ b/include/utilities/CSV_Reader.h @@ -1,6 +1,6 @@ #ifndef CSV_Reader_H #define CSV_Reader_H -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/utilities/FileChecker.h b/include/utilities/FileChecker.h index b9304587a0..92b2ee1045 100644 --- a/include/utilities/FileChecker.h +++ b/include/utilities/FileChecker.h @@ -6,7 +6,8 @@ #include #include #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" namespace utils { @@ -122,7 +123,7 @@ namespace utils { else { std::stringstream ss; ss << description << " path " << path << " not readable" << std::endl; - LOG(ss.str(), LogLevel::INFO); ss.str(""); + LOG(LogLevel::INFO, ss.str()); ss.str(""); return false; } } diff --git a/include/utilities/Logger.hpp b/include/utilities/Logger.hpp deleted file mode 100644 index 50f6c91220..0000000000 --- a/include/utilities/Logger.hpp +++ /dev/null @@ -1,84 +0,0 @@ -#ifndef LOGGER_HPP -#define LOGGER_HPP - -#include -#include -#include -#include -#include -#include -#include -#include - -enum class LogLevel { - NONE = 0, - DEBUG = 1, - INFO = 2, - WARNING = 3, - SEVERE = 4, - FATAL = 5, -}; - -/** -* Logger Class Used to Output Details of Current Application Flow -*/ -class Logger { - public: - Logger(); - ~Logger() = default; - - // Methods - static void Log(std::string message, LogLevel messageLevel=LogLevel::INFO); - static void Log(LogLevel messageLevel, const char* message, ...); - static void Log(LogLevel messageLevel, std::string message); - bool IsLoggingEnabled(void); - LogLevel GetLogLevel(void); - void SetLogPreferences(LogLevel level=LogLevel::INFO); - - static __always_inline void logMsgAndThrowError(const std::string& message) { - Log(message, LogLevel::SEVERE); - throw std::runtime_error(message); - }; - - static Logger* GetLogger(); - - private: - // Methods - static std::string ConvertLogLevelToString(LogLevel level); - static LogLevel ConvertStringToLogLevel(const std::string& logLevel); - std::string CreateDateString(void); - bool CreateDirectory(const std::string& path); - static std::string CreateTimestamp(bool appendMS=true, bool iso=true); - bool DirectoryExists(const std::string& path); - std::string ExtractFirstNDirs(const std::string& path, int numDirs); - bool FileExists(const std::string& path); - bool FindAndOpenLogConfigFile(std::string path, std::ifstream& configFileStream); - std::string GetLogFilePath(void); - std::string GetParentDirName(const std::string& path); - bool IsValidEnvVarName(const std::string& name); - bool LogFileReady(void); - bool ParseLoggerConfigFile(std::ifstream& jsonFile); - void ReadConfigFile(std::string searchPath); - void SetupLogFile(void); - void ManageLoggingEnvVars(bool set=true); - static std::string ToUpper(const std::string& str); - static std::string TrimString(const std::string& str); - - // Variables - bool loggerInitialized = false; - bool loggingEnabled = true; - std::fstream logFile; - std::string logFileDir = ""; - std::string logFilePath = ""; - LogLevel logLevel = LogLevel::INFO; - std::string moduleName = ""; - std::string ngenResultsDir = ""; - bool openedOnce = false; - - std::unordered_map moduleLogLevels; -}; - -// Placed here to ensure the class is declared before setting this preprocessor symbol -#define LOG Logger::Log - -#endif diff --git a/include/utilities/bmi_utilities.hpp b/include/utilities/bmi_utilities.hpp index eea56f6de2..b33f4390bf 100644 --- a/include/utilities/bmi_utilities.hpp +++ b/include/utilities/bmi_utilities.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_BMI_UTILITIES_HPP #define NGEN_BMI_UTILITIES_HPP -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/utilities/mdarray/mdarray.hpp b/include/utilities/mdarray/mdarray.hpp index e5a6fa867e..7a229e0e93 100644 --- a/include/utilities/mdarray/mdarray.hpp +++ b/include/utilities/mdarray/mdarray.hpp @@ -5,7 +5,7 @@ #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" namespace ngen { diff --git a/include/utilities/mdframe/mdframe.hpp b/include/utilities/mdframe/mdframe.hpp index 4fa97dd5f3..79f66c332b 100644 --- a/include/utilities/mdframe/mdframe.hpp +++ b/include/utilities/mdframe/mdframe.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_MDFRAME_DEFINITION_HPP #define NGEN_MDFRAME_DEFINITION_HPP -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/include/utilities/python/HydrofabricSubsetter.hpp b/include/utilities/python/HydrofabricSubsetter.hpp index eaf71e42e4..0c0bfc1f39 100644 --- a/include/utilities/python/HydrofabricSubsetter.hpp +++ b/include/utilities/python/HydrofabricSubsetter.hpp @@ -1,6 +1,6 @@ #ifndef NGEN_HYDROFABRICSUBSETTER_HPP #define NGEN_HYDROFABRICSUBSETTER_HPP -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include diff --git a/src/NGen.cpp b/src/NGen.cpp index 67177864ef..128f7a0871 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -17,7 +17,7 @@ #include "NGenConfig.h" -#include +#include "ewts_ngen/logger.hpp" #include #include @@ -456,7 +456,8 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { throw; } #else - Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); + LOG(LogLevel::FATAL, "SQLite3 support required to read GeoPackage files."); + throw std::runtime_error("SQLite3 support required to read GeoPackage files."); #endif } else { nexus_collection = geojson::read(nexusDataFile, nexus_subset_ids); @@ -489,7 +490,8 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } #else - Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); + LOG(LogLevel::FATAL, "SQLite3 support required to read GeoPackage files."); + throw std::runtime_error("SQLite3 support required to read GeoPackage files."); #endif } else { catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); diff --git a/src/bmi/AbstractCLibBmiAdapter.cpp b/src/bmi/AbstractCLibBmiAdapter.cpp index dd7565a272..058aa07ab4 100644 --- a/src/bmi/AbstractCLibBmiAdapter.cpp +++ b/src/bmi/AbstractCLibBmiAdapter.cpp @@ -5,7 +5,8 @@ #include "utilities/logging_utils.h" #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" namespace models { namespace bmi { @@ -37,7 +38,8 @@ void AbstractCLibBmiAdapter::dynamic_library_load() { if (bmi_registration_function.empty()) { this->init_exception_msg = "Can't init " + this->model_name + "; empty name given for library's registration function."; - Logger::logMsgAndThrowError(this->init_exception_msg); + LOG(LogLevel::FATAL, this->init_exception_msg); + throw std::runtime_error(this->init_exception_msg); } if (dyn_lib_handle != nullptr) { std::string message = "AbstractCLibBmiAdapter::dynamic_library_load: ignoring attempt to reload dynamic shared library '" + bmi_lib_file + "' for " + this->model_name; @@ -54,7 +56,8 @@ void AbstractCLibBmiAdapter::dynamic_library_load() { if (bmi_lib_file.length() == 0) { this->init_exception_msg = "Can't init " + this->model_name + "; library file path is empty"; - Logger::logMsgAndThrowError(this->init_exception_msg); + LOG(LogLevel::FATAL, this->init_exception_msg); + throw std::runtime_error(this->init_exception_msg); } if (bmi_lib_file.substr(idx) == ".so") { alt_bmi_lib_file = bmi_lib_file.substr(0, idx) + ".dylib"; @@ -77,7 +80,8 @@ void AbstractCLibBmiAdapter::dynamic_library_load() { } else { this->init_exception_msg = "Can't init " + this->model_name + "; unreadable shared library file '" + bmi_lib_file + "'"; - Logger::logMsgAndThrowError(this->init_exception_msg); + LOG(LogLevel::FATAL, this->init_exception_msg); + throw std::runtime_error(this->init_exception_msg); } } @@ -102,10 +106,10 @@ void* AbstractCLibBmiAdapter::dynamic_load_symbol( bool is_null_valid ) { if (dyn_lib_handle == nullptr) { - Logger::logMsgAndThrowError( - "Cannot load symbol '" + symbol_name + - "' without handle to shared library (bmi_lib_file = '" + bmi_lib_file + "')" - ); + this->init_exception_msg = "Cannot load symbol '" + symbol_name + + "' without handle to shared library (bmi_lib_file = '" + bmi_lib_file + "')"; + LOG(LogLevel::FATAL, this->init_exception_msg); + throw std::runtime_error(this->init_exception_msg); } // Call first to ensure any previous error is cleared before trying to load the symbol dlerror(); diff --git a/src/bmi/Bmi_Adapter.cpp b/src/bmi/Bmi_Adapter.cpp index 516efb1da2..a527efd091 100644 --- a/src/bmi/Bmi_Adapter.cpp +++ b/src/bmi/Bmi_Adapter.cpp @@ -1,8 +1,10 @@ +#include #include "bmi/Bmi_Adapter.hpp" #include "bmi/State_Exception.hpp" #include "utilities/FileChecker.h" #include "utilities/logging_utils.h" -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" using namespace std; std::stringstream str_stream; @@ -27,7 +29,8 @@ Bmi_Adapter::Bmi_Adapter( init_exception_msg = "Cannot create and initialize " + this->model_name + " using unreadable file '" + this->bmi_init_config + "'. Error: " + std::strerror(errno); - Logger::logMsgAndThrowError(init_exception_msg); + LOG(LogLevel::FATAL, init_exception_msg); + throw std::runtime_error(init_exception_msg); } str_stream << "Bmi_Adapter: Model name: " << this->model_name << std::endl; LOG(str_stream.str(), LogLevel::INFO); str_stream.str(""); @@ -43,7 +46,7 @@ double Bmi_Adapter::get_time_convert_factor() { input_units = GetTimeUnits(); } catch(std::exception &e){ - //Re-throwing any exception as a runtime_error so we don't lose + //Re-throwing any exception as a std::runtime_error so we don't lose //the error context/message. We will lose the original exception type, though //When a python exception is raised from the py adapter subclass, the //pybind exception is lost and all we see is a generic "uncaught exception" @@ -70,9 +73,8 @@ void Bmi_Adapter::Initialize() { // previous message errno = 0; if (model_initialized && !init_exception_msg.empty()) { - Logger::logMsgAndThrowError( - "Previous " + model_name + " init attempt had exception: \n\t" + init_exception_msg - ); + LOG(LogLevel::FATAL, init_exception_msg); + throw std::runtime_error(init_exception_msg); } // If there was previous init attempt w/ (implicitly) no exception on previous attempt, just // return @@ -82,7 +84,8 @@ void Bmi_Adapter::Initialize() { init_exception_msg = "Cannot initialize " + model_name + " using unreadable file '" + bmi_init_config + "'. Error: " + std::strerror(errno); ; - Logger::logMsgAndThrowError(init_exception_msg); + LOG(LogLevel::FATAL, init_exception_msg); + throw std::runtime_error(init_exception_msg); } else { try { // TODO: make this same name as used with other testing (adjust name in docstring above @@ -104,10 +107,10 @@ void Bmi_Adapter::Initialize() { void Bmi_Adapter::Initialize(std::string config_file) { if (config_file != bmi_init_config && model_initialized) { - Logger::logMsgAndThrowError( - "Model init previously attempted; cannot change config from " + bmi_init_config + - " to " + config_file - ); + LOG(LogLevel::FATAL, "Model init previously attempted; cannot change config from " + bmi_init_config + + " to " + config_file); + throw std::runtime_error("Model init previously attempted; cannot change config from " + bmi_init_config + + " to " + config_file); } str_stream << __FILE__ << ":" << __LINE__ << " Bmi_Adapter::Initialize: config_file = " << config_file << std::endl; LOG(str_stream.str(), LogLevel::INFO); str_stream.str(""); @@ -123,7 +126,8 @@ void Bmi_Adapter::Initialize(std::string config_file) { } catch (models::external::State_Exception& e) { throw e; } catch (std::exception& e) { - Logger::logMsgAndThrowError(e.what()); + LOG(LogLevel::FATAL, e.what()); + throw std::runtime_error(e.what()); } } diff --git a/src/bmi/Bmi_C_Adapter.cpp b/src/bmi/Bmi_C_Adapter.cpp index e85678946f..02ce6bcb71 100644 --- a/src/bmi/Bmi_C_Adapter.cpp +++ b/src/bmi/Bmi_C_Adapter.cpp @@ -2,7 +2,8 @@ #include #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" using namespace models::bmi; @@ -120,7 +121,8 @@ Bmi_C_Adapter::Bmi_C_Adapter(Bmi_C_Adapter &adapter) : model_name(adapter.model_ std::string Bmi_C_Adapter::GetComponentName() { char component_name[BMI_MAX_COMPONENT_NAME]; if (bmi_model->get_component_name(bmi_model.get(), component_name) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model component name."); + LOG(LogLevel::FATAL, "failed to get model component name."); + std::runtime_error("failed to get model component name."); } return {component_name}; } @@ -129,7 +131,8 @@ double Bmi_C_Adapter::GetCurrentTime() { double current_time; int result = bmi_model->get_current_time(bmi_model.get(), ¤t_time); if (result != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get current model time."); + LOG(LogLevel::FATAL, "failed to get current model time."); + std::runtime_error("failed to get current model time."); } return current_time; } @@ -138,7 +141,8 @@ double Bmi_C_Adapter::GetEndTime() { double end_time; int result = bmi_model->get_end_time(bmi_model.get(), &end_time); if (result != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model end time."); + LOG(LogLevel::FATAL, " failed to get model end time."); + std::runtime_error("failed to get model end time."); } return end_time; } @@ -171,7 +175,8 @@ double Bmi_C_Adapter::GetStartTime() { double start_time; int result = bmi_model->get_start_time(bmi_model.get(), &start_time); if (result != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model start time."); + LOG(LogLevel::FATAL, "failed to get model start time."); + std::runtime_error("failed to get model start time."); } return start_time; } @@ -180,7 +185,8 @@ std::string Bmi_C_Adapter::GetTimeUnits() { char time_units_cstr[BMI_MAX_UNITS_NAME]; int result = bmi_model->get_time_units(bmi_model.get(), time_units_cstr); if (result != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to read time units from model."); + LOG(LogLevel::FATAL, "failed to read time units from model."); + std::runtime_error("failed to read time units from model."); } return std::string(time_units_cstr); } @@ -195,7 +201,8 @@ int Bmi_C_Adapter::GetVarItemsize(std::string name) { int size; int success = bmi_model->get_var_itemsize(bmi_model.get(), name.c_str(), &size); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable item size for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable item size for " + name + "."); + std::runtime_error("failed to get variable item size for " + name + "."); } return size; } @@ -204,7 +211,8 @@ int Bmi_C_Adapter::GetVarNbytes(std::string name) { int size; int success = bmi_model->get_var_nbytes(bmi_model.get(), name.c_str(), &size); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable array size (i.e., nbytes) for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable array size (i.e., nbytes) for " + name + "."); + std::runtime_error("failed to get variable array size (i.e., nbytes) for " + name + "."); } return size; } @@ -213,7 +221,8 @@ std::string Bmi_C_Adapter::GetVarType(std::string name) { char type_c_str[BMI_MAX_TYPE_NAME]; int success = bmi_model->get_var_type(bmi_model.get(), name.c_str(), type_c_str); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable type for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable type for " + name + "."); + std::runtime_error("failed to get variable type for " + name + "."); } return std::string(type_c_str); } @@ -222,7 +231,8 @@ std::string Bmi_C_Adapter::GetVarUnits(std::string name) { char units_c_str[BMI_MAX_UNITS_NAME]; int success = bmi_model->get_var_units(bmi_model.get(), name.c_str(), units_c_str); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable units for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable units for " + name + "."); + std::runtime_error("failed to get variable units for " + name + "."); } return std::string(units_c_str); } @@ -231,7 +241,7 @@ std::string Bmi_C_Adapter::GetVarLocation(std::string name) { char location_c_str[BMI_MAX_LOCATION_NAME]; int success = bmi_model->get_var_location(bmi_model.get(), name.c_str(), location_c_str); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable location for " + name + "."); + std::runtime_error("failed to get variable location for " + name + "."); } return std::string(location_c_str); } @@ -240,7 +250,8 @@ int Bmi_C_Adapter::GetVarGrid(std::string name) { int grid; int success = bmi_model->get_var_grid(bmi_model.get(), name.c_str(), &grid); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable grid for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable grid for " + name + "."); + std::runtime_error("failed to get variable grid for " + name + "."); } return grid; } @@ -249,7 +260,8 @@ std::string Bmi_C_Adapter::GetGridType(int grid_id) { char gridtype_c_str[BMI_MAX_TYPE_NAME]; int success = bmi_model->get_grid_type(bmi_model.get(), grid_id, gridtype_c_str); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid type for grid ID " + std::to_string(grid_id) + "."); + LOG(LogLevel::FATAL, "failed to get grid type for grid ID " + std::to_string(grid_id) + "."); + std::runtime_error("failed to get grid type for grid ID " + std::to_string(grid_id) + "."); } return std::string(gridtype_c_str); } @@ -258,7 +270,8 @@ int Bmi_C_Adapter::GetGridRank(int grid_id) { int gridrank; int success = bmi_model->get_grid_rank(bmi_model.get(), grid_id, &gridrank); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid rank for grid ID " + std::to_string(grid_id) + "."); + LOG(LogLevel::FATAL, "failed to get grid rank for grid ID " + std::to_string(grid_id) + "."); + std::runtime_error("failed to get grid rank for grid ID " + std::to_string(grid_id) + "."); } return gridrank; } @@ -267,7 +280,8 @@ int Bmi_C_Adapter::GetGridSize(int grid_id) { int gridsize; int success = bmi_model->get_grid_size(bmi_model.get(), grid_id, &gridsize); if (success != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid size for grid ID " + std::to_string(grid_id) + "."); + LOG(LogLevel::FATAL, "failed to get grid size for grid ID " + std::to_string(grid_id) + "."); + std::runtime_error("failed to get grid size for grid ID " + std::to_string(grid_id) + "."); } return gridsize; } @@ -282,7 +296,8 @@ std::shared_ptr> Bmi_C_Adapter::inner_get_variable_name variableCount = (is_input_variables) ? inner_get_input_item_count() : inner_get_output_item_count(); } catch (const std::exception &e) { - Logger::logMsgAndThrowError(model_name + " failed to count of " + varType + " variable names array."); + LOG(LogLevel::FATAL, "failed to count of " + varType + " variable names array."); + std::runtime_error("failed to count of " + varType + " variable names array."); } // With variable count now obtained, create the vector @@ -305,7 +320,8 @@ std::shared_ptr> Bmi_C_Adapter::inner_get_variable_name names_result = bmi_model->get_output_var_names(bmi_model.get(), names_array.data()); } if (names_result != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get array of output variables names."); + LOG(LogLevel::FATAL, "failed to get array of output variables names."); + std::runtime_error("failed to get array of output variables names."); } // Then convert from array of C strings to vector of strings, freeing the allocated space as we go @@ -369,44 +385,51 @@ void Bmi_C_Adapter::UpdateUntil(double time) { void Bmi_C_Adapter::GetGridShape(const int grid, int *shape) { if (bmi_model->get_grid_shape(bmi_model.get(), grid, shape) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " shape."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " shape."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " shape."); } } void Bmi_C_Adapter::GetGridSpacing(const int grid, double *spacing) { if (bmi_model->get_grid_spacing(bmi_model.get(), grid, spacing) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " spacing."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " spacing."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " spacing."); } } void Bmi_C_Adapter::GetGridOrigin(const int grid, double *origin) { if (bmi_model->get_grid_origin(bmi_model.get(), grid, origin) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " origin."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " origin."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " origin."); } } void Bmi_C_Adapter::GetGridX(const int grid, double *x) { if (bmi_model->get_grid_x(bmi_model.get(), grid, x) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " x."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " x."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " x."); } } void Bmi_C_Adapter::GetGridY(const int grid, double *y) { if (bmi_model->get_grid_y(bmi_model.get(), grid, y) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " y."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " y."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " y."); } } void Bmi_C_Adapter::GetGridZ(const int grid, double *z) { if (bmi_model->get_grid_z(bmi_model.get(), grid, z) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " z."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " z."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " z."); } } int Bmi_C_Adapter::GetGridNodeCount(const int grid) { int count; if (bmi_model->get_grid_node_count(bmi_model.get(), grid, &count) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " node count."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " node count."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " node count."); } return count; } @@ -414,7 +437,8 @@ int Bmi_C_Adapter::GetGridNodeCount(const int grid) { int Bmi_C_Adapter::GetGridEdgeCount(const int grid) { int count; if (bmi_model->get_grid_edge_count(bmi_model.get(), grid, &count) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " edge count."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " edge count."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " edge count."); } return count; } @@ -422,31 +446,36 @@ int Bmi_C_Adapter::GetGridEdgeCount(const int grid) { int Bmi_C_Adapter::GetGridFaceCount(const int grid) { int count; if (bmi_model->get_grid_face_count(bmi_model.get(), grid, &count) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " face count."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " face count."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " face count."); } return count; } void Bmi_C_Adapter::GetGridEdgeNodes(const int grid, int *edge_nodes) { if (bmi_model->get_grid_edge_nodes(bmi_model.get(), grid, edge_nodes) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " edge nodes."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " edge nodes."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " edge nodes."); } } void Bmi_C_Adapter::GetGridFaceEdges(const int grid, int *face_edges) { if (bmi_model->get_grid_face_edges(bmi_model.get(), grid, face_edges) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " face edges."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " face edges."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " face edges."); } } void Bmi_C_Adapter::GetGridFaceNodes(const int grid, int *face_nodes) { if (bmi_model->get_grid_face_nodes(bmi_model.get(), grid, face_nodes) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " face nodes."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " face nodes."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " face nodes."); } } void Bmi_C_Adapter::GetGridNodesPerFace(const int grid, int *nodes_per_face) { if (bmi_model->get_grid_nodes_per_face(bmi_model.get(), grid, nodes_per_face) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " nodes per face."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " nodes per face."); + std::runtime_error("failed to get grid " + std::to_string(grid) + " nodes per face."); } } diff --git a/src/bmi/Bmi_Fortran_Adapter.cpp b/src/bmi/Bmi_Fortran_Adapter.cpp index de507b4721..4b0c61ff24 100644 --- a/src/bmi/Bmi_Fortran_Adapter.cpp +++ b/src/bmi/Bmi_Fortran_Adapter.cpp @@ -1,5 +1,5 @@ #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_BMI_FORTRAN #include "bmi/Bmi_Fortran_Adapter.hpp" @@ -9,7 +9,8 @@ using namespace models::bmi; std::string Bmi_Fortran_Adapter::GetComponentName() { char component_name[BMI_MAX_COMPONENT_NAME]; if (get_component_name(&bmi_model->handle, component_name) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model component name."); + LOG(LogLevel::FATAL, "failed to get model component name."); + throw std::runtime_error("failed to get model component name."); } return {component_name}; } @@ -17,7 +18,8 @@ std::string Bmi_Fortran_Adapter::GetComponentName() { double Bmi_Fortran_Adapter::GetCurrentTime() { double current_time; if (get_current_time(&bmi_model->handle, ¤t_time) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get current model time."); + LOG(LogLevel::FATAL, "failed to get current model time."); + throw std::runtime_error("failed to get current model time."); } return current_time; } @@ -25,7 +27,8 @@ double Bmi_Fortran_Adapter::GetCurrentTime() { double Bmi_Fortran_Adapter::GetEndTime() { double end_time; if (get_end_time(&bmi_model->handle, &end_time) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model end time."); + LOG(LogLevel::FATAL, "failed to get model end time."); + throw std::runtime_error("failed to get model end time."); } return end_time; } @@ -47,7 +50,8 @@ std::vector Bmi_Fortran_Adapter::GetOutputVarNames() { double Bmi_Fortran_Adapter::GetStartTime() { double start_time; if (get_start_time(&bmi_model->handle, &start_time) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model start time."); + LOG(LogLevel::FATAL, "failed to get model start time."); + throw std::runtime_error("failed to get model start time."); } return start_time; } @@ -57,7 +61,8 @@ double Bmi_Fortran_Adapter::GetTimeStep() { //return *get_bmi_model_time_step_size_ptr(); double ts; if (get_time_step(&bmi_model->handle, &ts) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get model time step size."); + LOG(LogLevel::FATAL, "failed to get model time step size."); + throw std::runtime_error("failed to get model time step size."); } return ts; } @@ -65,7 +70,8 @@ double Bmi_Fortran_Adapter::GetTimeStep() { std::string Bmi_Fortran_Adapter::GetTimeUnits() { char time_units_cstr[BMI_MAX_UNITS_NAME]; if (get_time_units(&bmi_model->handle, time_units_cstr) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to read time units from model."); + LOG(LogLevel::FATAL, "failed to read time units from model."); + throw std::runtime_error("failed to read time units from model."); } return {time_units_cstr}; } @@ -79,13 +85,15 @@ void Bmi_Fortran_Adapter::GetValueAtIndices(std::string name, void *dest, int *i " indices."); } */ - Logger::logMsgAndThrowError("Fortran BMI module integration does not currently support getting values by index"); + LOG(LogLevel::FATAL, "Fortran BMI module integration does not currently support getting values by index"); + throw std::runtime_error("Fortran BMI module integration does not currently support getting values by index"); } int Bmi_Fortran_Adapter::GetVarItemsize(std::string name) { int size; if (get_var_itemsize(&bmi_model->handle, name.c_str(), &size) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable item size for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable item size for " + name + "."); + throw std::runtime_error("failed to get variable item size for " + name + "."); } return size; } @@ -93,7 +101,8 @@ int Bmi_Fortran_Adapter::GetVarItemsize(std::string name) { int Bmi_Fortran_Adapter::GetVarNbytes(std::string name) { int size; if (get_var_nbytes(&bmi_model->handle, name.c_str(), &size) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable array size (i.e., nbytes) for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable array size (i.e., nbytes) for " + name + "."); + throw std::runtime_error("failed to get variable array size (i.e., nbytes) for " + name + "."); } return size; } @@ -105,7 +114,8 @@ std::string Bmi_Fortran_Adapter::GetVarType(std::string name) { std::string Bmi_Fortran_Adapter::GetVarUnits(std::string name) { char units_c_str[BMI_MAX_UNITS_NAME]; if (get_var_units(&bmi_model->handle, name.c_str(), units_c_str) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable units for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable units for " + name + "."); + throw std::runtime_error("failed to get variable units for " + name + "."); } return std::string(units_c_str); } @@ -113,7 +123,8 @@ std::string Bmi_Fortran_Adapter::GetVarUnits(std::string name) { std::string Bmi_Fortran_Adapter::GetVarLocation(std::string name) { char location_c_str[BMI_MAX_LOCATION_NAME]; if (get_var_location(&bmi_model->handle, name.c_str(), location_c_str) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable location for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable location for " + name + "."); + throw std::runtime_error("failed to get variable location for " + name + "."); } return std::string(location_c_str); } @@ -121,7 +132,8 @@ std::string Bmi_Fortran_Adapter::GetVarLocation(std::string name) { int Bmi_Fortran_Adapter::GetVarGrid(std::string name) { int grid; if (get_var_grid(&bmi_model->handle, name.c_str(), &grid) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get variable grid for " + name + "."); + LOG(LogLevel::FATAL, "failed to get variable grid for " + name + "."); + throw std::runtime_error("failed to get variable grid for " + name + "."); } return grid; } @@ -129,7 +141,8 @@ int Bmi_Fortran_Adapter::GetVarGrid(std::string name) { std::string Bmi_Fortran_Adapter::GetGridType(int grid_id) { char gridtype_c_str[BMI_MAX_TYPE_NAME]; if (get_grid_type(&bmi_model->handle, &grid_id, gridtype_c_str) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid type for grid ID " + std::to_string(grid_id) + "."); + LOG(LogLevel::FATAL, "failed to get grid type for grid ID " + std::to_string(grid_id) + "."); + throw std::runtime_error("failed to get grid type for grid ID " + std::to_string(grid_id) + "."); } return std::string(gridtype_c_str); } @@ -137,7 +150,8 @@ std::string Bmi_Fortran_Adapter::GetGridType(int grid_id) { int Bmi_Fortran_Adapter::GetGridRank(int grid_id) { int gridrank; if (get_grid_rank(&bmi_model->handle, &grid_id, &gridrank) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid rank for grid ID " + std::to_string(grid_id) + "."); + LOG(LogLevel::FATAL, "failed to get grid rank for grid ID " + std::to_string(grid_id) + "."); + throw std::runtime_error("failed to get grid rank for grid ID " + std::to_string(grid_id) + "."); } return gridrank; } @@ -145,7 +159,8 @@ int Bmi_Fortran_Adapter::GetGridRank(int grid_id) { int Bmi_Fortran_Adapter::GetGridSize(int grid_id) { int gridsize; if (get_grid_size(&bmi_model->handle, &grid_id, &gridsize) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid size for grid ID " + std::to_string(grid_id) + "."); + LOG(LogLevel::FATAL, "failed to get grid size for grid ID " + std::to_string(grid_id) + "."); + throw std::runtime_error("failed to get grid size for grid ID " + std::to_string(grid_id) + "."); } return gridsize; } @@ -167,7 +182,8 @@ void Bmi_Fortran_Adapter::SetValueAtIndices(std::string name, int *inds, int cou "Failed to set specified indexes for " + name + " variable of " + model_name); } */ - Logger::logMsgAndThrowError("Fortran BMI module integration does not currently support setting values by index"); + LOG(LogLevel::FATAL, "Fortran BMI module integration does not currently support setting values by index"); + throw std::runtime_error("Fortran BMI module integration does not currently support setting values by index"); } /** @@ -201,44 +217,51 @@ void Bmi_Fortran_Adapter::UpdateUntil(double time) { void Bmi_Fortran_Adapter::GetGridShape(int grid, int *shape) { if (get_grid_shape(&bmi_model->handle, &grid, shape) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " shape."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " shape."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " shape."); } } void Bmi_Fortran_Adapter::GetGridSpacing(int grid, double *spacing) { if (get_grid_spacing(&bmi_model->handle, &grid, spacing) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " spacing."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " spacing."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " spacing."); } } void Bmi_Fortran_Adapter::GetGridOrigin(int grid, double *origin) { if (get_grid_origin(&bmi_model->handle, &grid, origin) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " origin."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " origin."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " origin."); } } void Bmi_Fortran_Adapter::GetGridX(int grid, double *x) { if (get_grid_x(&bmi_model->handle, &grid, x) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " x."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " x."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " x."); } } void Bmi_Fortran_Adapter::GetGridY(int grid, double *y) { if (get_grid_y(&bmi_model->handle, &grid, y) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " y."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " y."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " y."); } } void Bmi_Fortran_Adapter::GetGridZ(int grid, double *z) { if (get_grid_z(&bmi_model->handle, &grid, z) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " z."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " z."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " z."); } } int Bmi_Fortran_Adapter::GetGridNodeCount(int grid) { int count; if (get_grid_node_count(&bmi_model->handle, &grid, &count) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " node count."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " node count."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " node count."); } return count; } @@ -246,7 +269,8 @@ int Bmi_Fortran_Adapter::GetGridNodeCount(int grid) { int Bmi_Fortran_Adapter::GetGridEdgeCount(int grid) { int count; if (get_grid_edge_count(&bmi_model->handle, &grid, &count) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " edge count."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " edge count."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " edge count."); } return count; } @@ -254,32 +278,37 @@ int Bmi_Fortran_Adapter::GetGridEdgeCount(int grid) { int Bmi_Fortran_Adapter::GetGridFaceCount(int grid) { int count; if (get_grid_face_count(&bmi_model->handle, &grid, &count) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " face count."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " face count."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " face count."); } return count; } void Bmi_Fortran_Adapter::GetGridEdgeNodes(int grid, int *edge_nodes) { if (get_grid_edge_nodes(&bmi_model->handle, &grid, edge_nodes) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " edge nodes."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " edge nodes."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " edge nodes."); } } void Bmi_Fortran_Adapter::GetGridFaceEdges(int grid, int *face_edges) { if (get_grid_face_edges(&bmi_model->handle, &grid, face_edges) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " face edges."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " face edges."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " face edges."); } } void Bmi_Fortran_Adapter::GetGridFaceNodes(int grid, int *face_nodes) { if (get_grid_face_nodes(&bmi_model->handle, &grid, face_nodes) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " face nodes."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " face nodes."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " face nodes."); } } void Bmi_Fortran_Adapter::GetGridNodesPerFace(int grid, int *nodes_per_face) { if (get_grid_nodes_per_face(&bmi_model->handle, &grid, nodes_per_face) != BMI_SUCCESS) { - Logger::logMsgAndThrowError(model_name + " failed to get grid " + std::to_string(grid) + " nodes per face."); + LOG(LogLevel::FATAL, "failed to get grid " + std::to_string(grid) + " nodes per face."); + throw std::runtime_error("failed to get grid " + std::to_string(grid) + " nodes per face."); } } diff --git a/src/bmi/Bmi_Py_Adapter.cpp b/src/bmi/Bmi_Py_Adapter.cpp index 5fdc3eb8ec..d4d667fedc 100644 --- a/src/bmi/Bmi_Py_Adapter.cpp +++ b/src/bmi/Bmi_Py_Adapter.cpp @@ -5,7 +5,7 @@ #include #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include "bmi/Bmi_Py_Adapter.hpp" @@ -102,7 +102,8 @@ void Bmi_Py_Adapter::GetValue(std::string name, void *dest) { catch (std::runtime_error &e) { std::string msg = "Encountered error getting C++ type during call to GetValue: \n"; msg += e.what(); - Logger::logMsgAndThrowError(msg); + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } if (cxx_type == "signed char") { this->copy_to_array(name, static_cast(dest)); @@ -131,7 +132,8 @@ void Bmi_Py_Adapter::GetValue(std::string name, void *dest) { } else if (cxx_type == "long double") { this->copy_to_array(name, static_cast(dest)); } else { - Logger::logMsgAndThrowError("Bmi_Py_Adapter can't get value of unsupported type: " + cxx_type); + LOG(LogLevel::FATAL, "Bmi_Py_Adapter can't get value of unsupported type: " + cxx_type); + throw std::runtime_error("Bmi_Py_Adapter can't get value of unsupported type: " + cxx_type); } } @@ -186,13 +188,13 @@ std::string Bmi_Py_Adapter::get_bmi_type_simple_name() const { * type and size of the variable in question via @ref GetVarType and @ref GetVarItemsize to infer the native * type for this variable (i.e., the actual type for the values pointed to by ``src``). It then uses this * as the type param in a nested called to the template-based @ref set_value_at_indices. If such a type - * param cannot be determined, a ``runtime_error`` is thrown. + * param cannot be determined, a ``std::runtime_error`` is thrown. * * @param name The name of the involved BMI variable. * @param inds A C++ integer array of indices to update, corresponding to each value in ``src``. * @param count Number of elements in the ``inds`` and ``src`` arrays. * @param src A C++ array containing the new values to be set in the BMI variable. - * @throws runtime_error Thrown if @ref GetVarType and @ref GetVarItemsize functions return a combination for + * @throws std::runtime_error Thrown if @ref GetVarType and @ref GetVarItemsize functions return a combination for * which there is not support for mapping to a native type in the framework. * @see set_value_at_indices */ @@ -217,10 +219,11 @@ void Bmi_Py_Adapter::SetValueAtIndices(std::string name, int *inds, int count, v else BMI_PY_SET_VALUE_INDEX(double) else BMI_PY_SET_VALUE_INDEX(long double) else { - Logger::logMsgAndThrowError( - "(Bmi_Py_Adapter) Failed attempt to SET values of BMI variable '" + name + "' from '" + + std::string msg = "(Bmi_Py_Adapter) Failed attempt to SET values of BMI variable '" + name + "' from '" + model_name + "' model: model advertises unsupported combination of type (" + val_type + - ") and size (" + std::to_string(val_item_size) + ")."); + ") and size (" + std::to_string(val_item_size) + ")."; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } #undef BMI_PY_SET_VALUE_INDEX } diff --git a/src/bmi/CMakeLists.txt b/src/bmi/CMakeLists.txt index aafcc1b6f4..d98d201d42 100644 --- a/src/bmi/CMakeLists.txt +++ b/src/bmi/CMakeLists.txt @@ -16,6 +16,8 @@ target_link_libraries(ngen_bmi NGen::core_mediator ) +target_link_libraries(ngen_bmi PRIVATE ewts::ewts_ngen_bridge) + target_sources(ngen_bmi PRIVATE "${CMAKE_CURRENT_LIST_DIR}/Bmi_Adapter.cpp" diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1d28f9f628..3d0126e15d 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -11,6 +11,10 @@ target_link_libraries(core PRIVATE NGen::parallel ) +target_link_libraries(core PRIVATE + ewts::ewts_ngen_bridge + ) + target_include_directories(core PUBLIC ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/include/simulation_time diff --git a/src/core/HY_Features.cpp b/src/core/HY_Features.cpp index b3653ce7e0..3462c6292e 100644 --- a/src/core/HY_Features.cpp +++ b/src/core/HY_Features.cpp @@ -1,7 +1,7 @@ #include #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" using namespace hy_features; diff --git a/src/core/HY_Features_MPI.cpp b/src/core/HY_Features_MPI.cpp index 3490e4f1e6..785cd0e3ec 100644 --- a/src/core/HY_Features_MPI.cpp +++ b/src/core/HY_Features_MPI.cpp @@ -1,7 +1,7 @@ #include #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_MPI diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index 05d4a8076a..9addca80d4 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_MPI #include "HY_Features_MPI.hpp" diff --git a/src/core/SurfaceLayer.cpp b/src/core/SurfaceLayer.cpp index 312f37d3ba..fd06c11c79 100644 --- a/src/core/SurfaceLayer.cpp +++ b/src/core/SurfaceLayer.cpp @@ -1,5 +1,5 @@ #include "SurfaceLayer.hpp" -#include +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_MPI #include "HY_Features_MPI.hpp" diff --git a/src/core/mediator/CMakeLists.txt b/src/core/mediator/CMakeLists.txt index 66b0bc28c8..80374fee2a 100644 --- a/src/core/mediator/CMakeLists.txt +++ b/src/core/mediator/CMakeLists.txt @@ -3,6 +3,8 @@ dynamic_sourced_cxx_library(core_mediator "${CMAKE_CURRENT_SOURCE_DIR}") add_library(NGen::core_mediator ALIAS core_mediator) +target_link_libraries(core_mediator PRIVATE ewts::ewts_ngen_bridge) + if(NGEN_WITH_UDUNITS) target_link_libraries(core_mediator PUBLIC libudunits2) endif() diff --git a/src/core/nexus/CMakeLists.txt b/src/core/nexus/CMakeLists.txt index b35a8c1f08..759a5c3c23 100644 --- a/src/core/nexus/CMakeLists.txt +++ b/src/core/nexus/CMakeLists.txt @@ -3,6 +3,8 @@ dynamic_sourced_cxx_library(core_nexus "${CMAKE_CURRENT_SOURCE_DIR}") add_library(NGen::core_nexus ALIAS core_nexus) +target_link_libraries(core_nexus PRIVATE ewts::ewts_ngen_bridge) + target_include_directories(core_nexus PUBLIC ${PROJECT_SOURCE_DIR}/include/core ${PROJECT_SOURCE_DIR}/include/core/nexus diff --git a/src/core/nexus/HY_PointHydroNexus.cpp b/src/core/nexus/HY_PointHydroNexus.cpp index b38561e2b8..26dd15bbbe 100644 --- a/src/core/nexus/HY_PointHydroNexus.cpp +++ b/src/core/nexus/HY_PointHydroNexus.cpp @@ -2,7 +2,7 @@ #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" typedef boost::error_info errmsg_info; diff --git a/src/core/nexus/HY_PointHydroNexusRemote.cpp b/src/core/nexus/HY_PointHydroNexusRemote.cpp index 1f1e0ab544..3dd9323949 100644 --- a/src/core/nexus/HY_PointHydroNexusRemote.cpp +++ b/src/core/nexus/HY_PointHydroNexusRemote.cpp @@ -6,6 +6,7 @@ #include #include +#include #include // TODO add loggin to this function @@ -126,7 +127,8 @@ double HY_PointHydroNexusRemote::get_downstream_flow(std::string catchment_id, t //because the `add_upstream_flow` call triggers a `send` which removes from the local accounting //all available water and sends it to the remote counterpart for this nexus. std::string msg = "Nexus "+id+" attempted to get_downstream_flow, but its communicator type is sender only."; - Logger::logMsgAndThrowError(msg); + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } else if ( type == receiver || type == sender_receiver ) { @@ -281,4 +283,4 @@ int HY_PointHydroNexusRemote::get_world_rank() return world_rank; } -#endif // NGEN_WITH_MPI \ No newline at end of file +#endif // NGEN_WITH_MPI diff --git a/src/forcing/CMakeLists.txt b/src/forcing/CMakeLists.txt index f64456c873..130ff2508b 100644 --- a/src/forcing/CMakeLists.txt +++ b/src/forcing/CMakeLists.txt @@ -19,6 +19,7 @@ target_link_libraries(forcing PUBLIC NGen::config_header Threads::Threads ) +target_link_libraries(forcing PRIVATE ewts::ewts_ngen_bridge) target_sources(forcing PRIVATE "${CMAKE_CURRENT_LIST_DIR}/NullForcingProvider.cpp") diff --git a/src/forcing/ForcingsEngineDataProvider.cpp b/src/forcing/ForcingsEngineDataProvider.cpp index 91a732f42d..be87ace619 100644 --- a/src/forcing/ForcingsEngineDataProvider.cpp +++ b/src/forcing/ForcingsEngineDataProvider.cpp @@ -3,7 +3,8 @@ #include // timegm #include // std::get_time -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" namespace data_access { namespace detail { @@ -30,9 +31,10 @@ void assert_forcings_engine_requirements() auto mod = interpreter_->getModule(forcings_engine_python_module); auto cls = mod.attr(forcings_engine_python_class).cast(); } catch(std::exception& e) { - Logger::logMsgAndThrowError( - "Failed to initialize ForcingsEngine: ForcingsEngine python module is not installed or is not properly configured. (" + std::string{e.what()} + ")" - ); + std::string msg = "Failed to initialize ForcingsEngine: ForcingsEngine python module is not installed or is not properly configured. (" + + std::string{e.what()} + ")"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } } @@ -40,7 +42,9 @@ void assert_forcings_engine_requirements() { const auto* wgrib2_exec = std::getenv("WGRIB2"); if (wgrib2_exec == nullptr) { - Logger::logMsgAndThrowError("Failed to initialize ForcingsEngine: $WGRIB2 is not defined"); + std::string msg = "Failed to initialize ForcingsEngine: $WGRIB2 is not defined"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } } } diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index bb8507145d..cc41abb7dd 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -8,7 +8,8 @@ #include #include #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" using namespace std; std::stringstream netcdf_ss; @@ -103,14 +104,18 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat // some sanity checks if ( id_dim_count > 1) { - Logger::logMsgAndThrowError("Provided NetCDF file has an \"ids\" variable with more than 1 dimension"); + std::string msg = "Provided NetCDF file has an \"ids\" variable with more than 1 dimension"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } auto id_dim = ids.getDim(0); if (id_dim.isNull() ) { - Logger::logMsgAndThrowError("Provided NetCDF file has a NULL dimension for variable \"ids\""); + std::string msg = "Provided NetCDF file has a NULL dimension for variable \"ids\""; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } auto num_ids = id_dim.getSize(); @@ -235,7 +240,9 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat netcdf_ss << "Error: Time intervals are not constant in forcing file\n" << std::endl; log_stream << netcdf_ss.str(); LOG(netcdf_ss.str(), LogLevel::WARNING); netcdf_ss.str(""); - Logger::logMsgAndThrowError("Time intervals in forcing file are not constant"); + std::string msg = "Time intervals in forcing file are not constant"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } } #endif diff --git a/src/forcing/NullForcingProvider.cpp b/src/forcing/NullForcingProvider.cpp index 54b244a4ff..2a699e6ec1 100644 --- a/src/forcing/NullForcingProvider.cpp +++ b/src/forcing/NullForcingProvider.cpp @@ -1,5 +1,5 @@ #include "NullForcingProvider.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include #include diff --git a/src/geojson/CMakeLists.txt b/src/geojson/CMakeLists.txt index c1f6ab8804..f22dfad944 100644 --- a/src/geojson/CMakeLists.txt +++ b/src/geojson/CMakeLists.txt @@ -12,3 +12,7 @@ target_link_libraries(geojson PUBLIC Boost::boost # Headers-only Boost NGen::logging ) +target_link_libraries(geojson PRIVATE ewts::ewts_ngen_bridge) + + + diff --git a/src/geojson/JSONProperty.cpp b/src/geojson/JSONProperty.cpp index be8bba7f56..32044dd5f7 100644 --- a/src/geojson/JSONProperty.cpp +++ b/src/geojson/JSONProperty.cpp @@ -1,5 +1,5 @@ #include "JSONProperty.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" using namespace geojson; diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 0e544b800b..2a10f58ced 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -10,3 +10,5 @@ add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/utilities) target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost SQLite::SQLite3 NGen::logging) +target_link_libraries(geopackage PRIVATE ewts::ewts_ngen_bridge) + diff --git a/src/geopackage/feature.cpp b/src/geopackage/feature.cpp index 2d20014735..202f4a45c2 100644 --- a/src/geopackage/feature.cpp +++ b/src/geopackage/feature.cpp @@ -1,5 +1,5 @@ #include "geopackage.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" // Points don't have a bounding box, so we can say its bbox is itself inline void build_point_bbox(const geojson::geometry& geom, std::vector& bbox) diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index 5f77072d4b..602ec25db8 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -2,7 +2,8 @@ #include "EndianCopy.hpp" #include "wkb.hpp" #include "proj.hpp" -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" geojson::geometry ngen::geopackage::build_geometry( const ngen::sqlite::database::iterator& row, @@ -12,7 +13,8 @@ geojson::geometry ngen::geopackage::build_geometry( { const std::vector geometry_blob = row.get>(geom_col); if (geometry_blob[0] != 'G' || geometry_blob[1] != 'P') { - Logger::logMsgAndThrowError("expected geopackage WKB, but found invalid format instead"); + LOG(LogLevel::FATAL, "expected geopackage WKB, but found invalid format instead"); + throw std::runtime_error("expected geopackage WKB, but found invalid format instead"); } int index = 3; // skip version diff --git a/src/geopackage/ngen_sqlite.cpp b/src/geopackage/ngen_sqlite.cpp index 0bf28bb608..81a136e479 100644 --- a/src/geopackage/ngen_sqlite.cpp +++ b/src/geopackage/ngen_sqlite.cpp @@ -2,7 +2,7 @@ #include #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" namespace ngen { namespace sqlite { diff --git a/src/geopackage/proj.cpp b/src/geopackage/proj.cpp index f1bd075485..70d7e2db22 100644 --- a/src/geopackage/proj.cpp +++ b/src/geopackage/proj.cpp @@ -1,5 +1,5 @@ #include "proj.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" namespace ngen { namespace srs { diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 2b41cbb4e0..c967560ea4 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -2,19 +2,24 @@ #include #include -#include +#include +#include "ewts_ngen/logger.hpp" std::stringstream read_ss(""); void check_table_name(const std::string& table) { if (boost::algorithm::starts_with(table, "sqlite_")) { - Logger::logMsgAndThrowError("table `" + table + "` is not queryable"); + std::string msg = "table `" + table + "` is not queryable"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } std::regex allowed("[^-A-Za-z0-9_ ]+"); if (std::regex_match(table, allowed)) { - Logger::logMsgAndThrowError("table `" + table + "` contains invalid characters"); + std::string msg = "table `" + table + "` contains invalid characters"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } } @@ -47,8 +52,8 @@ std::shared_ptr ngen::geopackage::read( errmsg += ", "; errquery.next(); } - - Logger::logMsgAndThrowError(errmsg); + LOG(LogLevel::FATAL, errmsg); + throw std::runtime_error(errmsg); } std::string id_column; diff --git a/src/geopackage/wkb.cpp b/src/geopackage/wkb.cpp index cdcb6f1130..1f2d2fd0bf 100644 --- a/src/geopackage/wkb.cpp +++ b/src/geopackage/wkb.cpp @@ -1,6 +1,7 @@ #include "wkb.hpp" #include "proj.hpp" -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" namespace ngen { namespace geopackage { @@ -19,12 +20,12 @@ enum wkb_geom_t { void throw_if_not_type(uint32_t given, wkb_geom_t expected) { if (given != expected) { - Logger::logMsgAndThrowError( - "expected WKB geometry type " + + std::string msg = "expected WKB geometry type " + std::to_string(expected) + ", but received " + - std::to_string(given) - ); + std::to_string(given); + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } } @@ -164,7 +165,8 @@ typename wkb::multipolygon_t wkb::read_multipolygon(const boost::span buffer) { if (buffer.size() < 5) { - Logger::logMsgAndThrowError("buffer reached end before encountering WKB"); + LOG(LogLevel::FATAL, "buffer reached end before encountering WKB"); + throw std::runtime_error("buffer reached end before encountering WKB"); } int index = 0; @@ -192,7 +194,7 @@ typename wkb::geometry wkb::read(const boost::span buffer) "this reader only implements OGC geometry types 1-6, " "but received type " + std::to_string(type) ); - LOG(throw_msg, LogLevel::WARNING); + LOG(LogLevel::FATAL, throw_msg); throw std::runtime_error(throw_msg); } diff --git a/src/partitionGenerator.cpp b/src/partitionGenerator.cpp index d724c1623a..924e7e3d4f 100644 --- a/src/partitionGenerator.cpp +++ b/src/partitionGenerator.cpp @@ -20,7 +20,7 @@ #endif #include "core/Partition_Parser.hpp" -#include +#include "ewts_ngen/logger.hpp" std::stringstream partgen_ss(""); @@ -218,7 +218,7 @@ void generate_partitions(network::Network& network, const int& num_partitions, P * @param catchment_partitions The global set of partitions * @return int partition number containing the id * - * @throws runtime_error if no partition contains the requested id + * @throws std::runtime_error if no partition contains the requested id */ int find_remote_rank(const std::string& id, const PartitionVSet& catchment_partitions) { @@ -237,7 +237,8 @@ int find_remote_rank(const std::string& id, const PartitionVSet& catchment_parti } if(pos < 0){ std::string msg = "find_remote_rank: Could not find feature id "+id+" in any partition"; - Logger::logMsgAndThrowError(msg); + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } return pos; } @@ -430,7 +431,7 @@ int main(int argc, char* argv[]) geojson::GeoJSON catchment_collection; if (boost::algorithm::ends_with(catchmentDataFile, "gpkg")) { - #if NGEN_WITH_SQLITE3 +#if NGEN_WITH_SQLITE3 try { catchment_collection = ngen::geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); } catch (std::exception &e) { @@ -440,9 +441,10 @@ int main(int argc, char* argv[]) LOG(LogLevel::FATAL, e.what()); throw; } - #else - Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); - #endif +#else + LOG(LogLevel::FATAL, "SQLite3 support required to read GeoPackage files"); + throw std::runtime_error("SQLite3 support required to read GeoPackage files"); +#endif } else { @@ -454,8 +456,10 @@ int main(int argc, char* argv[]) //Check that the number of partitions is less or equal to the number of catchment if (num_catchments < num_partitions) { - Logger::logMsgAndThrowError("Input error: total number of catchments: " + std::to_string(num_catchments) + \ - ", cannot be less than the number of partitions: " + std::to_string(num_partitions)); + std::string msg = "Input error: total number of catchments: " + std::to_string(num_catchments) + \ + ", cannot be less than the number of partitions: " + std::to_string(num_partitions); + LOG(msg,LogLevel::FATAL, msg); + throw std::runtime_error(msg); } std::string link_key = "toid"; @@ -468,7 +472,7 @@ int main(int argc, char* argv[]) geojson::GeoJSON global_nexus_collection; if (boost::algorithm::ends_with(nexusDataFile, "gpkg")) { - #if NGEN_WITH_SQLITE3 +#if NGEN_WITH_SQLITE3 try { global_nexus_collection = ngen::geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); } catch (std::exception &e) { @@ -478,9 +482,10 @@ int main(int argc, char* argv[]) LOG(LogLevel::FATAL, e.what()); throw; } - #else - Logger::logMsgAndThrowError("SQLite3 support required to read GeoPackage files."); - #endif +#else + LOG(msg,LogLevel::FATAL, "SQLite3 support required to read GeoPackage files."); + throw std::runtime_error("SQLite3 support required to read GeoPackage files."); +#endif } else { diff --git a/src/realizations/catchment/Bmi_C_Formulation.cpp b/src/realizations/catchment/Bmi_C_Formulation.cpp index c7d4398941..cfc2eaa82c 100644 --- a/src/realizations/catchment/Bmi_C_Formulation.cpp +++ b/src/realizations/catchment/Bmi_C_Formulation.cpp @@ -1,7 +1,8 @@ #include "Bmi_C_Formulation.hpp" using namespace realization; using namespace models::bmi; -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" Bmi_C_Formulation::Bmi_C_Formulation(std::string id, std::shared_ptr forcing_provider, utils::StreamHandler output_stream) : Bmi_Module_Formulation(id, forcing_provider, output_stream) { } @@ -19,7 +20,9 @@ std::string Bmi_C_Formulation::get_formulation_type() const { std::shared_ptr Bmi_C_Formulation::construct_model(const geojson::PropertyMap& properties) { auto library_file_iter = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__LIB_FILE); if (library_file_iter == properties.end()) { - Logger::logMsgAndThrowError("BMI C formulation requires path to library file, but none provided in config"); + std::string msg = "BMI C formulation requires path to library file, but none provided in config"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } std::string lib_file = library_file_iter->second.as_string(); auto reg_func_itr = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__REGISTRATION_FUNC); @@ -72,7 +75,11 @@ double Bmi_C_Formulation::get_var_value_as_double(const int& index, const std::s if (type == "unsigned long long" || type == "unsigned long long int") return (double) (model->GetValuePtr(var_name))[index]; - Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + " as double: no logic for converting variable type " + type); + std::string msg = "Unable to get value of variable " + var_name + " from " + + get_model_type_name() + " as double: no logic for converting variable type " + type; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); + return 1.0; } diff --git a/src/realizations/catchment/Bmi_Cpp_Formulation.cpp b/src/realizations/catchment/Bmi_Cpp_Formulation.cpp index 921675ffe7..dffb0e2649 100644 --- a/src/realizations/catchment/Bmi_Cpp_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Cpp_Formulation.cpp @@ -1,5 +1,6 @@ #include "Bmi_Cpp_Formulation.hpp" -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" using namespace realization; using namespace models::bmi; @@ -23,7 +24,9 @@ std::string Bmi_Cpp_Formulation::get_formulation_type() const { std::shared_ptr Bmi_Cpp_Formulation::construct_model(const geojson::PropertyMap& properties) { auto json_prop_itr = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__LIB_FILE); if (json_prop_itr == properties.end()) { - Logger::logMsgAndThrowError("BMI C++ formulation requires path to library file, but none provided in config"); + std::string msg = "BMI C++ formulation requires path to library file, but none provided in config"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } std::string lib_file = json_prop_itr->second.as_string(); @@ -84,8 +87,11 @@ double Bmi_Cpp_Formulation::get_var_value_as_double(const int& index, const std: if (type == "unsigned long long" || type == "unsigned long long int") return (double) (model->GetValuePtr(var_name))[index]; - Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + - " as double: no logic for converting variable type " + type); + std::string msg = "Unable to get value of variable " + var_name + " from " + get_model_type_name() + + " as double: no logic for converting variable type " + type; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); + return 1.0; } diff --git a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp index bfd59b0795..04b7e7e51a 100644 --- a/src/realizations/catchment/Bmi_Fortran_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Fortran_Formulation.cpp @@ -1,5 +1,6 @@ #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" #if NGEN_WITH_BMI_FORTRAN @@ -29,7 +30,9 @@ Bmi_Fortran_Formulation::Bmi_Fortran_Formulation(std::string id, std::shared_ptr std::shared_ptr Bmi_Fortran_Formulation::construct_model(const geojson::PropertyMap& properties) { auto library_file_iter = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__LIB_FILE); if (library_file_iter == properties.end()) { - Logger::logMsgAndThrowError("BMI C formulation requires path to library file, but none provided in config"); + std::string msg = "BMI C formulation requires path to library file, but none provided in config."; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } std::string lib_file = library_file_iter->second.as_string(); auto reg_func_itr = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__REGISTRATION_FUNC); @@ -54,7 +57,7 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const // don't fit or might convert inappropriately std::string type = model->GetVarType(var_name); //Can cause a segfault here if GetValue returns an empty vector...a "fix" in bmi_utilities GetValue - //will throw a relevant runtime_error if the vector is empty, so this is safe to use this way for now... + //will throw a relevant std::runtime_error if the vector is empty, so this is safe to use this way for now... if (type == "long double") return (double) (models::bmi::GetValue(*model, var_name))[index]; @@ -88,8 +91,10 @@ double Bmi_Fortran_Formulation::get_var_value_as_double(const int &index, const if (type == "unsigned long long" || type == "unsigned long long int") return (double) (models::bmi::GetValue(*model, var_name))[index]; - Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + - " as double: no logic for converting variable type " + type); + std::string msg = "Unable to get value of variable " + var_name + " from " + get_model_type_name() + + " as double: no logic for converting variable type " + type; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); return 1.0; } diff --git a/src/realizations/catchment/Bmi_Module_Formulation.cpp b/src/realizations/catchment/Bmi_Module_Formulation.cpp index 3fd0b3dcd7..359a257f88 100644 --- a/src/realizations/catchment/Bmi_Module_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Module_Formulation.cpp @@ -1,7 +1,8 @@ #include "Bmi_Module_Formulation.hpp" #include "utilities/logging_utils.h" #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" #include "state_save_restore/State_Save_Utils.hpp" #include @@ -247,7 +248,7 @@ namespace realization { // consistent with times /* if (last_model_response_delta == 0 && last_model_response_start_time == 0) { - throw runtime_error(get_formulation_type() + " does not properly set output time validity ranges " + throw std::runtime_error(get_formulation_type() + " does not properly set output time validity ranges " "needed to provide outputs as forcings"); } */ @@ -308,7 +309,7 @@ namespace realization { // consistent with times /* if (last_model_response_delta == 0 && last_model_response_start_time == 0) { - throw runtime_error(get_formulation_type() + " does not properly set output time validity ranges " + throw std::runtime_error(get_formulation_type() + " does not properly set output time validity ranges " "needed to provide outputs as forcings"); } */ diff --git a/src/realizations/catchment/Bmi_Multi_Formulation.cpp b/src/realizations/catchment/Bmi_Multi_Formulation.cpp index 1f4fbbf245..ed5b296193 100644 --- a/src/realizations/catchment/Bmi_Multi_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Multi_Formulation.cpp @@ -11,7 +11,8 @@ #include "Bmi_C_Formulation.hpp" #include "Bmi_Fortran_Formulation.hpp" #include "Bmi_Py_Formulation.hpp" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" +#include #include "state_save_restore/vecbuf.hpp" #include "state_save_restore/State_Save_Utils.hpp" @@ -128,12 +129,15 @@ void Bmi_Multi_Formulation::create_multi_formulation(geojson::PropertyMap proper #endif // NGEN_WITH_PYTHON } if (inactive_type_requested) { - Logger::logMsgAndThrowError( - get_formulation_type() + " could not initialize sub formulation of type " + type_name + - " due to support for this type not being activated."); + std::string msg = get_formulation_type() + " could not initialize sub formulation of type " + type_name + + " due to support for this type not being activated."; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } if (module == nullptr) { - Logger::logMsgAndThrowError(get_formulation_type() + " received unexpected subtype formulation " + type_name); + std::string msg = get_formulation_type() + " received unexpected subtype formulation " + type_name; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } modules[i] = module; @@ -527,7 +531,9 @@ std::string Bmi_Multi_Formulation::get_output_line_for_timestep(int timestep, st void Bmi_Multi_Formulation::update(time_step_t t_index, time_step_t t_delta) { if (modules.empty()) { - Logger::logMsgAndThrowError("Trying to get response of improperly created empty BMI multi-module formulation."); + std::string msg = "Trying to get response of improperly created empty BMI multi-module formulation."; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } if (t_index < 0) { throw std::invalid_argument( diff --git a/src/realizations/catchment/Bmi_Py_Formulation.cpp b/src/realizations/catchment/Bmi_Py_Formulation.cpp index 1712ec9cc7..c6f14b3ff9 100644 --- a/src/realizations/catchment/Bmi_Py_Formulation.cpp +++ b/src/realizations/catchment/Bmi_Py_Formulation.cpp @@ -1,5 +1,6 @@ #include -#include "Logger.hpp" +#include +#include "ewts_ngen/logger.hpp" #include "state_save_restore/State_Save_Utils.hpp" #if NGEN_WITH_PYTHON @@ -17,7 +18,9 @@ Bmi_Py_Formulation::Bmi_Py_Formulation(std::string id, std::shared_ptr Bmi_Py_Formulation::construct_model(const geojson::PropertyMap &properties) { auto python_type_name_iter = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__PYTHON_TYPE_NAME); if (python_type_name_iter == properties.end()) { - Logger::logMsgAndThrowError("BMI Python formulation requires Python model class type, but none given in config"); + std::string msg = "BMI Python formulation requires Python model class type, but none given in config"; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); } //Load a custom module path, if provided auto python_module_path_iter = properties.find(BMI_REALIZATION_CFG_PARAM_OPT__PYTHON_MODULE_PATH); @@ -78,8 +81,10 @@ double Bmi_Py_Formulation::get_var_value_as_double(const int &index, const std:: else PY_BMI_DOUBLE_AT_INDEX(double) else PY_BMI_DOUBLE_AT_INDEX(long double) #undef PY_BMI_DOUBLE_AT_INDEX - Logger::logMsgAndThrowError("Unable to get value of variable " + var_name + " from " + get_model_type_name() + - " as double: no logic for converting variable type " + val_type); + std::string msg = "Unable to get value of variable " + var_name + " from " + get_model_type_name() + + " as double: no logic for converting variable type " + val_type; + LOG(LogLevel::FATAL, msg); + throw std::runtime_error(msg); return 1.0; } diff --git a/src/realizations/catchment/CMakeLists.txt b/src/realizations/catchment/CMakeLists.txt index 6df221d6af..0cc951afd0 100644 --- a/src/realizations/catchment/CMakeLists.txt +++ b/src/realizations/catchment/CMakeLists.txt @@ -33,3 +33,6 @@ target_link_libraries(realizations_catchment PUBLIC NGen::bmi_protocols ) +target_link_libraries(realizations_catchment PRIVATE ewts::ewts_ngen_bridge) + + diff --git a/src/utilities/CMakeLists.txt b/src/utilities/CMakeLists.txt index 1b0788f3b6..b1a2aa0c3d 100644 --- a/src/utilities/CMakeLists.txt +++ b/src/utilities/CMakeLists.txt @@ -12,6 +12,8 @@ target_link_libraries(ngen_parallel NGen::logging ) +target_link_libraries(ngen_parallel PRIVATE ewts::ewts_ngen_bridge) + if(NGEN_WITH_MPI) target_link_libraries(ngen_parallel PUBLIC diff --git a/src/utilities/bmi/CMakeLists.txt b/src/utilities/bmi/CMakeLists.txt index 5d86e87f70..ca4bb2bb4b 100644 --- a/src/utilities/bmi/CMakeLists.txt +++ b/src/utilities/bmi/CMakeLists.txt @@ -31,6 +31,8 @@ target_link_libraries(ngen_bmi_protocols NGen::logging ) +target_link_libraries(ngen_bmi_protocols PRIVATE ewts::ewts_ngen_bridge) + target_sources(ngen_bmi_protocols PRIVATE "${PROJECT_SOURCE_DIR}/src/bmi/Bmi_Adapter.cpp" diff --git a/src/utilities/bmi/mass_balance.cpp b/src/utilities/bmi/mass_balance.cpp index f542a682a0..05cfb04810 100644 --- a/src/utilities/bmi/mass_balance.cpp +++ b/src/utilities/bmi/mass_balance.cpp @@ -29,7 +29,7 @@ namespace models { namespace bmi { namespace protocols { NgenMassBalance::NgenMassBalance(const ModelPtr& model, const Properties& properties) : check(false), is_fatal(false), tolerance(1.0E-16), frequency(1){ - initialize(model, properties); + auto init = initialize(model, properties); } NgenMassBalance::NgenMassBalance() : check(false) {} @@ -183,7 +183,7 @@ auto NgenMassBalance::initialize(const ModelPtr& model, const Properties& proper } if ( check ) { //Ensure the model is capable of mass balance using the protocol - check_support(model).or_else( error_or_warning ); + auto result = check_support(model).or_else( error_or_warning ); } return {}; // important to return for the expected to be properly created! } diff --git a/src/utilities/logging/CMakeLists.txt b/src/utilities/logging/CMakeLists.txt index 465cff20b0..0080d26680 100644 --- a/src/utilities/logging/CMakeLists.txt +++ b/src/utilities/logging/CMakeLists.txt @@ -1,7 +1,9 @@ -add_library(logging logging_utils.cpp Logger.cpp) +add_library(logging logging_utils.cpp) add_library(NGen::logging ALIAS logging) target_include_directories(logging PUBLIC ${PROJECT_SOURCE_DIR}/include/utilities) +target_link_libraries(logging PRIVATE ewts::ewts_ngen_bridge) + target_link_libraries(logging PUBLIC Boost::boost ) diff --git a/src/utilities/logging/Logger.cpp b/src/utilities/logging/Logger.cpp deleted file mode 100644 index fb4c8c785a..0000000000 --- a/src/utilities/logging/Logger.cpp +++ /dev/null @@ -1,671 +0,0 @@ -#include "Logger.hpp" -#include -#include -#include -#include -#include // For getenv() -#include -#include -#include // For file handling -#include -#include -#include -#include // For std::string -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -const std::string MODULE_NAME = "ngen"; -const std::string LOG_DIR_NGENCERF = "/ngencerf/data"; // ngenCERF log directory string if environement var empty. -const std::string LOG_DIR_DEFAULT = "run-logs"; // Default parent log directory string if env var empty & ngencerf dosn't exist -const std::string LOG_FILE_EXT = "log"; // Log file name extension -const std::string DS = "/"; // Directory separator -const unsigned int LOG_MODULE_NAME_LEN = 8; // Width of module name for log entries - -const std::string EV_EWTS_LOGGING = "NGEN_EWTS_LOGGING"; // Enable/disable of Error Warning and Trapping System -const std::string EV_NGEN_LOGFILEPATH = "NGEN_LOG_FILE_PATH"; // ngen log file - -const std::string CONFIG_FILENAME = "ngen_logging.json"; // ngen logging config file - -// String to LogLevel map -static const std::unordered_map logLevelMap = { - {"NONE", LogLevel::NONE}, {"0", LogLevel::NONE}, - {"DEBUG", LogLevel::DEBUG}, {"1", LogLevel::DEBUG}, - {"INFO", LogLevel::INFO}, {"2", LogLevel::INFO}, - {"WARNING", LogLevel::WARNING}, {"3", LogLevel::WARNING}, - {"SEVERE", LogLevel::SEVERE}, {"4", LogLevel::SEVERE}, - {"FATAL", LogLevel::FATAL}, {"5", LogLevel::FATAL}, -}; - -// Reverse map: LogLevel to String -static const std::unordered_map logLevelToStringMap = { - {LogLevel::NONE, "NONE "}, - {LogLevel::DEBUG, "DEBUG "}, - {LogLevel::INFO, "INFO "}, - {LogLevel::WARNING, "WARNING"}, - {LogLevel::SEVERE, "SEVERE "}, - {LogLevel::FATAL, "FATAL "}, -}; - -const std::unordered_map moduleNamesMap = { - {"NGEN", "NGEN"}, - {"CFE-S", "CFE"}, - {"CFE-X", "CFE"}, - {"LASAM", "LASAM"}, - {"NOAH-OWP-MODULAR", "NOAHOWP"}, - {"PET", "PET"}, - {"SAC-SMA", "SACSMA"}, - {"SFT", "SFT"}, - {"SMP", "SMP"}, - {"SNOW-17", "SNOW17"}, - {"TOPMODEL", "TOPMODEL"}, - {"TOPOFLOW-GLACIER", "TFGLACR"}, - {"T-ROUTE", "TROUTE"}, - {"UEB", "UEB_BMI"}, - {"LSTM", "LSTM"}, - {"FORCING", "FORCING"} -}; - -bool Logger::DirectoryExists(const std::string& path) { - struct stat info; - if (stat(path.c_str(), &info) != 0) { - return false; // Cannot access path - } - return (info.st_mode & S_IFDIR) != 0; -} - -/** - * Create the directory checking both the call - * to execute the command and the result of the command - */ -bool Logger::CreateDirectory(const std::string& path) { - - if (!DirectoryExists(path)) { - std::string mkdir_cmd = "mkdir -p " + path; - int status = system(mkdir_cmd.c_str()); - - if (status == -1) { - std::cerr << "[CRITICAL] " << MODULE_NAME << " system() failed to run mkdir.\n"; - return false; - } else if (WIFEXITED(status)) { - int exitCode = WEXITSTATUS(status); - if (exitCode != 0) { - std::cerr << "[CRITICAL] " << MODULE_NAME << " mkdir command failed with exit code: " << exitCode << "\n"; - return false; - } - } else { - std::cerr << "[CRITICAL] " << MODULE_NAME << " mkdir terminated abnormally.\n"; - return false; - } - } - return true; -} - -/** - * Open log file and return open status. If already open, - * ensure the write pointer is at the end of the file. - * - * return bool true if open and good, false otherwise - */ -bool Logger::LogFileReady(void) { - - if (openedOnce && logFile.is_open() && logFile.good()) { - logFile.seekp(0, std::ios::end); // Ensure write pointer is at the actual file end - return true; - } - else if (openedOnce) { - // Somehow the logfile was closed. Open in append mode so - // previosly logged messages are not lost - logFile.open(logFilePath, ios::out | ios::app); // This will silently fail if already open. - if (logFile.good()) return true; - } - return false; -} - -void Logger::SetupLogFile(void) { - - // Determine the log file directory and log file name. - // Use name from environment variable if set, otherwise use a default - if (!ngenResultsDir.empty()) { - logFileDir = ngenResultsDir + DS + "logs"; - if (CreateDirectory(logFileDir)) - logFilePath = logFileDir + DS + MODULE_NAME + "." + LOG_FILE_EXT; - } - if (logFilePath.empty()) { - // Get parent log directory - if (DirectoryExists(LOG_DIR_NGENCERF)) { - logFileDir = LOG_DIR_NGENCERF + DS + LOG_DIR_DEFAULT; - } - else { - const char *home = getenv("HOME"); // Get users home directory pathname - std::string dir = (home) ? home : "~"; - logFileDir = dir + DS + LOG_DIR_DEFAULT; - } - - // Ensure parent log direcotry exists - if (CreateDirectory(logFileDir)) { - // Get full log directory path - const char* envUsername = std::getenv("USER"); - std::string dirName = (envUsername) ? envUsername : CreateDateString(); - logFileDir = logFileDir + DS + dirName; - - // Set the full path if log directory exists/created - if (CreateDirectory(logFileDir)) - logFilePath = logFileDir + DS + MODULE_NAME + "_" + CreateTimestamp(false,false) + "." + LOG_FILE_EXT; - } - } - - // Attempt to open log file - if (!logFilePath.empty()) { - logFile.open(logFilePath, ios::out | ios::trunc); // Truncating ensures keeping only the last calibration iteration. - if (logFile.is_open()) { - openedOnce = true; - std::cout << "[DEBUG] " << MODULE_NAME << " Log File: " << logFilePath << std::endl; - return; - } - } - std::cout << "[WARNING] " << MODULE_NAME << " Unable to create log file "; - if (!logFilePath.empty()) { - std::cout << logFilePath; - } - else if (!logFileDir.empty()) { - std::cout << logFileDir; - } - std::cout << " (Perhaps check permissions)" << std::endl; - std::cout << "[WARNING] " << MODULE_NAME << " Log entries will be written to stdout" << std::endl; -} - -std::string Logger::ToUpper(const std::string& input) { - std::string result = input; - std::transform(result.begin(), result.end(), result.begin(), - [](unsigned char c){ return std::toupper(c); }); - return result; -} - -std::string Logger::ExtractFirstNDirs(const std::string& path, int numDirs) { - size_t pos = 0; - int slashCount = 0; - - while (pos < path.length() && slashCount < numDirs) { - if (path[pos] == '/') { - ++slashCount; - } - ++pos; - } - - // If the path starts with '/', keep it as is; otherwise return substring - return path.substr(0, pos); -} - -std::string CleanJsonToken(const std::string& token) { - std::string s = token; - if (!s.empty() && s.front() == '"') s.erase(0, 1); - if (!s.empty() && s.back() == ',') s.pop_back(); - if (!s.empty() && s.back() == '"') s.pop_back(); - return s; -} - -/* - JSON file format exmaple: - { - "logging_enabled": true, - "modules": { - "ngen": "INFO", - "CFE-S": "INFO", - "UEB": "INFO", - "Noah-OWP-Modular": "DEBUG", - "T-Route": "INFO" - } - } -*/ - -bool Logger::ParseLoggerConfigFile(std::ifstream& jsonFile) -{ - // Rewind file in case it's been partially read - jsonFile.clear(); - jsonFile.seekg(0, std::ios::beg); - - try { - // Read the JSON into a property tree - boost::property_tree::ptree config; - boost::property_tree::read_json(jsonFile, config); - - // Read logging_enabled flag - try { - loggingEnabled = config.get("logging_enabled", true); // default true if missing - std::cout << "[DEBUG] " << MODULE_NAME << " Logging " - << (loggingEnabled ? "ENABLED" : "DISABLED") << std::endl; - } - catch (const boost::property_tree::ptree_bad_data& e) { - std::cout << "[ERROR] " << MODULE_NAME << " JSON data error: " << e.what() << std::endl; - return false; - } - - // Read modules subtree only if logging enabled - if (loggingEnabled) { - bool atLeastOneModuleFound = false; - if (auto modulesOpt = config.get_child_optional("modules")) { - for (const auto& kv : *modulesOpt) { - std::string moduleName = ToUpper(kv.first); - std::string levelStr = ToUpper(kv.second.get_value()); - - auto it = moduleNamesMap.find(moduleName); - if (it != moduleNamesMap.end()) { - atLeastOneModuleFound = true; - moduleLogLevels[moduleName] = ConvertStringToLogLevel(levelStr); - std::cout << "[DEBUG] " << MODULE_NAME << " Found Log level " - << kv.first << "=" - << ConvertLogLevelToString(moduleLogLevels[moduleName]) - << std::endl; - if (moduleName == "NGEN") logLevel = moduleLogLevels[moduleName]; - } else { - std::cout << "[ERROR] " << MODULE_NAME << " Ignoring unknown module " << moduleName << std::endl; - } - } - } else { - std::cout << "[ERROR] " << MODULE_NAME << " Missing 'modules' section in logging.json." << std::endl; - } - return atLeastOneModuleFound; - } - return true; - } - catch (const boost::property_tree::json_parser_error& e) { - std::cout << "[ERROR] " << MODULE_NAME << " JSON parse error: " << e.what() << std::endl; - } - catch (const std::exception& e) { - std::cout << "[ERROR] " << MODULE_NAME << " Exception while parsing config: " << e.what() << std::endl; - } - return false; -} - -void Logger::ReadConfigFile(std::string searchPath) { - - bool success = false; - std::ifstream jsonFile; - - // Set logger defaults - moduleLogLevels.clear(); - loggingEnabled = true; - - // Open and Parse config file - if (searchPath.empty()) { - std::cout << "[WARNING] " << MODULE_NAME << " Logging config file cannot be read from NGEN_RESULTS_DIR environment variable because not set or empty." << std::endl; - std::cout << "[WARNING] " << MODULE_NAME << " Using defaults for logging." << std::endl; - } else { - if (FindAndOpenLogConfigFile(searchPath, jsonFile)) { - if (jsonFile.peek() != std::ifstream::traits_type::eof()) { - std::cout << "[DEBUG] " << MODULE_NAME << " parsing logging config file " << searchPath << std::endl; - success = ParseLoggerConfigFile(jsonFile); - } - } - } - if (loggingEnabled && !success) { - std::cout << "[WARNING] " << MODULE_NAME << " Issue with logging config file " << CONFIG_FILENAME << " in " << ((searchPath.empty())?"undefined path":searchPath) << "." << std::endl; - std::cout << "[WARNING] " << MODULE_NAME << " Using default logging configuration of enabled and log level INFO for all known modules" << std::endl; - for (const auto kv : moduleNamesMap) { - std::string moduleName = ToUpper(kv.first); - moduleLogLevels[moduleName] = LogLevel::INFO; - } - } -} - -bool Logger::IsValidEnvVarName(const std::string& name) { - if (name.empty()) return false; - - // First character must be a letter or underscore - if (!std::isalpha(name[0]) && name[0] != '_') return false; - - // All other characters must be alphanumeric or underscore - for (size_t i = 1; i < name.size(); ++i) { - if (!std::isalnum(name[i]) && name[i] != '_') return false; - } - - return true; -} - -void Logger::ManageLoggingEnvVars(bool set) { - - // Set logger env vars common to all modules - if (set) { - Log("ngen Logger setup: Setting Module Logger Environment Variables", LogLevel::DEBUG); - if (!logFilePath.empty()) { - // Set the log file env var - setenv("NGEN_LOG_FILE_PATH", logFilePath.c_str(), 1); - std::cout << "[DEBUG] " << MODULE_NAME << " Set env var NGEN_LOG_FILE_PATH=" << logFilePath << std::endl; - } - else { - std::cout << "[WARNING] " << MODULE_NAME << " NGEN_LOG_FILE_PATH env var not set. Modules writing to their default logs." << std::endl; - } - - // Set the logging enabled/disabled env var - setenv((EV_EWTS_LOGGING).c_str(), ((loggingEnabled)?"ENABLED":"DISABLED"), 1); - std::string logMsg = std::string("Set Logging ") + ((loggingEnabled)?"ENABLED":"DISABLED"); - std::cout << logMsg << "(envVar=" << EV_EWTS_LOGGING << ")" << std::endl; - if (!logFilePath.empty()) { - LogLevel saveLevel = logLevel; - logLevel = LogLevel::INFO; // Ensure this INFO message is always logged - Log(logMsg, logLevel); - logLevel = saveLevel; - } - } - else { - Log("Logger setup: Unset existing Module Logger Environment Variables", LogLevel::DEBUG); - } - - // Set logger env vars unique to each module in the formulation - // Note: moduleLogLevels is populated from the logging config file which - // only contains the log levels for module in the formulation - for (const auto& modulePair : moduleLogLevels) { - std::string envVar; - std::string moduleNameForEnvVar; - - const std::string& moduleName = modulePair.first; - LogLevel level = modulePair.second; - - // Look up the module env var name in moduleNamesMap - auto it = moduleNamesMap.find(moduleName); - if (it != moduleNamesMap.end()) { - moduleNameForEnvVar = it->second; - if (!IsValidEnvVarName(moduleNameForEnvVar)) { - std::string logMsg = std::string("Invalid env var name ") + moduleNameForEnvVar + - std::string(" for module ") + moduleName; - Log(logMsg, LogLevel::WARNING); - continue; - } - } - else { - std::string logMsg = std::string("Unknown module in logLevels: ") + moduleName; - Log(logMsg, LogLevel::WARNING); - continue; - } - - if (set) { - // Sets the log level envirnoment variable - envVar = moduleNameForEnvVar + "_LOGLEVEL"; - std::string ll = ConvertLogLevelToString(level); - setenv(envVar.c_str(), ll.c_str(), 1); - std::string logMsg = std::string("Set ") + moduleName - + ((moduleName != "NGEN")?" Log Level env var to ":" Log Level to ") - + TrimString(ll); - std::cout << logMsg; - if (moduleName != "NGEN") std::cout << " (" << envVar << ")"; - std::cout << std::endl; - if (!logFilePath.empty()) { - LogLevel saveLevel = logLevel; - logLevel = LogLevel::INFO; // Ensure this INFO message is always logged - Log(logMsg, logLevel); - logLevel = saveLevel; - } - } - else { - if (moduleName != "NGEN") { - // It is possible that individual submodules may be writing to their own - // logs if there was an issue accessing the ngen log file. The log file used - // by each module is stored in its own environment variable. Unsetting this - // environment variable will cause the modules to truncate their logs. - // This is important when running calibrations since iterative runs of ngen - // can number in the 1000's and it is only necessary to retain the last ngen run - envVar = moduleNameForEnvVar + "_LOGFILEPATH"; - unsetenv(envVar.c_str()); - envVar = moduleNameForEnvVar + "_LOGLEVEL"; - unsetenv(envVar.c_str()); - } - } - } -} - -/** -* Configure Logger Preferences and open log file -* @param level: LogLevel::WARNING by Default -* @return void -*/ -void Logger::SetLogPreferences(LogLevel level) { - - if (!loggerInitialized) { - loggerInitialized = true; // Only call this once - - // Unset any existing related environment vars - ManageLoggingEnvVars(false); - - // Determine the log file directory and log file name. - // Use name from environment variable if set, otherwise use a default - const char* envVar = std::getenv("NGEN_RESULTS_DIR"); // Currently set by ngen-cal but envision set for WCOSS at some point - if (envVar != nullptr && envVar[0] != '\0') { - ngenResultsDir = envVar; - std::cout << "[DEBUG] " << MODULE_NAME << " Found envVar NGEN_RESULTS_DIR = " << ngenResultsDir << std::endl; - } - - ReadConfigFile(ngenResultsDir); - - if (loggingEnabled) { - - // Make sure the module name used for logging is all uppercase and LOG_MODULE_NAME_LEN characters wide. - moduleName = MODULE_NAME; - std::string upperName = moduleName.substr(0, LOG_MODULE_NAME_LEN); // Truncate to LOG_MODULE_NAME_LEN chars max - std::transform(upperName.begin(), upperName.end(), upperName.begin(), ::toupper); - - std::ostringstream oss; - oss << std::left << std::setw(LOG_MODULE_NAME_LEN) << std::setfill(' ') << upperName; - moduleName = oss.str(); - - SetupLogFile(); - - // Set the environment variables for the module loggers - ManageLoggingEnvVars(true); - } - } -} - -void Logger::Log(LogLevel messageLevel, const char* message, ...) { - va_list args; - va_start(args, message); - - // Make a copy to calculate required size - va_list args_copy; - va_copy(args_copy, args); - int requiredLen = vsnprintf(nullptr, 0, message, args_copy); - va_end(args_copy); - - if (requiredLen > 0) { - std::vector buffer(requiredLen + 1); // +1 for null terminator - vsnprintf(buffer.data(), buffer.size(), message, args); - - va_end(args); - - Log(std::string(buffer.data()), messageLevel); - } else { - va_end(args); // still need to clean up - } -} - -/** - * Log given message with defined parameters and generate message to pass on Console or File - * @param message: Log Message - * @param messageLevel: Log Level, LogLevel::INFO by default - */ -void Logger::Log(LogLevel messageLevel, std::string message) { - Log(message, messageLevel); -} -/** -* Log given message with defined parameters and generate message to pass on Console or File -* @param message: Log Message -* @param messageLevel: Log Level, LogLevel::INFO by default -*/ -void Logger::Log(std::string message, LogLevel messageLevel) { - Logger *logger = GetLogger(); - - // Log only when appropriate - if ((logger->loggingEnabled) && (messageLevel >= logger->logLevel)) { - std::string logType = ConvertLogLevelToString(messageLevel); - std::string logPrefix = CreateTimestamp() + " " + logger->moduleName + " " + logType; - - // Log message, creating individual entries for a multi-line message - std::istringstream logMsg(message); - std::string line; - if (logger->LogFileReady()) { - while (std::getline(logMsg, line)) { - logger->logFile << logPrefix + " " + line << std::endl; - } - logger->logFile.flush(); - } - else { - // Log file not found. Write to stdout. - while (std::getline(logMsg, line)) { - std::cout << logPrefix + " " + line << std::endl; - } - std::cout << std::flush; - } - } -} - -Logger* Logger::GetLogger() -{ - static Logger* logger = nullptr; - if (logger == nullptr) { - logger = new Logger; - logger->SetLogPreferences(); - } - return logger; -} - -Logger::Logger() -{ - -} - -// Function to trim leading and trailing spaces -std::string Logger::TrimString(const std::string& str) { - // Trim leading spaces - size_t first = str.find_first_not_of(" \t\n\r\f\v"); - if (first == std::string::npos) { - return ""; // No non-whitespace characters - } - - // Trim trailing spaces - size_t last = str.find_last_not_of(" \t\n\r\f\v"); - - // Return the trimmed string - return str.substr(first, last - first + 1); -} - -std::string Logger::ConvertLogLevelToString(LogLevel level) { - auto it = logLevelToStringMap.find(level); - if (it != logLevelToStringMap.end()) { - return it->second; // Found valid named or numeric log level - } - return "NONE"; -} - -/** -* Convert String Representation of Log Level to LogLevel Type -* @param levelStr : String log level -* @return LogLevel -*/ -LogLevel Logger::ConvertStringToLogLevel(const std::string& levelStr) { - std::string level = TrimString(levelStr); - if (!level.empty()) { - // Convert string to LogLevel (supports both names and numbers) - auto it = logLevelMap.find(level); - if (it != logLevelMap.end()) { - return it->second; // Found valid named or numeric log level - } - - // Try parsing as an integer (for cases where an invalid numeric value is given) - try { - int levelNum = std::stoi(level); - if (levelNum >= 0 && levelNum <= 5) { - return static_cast(levelNum); - } - } catch (...) { - // Ignore errors (e.g., if std::stoi fails for non-numeric input) - } - } - return LogLevel::NONE; -} - -std::string Logger::CreateTimestamp(bool appendMS, bool iso) { - using namespace std::chrono; - - // Get current time point - auto now = system_clock::now(); - auto now_time_t = system_clock::to_time_t(now); - - // Get milliseconds - auto ms = duration_cast(now.time_since_epoch()) % 1000; - - // Convert to UTC time - std::tm utc_tm; - gmtime_r(&now_time_t, &utc_tm); - - // Format date/time with strftime - char buffer[32]; - if (iso) { - std::strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &utc_tm); - } - else { - std::strftime(buffer, sizeof(buffer), "%Y%m%dT%H%M%S", &utc_tm); - } - - if (appendMS) { - // Combine with milliseconds - std::ostringstream oss; - oss << buffer << '.' << std::setw(3) << std::setfill('0') << ms.count(); - return oss.str(); - } - return std::string(buffer); -} - -std::string Logger::CreateDateString(void) { - std::time_t tt = std::time(0); - std::tm* timeinfo = std::gmtime(&tt); // Use std::localtime(&tt) if you want local time - - char buffer[11]; // Enough for "YYYY-MM-DD" + null terminator - std::strftime(buffer, sizeof(buffer), "%F", timeinfo); // %F == %Y-%m-%d - - std::stringstream ss; - ss << buffer; - - return ss.str(); -} - -std::string Logger::GetLogFilePath(void) { - return logFilePath; -} - -bool Logger::FileExists(const std::string& path) { - struct stat statbuf{}; - return stat(path.c_str(), &statbuf) == 0 && S_ISREG(statbuf.st_mode); -} - -std::string Logger::GetParentDirName(const std::string& path) { - size_t pos = path.find_last_of('/'); - if (pos == std::string::npos || pos == 0) return "/"; - return path.substr(0, pos); -} - -bool Logger::FindAndOpenLogConfigFile(std::string path, std::ifstream& configFileStream) { - while (!path.empty() && path != "/") { - std::string candidate = path + DS + CONFIG_FILENAME; - if (FileExists(candidate)) { - std::cout << "[DEBUG] " << MODULE_NAME << " Opening logger config file " << candidate << std::endl; - configFileStream.open(candidate); - return configFileStream.is_open(); - } - path = GetParentDirName(path); - } - return false; -} - -LogLevel Logger::GetLogLevel(void) { - return logLevel; -} - -bool Logger::IsLoggingEnabled(void) { - return loggingEnabled; -} diff --git a/src/utilities/logging/logging_utils.cpp b/src/utilities/logging/logging_utils.cpp index 94a0f80567..43f4fc8620 100644 --- a/src/utilities/logging/logging_utils.cpp +++ b/src/utilities/logging/logging_utils.cpp @@ -2,7 +2,7 @@ #include #include #include "logging_utils.h" -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" namespace logging { diff --git a/src/utilities/mdframe/CMakeLists.txt b/src/utilities/mdframe/CMakeLists.txt index cafdf651fc..0073046398 100644 --- a/src/utilities/mdframe/CMakeLists.txt +++ b/src/utilities/mdframe/CMakeLists.txt @@ -7,6 +7,7 @@ NGen::config_header NGen::mdarray NGen::logging ) +target_link_libraries(mdframe PRIVATE ewts::ewts_ngen_bridge) if(NGEN_WITH_NETCDF) target_link_libraries(mdframe PUBLIC NetCDF) diff --git a/src/utilities/mdframe/handler_csv.cpp b/src/utilities/mdframe/handler_csv.cpp index 6f9d291943..8e4526ee97 100644 --- a/src/utilities/mdframe/handler_csv.cpp +++ b/src/utilities/mdframe/handler_csv.cpp @@ -1,5 +1,5 @@ #include -#include "Logger.hpp" +#include "ewts_ngen/logger.hpp" #include "mdframe/mdframe.hpp" #include @@ -41,9 +41,10 @@ void cartesian_indices(const boost::span shape, std::vector variable_subset; @@ -65,7 +66,8 @@ void mdframe::to_csv(const std::string& path, bool header) const } if (variable_subset.empty()) { - Logger::logMsgAndThrowError("cannot output CSV with no output variables"); + LOG(LogLevel::FATAL, "cannot output CSV with no output variables"); + throw std::runtime_error("cannot output CSV with no output variables"); } // Calculate total number of rows across all subdimensions (not including header) diff --git a/src/utilities/mdframe/handler_netcdf.cpp b/src/utilities/mdframe/handler_netcdf.cpp index d36c5a9a65..0858a080d8 100644 --- a/src/utilities/mdframe/handler_netcdf.cpp +++ b/src/utilities/mdframe/handler_netcdf.cpp @@ -82,7 +82,8 @@ void mdframe::to_netcdf(const std::string& path) const namespace ngen { void mdframe::to_netcdf(const std::string& path) const { - Logger::logMsgAndThrowError("This functionality isn't available. Compile NGen with NGEN_WITH_NETCDF=ON to enable NetCDF support"); + LOG(LogLevel::FATAL, "This functionality isn't available. Compile NGen with NGEN_WITH_NETCDF=ON to enable NetCDF support"); + throw std::runtime_error("This functionality isn't available. Compile NGen with NGEN_WITH_NETCDF=ON to enable NetCDF support"); } } diff --git a/src/utilities/python/CMakeLists.txt b/src/utilities/python/CMakeLists.txt index 692bbe8c10..2347625dd2 100644 --- a/src/utilities/python/CMakeLists.txt +++ b/src/utilities/python/CMakeLists.txt @@ -1,6 +1,9 @@ add_library(ngen_python InterpreterUtil.cpp) add_library(NGen::python ALIAS ngen_python) +find_package(ewts CONFIG REQUIRED) # if not already in scope +target_link_libraries(ngen_python PRIVATE ewts::ewts_ngen_bridge) + target_include_directories(ngen_python PUBLIC ${PROJECT_SOURCE_DIR}/include/) target_link_libraries(ngen_python PUBLIC diff --git a/src/utilities/python/InterpreterUtil.cpp b/src/utilities/python/InterpreterUtil.cpp index a4555f3fea..be292349c5 100644 --- a/src/utilities/python/InterpreterUtil.cpp +++ b/src/utilities/python/InterpreterUtil.cpp @@ -4,7 +4,7 @@ #if NGEN_WITH_PYTHON #include -#include +#include "ewts_ngen/logger.hpp" #include #include From 5db2463818d7745b2bb9fb4725efbeb6af59ad3e Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 02:23:23 -0700 Subject: [PATCH 092/109] updated Dockerfile to include EWTS --- Dockerfile | 267 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 253 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca6b38dbfc..cb880b91d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -222,17 +222,124 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ WORKDIR /ngen-app/ -# TODO This will invalidate the cache for all subsequent stages so we don't really want to do this -# Copy the remainder of your application code -COPY . /ngen-app/ngen/ +############################## +# Stage: EWTS Build – Error, Warning and Trapping System +############################## +# EWTS is built in its own stage so that: +# - It is cached independently from ngen source changes (COPY . /ngen-app/ngen/ +# happens later in the submodules stage). +# - Iterative ngen/submodule development doesn't re-trigger the EWTS clone+build. +# - EWTS_ORG / EWTS_REF can be pinned without affecting other stages' caches. +# +# EWTS provides a unified logging library used by ngen core and ALL Fortran/C/C++ +# submodules (LASAM, snow17, sac-sma, SoilMoistureProfiles, SoilFreezeThaw, +# cfe, topmodel, noah-owp-modular, ueb-bmi) plus a Python package used by lstm. +# +# How the plumbing works: +# 1. We build EWTS here and install it to /opt/ewts. +# 2. Every cmake call in the submodules stage passes +# -DCMAKE_PREFIX_PATH=/opt/ewts so that +# find_package(ewts CONFIG REQUIRED) in each submodule's CMakeLists.txt +# can locate the ewtsConfig.cmake package file. +# 3. That gives each submodule access to the EWTS targets: +# ewts::ewts_cpp – C++ runtime logger (used by LASAM) +# ewts::ewts_fortran – Fortran runtime (snow17, sac-sma, SoilMoistureProfiles, SoilFreezeThaw, noah-owp-modular) +# ewts::ewts_c – C runtime (cfe, topmodel) +# ewts::ewts_ngen_bridge – ngen↔EWTS bridge lib (linked by ngen itself) +# 4. The EWTS Python wheel is pip-installed so that lstm's bmi_lstm.py can +# "import ewts" at runtime. +# +# Build args – override at build time to pin a branch, tag, or full commit SHA: +# docker build --build-arg EWTS_REF=v1.2.3 ... +# docker build --build-arg EWTS_REF=abc123def456 ... +############################## +FROM base AS ewts-build + +SHELL [ "/usr/bin/scl", "enable", "gcc-toolset-10" ] + +ARG EWTS_ORG=NGWPC +ARG EWTS_REF=development + +# Install path for the built EWTS libraries, headers, cmake config, and +# Fortran .mod files. /opt/ewts follows the FHS convention for add-on +# packages (same pattern as /opt/boost already in this image) and avoids +# /tmp which can be cleaned unexpectedly. +ENV EWTS_PREFIX=/opt/ewts + +# Clone nwm-ewts with minimal data (blobless clone), build, install, capture +# git metadata for provenance, then remove the source tree. +# The four-way fetch fallback handles branches, tags, AND bare commit SHAs +RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ewts \ + set -eux && \ + git clone --filter=blob:none --no-checkout \ + "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts && \ + cd /tmp/nwm-ewts && \ + (git fetch --depth 1 origin "${EWTS_REF}" \ + || git fetch --depth 1 origin "refs/tags/${EWTS_REF}:refs/tags/${EWTS_REF}" \ + || git fetch origin "${EWTS_REF}" \ + || git fetch origin "refs/tags/${EWTS_REF}:refs/tags/${EWTS_REF}") && \ + git checkout FETCH_HEAD && \ + # ── Build EWTS ── + # This produces: C, C++, Fortran shared libs + ngen bridge + Python wheel. + # -DEWTS_WITH_NGEN=ON enables the ngen bridge (ewts_ngen_bridge.so) which + # provides the C shim ewts_ngen_log() that ngen's core calls into. + # -DEWTS_BUILD_SHARED=ON builds .so's so submodule DSOs can link at runtime. + cmake -S . -B cmake_build \ + -DCMAKE_BUILD_TYPE=Release \ + -DEWTS_WITH_NGEN=ON \ + -DEWTS_BUILD_SHARED=ON && \ + cmake --build cmake_build -j "$(nproc)" && \ + cmake --install cmake_build --prefix ${EWTS_PREFIX} && \ + # ── Capture EWTS git provenance ── + # Saved as a JSON sidecar so the git-info merge step at the bottom of this + # Dockerfile can include EWTS metadata alongside ngen + submodules. + jq -n \ + --arg commit_hash "$(git rev-parse HEAD)" \ + --arg branch "$(git branch -r --contains HEAD 2>/dev/null | grep -v '\->' | sed 's|origin/||' | head -n1 | xargs || echo "${EWTS_REF}")" \ + --arg tags "$(git tag --points-at HEAD 2>–/dev/null | tr '\n' ' ')" \ + --arg author "$(git log -1 --pretty=format:'%an')" \ + --arg commit_date "$(date -u -d @$(git log -1 --pretty=format:'%ct') +'%Y-%m-%d %H:%M:%S UTC')" \ + --arg message "$(git log -1 --pretty=format:'%s' | tr '\n' ';')" \ + --arg build_date "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" \ + '{"nwm-ewts": {commit_hash: $commit_hash, branch: $branch, tags: $tags, author: $author, commit_date: $commit_date, message: $message, build_date: $build_date}}' \ + > /ngen-app/nwm-ewts_git_info.json && \ + # ── Cleanup source ── + cd / && \ + rm -rf /tmp/nwm-ewts + +# Install the EWTS Python wheel into the venv. +# This is what makes "import ewts" work for Python-based submodules (lstm). +# lstm's bmi_lstm.py does: import ewts; LOG = ewts.get_logger(ewts.LSTM_ID) +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ + set -eux && \ + pip install ${EWTS_PREFIX}/python/dist/ewts-*.whl + +# Make EWTS shared libraries (.so) discoverable at runtime. +# Without this, ngen and every submodule DSO would fail with: +# "error while loading shared libraries: libewts_ngen_bridge.so: cannot open" +# We include both lib/ and lib64/ because cmake may install to either depending +# on the platform/distro convention. +ENV LD_LIBRARY_PATH="${EWTS_PREFIX}/lib:${EWTS_PREFIX}/lib64:${LD_LIBRARY_PATH}" ############################## # Stage: Submodules Build ############################## -FROM base AS submodules +# Inherits from ewts-build so /opt/ewts is already present. +# The ngen source COPY happens here — changing ngen code only invalidates +# this stage and below, not the EWTS build above. +############################## +FROM ewts-build AS submodules SHELL [ "/usr/bin/scl", "enable", "gcc-toolset-10" ] +WORKDIR /ngen-app/ + +# Copy the ngen application source. +# This is placed here (not in base) so that +# ngen code changes only invalidate the submodules stage, leaving the base and +# ewts-build stages cached. +COPY . /ngen-app/ngen/ + WORKDIR /ngen-app/ngen/ # Copy only the requirements files first for dependency installation caching @@ -260,12 +367,17 @@ RUN --mount=type=cache,target=/root/.cache/t-route,id=t-route-build \ find /ngen-app/ngen/extern/t-route -name "*.a" -exec rm -f {} + # Configure the build with cache for CMake +# -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} tells cmake where +# to find the ewtsConfig.cmake package file so that ngen's +# CMakeLists.txt line find_package(ewts CONFIG REQUIRED) succeeds. +# ngen links against ewts::ewts_ngen_bridge (the C++/MPI bridge). RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ngen \ set -eux && \ export FFLAGS="-fPIC" && \ export FCFLAGS="-fPIC" && \ export CMAKE_Fortran_FLAGS="-fPIC" && \ cmake -B cmake_build -S . \ + -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} \ -DNGEN_WITH_MPI=ON \ -DNGEN_WITH_NETCDF=ON \ -DNGEN_WITH_SQLITE=ON \ @@ -284,48 +396,72 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ngen \ find /ngen-app/ngen/cmake_build -name "*.a" -exec rm -f {} + && \ find /ngen-app/ngen/cmake_build -name "*.o" -exec rm -f {} + +# ────────────────────────────────────────────────────────────────────────────── # Build each submodule in a separate layer, using cache for CMake as well +# +# IMPORTANT: Every submodule's CMakeLists.txt now contains: +# find_package(ewts CONFIG REQUIRED) +# so we MUST pass -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} to each cmake call. +# Without it, cmake cannot locate ewtsConfig.cmake and the build fails with: +# "Could not find a package configuration file provided by "ewts"..." +# +# What each submodule links: +# LASAM → ewts::ewts_cpp + ewts::ewts_ngen_bridge +# snow17 → ewts::ewts_fortran + ewts::ewts_ngen_bridge +# sac-sma → ewts::ewts_fortran + ewts::ewts_ngen_bridge +# SoilMoistureProfiles → (check its CMakeLists.txt for specifics) +# SoilFreezeThaw → (check its CMakeLists.txt for specifics) +# cfe → (check its CMakeLists.txt for specifics) +# ueb-bmi → (check its CMakeLists.txt for specifics) +# ────────────────────────────────────────────────────────────────────────────── + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-lasam \ set -eux && \ - cmake -B extern/LASAM/cmake_build -S extern/LASAM/ -DNGEN=ON -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/LASAM/cmake_build -S extern/LASAM/ \ + -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} -DNGEN=ON -DBOOST_ROOT=/opt/boost && \ cmake --build extern/LASAM/cmake_build/ && \ find /ngen-app/ngen/extern/LASAM -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-snow17 \ set -eux && \ - cmake -B extern/snow17/cmake_build -S extern/snow17/ -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/snow17/cmake_build -S extern/snow17/ \ + -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} -DBOOST_ROOT=/opt/boost && \ cmake --build extern/snow17/cmake_build/ && \ find /ngen-app/ngen/extern/snow17 -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-sac-sma \ set -eux && \ - cmake -B extern/sac-sma/cmake_build -S extern/sac-sma/ -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/sac-sma/cmake_build -S extern/sac-sma/ \ + -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} -DBOOST_ROOT=/opt/boost && \ cmake --build extern/sac-sma/cmake_build/ && \ find /ngen-app/ngen/extern/sac-sma -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-soilmoistureprofiles \ set -eux && \ - cmake -B extern/SoilMoistureProfiles/cmake_build -S extern/SoilMoistureProfiles/SoilMoistureProfiles/ -DNGEN=ON -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/SoilMoistureProfiles/cmake_build -S extern/SoilMoistureProfiles/SoilMoistureProfiles/ \ + -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} -DNGEN=ON -DBOOST_ROOT=/opt/boost && \ cmake --build extern/SoilMoistureProfiles/cmake_build/ && \ find /ngen-app/ngen/extern/SoilMoistureProfiles -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-soilfreezethaw \ set -eux && \ - cmake -B extern/SoilFreezeThaw/cmake_build -S extern/SoilFreezeThaw/SoilFreezeThaw/ -DNGEN=ON -DBOOST_ROOT=/opt/boost && \ + cmake -B extern/SoilFreezeThaw/cmake_build -S extern/SoilFreezeThaw/SoilFreezeThaw/ \ + -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} -DNGEN=ON -DBOOST_ROOT=/opt/boost && \ cmake --build extern/SoilFreezeThaw/cmake_build/ && \ find /ngen-app/ngen/extern/SoilFreezeThaw -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ueb-bmi \ set -eux && \ cmake -B extern/ueb-bmi/cmake_build -S extern/ueb-bmi/ \ - -DUEB_SUPPRESS_OUTPUTS=ON -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \ + -DUEB_SUPPRESS_OUTPUTS=ON -DCMAKE_PREFIX_PATH=${EWTS_PREFIX} \ + -DBMICXX_INCLUDE_DIRS=/ngen-app/ngen/extern/bmi-cxx/ -DBOOST_ROOT=/opt/boost && \ cmake --build extern/ueb-bmi/cmake_build/ && \ find /ngen-app/ngen/extern/ueb-bmi/ -name '*.o' -exec rm -f {} + RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ set -eux; \ cd extern/lstm; \ - pip install . ./lstm_ewts + pip install . RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ set -eux; \ @@ -415,10 +551,12 @@ RUN set -eux && \ echo "$info" > /ngen-app/submodules-json/git_info_"$sub_key".json; \ done; \ \ - # Merge the main repository JSON with all submodule JSON files as top-level objects - jq -s 'add' $GIT_INFO_PATH /ngen-app/submodules-json/*.json > /ngen-app/merged_git_info.json && \ + # Merge the main repository JSON + submodule JSONs + EWTS provenance into one file. + # The EWTS json was created during the EWTS build step above; including it + # here means `cat /ngen-app/ngen_git_info.json` shows EWTS version info too. + jq -s 'add' $GIT_INFO_PATH /ngen-app/submodules-json/*.json /ngen-app/nwm-ewts_git_info.json > /ngen-app/merged_git_info.json && \ mv /ngen-app/merged_git_info.json $GIT_INFO_PATH && \ - rm -rf /ngen-app/submodules-json + rm -rf /ngen-app/submodules-json /ngen-app/nwm-ewts_git_info.json # Extend PYTHONPATH for LSTM models (preserve venv path from ngen-bmi-forcing) ENV PYTHONPATH="${PYTHONPATH}:/ngen-app/ngen/extern/lstm:/ngen-app/ngen/extern/lstm/lstm" @@ -432,3 +570,104 @@ SHELL ["/bin/bash", "-c"] ENTRYPOINT [ "/ngen-app/bin/run-ngen.sh" ] CMD [ "--info" ] +############################## +# Stage: EWTS Verification (optional, build with --target ewts-verify) +############################## +# Usage: +# docker build --target ewts-verify -t ngen-ewts-verify -f Dockerfile . +# docker run --rm ngen-ewts-verify +# +# This stage runs a comprehensive check that EWTS is properly wired into +# ngen and all submodules. It does NOT run ngen itself — just verifies +# that libraries, headers, cmake config, Python packages, and shared +# object linkages are all in place. +############################## +FROM submodules AS ewts-verify + +SHELL [ "/usr/bin/scl", "enable", "gcc-toolset-10" ] + +RUN set -eux && \ + echo "" && \ + echo "============================================" && \ + echo " EWTS Integration Verification" && \ + echo "============================================" && \ + echo "" && \ + \ + # ── 1. Check EWTS install tree exists ── + echo "--- 1. EWTS install tree ---" && \ + echo "EWTS_PREFIX=${EWTS_PREFIX}" && \ + ls -la ${EWTS_PREFIX}/lib/ && \ + echo "" && \ + \ + # ── 2. Check EWTS shared libraries are present ── + echo "--- 2. EWTS shared libraries ---" && \ + echo "Looking for libewts_*.so files..." && \ + find ${EWTS_PREFIX} -name '*.so' -o -name '*.so.*' | sort && \ + echo "" && \ + \ + # ── 3. Verify cmake package config is findable ── + echo "--- 3. EWTS cmake config ---" && \ + ls ${EWTS_PREFIX}/lib/cmake/ewts/ewtsConfig.cmake && \ + echo "ewtsConfig.cmake found OK" && \ + echo "" && \ + \ + # ── 4. Check Fortran .mod files (needed by snow17, sac-sma, etc.) ── + echo "--- 4. EWTS Fortran .mod files ---" && \ + find ${EWTS_PREFIX} -name '*.mod' | sort && \ + echo "" && \ + \ + # ── 5. Verify ngen executable exists and links to EWTS ── + echo "--- 5. ngen binary – EWTS linkage ---" && \ + NGEN_BIN=/ngen-app/ngen/cmake_build/ngen && \ + file "$NGEN_BIN" && \ + echo "Checking ldd for ewts symbols..." && \ + ldd "$NGEN_BIN" | grep -i ewts && \ + echo "ngen links to EWTS OK" && \ + echo "" && \ + \ + # ── 6. Check each submodule .so links to EWTS ── + echo "--- 6. Submodule .so files – EWTS linkage ---" && \ + for so in \ + extern/LASAM/cmake_build/*.so \ + extern/snow17/cmake_build/*.so \ + extern/sac-sma/cmake_build/*.so \ + extern/SoilMoistureProfiles/cmake_build/*.so \ + extern/SoilFreezeThaw/cmake_build/*.so \ + extern/ueb-bmi/cmake_build/*.so; \ + do \ + if [ -f "$so" ]; then \ + echo "Checking: $so"; \ + if ldd "$so" | grep -qi ewts; then \ + echo " ✓ links to EWTS"; \ + else \ + echo " ⚠ WARNING: no EWTS linkage found (may be expected if submodule doesn't use EWTS directly)"; \ + fi; \ + fi; \ + done && \ + echo "" && \ + \ + # ── 7. Verify EWTS Python package is importable ── + echo "--- 7. EWTS Python package ---" && \ + python3 -c "import ewts; print(f'ewts version: {ewts.__version__}')" && \ + python3 -c "import ewts; print(f'EWTS module keys available: {dir(ewts)}')" && \ + echo "Python ewts import OK" && \ + echo "" && \ + \ + # ── 8. Verify lstm can import ewts (this is the runtime dependency) ── + echo "--- 8. lstm → ewts Python integration ---" && \ + python3 -c "from lstm.bmi_lstm import *; print('lstm.bmi_lstm imports OK (includes ewts)')" && \ + echo "" && \ + \ + # ── 9. Show git provenance ── + echo "--- 9. Git provenance (nwm-ewts entry) ---" && \ + GIT_INFO=$(find /ngen-app -name '*_git_info.json' | head -1) && \ + if [ -n "$GIT_INFO" ]; then \ + jq '."nwm-ewts"' "$GIT_INFO"; \ + else \ + echo "No git_info.json found"; \ + fi && \ + echo "" && \ + echo "============================================" && \ + echo " EWTS verification complete" && \ + echo "============================================" + From a29731cbb67f6b87fb9da3500afc2a68283d7a4a Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Fri, 13 Mar 2026 08:00:18 -0700 Subject: [PATCH 093/109] Updates to Dockerfile comments for clarification --- Dockerfile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb880b91d7..3f91155216 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,12 +5,12 @@ ############################## ARG ORG=ngwpc ARG NGEN_FORCING_IMAGE_TAG=latest -ARG NGEN_FORCING_IMAGE=ghcr.io/ngwpc/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG} +ARG NGEN_FORCING_IMAGE=ghcr.io/${ORG}/ngen-bmi-forcing:${NGEN_FORCING_IMAGE_TAG} -#FROM ${NGEN_FORCING_IMAGE} AS base +FROM ${NGEN_FORCING_IMAGE} AS base # Uncomment when building locally -FROM ngen-bmi-forcing AS base +#FROM ngen-bmi-forcing AS base # OCI Metadata Arguments ARG NGEN_FORCING_IMAGE @@ -231,23 +231,24 @@ WORKDIR /ngen-app/ # - Iterative ngen/submodule development doesn't re-trigger the EWTS clone+build. # - EWTS_ORG / EWTS_REF can be pinned without affecting other stages' caches. # -# EWTS provides a unified logging library used by ngen core and ALL Fortran/C/C++ -# submodules (LASAM, snow17, sac-sma, SoilMoistureProfiles, SoilFreezeThaw, -# cfe, topmodel, noah-owp-modular, ueb-bmi) plus a Python package used by lstm. +# EWTS provides a unified logging framework used by ngen core and ALL C, C++, Fortran, +# and Python submodules. Libraries are created for C, C++ and Fortran submodules +# (cfe, evapotranspiration, LASAM, noah-owp-modular, snow17, sac-sma, +# SoilFreezeThaw, SoilMoistureProfiles, topmodel, ueb-bmi) and a Python package is +# used by Python sumbodules (lstm, topoflow-glacier and t-route). # # How the plumbing works: # 1. We build EWTS here and install it to /opt/ewts. # 2. Every cmake call in the submodules stage passes -# -DCMAKE_PREFIX_PATH=/opt/ewts so that +# -DCMAKE_PREFIX_PATH=/opt/ewts so that # find_package(ewts CONFIG REQUIRED) in each submodule's CMakeLists.txt # can locate the ewtsConfig.cmake package file. -# 3. That gives each submodule access to the EWTS targets: -# ewts::ewts_cpp – C++ runtime logger (used by LASAM) -# ewts::ewts_fortran – Fortran runtime (snow17, sac-sma, SoilMoistureProfiles, SoilFreezeThaw, noah-owp-modular) -# ewts::ewts_c – C runtime (cfe, topmodel) +# 3. The following gives each submodule access to the EWTS targets: +# ewts::ewts_c – C runtime (cfe, evapotranspiration, topmodel) +# ewts::ewts_cpp – C++ runtime logger (used by LASAM, SoilFreezeThaw, SoilMoistureProfiles) +# ewts::ewts_fortran – Fortran runtime (noah-owp-modular sac-sma,, snow17) # ewts::ewts_ngen_bridge – ngen↔EWTS bridge lib (linked by ngen itself) -# 4. The EWTS Python wheel is pip-installed so that lstm's bmi_lstm.py can -# "import ewts" at runtime. +# EWTS Python wheel – pip intalled package (lstm, topoflow-glacier, t-route) # # Build args – override at build time to pin a branch, tag, or full commit SHA: # docker build --build-arg EWTS_REF=v1.2.3 ... @@ -308,8 +309,8 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ewts \ rm -rf /tmp/nwm-ewts # Install the EWTS Python wheel into the venv. -# This is what makes "import ewts" work for Python-based submodules (lstm). -# lstm's bmi_lstm.py does: import ewts; LOG = ewts.get_logger(ewts.LSTM_ID) +# This is what makes "import ewts" work for Python-based submodules. +# For example, lstm's bmi_lstm.py does: import ewts; LOG = ewts.get_logger(ewts.LSTM_ID) RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ set -eux && \ pip install ${EWTS_PREFIX}/python/dist/ewts-*.whl From 1fd175803e206fe615b6fd16b74b32c95b29d86a Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 08:47:09 -0700 Subject: [PATCH 094/109] fixing cloning issue with ewts --- Dockerfile | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f91155216..3b24043efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -231,7 +231,7 @@ WORKDIR /ngen-app/ # - Iterative ngen/submodule development doesn't re-trigger the EWTS clone+build. # - EWTS_ORG / EWTS_REF can be pinned without affecting other stages' caches. # -# EWTS provides a unified logging framework used by ngen core and ALL C, C++, Fortran, +# EWTS provides a unified logging framework used by ngen core and ALL C, C++, Fortran, # and Python submodules. Libraries are created for C, C++ and Fortran submodules # (cfe, evapotranspiration, LASAM, noah-owp-modular, snow17, sac-sma, # SoilFreezeThaw, SoilMoistureProfiles, topmodel, ueb-bmi) and a Python package is @@ -248,7 +248,7 @@ WORKDIR /ngen-app/ # ewts::ewts_cpp – C++ runtime logger (used by LASAM, SoilFreezeThaw, SoilMoistureProfiles) # ewts::ewts_fortran – Fortran runtime (noah-owp-modular sac-sma,, snow17) # ewts::ewts_ngen_bridge – ngen↔EWTS bridge lib (linked by ngen itself) -# EWTS Python wheel – pip intalled package (lstm, topoflow-glacier, t-route) +# EWTS Python wheel – pip intalled package (lstm, topoflow-glacier, t-route) # # Build args – override at build time to pin a branch, tag, or full commit SHA: # docker build --build-arg EWTS_REF=v1.2.3 ... @@ -267,19 +267,17 @@ ARG EWTS_REF=development # /tmp which can be cleaned unexpectedly. ENV EWTS_PREFIX=/opt/ewts -# Clone nwm-ewts with minimal data (blobless clone), build, install, capture -# git metadata for provenance, then remove the source tree. -# The four-way fetch fallback handles branches, tags, AND bare commit SHAs +# Clone nwm-ewts, build, install, capture git metadata for provenance, +# then remove the source tree. +# Try shallow clone by branch/tag name first; fall back to full clone + checkout +# for bare commit SHAs (which git clone -b doesn't support). RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ewts \ set -eux && \ - git clone --filter=blob:none --no-checkout \ - "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts && \ + (git clone --depth 1 -b "${EWTS_REF}" \ + "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts \ + || (git clone "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts && \ + cd /tmp/nwm-ewts && git checkout "${EWTS_REF}")) && \ cd /tmp/nwm-ewts && \ - (git fetch --depth 1 origin "${EWTS_REF}" \ - || git fetch --depth 1 origin "refs/tags/${EWTS_REF}:refs/tags/${EWTS_REF}" \ - || git fetch origin "${EWTS_REF}" \ - || git fetch origin "refs/tags/${EWTS_REF}:refs/tags/${EWTS_REF}") && \ - git checkout FETCH_HEAD && \ # ── Build EWTS ── # This produces: C, C++, Fortran shared libs + ngen bridge + Python wheel. # -DEWTS_WITH_NGEN=ON enables the ngen bridge (ewts_ngen_bridge.so) which From cffad40d4d9ad25a6f664e59c97de2cd9e9a8e63 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 09:10:40 -0700 Subject: [PATCH 095/109] added ewts path to t route build --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 3b24043efa..fdd31b0934 100644 --- a/Dockerfile +++ b/Dockerfile @@ -355,10 +355,14 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ ENV LD_LIBRARY_PATH=/usr/local/lib64:$LD_LIBRARY_PATH # Use cache for building the t-route submodule +# EWTS_INSTALL_PREFIX tells t-route's compiler.sh where to find the EWTS +# Python wheel (at $EWTS_INSTALL_PREFIX/python/dist/ewts-*.whl). +# Without this it defaults to /tmp/ewts_install which doesn't exist. RUN --mount=type=cache,target=/root/.cache/t-route,id=t-route-build \ set -eux && \ cd extern/t-route && \ echo "Running compiler.sh" && \ + export EWTS_INSTALL_PREFIX=${EWTS_PREFIX} && \ LDFLAGS='-Wl,-L/usr/local/lib64/,-L/usr/local/lib/,-rpath,/usr/local/lib64/,-rpath,/usr/local/lib/' && \ ./compiler.sh no-e && \ rm -rf /ngen-app/ngen/extern/t-route/test/LowerColorado_TX_v4 && \ From 7f834a75dfbb37e96aa01ab2132e99c7ab6d9dc6 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 10:41:19 -0700 Subject: [PATCH 096/109] dockerfile updates --- Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fdd31b0934..3b24043efa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -355,14 +355,10 @@ RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ ENV LD_LIBRARY_PATH=/usr/local/lib64:$LD_LIBRARY_PATH # Use cache for building the t-route submodule -# EWTS_INSTALL_PREFIX tells t-route's compiler.sh where to find the EWTS -# Python wheel (at $EWTS_INSTALL_PREFIX/python/dist/ewts-*.whl). -# Without this it defaults to /tmp/ewts_install which doesn't exist. RUN --mount=type=cache,target=/root/.cache/t-route,id=t-route-build \ set -eux && \ cd extern/t-route && \ echo "Running compiler.sh" && \ - export EWTS_INSTALL_PREFIX=${EWTS_PREFIX} && \ LDFLAGS='-Wl,-L/usr/local/lib64/,-L/usr/local/lib/,-rpath,/usr/local/lib64/,-rpath,/usr/local/lib/' && \ ./compiler.sh no-e && \ rm -rf /ngen-app/ngen/extern/t-route/test/LowerColorado_TX_v4 && \ From e28c742e4a2d01047ad96c56a92c6cf84330ed3f Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 10:52:46 -0700 Subject: [PATCH 097/109] dockerfile updates --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b24043efa..40168627ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -267,6 +267,10 @@ ARG EWTS_REF=development # /tmp which can be cleaned unexpectedly. ENV EWTS_PREFIX=/opt/ewts +# Point the development fallback to the cloned source tree so that +# compiler.sh can pip-install EWTS from source if the wheel is missing. +ENV EWTS_PY_ROOT=/tmp/nwm-ewts/runtime/python/ewts + # Clone nwm-ewts, build, install, capture git metadata for provenance, # then remove the source tree. # Try shallow clone by branch/tag name first; fall back to full clone + checkout @@ -302,9 +306,9 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ewts \ --arg build_date "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" \ '{"nwm-ewts": {commit_hash: $commit_hash, branch: $branch, tags: $tags, author: $author, commit_date: $commit_date, message: $message, build_date: $build_date}}' \ > /ngen-app/nwm-ewts_git_info.json && \ - # ── Cleanup source ── + # ── Cleanup source (keep Python source as fallback for compiler.sh) ── cd / && \ - rm -rf /tmp/nwm-ewts + rm -rf /tmp/nwm-ewts/cmake_build /tmp/nwm-ewts/.git # Install the EWTS Python wheel into the venv. # This is what makes "import ewts" work for Python-based submodules. From c7182c26726d7edeab83c92e175f0e7d124f7fed Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 11:42:00 -0700 Subject: [PATCH 098/109] dockerfile updates --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 40168627ab..1a5a2c1a8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -299,7 +299,7 @@ RUN --mount=type=cache,target=/root/.cache/cmake,id=cmake-ewts \ jq -n \ --arg commit_hash "$(git rev-parse HEAD)" \ --arg branch "$(git branch -r --contains HEAD 2>/dev/null | grep -v '\->' | sed 's|origin/||' | head -n1 | xargs || echo "${EWTS_REF}")" \ - --arg tags "$(git tag --points-at HEAD 2>–/dev/null | tr '\n' ' ')" \ + --arg tags "$(git tag --points-at HEAD 2>/dev/null | tr '\n' ' ')" \ --arg author "$(git log -1 --pretty=format:'%an')" \ --arg commit_date "$(date -u -d @$(git log -1 --pretty=format:'%ct') +'%Y-%m-%d %H:%M:%S UTC')" \ --arg message "$(git log -1 --pretty=format:'%s' | tr '\n' ';')" \ From 91964260e52e84a7455503464f67b784a7511eac Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 12:22:01 -0700 Subject: [PATCH 099/109] fix: change ewts linkage from PRIVATE to PUBLIC in all CMakeLists ewts_ngen/logger.hpp is included in public headers, so the EWTS include directories must propagate to downstream consumers (tests, etc). PRIVATE linkage caused 'ewts_ngen/logger.hpp: No such file' when compiling test targets. --- src/bmi/CMakeLists.txt | 2 +- src/core/CMakeLists.txt | 2 +- src/core/mediator/CMakeLists.txt | 2 +- src/core/nexus/CMakeLists.txt | 2 +- src/forcing/CMakeLists.txt | 2 +- src/geojson/CMakeLists.txt | 2 +- src/geopackage/CMakeLists.txt | 2 +- src/realizations/catchment/CMakeLists.txt | 4 ++-- src/utilities/CMakeLists.txt | 2 +- src/utilities/bmi/CMakeLists.txt | 4 ++-- src/utilities/logging/CMakeLists.txt | 2 +- src/utilities/mdframe/CMakeLists.txt | 2 +- src/utilities/python/CMakeLists.txt | 2 +- 13 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/bmi/CMakeLists.txt b/src/bmi/CMakeLists.txt index d98d201d42..7c8bbd9d1c 100644 --- a/src/bmi/CMakeLists.txt +++ b/src/bmi/CMakeLists.txt @@ -16,7 +16,7 @@ target_link_libraries(ngen_bmi NGen::core_mediator ) -target_link_libraries(ngen_bmi PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(ngen_bmi PUBLIC ewts::ewts_ngen_bridge) target_sources(ngen_bmi PRIVATE diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 3d0126e15d..8793567a65 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -11,7 +11,7 @@ target_link_libraries(core PRIVATE NGen::parallel ) -target_link_libraries(core PRIVATE +target_link_libraries(core PUBLIC ewts::ewts_ngen_bridge ) diff --git a/src/core/mediator/CMakeLists.txt b/src/core/mediator/CMakeLists.txt index 80374fee2a..dcfb2fde6e 100644 --- a/src/core/mediator/CMakeLists.txt +++ b/src/core/mediator/CMakeLists.txt @@ -3,7 +3,7 @@ dynamic_sourced_cxx_library(core_mediator "${CMAKE_CURRENT_SOURCE_DIR}") add_library(NGen::core_mediator ALIAS core_mediator) -target_link_libraries(core_mediator PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(core_mediator PUBLIC ewts::ewts_ngen_bridge) if(NGEN_WITH_UDUNITS) target_link_libraries(core_mediator PUBLIC libudunits2) diff --git a/src/core/nexus/CMakeLists.txt b/src/core/nexus/CMakeLists.txt index 759a5c3c23..2e45b39e02 100644 --- a/src/core/nexus/CMakeLists.txt +++ b/src/core/nexus/CMakeLists.txt @@ -3,7 +3,7 @@ dynamic_sourced_cxx_library(core_nexus "${CMAKE_CURRENT_SOURCE_DIR}") add_library(NGen::core_nexus ALIAS core_nexus) -target_link_libraries(core_nexus PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(core_nexus PUBLIC ewts::ewts_ngen_bridge) target_include_directories(core_nexus PUBLIC ${PROJECT_SOURCE_DIR}/include/core diff --git a/src/forcing/CMakeLists.txt b/src/forcing/CMakeLists.txt index 130ff2508b..b02e3c4ff8 100644 --- a/src/forcing/CMakeLists.txt +++ b/src/forcing/CMakeLists.txt @@ -19,7 +19,7 @@ target_link_libraries(forcing PUBLIC NGen::config_header Threads::Threads ) -target_link_libraries(forcing PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(forcing PUBLIC ewts::ewts_ngen_bridge) target_sources(forcing PRIVATE "${CMAKE_CURRENT_LIST_DIR}/NullForcingProvider.cpp") diff --git a/src/geojson/CMakeLists.txt b/src/geojson/CMakeLists.txt index f22dfad944..d9d26d586e 100644 --- a/src/geojson/CMakeLists.txt +++ b/src/geojson/CMakeLists.txt @@ -11,8 +11,8 @@ target_include_directories(geojson PUBLIC target_link_libraries(geojson PUBLIC Boost::boost # Headers-only Boost NGen::logging + ewts::ewts_ngen_bridge ) -target_link_libraries(geojson PRIVATE ewts::ewts_ngen_bridge) diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 2a10f58ced..c438e03095 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -10,5 +10,5 @@ add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/utilities) target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost SQLite::SQLite3 NGen::logging) -target_link_libraries(geopackage PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(geopackage PUBLIC ewts::ewts_ngen_bridge) diff --git a/src/realizations/catchment/CMakeLists.txt b/src/realizations/catchment/CMakeLists.txt index 0cc951afd0..ffe82683b2 100644 --- a/src/realizations/catchment/CMakeLists.txt +++ b/src/realizations/catchment/CMakeLists.txt @@ -22,7 +22,7 @@ target_include_directories(realizations_catchment PUBLIC ${PROJECT_SOURCE_DIR}/include/geojson ${PROJECT_SOURCE_DIR}/include/bmi ) - + target_link_libraries(realizations_catchment PUBLIC ${CMAKE_DL_LIBS} NGen::config_header @@ -33,6 +33,6 @@ target_link_libraries(realizations_catchment PUBLIC NGen::bmi_protocols ) -target_link_libraries(realizations_catchment PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(realizations_catchment PUBLIC ewts::ewts_ngen_bridge) diff --git a/src/utilities/CMakeLists.txt b/src/utilities/CMakeLists.txt index b1a2aa0c3d..c9b340fa46 100644 --- a/src/utilities/CMakeLists.txt +++ b/src/utilities/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries(ngen_parallel NGen::logging ) -target_link_libraries(ngen_parallel PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(ngen_parallel PUBLIC ewts::ewts_ngen_bridge) if(NGEN_WITH_MPI) target_link_libraries(ngen_parallel diff --git a/src/utilities/bmi/CMakeLists.txt b/src/utilities/bmi/CMakeLists.txt index ca4bb2bb4b..a98ad04af5 100644 --- a/src/utilities/bmi/CMakeLists.txt +++ b/src/utilities/bmi/CMakeLists.txt @@ -17,7 +17,7 @@ add_library(ngen_bmi_protocols protocols.cpp mass_balance.cpp) add_library(NGen::bmi_protocols ALIAS ngen_bmi_protocols) -target_include_directories(ngen_bmi_protocols PUBLIC +target_include_directories(ngen_bmi_protocols PUBLIC ${PROJECT_SOURCE_DIR}/include/bmi ${PROJECT_SOURCE_DIR}/include/utilities/bmi ${PROJECT_SOURCE_DIR}/include/geojson @@ -31,7 +31,7 @@ target_link_libraries(ngen_bmi_protocols NGen::logging ) -target_link_libraries(ngen_bmi_protocols PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(ngen_bmi_protocols PUBLIC ewts::ewts_ngen_bridge) target_sources(ngen_bmi_protocols PRIVATE diff --git a/src/utilities/logging/CMakeLists.txt b/src/utilities/logging/CMakeLists.txt index 0080d26680..c6745c01ea 100644 --- a/src/utilities/logging/CMakeLists.txt +++ b/src/utilities/logging/CMakeLists.txt @@ -2,7 +2,7 @@ add_library(logging logging_utils.cpp) add_library(NGen::logging ALIAS logging) target_include_directories(logging PUBLIC ${PROJECT_SOURCE_DIR}/include/utilities) -target_link_libraries(logging PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(logging PUBLIC ewts::ewts_ngen_bridge) target_link_libraries(logging PUBLIC Boost::boost diff --git a/src/utilities/mdframe/CMakeLists.txt b/src/utilities/mdframe/CMakeLists.txt index 0073046398..a254268f54 100644 --- a/src/utilities/mdframe/CMakeLists.txt +++ b/src/utilities/mdframe/CMakeLists.txt @@ -7,7 +7,7 @@ NGen::config_header NGen::mdarray NGen::logging ) -target_link_libraries(mdframe PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(mdframe PUBLIC ewts::ewts_ngen_bridge) if(NGEN_WITH_NETCDF) target_link_libraries(mdframe PUBLIC NetCDF) diff --git a/src/utilities/python/CMakeLists.txt b/src/utilities/python/CMakeLists.txt index 2347625dd2..1779770a8b 100644 --- a/src/utilities/python/CMakeLists.txt +++ b/src/utilities/python/CMakeLists.txt @@ -2,7 +2,7 @@ add_library(ngen_python InterpreterUtil.cpp) add_library(NGen::python ALIAS ngen_python) find_package(ewts CONFIG REQUIRED) # if not already in scope -target_link_libraries(ngen_python PRIVATE ewts::ewts_ngen_bridge) +target_link_libraries(ngen_python PUBLIC ewts::ewts_ngen_bridge) target_include_directories(ngen_python PUBLIC ${PROJECT_SOURCE_DIR}/include/) From a0c8b083141974ee8344907e377a35110beee748 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Fri, 13 Mar 2026 13:03:32 -0700 Subject: [PATCH 100/109] Fix ewts-verify stage to handle lib64 install path --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a5a2c1a8b..27ddbca009 100644 --- a/Dockerfile +++ b/Dockerfile @@ -597,9 +597,12 @@ RUN set -eux && \ echo "" && \ \ # ── 1. Check EWTS install tree exists ── + # cmake may install to lib/ or lib64/ depending on the platform echo "--- 1. EWTS install tree ---" && \ echo "EWTS_PREFIX=${EWTS_PREFIX}" && \ - ls -la ${EWTS_PREFIX}/lib/ && \ + EWTS_LIBDIR=$(find ${EWTS_PREFIX} -maxdepth 1 -type d \( -name lib -o -name lib64 \) | head -1) && \ + echo "EWTS_LIBDIR=${EWTS_LIBDIR}" && \ + ls -la ${EWTS_LIBDIR}/ && \ echo "" && \ \ # ── 2. Check EWTS shared libraries are present ── @@ -610,7 +613,7 @@ RUN set -eux && \ \ # ── 3. Verify cmake package config is findable ── echo "--- 3. EWTS cmake config ---" && \ - ls ${EWTS_PREFIX}/lib/cmake/ewts/ewtsConfig.cmake && \ + find ${EWTS_PREFIX} -name 'ewtsConfig.cmake' | head -1 | xargs ls && \ echo "ewtsConfig.cmake found OK" && \ echo "" && \ \ From 83bdfb48005b115718fba88ce280bec4d45bd87a Mon Sep 17 00:00:00 2001 From: "Miguel.Pena" Date: Fri, 13 Mar 2026 15:25:58 -0700 Subject: [PATCH 101/109] removed ewts verification stage --- Dockerfile | 105 ----------------------------------------------------- 1 file changed, 105 deletions(-) diff --git a/Dockerfile b/Dockerfile index 27ddbca009..c08f6cb206 100644 --- a/Dockerfile +++ b/Dockerfile @@ -572,108 +572,3 @@ SHELL ["/bin/bash", "-c"] ENTRYPOINT [ "/ngen-app/bin/run-ngen.sh" ] CMD [ "--info" ] - -############################## -# Stage: EWTS Verification (optional, build with --target ewts-verify) -############################## -# Usage: -# docker build --target ewts-verify -t ngen-ewts-verify -f Dockerfile . -# docker run --rm ngen-ewts-verify -# -# This stage runs a comprehensive check that EWTS is properly wired into -# ngen and all submodules. It does NOT run ngen itself — just verifies -# that libraries, headers, cmake config, Python packages, and shared -# object linkages are all in place. -############################## -FROM submodules AS ewts-verify - -SHELL [ "/usr/bin/scl", "enable", "gcc-toolset-10" ] - -RUN set -eux && \ - echo "" && \ - echo "============================================" && \ - echo " EWTS Integration Verification" && \ - echo "============================================" && \ - echo "" && \ - \ - # ── 1. Check EWTS install tree exists ── - # cmake may install to lib/ or lib64/ depending on the platform - echo "--- 1. EWTS install tree ---" && \ - echo "EWTS_PREFIX=${EWTS_PREFIX}" && \ - EWTS_LIBDIR=$(find ${EWTS_PREFIX} -maxdepth 1 -type d \( -name lib -o -name lib64 \) | head -1) && \ - echo "EWTS_LIBDIR=${EWTS_LIBDIR}" && \ - ls -la ${EWTS_LIBDIR}/ && \ - echo "" && \ - \ - # ── 2. Check EWTS shared libraries are present ── - echo "--- 2. EWTS shared libraries ---" && \ - echo "Looking for libewts_*.so files..." && \ - find ${EWTS_PREFIX} -name '*.so' -o -name '*.so.*' | sort && \ - echo "" && \ - \ - # ── 3. Verify cmake package config is findable ── - echo "--- 3. EWTS cmake config ---" && \ - find ${EWTS_PREFIX} -name 'ewtsConfig.cmake' | head -1 | xargs ls && \ - echo "ewtsConfig.cmake found OK" && \ - echo "" && \ - \ - # ── 4. Check Fortran .mod files (needed by snow17, sac-sma, etc.) ── - echo "--- 4. EWTS Fortran .mod files ---" && \ - find ${EWTS_PREFIX} -name '*.mod' | sort && \ - echo "" && \ - \ - # ── 5. Verify ngen executable exists and links to EWTS ── - echo "--- 5. ngen binary – EWTS linkage ---" && \ - NGEN_BIN=/ngen-app/ngen/cmake_build/ngen && \ - file "$NGEN_BIN" && \ - echo "Checking ldd for ewts symbols..." && \ - ldd "$NGEN_BIN" | grep -i ewts && \ - echo "ngen links to EWTS OK" && \ - echo "" && \ - \ - # ── 6. Check each submodule .so links to EWTS ── - echo "--- 6. Submodule .so files – EWTS linkage ---" && \ - for so in \ - extern/LASAM/cmake_build/*.so \ - extern/snow17/cmake_build/*.so \ - extern/sac-sma/cmake_build/*.so \ - extern/SoilMoistureProfiles/cmake_build/*.so \ - extern/SoilFreezeThaw/cmake_build/*.so \ - extern/ueb-bmi/cmake_build/*.so; \ - do \ - if [ -f "$so" ]; then \ - echo "Checking: $so"; \ - if ldd "$so" | grep -qi ewts; then \ - echo " ✓ links to EWTS"; \ - else \ - echo " ⚠ WARNING: no EWTS linkage found (may be expected if submodule doesn't use EWTS directly)"; \ - fi; \ - fi; \ - done && \ - echo "" && \ - \ - # ── 7. Verify EWTS Python package is importable ── - echo "--- 7. EWTS Python package ---" && \ - python3 -c "import ewts; print(f'ewts version: {ewts.__version__}')" && \ - python3 -c "import ewts; print(f'EWTS module keys available: {dir(ewts)}')" && \ - echo "Python ewts import OK" && \ - echo "" && \ - \ - # ── 8. Verify lstm can import ewts (this is the runtime dependency) ── - echo "--- 8. lstm → ewts Python integration ---" && \ - python3 -c "from lstm.bmi_lstm import *; print('lstm.bmi_lstm imports OK (includes ewts)')" && \ - echo "" && \ - \ - # ── 9. Show git provenance ── - echo "--- 9. Git provenance (nwm-ewts entry) ---" && \ - GIT_INFO=$(find /ngen-app -name '*_git_info.json' | head -1) && \ - if [ -n "$GIT_INFO" ]; then \ - jq '."nwm-ewts"' "$GIT_INFO"; \ - else \ - echo "No git_info.json found"; \ - fi && \ - echo "" && \ - echo "============================================" && \ - echo " EWTS verification complete" && \ - echo "============================================" - From d6024d2c95cf16fc2655680999cc8d63bae757a4 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Mon, 16 Mar 2026 19:41:53 -0700 Subject: [PATCH 102/109] added ewts build arguments to ngwpc cicd file --- .github/workflows/ngwpc-cicd.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 87ed350453..0b2a3019a8 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -23,6 +23,14 @@ on: description: 'NGEN_FORCING_IMAGE_TAG' required: false type: string + EWTS_ORG: + description: 'EWTS_ORG' + required: false + type: string + EWTS_REF: + description: 'EWTS_REF' + required: false + type: string permissions: contents: read @@ -241,6 +249,8 @@ jobs: build-args: | ORG=${{ needs.setup.outputs.org }} NGEN_FORCING_IMAGE_TAG=${{ inputs.NGEN_FORCING_IMAGE_TAG || 'latest' }} + EWTS_ORG=${{ inputs.EWTS_ORG || 'NGWPC' }} + EWTS_REF=${{ inputs.EWTS_REF || 'development' }} IMAGE_SOURCE=https://github.com/${{ github.repository }} IMAGE_VENDOR=${{ github.repository_owner }} IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} From 7e55bedf5d5b4756e4ca20d507fb0fb8e208651e Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Mar 2026 15:12:00 -0700 Subject: [PATCH 103/109] added ewts to ngwpc cicd file in codeql step --- .github/workflows/ngwpc-cicd.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 0b2a3019a8..e09e31c868 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -180,12 +180,34 @@ jobs: echo "BOOST_ROOT=${PREFIX}" >> "$GITHUB_ENV" echo "CMAKE_PREFIX_PATH=${PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "$GITHUB_ENV" + - name: Build and install EWTS + run: | + set -euo pipefail + EWTS_ORG="${{ inputs.EWTS_ORG || 'NGWPC' }}" + EWTS_REF="${{ inputs.EWTS_REF || 'development' }}" + EWTS_PREFIX=/opt/ewts + + git clone --depth 1 -b "${EWTS_REF}" \ + "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts \ + || (git clone "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts && \ + cd /tmp/nwm-ewts && git checkout "${EWTS_REF}") + + cd /tmp/nwm-ewts + cmake -S . -B cmake_build \ + -DCMAKE_BUILD_TYPE=Release \ + -DEWTS_WITH_NGEN=ON \ + -DEWTS_BUILD_SHARED=ON + cmake --build cmake_build -j "$(nproc)" + sudo cmake --install cmake_build --prefix "${EWTS_PREFIX}" + + echo "EWTS_PREFIX=${EWTS_PREFIX}" >> "$GITHUB_ENV" + - name: Build C++ code env: PYTHONPATH: ${{ env.PYTHONPATH }} run: | cmake -B cmake_build -S . \ - -DCMAKE_PREFIX_PATH="${BOOST_ROOT}" \ + -DCMAKE_PREFIX_PATH="${EWTS_PREFIX};${BOOST_ROOT}" \ -DBoost_NO_SYSTEM_PATHS=ON \ -DBOOST_ROOT="${BOOST_ROOT}" \ -DPYTHON_EXECUTABLE=$(which python3) \ From 700c46db6348126714601d5c8387e96452c37e59 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Mar 2026 15:29:25 -0700 Subject: [PATCH 104/109] cicd updates --- .github/workflows/ngwpc-cicd.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index e09e31c868..7824b7f1ab 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -196,7 +196,10 @@ jobs: cmake -S . -B cmake_build \ -DCMAKE_BUILD_TYPE=Release \ -DEWTS_WITH_NGEN=ON \ - -DEWTS_BUILD_SHARED=ON + -DEWTS_BUILD_SHARED=ON \ + -DBOOST_ROOT="${BOOST_ROOT}" \ + -DBoost_NO_SYSTEM_PATHS=ON \ + -DCMAKE_PREFIX_PATH="${BOOST_ROOT}" cmake --build cmake_build -j "$(nproc)" sudo cmake --install cmake_build --prefix "${EWTS_PREFIX}" From b48a387bf2d00f6a7f8a6625c07d5f88143400eb Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Tue, 17 Mar 2026 16:02:55 -0700 Subject: [PATCH 105/109] cicd updates --- .github/workflows/ngwpc-cicd.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ngwpc-cicd.yml b/.github/workflows/ngwpc-cicd.yml index 7824b7f1ab..03791a9e98 100644 --- a/.github/workflows/ngwpc-cicd.yml +++ b/.github/workflows/ngwpc-cicd.yml @@ -157,7 +157,7 @@ jobs: BOOST_VER=1.86.0 BOOST_UNDERSCORE=1_86_0 - PREFIX=/opt/boost-${BOOST_VER} + PREFIX=/usr/local curl -fL --retry 10 --retry-delay 2 --max-time 600 \ -o /tmp/boost.tar.bz2 \ @@ -169,7 +169,7 @@ jobs: ./bootstrap.sh --prefix="${PREFIX}" # Build Boost libraries - ./b2 -j"$(nproc)" install \ + sudo ./b2 -j"$(nproc)" install \ --with-system \ --with-filesystem \ --with-program_options \ @@ -178,7 +178,6 @@ jobs: --with-date_time \ --with-serialization echo "BOOST_ROOT=${PREFIX}" >> "$GITHUB_ENV" - echo "CMAKE_PREFIX_PATH=${PREFIX}:${CMAKE_PREFIX_PATH:-}" >> "$GITHUB_ENV" - name: Build and install EWTS run: | @@ -196,10 +195,7 @@ jobs: cmake -S . -B cmake_build \ -DCMAKE_BUILD_TYPE=Release \ -DEWTS_WITH_NGEN=ON \ - -DEWTS_BUILD_SHARED=ON \ - -DBOOST_ROOT="${BOOST_ROOT}" \ - -DBoost_NO_SYSTEM_PATHS=ON \ - -DCMAKE_PREFIX_PATH="${BOOST_ROOT}" + -DEWTS_BUILD_SHARED=ON cmake --build cmake_build -j "$(nproc)" sudo cmake --install cmake_build --prefix "${EWTS_PREFIX}" From 2e6cc9b833a652f3b18d8d7c79a359fc6c63f7f4 Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Wed, 18 Mar 2026 10:44:18 -0700 Subject: [PATCH 106/109] Add ewts to ngen state saving. --- src/forcing/NetCDFPerFeatureDataProvider.cpp | 20 +++++++++++++++---- src/state_save_restore/CMakeLists.txt | 2 ++ src/state_save_restore/File_Per_Unit.cpp | 2 +- src/state_save_restore/State_Save_Restore.cpp | 17 ++++++++++++---- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index cc41abb7dd..9940521252 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -155,7 +155,10 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat // files generated by the forcings engine have just (time) if (dim_count == 2) { if (time_var.getDim(0).getName() != "catchment-id" || time_var.getDim(1).getName() != "time") { - Logger::logMsgAndThrowError("In NetCDF file '" + input_path + "', 'Time' variable dimensions don't match expectations"); + std::string message = "In NetCDF file '" + input_path + "', 'Time' variable dimensions don't match expectations"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } time_var.getVar({0ul, 0ul}, {1ul, num_times}, raw_time.data()); } else if (dim_count == 1) { @@ -218,7 +221,10 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat time_unit = TIME_NANOSECONDS; time_scale_factor = 1.0e-9; } else { - Logger::logMsgAndThrowError("In NetCDF file '" + input_path + "', time unit '" + time_base_unit + "' could not be converted"); + std::string message = "In NetCDF file '" + input_path + "', time unit '" + time_base_unit + "' could not be converted"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } time_vals.resize(raw_time.size()); @@ -373,7 +379,10 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& int dim_time, dim_catchment; if (dims.size() != 2) { - Logger::logMsgAndThrowError("Variable dimension count isn't 2"); + std::string message = "Variable dimension count isn't 2"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } if (dims[0].getName() == "time" && dims[1].getName() == "catchment-id") { // Forcings Engine NetCDF output case @@ -384,7 +393,10 @@ double NetCDFPerFeatureDataProvider::get_value(const CatchmentAggrDataSelector& dim_time = 1; dim_catchment = 0; } else { - Logger::logMsgAndThrowError("Variable dimensions aren't 'time' and 'catchment-id'"); + std::string message = "Variable dimensions aren't 'time' and 'catchment-id'"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } size_t time_dim_size = dims[dim_time].getSize(); diff --git a/src/state_save_restore/CMakeLists.txt b/src/state_save_restore/CMakeLists.txt index cde58c6073..b068d6d4ab 100644 --- a/src/state_save_restore/CMakeLists.txt +++ b/src/state_save_restore/CMakeLists.txt @@ -7,9 +7,11 @@ target_link_libraries(state_save_restore PUBLIC Boost::boost # Headers-only Boost Boost::system Boost::filesystem + ewts::ewts_ngen_bridge ) target_include_directories(state_save_restore PUBLIC ${PROJECT_SOURCE_DIR}/include ) + diff --git a/src/state_save_restore/File_Per_Unit.cpp b/src/state_save_restore/File_Per_Unit.cpp index 80bce91418..6d355d4c93 100644 --- a/src/state_save_restore/File_Per_Unit.cpp +++ b/src/state_save_restore/File_Per_Unit.cpp @@ -1,5 +1,5 @@ #include -#include +#include "ewts_ngen/logger.hpp" #if __has_include() && __cpp_lib_filesystem >= 201703L #include diff --git a/src/state_save_restore/State_Save_Restore.cpp b/src/state_save_restore/State_Save_Restore.cpp index ee3f5ae3c9..4a54b1a21c 100644 --- a/src/state_save_restore/State_Save_Restore.cpp +++ b/src/state_save_restore/State_Save_Restore.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include "ewts_ngen/logger.hpp" #include #include @@ -98,13 +98,19 @@ State_Save_Config::instance::instance(std::string const& direction, std::string } else if (direction == "load") { direction_ = State_Save_Direction::Load; } else { - Logger::logMsgAndThrowError("Unrecognized state saving direction '" + direction + "'"); + std::string message = "Unrecognized state saving direction '" + direction + "'"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } if (mechanism == "FilePerUnit") { mechanism_ = State_Save_Mechanism::FilePerUnit; } else { - Logger::logMsgAndThrowError("Unrecognized state saving mechanism '" + mechanism + "'"); + std::string message = "Unrecognized state saving mechanism '" + mechanism + "'"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } if (timing == "EndOfRun") { @@ -114,7 +120,10 @@ State_Save_Config::instance::instance(std::string const& direction, std::string } else if (timing == "StartOfRun") { timing_ = State_Save_When::StartOfRun; } else { - Logger::logMsgAndThrowError("Unrecognized state saving timing '" + timing + "'"); + std::string message = "Unrecognized state saving timing '" + timing + "'"; + std::string throw_msg; throw_msg.assign(message); + LOG(throw_msg, LogLevel::WARNING); + throw std::runtime_error(throw_msg); } } From d18167a468ced7f7d3a20cb574698360b2ccde64 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 26 Mar 2026 14:49:47 -0400 Subject: [PATCH 107/109] EWST function name change --- src/geopackage/read.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index c967560ea4..8f9efa8283 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -88,7 +88,7 @@ std::shared_ptr ngen::geopackage::read( "LEFT JOIN flowpaths " "ON nexus.dn_fp_id = flowpaths.fp_id"; } else { - Logger::logMsgAndThrowError("Geopackage read only accepts layers `divides` and `nexus`. The layer entered was " + layer); + Logger::LogAndThrow("Geopackage read only accepts layers `divides` and `nexus`. The layer entered was " + layer); } std::string joined_ids = ""; @@ -107,7 +107,7 @@ std::shared_ptr ngen::geopackage::read( } int id_num = std::atoi(filter_id.c_str() + sep_index); if (id_num <= 0) - Logger::logMsgAndThrowError("Could not convert input " + layer + " ID into a number: " + filter_id); + Logger::LogAndThrow("Could not convert input " + layer + " ID into a number: " + filter_id); filter << id_num; } filter << ')'; From 74f3f57c50f27e1100729d4295107c8fbac12169 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 26 Mar 2026 15:49:40 -0400 Subject: [PATCH 108/109] Fix rebase problems --- CMakeLists.txt | 7 ++++++- .../catchment/Bmi_Multi_Formulation.hpp | 13 ------------- src/partitionGenerator.cpp | 4 ++-- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dbb9adcb48..a320eab3a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,7 +193,12 @@ add_compile_definitions(NGEN_SHARED_LIB_EXTENSION) set(Boost_USE_STATIC_LIBS OFF) set(Boost_USE_MULTITHREADED ON) set(Boost_USE_STATIC_RUNTIME OFF) -find_package(Boost 1.86.0 REQUIRED) +if(CMAKE_CXX_STANDARD LESS 17) + # requires non-header filesystem for state saving if C++ 11 or lower + find_package(Boost 1.86.0 REQUIRED COMPONENTS system filesystem) +else() + find_package(Boost 1.86.0 REQUIRED) +endif() # ----------------------------------------------------------------------------- if(NGEN_WITH_SQLITE) diff --git a/include/realizations/catchment/Bmi_Multi_Formulation.hpp b/include/realizations/catchment/Bmi_Multi_Formulation.hpp index 458b63949a..3387f9fa1e 100644 --- a/include/realizations/catchment/Bmi_Multi_Formulation.hpp +++ b/include/realizations/catchment/Bmi_Multi_Formulation.hpp @@ -57,19 +57,6 @@ namespace realization { } }; - void load_state(std::shared_ptr loader) override; - - void load_hot_start(std::shared_ptr loader) override; - - virtual void check_mass_balance(const int& iteration, const int& total_steps, const std::string& timestamp) const final { - for( const auto &module : modules ) { - // TODO may need to check on outputs form each module indepdently??? - // Right now, the assumption is that if each component is mass balanced - // then the entire formulation is mass balanced - module->check_mass_balance(iteration, total_steps, timestamp); - } - }; - void save_state(std::shared_ptr saver) override; void load_state(std::shared_ptr loader) override; diff --git a/src/partitionGenerator.cpp b/src/partitionGenerator.cpp index 924e7e3d4f..a4ae3940fb 100644 --- a/src/partitionGenerator.cpp +++ b/src/partitionGenerator.cpp @@ -433,7 +433,7 @@ int main(int argc, char* argv[]) { #if NGEN_WITH_SQLITE3 try { - catchment_collection = ngen::geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); + catchment_collection = ngen::geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); } catch (std::exception &e) { // Handle all exceptions std::string msg = "Geopackage error occurred reading divides: " + catchmentDataFile; @@ -483,7 +483,7 @@ int main(int argc, char* argv[]) throw; } #else - LOG(msg,LogLevel::FATAL, "SQLite3 support required to read GeoPackage files."); + LOG(LogLevel::FATAL, "SQLite3 support required to read GeoPackage files."); throw std::runtime_error("SQLite3 support required to read GeoPackage files."); #endif } From 79b44e1149fae4904d58540afaaea3024443fb84 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 26 Mar 2026 15:51:20 -0400 Subject: [PATCH 109/109] Update submodule references --- extern/cfe/cfe | 2 +- extern/evapotranspiration/evapotranspiration | 2 +- extern/sloth | 2 +- extern/t-route | 2 +- extern/topmodel/topmodel | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extern/cfe/cfe b/extern/cfe/cfe index f60afb6a8a..1e42e47d9e 160000 --- a/extern/cfe/cfe +++ b/extern/cfe/cfe @@ -1 +1 @@ -Subproject commit f60afb6a8a49e388e6263664239ff27742132d26 +Subproject commit 1e42e47d9edae96e337f0379f71deb4638cb45a6 diff --git a/extern/evapotranspiration/evapotranspiration b/extern/evapotranspiration/evapotranspiration index 096208ad62..fa10f12c46 160000 --- a/extern/evapotranspiration/evapotranspiration +++ b/extern/evapotranspiration/evapotranspiration @@ -1 +1 @@ -Subproject commit 096208ad624e07216617f770a3447eb829266112 +Subproject commit fa10f12c46bf6cda028f843292cac7b26d8b6ac0 diff --git a/extern/sloth b/extern/sloth index ee0d982ccc..fc9c09823c 160000 --- a/extern/sloth +++ b/extern/sloth @@ -1 +1 @@ -Subproject commit ee0d982ccc07663cfea7bf0ac4d645841e19ccc1 +Subproject commit fc9c09823c90ca70a388f032fa19d88fc4a96091 diff --git a/extern/t-route b/extern/t-route index 595d56cb6f..e2f7d5dcb7 160000 --- a/extern/t-route +++ b/extern/t-route @@ -1 +1 @@ -Subproject commit 595d56cb6fe79e724e9ce7045dfdd7c200fc01fc +Subproject commit e2f7d5dcb7efcc684523f759b438ad250543ccdd diff --git a/extern/topmodel/topmodel b/extern/topmodel/topmodel index fa4f7e56db..e608a31687 160000 --- a/extern/topmodel/topmodel +++ b/extern/topmodel/topmodel @@ -1 +1 @@ -Subproject commit fa4f7e56dbe46df8cc0d7ca9095102290170b866 +Subproject commit e608a31687c00835a5484d83df277a6587989ff1