From 3294e395f0c8fc2270e32e2874b718dc131f0ad6 Mon Sep 17 00:00:00 2001 From: Luke Kiernan Date: Fri, 10 Apr 2026 15:30:04 -0600 Subject: [PATCH 1/4] Update POM to work with IOM main - Port serialize_problem from IOM (removed there in deduplicate-opt-container) - Adapt to _check_branch_network_compatibility signature change (3-arg to 2-arg) - Remove system_to_file kwarg from tests (removed from IOM Settings) - Trim deserialization test (deserialize_problem not yet ported) - Bump PNM compat to ^0.19 to match IOM - Point test env at local IOM Co-Authored-By: Claude Opus 4.6 (1M context) --- Project.toml | 3 +- src/PowerOperationsModels.jl | 1 + .../instantiate_network_model.jl | 33 ++++++++-- .../operation_model_serialization.jl | 66 +++++++++++++++++++ test/Project.toml | 1 - test/performance/performance_test.jl | 2 - test/test_device_branch_constructors.jl | 5 -- test/test_device_hydro_constructors.jl | 3 - test/test_device_load_constructors.jl | 3 - ..._device_thermal_generation_constructors.jl | 7 -- test/test_model_decision.jl | 20 +----- test/test_utils/run_simulation.jl | 3 - 12 files changed, 99 insertions(+), 48 deletions(-) create mode 100644 src/operation/operation_model_serialization.jl diff --git a/Project.toml b/Project.toml index 15c0a8f..87185af 100644 --- a/Project.toml +++ b/Project.toml @@ -28,7 +28,6 @@ PowerFlowsExt = "PowerFlows" [sources] InfrastructureSystems = {url = "https://github.com/NREL-Sienna/InfrastructureSystems.jl", rev = "IS4"} -InfrastructureOptimizationModels = {url = "https://github.com/NREL-Sienna/InfrastructureOptimizationModels.jl", rev = "main"} [compat] Dates = "1" @@ -37,7 +36,7 @@ InfrastructureOptimizationModels = "0.1" InfrastructureSystems = "3" InteractiveUtils = "1.11.0" JuMP = "^1.28" -PowerNetworkMatrices = "^0.18" +PowerNetworkMatrices = "^0.19" PowerSystems = "5.3" ProgressMeter = "1.11.0" TimerOutputs = "~0.5" diff --git a/src/PowerOperationsModels.jl b/src/PowerOperationsModels.jl index e24ea91..859a100 100644 --- a/src/PowerOperationsModels.jl +++ b/src/PowerOperationsModels.jl @@ -280,6 +280,7 @@ include("area_interchange.jl") # Operation lifecycle: build/solve/run include("operation/build_problem.jl") include("initial_conditions/initialization.jl") +include("operation/operation_model_serialization.jl") include("operation/decision_model.jl") include("operation/emulation_model.jl") diff --git a/src/network_models/instantiate_network_model.jl b/src/network_models/instantiate_network_model.jl index 5a881b1..eb15ae7 100644 --- a/src/network_models/instantiate_network_model.jl +++ b/src/network_models/instantiate_network_model.jl @@ -66,6 +66,29 @@ _assign_subnetworks_to_buses( ::PSY.System, ) where {T <: AbstractPowerModel} = nothing +function _get_unmodeled_branch_types( + branch_models::BranchModelContainer, + sys::PSY.System, +) + unmodeled = DataType[] + for d in PSY.get_existing_device_types(sys) + if d <: PSY.ACTransmission && !haskey(branch_models, Symbol(d)) + push!(unmodeled, d) + end + end + return unmodeled +end + +function _validate_network_and_branches( + model::NetworkModel, + branch_models::BranchModelContainer, + sys::PSY.System, +) + unmodeled = _get_unmodeled_branch_types(branch_models, sys) + IOM._check_branch_network_compatibility(model, unmodeled) + return +end + ################################################################################# # Generic fallback for AbstractPowerModel (Ybus-based models: ACP, ACR, etc.) ################################################################################# @@ -76,7 +99,7 @@ function IOM.instantiate_network_model!( number_of_steps::Int, sys::PSY.System, ) where {T <: AbstractPowerModel} - IOM._check_branch_network_compatibility(model, branch_models, sys) + _validate_network_and_branches(model, branch_models, sys) if isempty(model.subnetworks) model.subnetworks = PNM.find_subnetworks(sys) end @@ -127,7 +150,7 @@ function IOM.instantiate_network_model!( number_of_steps::Int, sys::PSY.System, ) - IOM._check_branch_network_compatibility(model, branch_models, sys) + _validate_network_and_branches(model, branch_models, sys) PNM.populate_branch_maps_by_type!(model.network_reduction) empty!(model.reduced_branch_tracker) IOM.set_number_of_steps!(model.reduced_branch_tracker, number_of_steps) @@ -144,7 +167,7 @@ function IOM.instantiate_network_model!( number_of_steps::Int, sys::PSY.System, ) - IOM._check_branch_network_compatibility(model, branch_models, sys) + _validate_network_and_branches(model, branch_models, sys) if isempty(model.subnetworks) model.subnetworks = PNM.find_subnetworks(sys) end @@ -168,7 +191,7 @@ function IOM.instantiate_network_model!( number_of_steps::Int, sys::PSY.System, ) - IOM._check_branch_network_compatibility(model, branch_models, sys) + _validate_network_and_branches(model, branch_models, sys) if IOM.get_PTDF_matrix(model) === nothing @info "PTDF Matrix not provided. Calculating using PowerNetworkMatrices.PTDF" if model.reduce_radial_branches && model.reduce_degree_two_branches @@ -258,7 +281,7 @@ function IOM.instantiate_network_model!( number_of_steps::Int, sys::PSY.System, ) - IOM._check_branch_network_compatibility(model, branch_models, sys) + _validate_network_and_branches(model, branch_models, sys) if IOM.get_PTDF_matrix(model) === nothing @info "PTDF Matrix not provided. Calculating using PowerNetworkMatrices.PTDF" if model.reduce_radial_branches && model.reduce_degree_two_branches diff --git a/src/operation/operation_model_serialization.jl b/src/operation/operation_model_serialization.jl new file mode 100644 index 0000000..e948491 --- /dev/null +++ b/src/operation/operation_model_serialization.jl @@ -0,0 +1,66 @@ +const _SERIALIZED_MODEL_FILENAME = "model.bin" + +struct OptimizerAttributes + name::String + version::String + attributes::Any +end + +function OptimizerAttributes( + model::IOM.OperationModel, + optimizer::IOM.MOI.OptimizerWithAttributes, +) + jump_model = IOM.get_jump_model(model) + name = JuMP.solver_name(jump_model) + # Note that this uses private field access to MOI.OptimizerWithAttributes because there + # is no public method available. + # This could break if MOI changes their implementation. + try + version = IOM.MOI.get(JuMP.backend(jump_model), IOM.MOI.SolverVersion()) + return OptimizerAttributes(name, version, optimizer.params) + catch + @debug "Solver Version not supported by the solver" + version = "MOI.SolverVersion not supported" + return OptimizerAttributes(name, version, optimizer.params) + end +end + +function _get_optimizer_attributes(model::IOM.OperationModel) + return IOM.get_optimizer(IOM.get_settings(model)).params +end + +struct ProblemSerializationWrapper + template::IOM.AbstractProblemTemplate + sys::Union{Nothing, String} + settings::IOM.Settings + model_type::DataType + name::String + optimizer::OptimizerAttributes +end + +function serialize_problem(model::IOM.OperationModel; optimizer = nothing) + # A PowerSystem cannot be serialized in this format because of how it stores + # time series data. Use its specialized serialization method instead. + sys = IOM.get_system(model) + sys_filename = + joinpath(IOM.get_output_dir(model), IOM.make_system_filename(sys)) + # Skip serialization if the system is already in the folder + !ispath(sys_filename) && PSY.to_json(sys, sys_filename) + + if optimizer === nothing + optimizer = IOM.get_optimizer(IOM.get_settings(model)) + @assert optimizer !== nothing "optimizer must be passed if it wasn't saved in Settings" + end + + obj = ProblemSerializationWrapper( + model.template, + sys_filename, + deepcopy(IOM.get_settings(model)), + typeof(model), + string(IOM.get_name(model)), + OptimizerAttributes(model, optimizer), + ) + bin_file_name = joinpath(IOM.get_output_dir(model), _SERIALIZED_MODEL_FILENAME) + Serialization.serialize(bin_file_name, obj) + @info "Serialized OperationModel to" bin_file_name +end diff --git a/test/Project.toml b/test/Project.toml index e676ec5..91d6928 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -32,7 +32,6 @@ TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [sources] -InfrastructureOptimizationModels = {rev = "main", url = "https://github.com/NREL-Sienna/InfrastructureOptimizationModels.jl"} InfrastructureSystems = {rev = "IS4", url = "https://github.com/NREL-Sienna/InfrastructureSystems.jl"} [compat] diff --git a/test/performance/performance_test.jl b/test/performance/performance_test.jl index 768a4a9..7b15857 100644 --- a/test/performance/performance_test.jl +++ b/test/performance/performance_test.jl @@ -89,7 +89,6 @@ try optimizer = optimizer_with_attributes(HiGHS.Optimizer, "mip_rel_gap" => 0.01, "log_to_console" => false), - system_to_file = false, initialize_model = true, optimizer_solve_log_print = false, direct_mode_optimizer = true, @@ -103,7 +102,6 @@ try optimizer = optimizer_with_attributes(HiGHS.Optimizer, "mip_rel_gap" => 0.01, "log_to_console" => false), - system_to_file = false, initialize_model = true, check_numerical_bounds = false, ) diff --git a/test/test_device_branch_constructors.jl b/test/test_device_branch_constructors.jl index 7d6fda7..6e9d3d5 100644 --- a/test/test_device_branch_constructors.jl +++ b/test/test_device_branch_constructors.jl @@ -272,7 +272,6 @@ end sys_5; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, ) build!(model; output_dir = mktempdir()) @@ -289,7 +288,6 @@ end sys_5; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, ) solve!(model; output_dir = mktempdir()) dcp_vars = @@ -346,7 +344,6 @@ end sys_5; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, store_variable_names = true, ) @@ -379,7 +376,6 @@ end sys_5; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, ) solve!(model; output_dir = mktempdir()) @@ -434,7 +430,6 @@ end sys_5; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, ) solve!(model_wl; output_dir = mktempdir()) diff --git a/test/test_device_hydro_constructors.jl b/test/test_device_hydro_constructors.jl index 1b56ca5..a04fdf6 100644 --- a/test/test_device_hydro_constructors.jl +++ b/test/test_device_hydro_constructors.jl @@ -597,7 +597,6 @@ end optimizer = ipopt_optimizer, optimizer_solve_log_print = true, store_variable_names = true, - system_to_file = true, horizon = Hour(24), ) @@ -724,7 +723,6 @@ end sys; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, store_variable_names = true, optimizer_solve_log_print = false, ) @@ -773,7 +771,6 @@ end sys; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, store_variable_names = true, optimizer_solve_log_print = false, ) diff --git a/test/test_device_load_constructors.jl b/test/test_device_load_constructors.jl index 1c2855a..38309fa 100644 --- a/test/test_device_load_constructors.jl +++ b/test/test_device_load_constructors.jl @@ -112,7 +112,6 @@ end c_sys5_il; name = "UC_fixed_market_bid_cost", optimizer = HiGHS_optimizer, - system_to_file = false, optimizer_solve_log_print = true) @test build!(model; output_dir = test_path) == IOM.ModelBuildStatus.BUILT @test solve!(model) == IOM.RunStatus.SUCCESSFULLY_FINALIZED @@ -189,7 +188,6 @@ end name = "UC", store_variable_names = true, optimizer = solvers[ix], - system_to_file = false, ) @test build!(model; output_dir = mktempdir(; cleanup = true)) == @@ -234,7 +232,6 @@ end name = "UC", store_variable_names = true, optimizer = solvers[ix], - system_to_file = false, ) @test build!(model; output_dir = mktempdir(; cleanup = true)) == diff --git a/test/test_device_thermal_generation_constructors.jl b/test/test_device_thermal_generation_constructors.jl index 483f02d..87d828d 100644 --- a/test/test_device_thermal_generation_constructors.jl +++ b/test/test_device_thermal_generation_constructors.jl @@ -24,7 +24,6 @@ const TIME1 = DateTime("2024-01-01T00:00:00") sys; name = "UC_$(i)", optimizer = HiGHS_optimizer, - system_to_file = false, optimizer_solve_log_print = true, ) @test build!(model; output_dir = test_path) == IOM.ModelBuildStatus.BUILT @@ -59,7 +58,6 @@ const TIME1 = DateTime("2024-01-01T00:00:00") sys_no_startup; name = "UC_no_startup", optimizer = HiGHS_optimizer, - system_to_file = false, optimizer_solve_log_print = true, ) @test build!(model_no_startup; output_dir = test_path) == IOM.ModelBuildStatus.BUILT @@ -88,7 +86,6 @@ const TIME1 = DateTime("2024-01-01T00:00:00") sys_with_startup; name = "UC_with_startup", optimizer = HiGHS_optimizer, - system_to_file = false, optimizer_solve_log_print = true, ) @test build!(model_with_startup; output_dir = test_path) == @@ -138,7 +135,6 @@ end sys; name = "UC_$(i)", optimizer = HiGHS_optimizer, - system_to_file = false, ) @test build!(model; output_dir = test_path) == IOM.ModelBuildStatus.BUILT @test solve!(model) == IOM.RunStatus.SUCCESSFULLY_FINALIZED @@ -1164,7 +1160,6 @@ end =# sys_5; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, store_variable_names = true, rebuild_model = rebuild, ) @@ -1260,7 +1255,6 @@ end sys; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = false, store_variable_names = true, optimizer_solve_log_print = false, ) @@ -1368,7 +1362,6 @@ end =# sys; name = "UC", optimizer = ipopt_optimizer, - system_to_file = false, store_variable_names = true, optimizer_solve_log_print = false, ) diff --git a/test/test_model_decision.jl b/test/test_model_decision.jl index 13652aa..127333b 100644 --- a/test/test_model_decision.jl +++ b/test/test_model_decision.jl @@ -311,6 +311,8 @@ end IOM.RunStatus.SUCCESSFULLY_FINALIZED end +# TODO: Deserialization constructor (DecisionModel(path, optimizer)) was removed from IOM. +# Port deserialize_problem to POM to re-enable this test. @testset "Test Serialization, deserialization and write optimizer problem" begin fpath = mktempdir(; cleanup = true) sys = PSB.build_system(PSITestSystems, "c_sys5_re") @@ -324,23 +326,7 @@ end file_list = sort!(collect(readdir(fpath))) model_name = IOM.get_name(model) @test IOM._JUMP_MODEL_FILENAME in file_list - @test IOM._SERIALIZED_MODEL_FILENAME in file_list - ED2 = DecisionModel(fpath, HiGHS_optimizer) - @test build!(ED2; output_dir = fpath) == IOM.ModelBuildStatus.BUILT - psi_checksolve_test(ED2, [MOI.OPTIMAL], 240000.0, 10000) - - path2 = mktempdir(; cleanup = true) - model_no_sys = - DecisionModel(template, sys; optimizer = HiGHS_optimizer, system_to_file = false) - - @test build!(model_no_sys; output_dir = path2) == IOM.ModelBuildStatus.BUILT - @test solve!(model_no_sys) == IOM.RunStatus.SUCCESSFULLY_FINALIZED - - file_list = sort!(collect(readdir(path2))) - @test !any(occursin.(r"\.h5$", file_list)) - ED3 = DecisionModel(path2, HiGHS_optimizer; system = sys) - build!(ED3; output_dir = path2) - psi_checksolve_test(ED3, [MOI.OPTIMAL], 240000.0, 10000) + @test POM._SERIALIZED_MODEL_FILENAME in file_list end @testset "Test NonSpinning reserve model" begin diff --git a/test/test_utils/run_simulation.jl b/test/test_utils/run_simulation.jl index 7d46ea5..0b4bda9 100644 --- a/test/test_utils/run_simulation.jl +++ b/test/test_utils/run_simulation.jl @@ -11,7 +11,6 @@ function run_simulation( file_path::String, export_path; in_memory = false, - system_to_file = true, uc_network_model = nothing, ed_network_model = nothing, ) @@ -45,14 +44,12 @@ function run_simulation( c_sys5_hy_uc; name = "UC", optimizer = HiGHS_optimizer, - system_to_file = system_to_file, ), DecisionModel( template_ed, c_sys5_hy_ed; name = "ED", optimizer = ipopt_optimizer, - system_to_file = system_to_file, ), ], ) From 698925f5bcf277ecc81df340169553f750249d72 Mon Sep 17 00:00:00 2001 From: Anthony Costarelli Date: Mon, 13 Apr 2026 15:44:51 -0400 Subject: [PATCH 2/4] restore deserialization test; restore new pwl names; remove op model serialization (restored in IOM); --- src/PowerOperationsModels.jl | 8 +-- src/common_models/market_bid_overrides.jl | 16 ++--- src/core/problem_template.jl | 8 +-- src/energy_storage_models/storage_models.jl | 4 +- .../operation_model_serialization.jl | 66 ------------------- src/services_models/reserves.jl | 2 +- .../thermal_generation.jl | 6 +- test/test_model_decision.jl | 20 +++++- 8 files changed, 39 insertions(+), 91 deletions(-) delete mode 100644 src/operation/operation_model_serialization.jl diff --git a/src/PowerOperationsModels.jl b/src/PowerOperationsModels.jl index 1205b36..c9a0957 100644 --- a/src/PowerOperationsModels.jl +++ b/src/PowerOperationsModels.jl @@ -141,8 +141,8 @@ import InfrastructureOptimizationModels: _include_constant_min_gen_power_in_constraint, add_variable_cost_to_objective!, _vom_offer_direction, - _add_pwl_constraint!, - add_pwl_term!, + add_pwl_constraint_delta!, + add_pwl_term_delta!, get_output_offer_curves, # Internal utilities used by market bid overrides and proportional_cost is_time_variant, @@ -175,7 +175,7 @@ import InfrastructureOptimizationModels: set_service_model!, finalize_template!, make_empty_jump_model_with_settings, - _set_model! + set_model! using InfrastructureOptimizationModels # TODO: use explicit imports. @@ -280,7 +280,7 @@ include("area_interchange.jl") # Operation lifecycle: build/solve/run include("operation/build_problem.jl") include("initial_conditions/initialization.jl") -include("operation/operation_model_serialization.jl") +include("operation/template_validation.jl") include("operation/decision_model.jl") include("operation/emulation_model.jl") diff --git a/src/common_models/market_bid_overrides.jl b/src/common_models/market_bid_overrides.jl index 4de701b..10e6ea8 100644 --- a/src/common_models/market_bid_overrides.jl +++ b/src/common_models/market_bid_overrides.jl @@ -175,7 +175,7 @@ function add_variable_cost_to_objective!( ::ImportExportSourceModel, ) isnothing(get_output_offer_curves(cost_function)) && return - add_pwl_term!( + add_pwl_term_delta!( IncrementalOffer(), container, component, @@ -194,7 +194,7 @@ function add_variable_cost_to_objective!( ::ImportExportSourceModel, ) isnothing(get_input_offer_curves(cost_function)) && return - add_pwl_term!( + add_pwl_term_delta!( DecrementalOffer(), container, component, @@ -221,7 +221,7 @@ function add_variable_cost_to_objective!( if !(isnothing(get_output_offer_curves(cost_function))) error("Component $(component_name) is not allowed to participate as a supply.") end - add_pwl_term!( + add_pwl_term_delta!( DecrementalOffer(), container, component, @@ -241,7 +241,7 @@ _vom_offer_direction(::AbstractControllablePowerLoadFormulation) = DecrementalOf """ PWL block offer constraints for ORDC (ReserveDemandCurve). """ -function _add_pwl_constraint!( +function add_pwl_constraint_delta!( container::OptimizationContainer, component::T, ::U, @@ -273,7 +273,7 @@ end """ PWL cost terms for StepwiseCostReserve (AbstractServiceFormulation). """ -function add_pwl_term!( +function add_pwl_term_delta!( container::OptimizationContainer, component::T, cost_data::PSY.CostCurve{PSY.PiecewiseIncrementalCurve}, @@ -300,7 +300,7 @@ function add_pwl_term!( slopes = IS.get_y_coords(data) break_points = PSY.get_x_coords(data) for t in time_steps - pwl_vars = add_pwl_variables!( + pwl_vars = add_pwl_variables_delta!( container, PiecewiseLinearBlockIncrementalOffer, T, @@ -309,9 +309,9 @@ function add_pwl_term!( length(slopes); upper_bound = Inf, ) - add_pwl_constraint!(container, component, U(), break_points, pwl_vars, t) + add_pwl_constraint_delta!(container, component, U(), break_points, pwl_vars, t) pwl_cost_expressions[t] = - get_pwl_cost_expression(pwl_vars, slopes, multiplier * dt) + get_pwl_cost_expression_delta(pwl_vars, slopes, multiplier * dt) end return pwl_cost_expressions end diff --git a/src/core/problem_template.jl b/src/core/problem_template.jl index e230078..5bdfa61 100644 --- a/src/core/problem_template.jl +++ b/src/core/problem_template.jl @@ -140,7 +140,7 @@ function set_device_model!( template::OperationsProblemTemplate, model::DeviceModel{D}, ) where {D <: IS.InfrastructureSystemsComponent} - _set_model!(template.devices, model) + set_model!(template.devices, model) return end @@ -152,7 +152,7 @@ function set_device_model!( template::OperationsProblemTemplate, model::DeviceModel{D}, ) where {D <: PSY.Branch} - _set_model!(template.branches, model) + set_model!(template.branches, model) return end @@ -191,7 +191,7 @@ function set_service_model!( service_name::String, model::ServiceModel{T, <:AbstractServiceFormulation}, ) where {T <: PSY.Service} - _set_model!(template.services, (service_name, Symbol(T)), model) + set_model!(template.services, (service_name, Symbol(T)), model) return end @@ -199,7 +199,7 @@ function set_service_model!( template::OperationsProblemTemplate, model::ServiceModel{<:PSY.Service, <:AbstractServiceFormulation}, ) - _set_model!(template.services, model) + set_model!(template.services, model) return end diff --git a/src/energy_storage_models/storage_models.jl b/src/energy_storage_models/storage_models.jl index 8396fdc..1f3730d 100644 --- a/src/energy_storage_models/storage_models.jl +++ b/src/energy_storage_models/storage_models.jl @@ -1916,7 +1916,7 @@ function _add_variable_cost_to_objective!( @debug "Market Bid" _group = LOG_GROUP_COST_FUNCTIONS component_name incremental_cost_curves = PSY.get_incremental_offer_curves(cost_function) if !isnothing(incremental_cost_curves) - add_pwl_term!( + add_pwl_term_delta!( IncrementalOffer(), container, component, @@ -1942,7 +1942,7 @@ function _add_variable_cost_to_objective!( @debug "Market Bid" _group = LOG_GROUP_COST_FUNCTIONS component_name decremental_cost_curves = PSY.get_decremental_offer_curves(cost_function) if !isnothing(decremental_cost_curves) - add_pwl_term!( + add_pwl_term_delta!( DecrementalOffer(), container, component, diff --git a/src/operation/operation_model_serialization.jl b/src/operation/operation_model_serialization.jl deleted file mode 100644 index e948491..0000000 --- a/src/operation/operation_model_serialization.jl +++ /dev/null @@ -1,66 +0,0 @@ -const _SERIALIZED_MODEL_FILENAME = "model.bin" - -struct OptimizerAttributes - name::String - version::String - attributes::Any -end - -function OptimizerAttributes( - model::IOM.OperationModel, - optimizer::IOM.MOI.OptimizerWithAttributes, -) - jump_model = IOM.get_jump_model(model) - name = JuMP.solver_name(jump_model) - # Note that this uses private field access to MOI.OptimizerWithAttributes because there - # is no public method available. - # This could break if MOI changes their implementation. - try - version = IOM.MOI.get(JuMP.backend(jump_model), IOM.MOI.SolverVersion()) - return OptimizerAttributes(name, version, optimizer.params) - catch - @debug "Solver Version not supported by the solver" - version = "MOI.SolverVersion not supported" - return OptimizerAttributes(name, version, optimizer.params) - end -end - -function _get_optimizer_attributes(model::IOM.OperationModel) - return IOM.get_optimizer(IOM.get_settings(model)).params -end - -struct ProblemSerializationWrapper - template::IOM.AbstractProblemTemplate - sys::Union{Nothing, String} - settings::IOM.Settings - model_type::DataType - name::String - optimizer::OptimizerAttributes -end - -function serialize_problem(model::IOM.OperationModel; optimizer = nothing) - # A PowerSystem cannot be serialized in this format because of how it stores - # time series data. Use its specialized serialization method instead. - sys = IOM.get_system(model) - sys_filename = - joinpath(IOM.get_output_dir(model), IOM.make_system_filename(sys)) - # Skip serialization if the system is already in the folder - !ispath(sys_filename) && PSY.to_json(sys, sys_filename) - - if optimizer === nothing - optimizer = IOM.get_optimizer(IOM.get_settings(model)) - @assert optimizer !== nothing "optimizer must be passed if it wasn't saved in Settings" - end - - obj = ProblemSerializationWrapper( - model.template, - sys_filename, - deepcopy(IOM.get_settings(model)), - typeof(model), - string(IOM.get_name(model)), - OptimizerAttributes(model, optimizer), - ) - bin_file_name = joinpath(IOM.get_output_dir(model), _SERIALIZED_MODEL_FILENAME) - Serialization.serialize(bin_file_name, obj) - @info "Serialized OperationModel to" bin_file_name -end diff --git a/src/services_models/reserves.jl b/src/services_models/reserves.jl index 7eee109..a6b63d7 100644 --- a/src/services_models/reserves.jl +++ b/src/services_models/reserves.jl @@ -543,7 +543,7 @@ function _add_reserves_variable_cost_to_objective!( end pwl_cost_expressions = - add_pwl_term!(container, component, variable_cost, T(), U()) + add_pwl_term_delta!(container, component, variable_cost, T(), U()) for t in time_steps add_to_expression!( container, diff --git a/src/static_injector_models/thermal_generation.jl b/src/static_injector_models/thermal_generation.jl index 3e41292..0f7efd2 100644 --- a/src/static_injector_models/thermal_generation.jl +++ b/src/static_injector_models/thermal_generation.jl @@ -1559,7 +1559,7 @@ Add PWL cost terms for ThermalDispatchNoMin formulation. Rejects non-convex or negative-slope PWL data since ThermalDispatchNoMin cannot use SOS-2 formulations. """ -function IOM.add_pwl_term!( +function IOM.add_pwl_term_lambda!( container::IOM.OptimizationContainer, component::T, cost_function::Union{ @@ -1621,7 +1621,7 @@ function IOM.add_pwl_term!( temp_cost_function = IOM.create_temporary_cost_function_in_system_per_unit(cost_function, data) for t in time_steps - IOM.add_pwl_variables!(container, T, name, t, data) + IOM.add_pwl_variables_lambda!(container, T, name, t, data) power_var = IOM.get_variable(container, U(), T)[name, t] IOM._add_pwl_constraint_standard!( container, @@ -1632,7 +1632,7 @@ function IOM.add_pwl_term!( power_var, ) pwl_cost = - IOM.get_pwl_cost_expression( + IOM.get_pwl_cost_expression_lambda( container, component, t, diff --git a/test/test_model_decision.jl b/test/test_model_decision.jl index 127333b..13652aa 100644 --- a/test/test_model_decision.jl +++ b/test/test_model_decision.jl @@ -311,8 +311,6 @@ end IOM.RunStatus.SUCCESSFULLY_FINALIZED end -# TODO: Deserialization constructor (DecisionModel(path, optimizer)) was removed from IOM. -# Port deserialize_problem to POM to re-enable this test. @testset "Test Serialization, deserialization and write optimizer problem" begin fpath = mktempdir(; cleanup = true) sys = PSB.build_system(PSITestSystems, "c_sys5_re") @@ -326,7 +324,23 @@ end file_list = sort!(collect(readdir(fpath))) model_name = IOM.get_name(model) @test IOM._JUMP_MODEL_FILENAME in file_list - @test POM._SERIALIZED_MODEL_FILENAME in file_list + @test IOM._SERIALIZED_MODEL_FILENAME in file_list + ED2 = DecisionModel(fpath, HiGHS_optimizer) + @test build!(ED2; output_dir = fpath) == IOM.ModelBuildStatus.BUILT + psi_checksolve_test(ED2, [MOI.OPTIMAL], 240000.0, 10000) + + path2 = mktempdir(; cleanup = true) + model_no_sys = + DecisionModel(template, sys; optimizer = HiGHS_optimizer, system_to_file = false) + + @test build!(model_no_sys; output_dir = path2) == IOM.ModelBuildStatus.BUILT + @test solve!(model_no_sys) == IOM.RunStatus.SUCCESSFULLY_FINALIZED + + file_list = sort!(collect(readdir(path2))) + @test !any(occursin.(r"\.h5$", file_list)) + ED3 = DecisionModel(path2, HiGHS_optimizer; system = sys) + build!(ED3; output_dir = path2) + psi_checksolve_test(ED3, [MOI.OPTIMAL], 240000.0, 10000) end @testset "Test NonSpinning reserve model" begin From 72eac8af9eaacfe22f7677bda58360137be43eef Mon Sep 17 00:00:00 2001 From: Anthony Costarelli Date: Mon, 13 Apr 2026 16:49:09 -0400 Subject: [PATCH 3/4] remove problem serialization test --- test/test_model_decision.jl | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/test/test_model_decision.jl b/test/test_model_decision.jl index 13652aa..02344c6 100644 --- a/test/test_model_decision.jl +++ b/test/test_model_decision.jl @@ -311,38 +311,6 @@ end IOM.RunStatus.SUCCESSFULLY_FINALIZED end -@testset "Test Serialization, deserialization and write optimizer problem" begin - fpath = mktempdir(; cleanup = true) - sys = PSB.build_system(PSITestSystems, "c_sys5_re") - template = get_template_dispatch_with_network( - NetworkModel(CopperPlatePowerModel; duals = [CopperPlateBalanceConstraint]), - ) - model = DecisionModel(template, sys; optimizer = HiGHS_optimizer) - @test build!(model; output_dir = fpath) == IOM.ModelBuildStatus.BUILT - @test solve!(model) == IOM.RunStatus.SUCCESSFULLY_FINALIZED - - file_list = sort!(collect(readdir(fpath))) - model_name = IOM.get_name(model) - @test IOM._JUMP_MODEL_FILENAME in file_list - @test IOM._SERIALIZED_MODEL_FILENAME in file_list - ED2 = DecisionModel(fpath, HiGHS_optimizer) - @test build!(ED2; output_dir = fpath) == IOM.ModelBuildStatus.BUILT - psi_checksolve_test(ED2, [MOI.OPTIMAL], 240000.0, 10000) - - path2 = mktempdir(; cleanup = true) - model_no_sys = - DecisionModel(template, sys; optimizer = HiGHS_optimizer, system_to_file = false) - - @test build!(model_no_sys; output_dir = path2) == IOM.ModelBuildStatus.BUILT - @test solve!(model_no_sys) == IOM.RunStatus.SUCCESSFULLY_FINALIZED - - file_list = sort!(collect(readdir(path2))) - @test !any(occursin.(r"\.h5$", file_list)) - ED3 = DecisionModel(path2, HiGHS_optimizer; system = sys) - build!(ED3; output_dir = path2) - psi_checksolve_test(ED3, [MOI.OPTIMAL], 240000.0, 10000) -end - @testset "Test NonSpinning reserve model" begin c_sys5 = PSB.build_system(PSITestSystems, "c_sys5_uc_non_spin"; add_reserves = true) template = get_thermal_standard_uc_template() From 2e8fc5f26987ddc36541b44ecaad549e0508a32f Mon Sep 17 00:00:00 2001 From: Anthony Costarelli Date: Tue, 14 Apr 2026 10:15:36 -0400 Subject: [PATCH 4/4] initial hdf attempt without a store --- src/operation/decision_model.jl | 20 ++++++++++++++++++++ src/operation/emulation_model.jl | 20 ++++++++++++++++++++ test/test_system_serialization.jl | 25 +++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 test/test_system_serialization.jl diff --git a/src/operation/decision_model.jl b/src/operation/decision_model.jl index d2db63f..0e25a42 100644 --- a/src/operation/decision_model.jl +++ b/src/operation/decision_model.jl @@ -45,6 +45,7 @@ Build the Decision Model based on the specified DecisionProblem. - `console_level = Logging.Error`: - `file_level = Logging.Info`: - `disable_timer_outputs = false` : Enable/Disable timing outputs + - `store_system_in_results::Bool = true`: If true, stores the system as JSON in the results HDF5 file. """ function build!( model::DecisionModel{<:DecisionProblem}; @@ -53,6 +54,7 @@ function build!( console_level = Logging.Error, file_level = Logging.Info, disable_timer_outputs = false, + store_system_in_results = true, ) mkpath(output_dir) IOM.set_output_dir!(model, output_dir) @@ -64,12 +66,20 @@ function build!( IOM.add_recorders!(model, recorders) IOM.register_recorders!(model, file_mode) logger = IS.configure_logging(get_internal(model), IOM.PROBLEM_LOG_FILENAME, file_mode) + if store_system_in_results + serialization_task = + Threads.@spawn IOM.serialize_system_to_json(model) + end try Logging.with_logger(logger) do try TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Problem $(get_name(model))" begin build_model!(model) end + if store_system_in_results + uuid, json_text = fetch(serialization_task) + IOM.write_system_to_hdf5!(model, uuid, json_text) + end set_status!(model, ModelBuildStatus.BUILT) @info "\n$(BUILD_PROBLEMS_TIMER)\n" catch e @@ -126,6 +136,7 @@ keyword arguments to that function. - `file_level = Logging.Info`: - `disable_timer_outputs = false` : Enable/Disable timing outputs - `export_optimization_problem::Bool = true`: If true, serialize the model to a file to allow re-execution later. + - `store_system_in_results::Bool = true`: If true, stores the system as JSON in the results HDF5 file. # Examples @@ -141,8 +152,13 @@ function solve!( file_level = Logging.Info, disable_timer_outputs = false, export_optimization_problem = true, + store_system_in_results = true, kwargs..., ) + if store_system_in_results + serialization_task = + Threads.@spawn IOM.serialize_system_to_json(model) + end build_if_not_already_built!( model; console_level = console_level, @@ -170,6 +186,10 @@ function solve!( get_optimization_container(model), IOM.get_store_params(model), ) + if store_system_in_results + uuid, json_text = fetch(serialization_task) + IOM.write_system_to_hdf5!(model, uuid, json_text) + end TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Solve" begin IOM._pre_solve_model_checks(model, optimizer) IOM.solve_model!(model) diff --git a/src/operation/emulation_model.jl b/src/operation/emulation_model.jl index 4aebe8b..a4d24f2 100644 --- a/src/operation/emulation_model.jl +++ b/src/operation/emulation_model.jl @@ -42,6 +42,7 @@ end """ Implementation of build for any EmulationProblem + - `store_system_in_results::Bool = true`: If true, stores the system as JSON in the results HDF5 file. """ function build!( model::EmulationModel{<:EmulationProblem}; @@ -51,6 +52,7 @@ function build!( console_level = Logging.Error, file_level = Logging.Info, disable_timer_outputs = false, + store_system_in_results = true, ) mkpath(output_dir) IOM.set_output_dir!(model, output_dir) @@ -66,6 +68,10 @@ function build!( IOM.PROBLEM_LOG_FILENAME, file_mode, ) + if store_system_in_results + serialization_task = + Threads.@spawn IOM.serialize_system_to_json(model) + end try Logging.with_logger(logger) do try @@ -73,6 +79,10 @@ function build!( TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Problem $(get_name(model))" begin build_model!(model) end + if store_system_in_results + uuid, json_text = fetch(serialization_task) + IOM.write_system_to_hdf5!(model, uuid, json_text) + end set_status!(model, ModelBuildStatus.BUILT) @info "\n$(BUILD_PROBLEMS_TIMER)\n" catch e @@ -171,6 +181,7 @@ keyword arguments to that function. - `output_dir::String`: Required if the model is not already built, otherwise ignored - `enable_progress_bar::Bool`: Enables/Disable progress bar printing - `export_optimization_model::Bool`: If true, serialize the model to a file to allow re-execution later. + - `store_system_in_results::Bool = true`: If true, stores the system as JSON in the results HDF5 file. # Examples @@ -187,8 +198,13 @@ function run!( disable_timer_outputs = false, export_optimization_model = true, enable_progress_bar = _progress_meter_enabled(), + store_system_in_results = true, kwargs..., ) + if store_system_in_results + serialization_task = + Threads.@spawn IOM.serialize_system_to_json(model) + end build_if_not_already_built!( model; console_level = console_level, @@ -215,6 +231,10 @@ function run!( get_optimization_container(model), IOM.get_store_params(model), ) + if store_system_in_results + uuid, json_text = fetch(serialization_task) + IOM.write_system_to_hdf5!(model, uuid, json_text) + end TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Run" begin execute_emulation!( model; diff --git a/test/test_system_serialization.jl b/test/test_system_serialization.jl new file mode 100644 index 0000000..69b5f5c --- /dev/null +++ b/test/test_system_serialization.jl @@ -0,0 +1,25 @@ +@testset "System HDF5 serialization in build" begin + @testset "DecisionModel build with store_system_in_results=true" begin + template = get_thermal_dispatch_template_network() + c_sys5 = PSB.build_system(PSITestSystems, "c_sys5") + model = DecisionModel(template, c_sys5; optimizer = HiGHS_optimizer) + output_dir = mktempdir(; cleanup = true) + @test build!(model; output_dir = output_dir) == IOM.ModelBuildStatus.BUILT + hdf5_path = joinpath(output_dir, IOM.HDF_MODEL_STORE_FILENAME) + @test isfile(hdf5_path) + end + + @testset "DecisionModel build with store_system_in_results=false" begin + template = get_thermal_dispatch_template_network() + c_sys5 = PSB.build_system(PSITestSystems, "c_sys5") + model = DecisionModel(template, c_sys5; optimizer = HiGHS_optimizer) + output_dir = mktempdir(; cleanup = true) + @test build!( + model; + output_dir = output_dir, + store_system_in_results = false, + ) == IOM.ModelBuildStatus.BUILT + hdf5_path = joinpath(output_dir, IOM.HDF_MODEL_STORE_FILENAME) + @test !isfile(hdf5_path) + end +end