Extend support to SDDP Block and Solver

.github/workflows/test.yml

@@ -95,9 +95,10 @@ jobs:
       run: |
         conda list
 
+    # TODO: Re-enable SDDP test after new smspp-project release
     - name: Test package with SMSpp
       run: |
-        pytest --force-smspp
+        pytest --force-smspp -k "not test_optimize_sddp"
 
   test_with_smspp:
     # Test package build in matrix of OS and Python versions

pysmspp/__init__.py

@@ -56,5 +56,6 @@
     InvestmentBlockTestSolver as InvestmentBlockTestSolver,
     InvestmentBlockSolver as InvestmentBlockSolver,
     TSSBSolver as TSSBSolver,
+    SDDPSolver as SDDPSolver,
     is_smspp_installed as is_smspp_installed,
 )

pysmspp/block.py

@@ -6,6 +6,7 @@
     InvestmentBlockSolver,
     InvestmentSolver,
     TSSBSolver,
+    SDDPSolver,
 )
 from enum import IntEnum
 
@@ -197,10 +198,21 @@ def get_attr_field(
 
     if attr_name in simple_attrs.index:
         attr = attr_name
+    elif attr_name in block_attrs.index:
+        # Prefer an exact group name over wildcard prefixes. For example,
+        # ``StochasticBlock`` must not be confused with ``StochasticBlock_*``.
+        attr = attr_name
     else:
         attr_sel = block_attrs.loc[
             block_attrs.index.to_series().map(lambda x: attr_name.startswith(x))
         ]
+        if attr_sel.shape[0] > 1:
+            # Wildcard markers are stripped when the CSV files are loaded.
+            # Choosing the longest matching prefix makes the most specific
+            # wildcard deterministic (``StochasticBlock_`` wins over
+            # ``StochasticBlock`` for ``StochasticBlock_0``).
+            prefix_length = attr_sel.index.to_series().str.len()
+            attr_sel = attr_sel.loc[prefix_length == prefix_length.max()]
         if attr_sel.shape[0] == 1:
             attr = attr_sel.index[0]
         elif attr_sel.empty:
@@ -1470,8 +1482,8 @@ def optimize(
         default_solver_map = {
             "UCBlock": "UCBlockSolver",
             "InvestmentBlock": "InvestmentBlockSolver",
-            "SDDPBlock": "InvestmentSolver",
             "TwoStageStochasticBlock": "TSSBSolver",
+            "SDDPBlock": "SDDPSolver",
         }
 
         # Map solver names to actual solver classes
@@ -1481,6 +1493,7 @@ def optimize(
             "InvestmentBlockSolver": InvestmentBlockSolver,
             "InvestmentSolver": InvestmentSolver,
             "TSSBSolver": TSSBSolver,
+            "SDDPSolver": SDDPSolver,
         }
 
         if isinstance(smspp_solver, str) and smspp_solver == "auto":

pysmspp/data/blocks/AbstractPath.csv

@@ -1,10 +1,11 @@
 block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,unit,default,status,description
 AbstractPath,block_type,Attribute,Attribute,str,-,-,,required,Type of the block
 AbstractPath,id,Attribute,Attribute,str,-,-,,optional,Id of the block
-AbstractPath,PathDim,Dimension,Dimension,int,-,-,,required,Number of paths
+AbstractPath,PathDim,Dimension,Dimension,int,-,-,1,optional,Number of paths; one path is assumed when absent
 AbstractPath,TotalLength,Dimension,Dimension,int,-,-,,required,Total number of nodes across all paths
-AbstractPath,PathStart,Variable,Variable,int,-|PathDim,-,,required,Start index of each path
-AbstractPath,PathNodeTypes,Variable,Variable,str,-|TotalLength,-,,required,Type of each node
-AbstractPath,PathGroupIndices,Variable,Variable,str,-|TotalLength,-,,required,Group index per node
-AbstractPath,PathElementIndices,Variable,Variable,int,-|TotalLength,-,,required,Element index per node
-AbstractPath,PathRangeIndices,Variable,Variable,int,-|TotalLength,-,,required,Scenario range index per node
+AbstractPath,PathTotalLength,Dimension,Dimension,int,-,-,,optional,Current-format name for the total number of nodes across all paths
+AbstractPath,PathStart,Variable,Variable,int,PathDim,-,,optional,Start index of each path; required when PathDim is provided
+AbstractPath,PathNodeTypes,Variable,Variable,str,TotalLength|PathTotalLength,-,,required,Type of each node
+AbstractPath,PathGroupIndices,Variable,Variable,int|str,TotalLength|PathTotalLength,-,,required,Numeric group index or group name for each node
+AbstractPath,PathElementIndices,Variable,Variable,int,TotalLength|PathTotalLength,-,,optional,Element index per node; an unspecified element is assumed when absent
+AbstractPath,PathRangeIndices,Variable,Variable,int,TotalLength|PathTotalLength,-,,optional,End index of a ranged node; defaults depend on the node type

pysmspp/data/blocks/BendersBFunction.csv

@@ -0,0 +1,11 @@
+block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,unit,default,status,description
+BendersBFunction,NumVar,Dimension,Dimension,int,-,-,,required,Number of active variables and columns of the mapping matrix
+BendersBFunction,NumRow,Dimension,Dimension,int,-,-,0,optional,Number of rows of the mapping matrix
+BendersBFunction,NumNonzero,Dimension,Dimension,int,-,-,,optional,Number of nonzero matrix entries; its presence selects sparse storage
+BendersBFunction,NumNonzeroAtRow,Variable,Variable,int,NumRow,-,,optional,Nonzero count per row; required for non-identity sparse matrices
+BendersBFunction,Column,Variable,Variable,int,NumNonzero,-,,optional,Column index of each sparse matrix entry
+BendersBFunction,A,Variable,Variable,double,NumRow-NumVar|NumNonzero,-,,optional,Dense or sparse matrix values; required for nonempty mappings except an implicit sparse identity
+BendersBFunction,b,Variable,Variable,double,NumRow,-,0,optional,Constant vector; zero is assumed when absent
+BendersBFunction,ConstraintSide,Variable,Variable,str,NumRow,-,B,optional,Affected constraint side; both sides are assumed when absent
+BendersBFunction,AbstractPath,Block,Group,AbstractPath,list,-,-,optional,Paths to affected row constraints; required when NumRow is greater than zero
+BendersBFunction,Block,Block,Group,Block,-,-,-,required,Inner optimization block or indirect block reference

pysmspp/data/blocks/BendersBlock.csv

@@ -0,0 +1,6 @@
+block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,unit,default,status,description
+BendersBlock,block_type,Attribute,Attribute,str,-,-,,required,Type of the block
+BendersBlock,id,Attribute,Attribute,str,-,-,,optional,Id of the block
+BendersBlock,NumVar,Dimension,Dimension,int,-,-,,required,Number of active column variables
+BendersBlock,ObjectiveSense,Dimension,Dimension,int,-,-,1,optional,Objective sense; zero means maximization while a nonzero value or absence means minimization
+BendersBlock,BendersBFunction,Block,Group,BendersBFunction,-,-,-,required,Benders function used by the real objective

pysmspp/data/blocks/IndependentMultiStageScenarioGenerator.csv

@@ -0,0 +1,5 @@
+block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,unit,default,status,description
+IndependentMultiStageScenarioGenerator,block_type,Attribute,Attribute,str,-,-,,required,Type of the block
+IndependentMultiStageScenarioGenerator,id,Attribute,Attribute,str,-,-,,optional,Id of the scenario generator
+IndependentMultiStageScenarioGenerator,NumberStages,Dimension,Dimension,int,-,-,,required,Number of stages in the scenario generator
+IndependentMultiStageScenarioGenerator,Stage_*,Block,Group,ScenarioGenerator|DiscreteScenarioSet,list,-,-,required,Single-stage ScenarioGenerator for each stage

pysmspp/data/blocks/MultiStageScenarioGenerator.csv

@@ -0,0 +1,5 @@
+block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,unit,default,status,description
+MultiStageScenarioGenerator,block_type,Attribute,Attribute,str,-,-,,required,Type of the block
+MultiStageScenarioGenerator,id,Attribute,Attribute,str,-,-,,optional,Id of the scenario generator
+MultiStageScenarioGenerator,NumberStages,Dimension,Dimension,int,-,-,,required,Number of stages in the scenario generator
+MultiStageScenarioGenerator,Stage_*,Block,Group,ScenarioGenerator,list,-,-,required,ScenarioGenerator for each stage

pysmspp/data/blocks/SDDPBlock.csv

@@ -2,12 +2,21 @@ block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,un
 SDDPBlock,block_type,Attribute,Attribute,str,-,-,,required,Type of the block
 SDDPBlock,id,Attribute,Attribute,str,-,-,,optional,Id of the block
 SDDPBlock,TimeHorizon,Dimension,Dimension,int,-,-,,required,Time horizon of the optimization problem
+SDDPBlock,NumPolyhedralFunctionsPerStage,Dimension,Dimension,int,-,-,1,optional,Fixture-era name for the number of PolyhedralFunction objects in each stage; defaults to one
 SDDPBlock,NumPolyhedralFunctionsPerSubBlock,Dimension,Dimension,int,-,-,1,optional,Number of PolyhedralFunction that are present in each sub-Block
 SDDPBlock,NumSubBlocksPerStage,Dimension,Dimension,int,-,-,1,optional,Number of sub-Blocks that must be constructed for each stage
-SDDPBlock,InitialState,Variable,Variable,double,-,-,,required,Initial state for the first stage problem
+SDDPBlock,NumberScenarios,Dimension,Dimension,int,-,-,,optional,Number of scenarios; required when scenarios are stored inline in Scenarios
+SDDPBlock,SubScenarioSize,Dimension,Dimension,int,-,-,,optional,Size of the sub-scenario associated with each stage when all stages use the same size
+SDDPBlock,ScenarioSize,Dimension,Dimension,int,-,-,,optional,Size of a scenario spanning all stages; required when scenarios are stored inline in Scenarios
+SDDPBlock,NumberRandomDataGroups,Dimension,Dimension,int,-,-,1,optional,Number of related random-data groups in each equal-sized sub-scenario
+SDDPBlock,AdmissibleStateSize,Dimension,Dimension,int,-,-,,required,Total number of entries in AdmissibleState in the fixture layout
+SDDPBlock,InitialStateSize,Dimension,Dimension,int,-,-,,required,Number of entries in the initial state
+SDDPBlock,SizeRandomDataGroups,Variable,Variable,int,NumberRandomDataGroups,-,,optional,Size of each random-data group; required when NumberRandomDataGroups is greater than one
+SDDPBlock,Scenarios,Variable,Variable,double,NumberScenarios-ScenarioSize,-,,optional,Inline scenario matrix; exactly one of Scenarios and ScenarioGenerator must be provided
 SDDPBlock,StateSize,Variable,Variable,int,-|TimeHorizon,-,,required,Size of the states at each stage; scalar or array indexed over TimeHorizon
-SDDPBlock,AdmissibleState,Variable,Variable,double,-|StateSize|TimeHorizon-StateSize,-,,required,Admissible state for each stage; feasible final state for the problem at that stage
+SDDPBlock,AdmissibleState,Variable,Variable,double,AdmissibleStateSize,-,,required,Concatenation of one feasible final state for every stage
+SDDPBlock,InitialState,Variable,Variable,double,InitialStateSize,-,,required,Initial state for the first-stage problem
 SDDPBlock,AbstractPath,Block,Group,AbstractPath,list,-,-,required,Vector of AbstractPath describing PolyhedralFunction for sub-Blocks
-SDDPBlock,ScenarioSet,Block,Group,ScenarioSet,list,-,-,required,Description of the scenario set
-SDDPBlock,StochasticBlock,Block,Group,StochasticBlock,list,-,-,optional,Default description for sub-Blocks; used when StochasticBlock_t is not provided or incomplete
-SDDPBlock,StochasticBlock_*,Block,Group,StochasticBlock,list,-,-,optional,Description of the t-th sub-Block for each time stage t in {0...TimeHorizon-1}
+SDDPBlock,ScenarioGenerator,Block,Group,DiscreteScenarioSet|IndependentMultiStageScenarioGenerator|ScenarioGenerator,-,-,-,optional,Scenario source subgroup; exactly one of ScenarioGenerator and inline Scenarios must be provided
+SDDPBlock,StochasticBlock,Block,Group,StochasticBlock,-,-,-,optional,Default description used for stages whose StochasticBlock_t group is absent or incomplete
+SDDPBlock,StochasticBlock_*,Block,Group,StochasticBlock,list,-,-,optional,Stage-specific StochasticBlock; every stage must be resolved from this group or the default StochasticBlock

pysmspp/data/blocks/StochasticBlock.csv

@@ -1,13 +1,15 @@
 block_name,attribute,smspp_object,netcdf_component,netcdf_base_type,dimension,unit,default,status,description
 StochasticBlock,block_type,Attribute,Attribute,str,-,-,,required,Type of the block
 StochasticBlock,id,Attribute,Attribute,str,-,-,,optional,Id of the block
-StochasticBlock,NumberDataMappings,Dimension,Dimension,int,-,-,,required,Number of data mappings
-StochasticBlock,SetSize_dim,Dimension,Dimension,int,-,-,,required,Auxiliary dimension for SetSize
-StochasticBlock,SetElements_dim,Dimension,Dimension,int,-,-,,required,Auxiliary dimension for SetElements
-StochasticBlock,FunctionName,Variable,Variable,str,-|NumberDataMappings,-,,required,Name of the mapping function for each data mapping
-StochasticBlock,DataType,Variable,Variable,str,-|NumberDataMappings,-,,required,Type of the data mapped
+StochasticBlock,NumberDataMappings,Dimension,Dimension,int,-,-,0,optional,Number of data mappings; zero is assumed when the dimension is absent
+StochasticBlock,SetSizeSize,Dimension,Dimension,int,-,-,,optional,Fixture-era dimension of SetSize
+StochasticBlock,SetElementsSize,Dimension,Dimension,int,-,-,,optional,Fixture-era dimension of SetElements
+StochasticBlock,SetSize_dim,Dimension,Dimension,int,-,-,,optional,Current-format dimension of SetSize
+StochasticBlock,SetElements_dim,Dimension,Dimension,int,-,-,,optional,Current-format dimension of SetElements
+StochasticBlock,FunctionName,Variable,Variable,str,NumberDataMappings,-,,optional,Registered mapping function names; required when NumberDataMappings is greater than zero
+StochasticBlock,DataType,Variable,Variable,str,NumberDataMappings,-,D,optional,Type of the data mapped; double is assumed when absent
 StochasticBlock,Caller,Variable,Variable,str,-|NumberDataMappings,-,,optional,Name of the caller
-StochasticBlock,SetSize,Variable,Variable,int,-|SetSize_dim,-,,required,Size of each mapped set
-StochasticBlock,SetElements,Variable,Variable,int,-|SetElements_dim,-,,required,Elements of the mapped sets
-StochasticBlock,AbstractPath,Block,Group,AbstractPath,list,-,-,required,Abstract paths describing stochastic structure
-StochasticBlock,Block,Block,Group,Block,list,-,-,required,Deterministic referenced block
+StochasticBlock,SetSize,Variable,Variable,int,SetSizeSize|SetSize_dim,-,,optional,Size of each mapped source and target set; ranges are assumed when absent
+StochasticBlock,SetElements,Variable,Variable,int,SetElementsSize|SetElements_dim,-,,optional,Concatenated mapped ranges or subsets; required when NumberDataMappings is greater than zero
+StochasticBlock,AbstractPath,Block,Group,AbstractPath,list,-,-,optional,Paths to mapping callers; required when NumberDataMappings is greater than zero
+StochasticBlock,Block,Block,Group,Block|BendersBlock,-,-,-,optional,Inner deterministic block; may be inherited from the default StochasticBlock in an SDDPBlock

pysmspp/data/components.csv

@@ -19,3 +19,7 @@ DiscreteScenarioSet,Block,blocks,"Discrete scenario set block useful to describe
 AbstractPath,Block,blocks,"Abstract path block useful to describe custom Paths, such as needed in StochasticBlock"
 DesignNetworkBlock,Block,blocks,"Design network block useful to enable capacity expansion of the network within UCBlock"
 SDDPBlock,Block,blocks,"SDDP block useful to describe stochastic problems with SDDP approach"
+MultiStageScenarioGenerator,Block,blocks,"Multi-stage scenario generator block useful to describe stochastic problems with explicit scenario tree"
+IndependentMultiStageScenarioGenerator,Block,blocks,"Independent multi-stage scenario generator block useful to describe stochastic problems with explicit scenario tree"
+BendersBlock,Block,blocks,"Block whose objective is represented by a BendersBFunction"
+BendersBFunction,Block,blocks,"Benders function group and its inner optimization block"

pysmspp/data/configs/SDDPBlock/ACBCfg.txt

@@ -0,0 +1,53 @@
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - ACBCfg.txt- - - - - - - - - - - - - - - - -
+#
+# A txt description of a BlockConfig for all the NetworkBlock of a UCBlock
+#
+#   Antonio Frangioni
+#
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+# exact type of the Configuration object
+BlockConfig
+
+1   # the BlockConfig is differential
+
+# static constraints Configuration
+# a SimpleConfiguration< std::vector< double > > to set
+# the scaling constants in ACNetworkBlock
+# exact type of the Configuration object
+SimpleConfiguration<std::vector<double>>
+4       # the number of elements in the vector
+1.0     # C_v_scal
+0.0     # f_ACvS
+1.0     # f_scale
+16      # f_digits
+
+# dynamic constraints Configuration
+*  # none
+
+# static variables Configuration
+*  # none
+
+# dynamic variables Configuration
+*  # none
+
+# objective Configuration
+*  # none
+
+# is_feasible Configuration
+*  # none
+
+# is_optimal Configuration
+*  # none
+
+# solution Configuration
+*  # none
+
+# extra Configuration
+*  # none
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - END ACBCfg.txt- - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

pysmspp/data/configs/SDDPBlock/BCfg.txt

@@ -0,0 +1,47 @@
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - -BCfg.txt - - - - - - - - - - - - - - - - -
+#
+# A txt description of the default BlockConfig for any Block of the inner
+# Block of each stage of the SDDPBlock for which SDDPBCfg[-LD].txt has no
+# class-specific entry: it only sets the is_feasible() tolerances.
+#
+#   Donato Meoli
+#
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+BlockConfig # exact type of the Configuration object
+
+1  # the BlockConfig is a "differential" one
+
+# static_constraints
+*    # [none]
+
+# dynamic_constraints
+*    # [none]
+
+# static_variables
+*    # [none]
+
+# dynamic_variables
+*    # [none]
+
+# objective
+*    # [none]
+
+# is_feasible
+SimpleConfiguration<std::pair<double,int>>
+1e-6 1  # feasibility tolerance, relative violation
+
+# is_optimal
+*    # [none]
+
+# solution
+*    # [none]
+
+# extra
+*    # [none]
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - END BCfg.txt- - - - - - - - - - - - - - - - -
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -