feat: overhead-aware ILP path selection (fixes #780)

problemreductions-cli/src/test_support.rs

@@ -180,6 +180,7 @@ problemreductions::inventory::submit! {
         }),
         capabilities: EdgeCapabilities::aggregate_only(),
         overhead_eval_fn: |_| ProblemSize::new(vec![]),
+        source_size_fn: |_| ProblemSize::new(vec![]),
     }
 }
 
@@ -202,6 +203,7 @@ problemreductions::inventory::submit! {
         }),
         capabilities: EdgeCapabilities::aggregate_only(),
         overhead_eval_fn: |_| ProblemSize::new(vec![]),
+        source_size_fn: |_| ProblemSize::new(vec![]),
     }
 }
 

problemreductions-macros/src/lib.rs

@@ -252,6 +252,47 @@ fn generate_overhead_eval_fn(
     })
 }
 
+/// Generate a function that extracts the source problem's size fields from `&dyn Any`.
+///
+/// Collects all variable names referenced in the overhead expressions, generates
+/// getter calls for each, and returns a `ProblemSize`.
+fn generate_source_size_fn(
+    fields: &[(String, String)],
+    source_type: &Type,
+) -> syn::Result<TokenStream2> {
+    let src_ident = syn::Ident::new("__src", proc_macro2::Span::call_site());
+
+    // Collect all unique variable names from overhead expressions
+    let mut var_names = std::collections::BTreeSet::new();
+    for (_, expr_str) in fields {
+        let parsed = parser::parse_expr(expr_str).map_err(|e| {
+            syn::Error::new(
+                proc_macro2::Span::call_site(),
+                format!("error parsing overhead expression \"{expr_str}\": {e}"),
+            )
+        })?;
+        for v in parsed.variables() {
+            var_names.insert(v.to_string());
+        }
+    }
+
+    let getter_tokens: Vec<_> = var_names
+        .iter()
+        .map(|var| {
+            let getter = syn::Ident::new(var, proc_macro2::Span::call_site());
+            let name_lit = var.as_str();
+            quote! { (#name_lit, #src_ident.#getter() as usize) }
+        })
+        .collect();
+
+    Ok(quote! {
+        |__any_src: &dyn std::any::Any| -> crate::types::ProblemSize {
+            let #src_ident = __any_src.downcast_ref::<#source_type>().unwrap();
+            crate::types::ProblemSize::new(vec![#(#getter_tokens),*])
+        }
+    })
+}
+
 /// Generate the reduction entry code
 fn generate_reduction_entry(
     attrs: &ReductionAttrs,
@@ -288,21 +329,27 @@ fn generate_reduction_entry(
     let source_variant_body = make_variant_fn_body(source_type, &type_generics)?;
     let target_variant_body = make_variant_fn_body(&target_type, &type_generics)?;
 
-    // Generate overhead and eval fn
-    let (overhead, overhead_eval_fn) = match &attrs.overhead {
+    // Generate overhead, eval fn, and source size fn
+    let (overhead, overhead_eval_fn, source_size_fn) = match &attrs.overhead {
         Some(OverheadSpec::Legacy(tokens)) => {
             let eval_fn = quote! {
                 |_: &dyn std::any::Any| -> crate::types::ProblemSize {
                     panic!("overhead_eval_fn not available for legacy overhead syntax; \
                             migrate to parsed syntax: field = \"expression\"")
                 }
             };
-            (tokens.clone(), eval_fn)
+            let size_fn = quote! {
+                |_: &dyn std::any::Any| -> crate::types::ProblemSize {
+                    crate::types::ProblemSize::new(vec![])
+                }
+            };
+            (tokens.clone(), eval_fn, size_fn)
         }
         Some(OverheadSpec::Parsed(fields)) => {
             let overhead_tokens = generate_parsed_overhead(fields)?;
             let eval_fn = generate_overhead_eval_fn(fields, source_type)?;
-            (overhead_tokens, eval_fn)
+            let size_fn = generate_source_size_fn(fields, source_type)?;
+            (overhead_tokens, eval_fn, size_fn)
         }
         None => {
             return Err(syn::Error::new(
@@ -337,6 +384,7 @@ fn generate_reduction_entry(
                 reduce_aggregate_fn: None,
                 capabilities: #capabilities,
                 overhead_eval_fn: #overhead_eval_fn,
+                source_size_fn: #source_size_fn,
             }
         }
 

src/rules/cost.rs

@@ -27,6 +27,39 @@ impl PathCostFn for MinimizeSteps {
     }
 }
 
+/// Minimize total output size (sum of all output field values).
+///
+/// Prefers reduction paths that produce smaller intermediate and final problems.
+/// Breaks ties that `MinimizeSteps` cannot resolve (e.g., two 2-step paths
+/// where one produces 144 ILP variables and the other 1,332).
+pub struct MinimizeOutputSize;
+
+impl PathCostFn for MinimizeOutputSize {
+    fn edge_cost(&self, overhead: &ReductionOverhead, size: &ProblemSize) -> f64 {
+        let output = overhead.evaluate_output_size(size);
+        output.total() as f64
+    }
+}
+
+/// Minimize steps first, then use output size as tiebreaker.
+///
+/// Each edge has a primary cost of `STEP_WEIGHT` (ensuring fewer-step paths
+/// always win) plus a small overhead-based cost that breaks ties between
+/// equal-step paths.
+pub struct MinimizeStepsThenOverhead;
+
+impl PathCostFn for MinimizeStepsThenOverhead {
+    fn edge_cost(&self, overhead: &ReductionOverhead, size: &ProblemSize) -> f64 {
+        // Use a large step weight to ensure step count dominates.
+        // The overhead tiebreaker uses log1p to compress the range,
+        // keeping it far smaller than STEP_WEIGHT for any realistic problem size.
+        const STEP_WEIGHT: f64 = 1e9;
+        let output = overhead.evaluate_output_size(size);
+        let overhead_tiebreaker = (1.0 + output.total() as f64).ln();
+        STEP_WEIGHT + overhead_tiebreaker
+    }
+}
+
 /// Custom cost function from closure.
 pub struct CustomCost<F>(pub F);
 

src/rules/graph.rs

@@ -904,6 +904,54 @@ impl ReductionGraph {
         result
     }
 
+    /// Evaluate the cumulative output size along a reduction path.
+    ///
+    /// Walks the path from start to end, applying each edge's overhead
+    /// expressions to transform the problem size at each step.
+    /// Returns `None` if any edge in the path cannot be found.
+    pub fn evaluate_path_overhead(
+        &self,
+        path: &ReductionPath,
+        input_size: &ProblemSize,
+    ) -> Option<ProblemSize> {
+        let mut current_size = input_size.clone();
+        for pair in path.steps.windows(2) {
+            let src = self.lookup_node(&pair[0].name, &pair[0].variant)?;
+            let dst = self.lookup_node(&pair[1].name, &pair[1].variant)?;
+            let edge_idx = self.graph.find_edge(src, dst)?;
+            let edge = &self.graph[edge_idx];
+            current_size = edge.overhead.evaluate_output_size(&current_size);
+        }
+        Some(current_size)
+    }
+
+    /// Compute the source problem's size from a type-erased instance.
+    ///
+    /// Iterates over all registered reduction entries with a matching source name
+    /// and merges their `source_size_fn` results to capture all size fields.
+    /// Different entries may reference different getter methods (e.g., one uses
+    /// `num_vertices` while another also uses `num_edges`).
+    pub fn compute_source_size(name: &str, instance: &dyn Any) -> ProblemSize {
+        let mut merged: Vec<(String, usize)> = Vec::new();
+        let mut seen: HashSet<String> = HashSet::new();
+
+        for entry in inventory::iter::<ReductionEntry> {
+            if entry.source_name == name {
+                let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                    (entry.source_size_fn)(instance)
+                }));
+                if let Ok(size) = result {
+                    for (k, v) in size.components {
+                        if seen.insert(k.clone()) {
+                            merged.push((k, v));
+                        }
+                    }
+                }
+            }
+        }
+        ProblemSize { components: merged }
+    }
+
     /// Get all incoming reductions to a problem (across all its variants).
     pub fn incoming_reductions(&self, name: &str) -> Vec<ReductionEdgeInfo> {
         let Some(indices) = self.name_to_nodes.get(name) else {

src/rules/mod.rs

@@ -3,7 +3,9 @@
 pub mod analysis;
 pub mod cost;
 pub mod registry;
-pub use cost::{CustomCost, Minimize, MinimizeSteps, PathCostFn};
+pub use cost::{
+    CustomCost, Minimize, MinimizeOutputSize, MinimizeSteps, MinimizeStepsThenOverhead, PathCostFn,
+};
 pub use registry::{EdgeCapabilities, ReductionEntry, ReductionOverhead};
 
 pub(crate) mod circuit_spinglass;

src/rules/registry.rs

@@ -157,6 +157,10 @@ pub struct ReductionEntry {
     /// Takes a `&dyn Any` (must be `&SourceType`), calls getter methods directly,
     /// and returns the computed target problem size.
     pub overhead_eval_fn: fn(&dyn Any) -> ProblemSize,
+    /// Extract source problem size from a type-erased instance.
+    /// Takes a `&dyn Any` (must be `&SourceType`), calls getter methods,
+    /// and returns the source problem's size fields as a `ProblemSize`.
+    pub source_size_fn: fn(&dyn Any) -> ProblemSize,
 }
 
 impl ReductionEntry {

src/solvers/ilp/solver.rs

@@ -239,18 +239,23 @@ impl ILPSolver {
         any.is::<ILP<bool>>() || any.is::<ILP<i32>>() || any.is::<TimetableDesign>()
     }
 
+    /// Two-level path selection:
+    /// 1. Dijkstra finds the cheapest path to each ILP variant using
+    ///    `MinimizeStepsThenOverhead` (additive edge costs: step count + log overhead).
+    /// 2. Across ILP variants, we pick the path whose composed final output size
+    ///    is smallest — this is the actual ILP problem size the solver will face.
     fn best_path_to_ilp(
         &self,
         graph: &crate::rules::ReductionGraph,
         name: &str,
         variant: &std::collections::BTreeMap<String, String>,
         mode: ReductionMode,
+        instance: &dyn std::any::Any,
     ) -> Option<crate::rules::ReductionPath> {
-        use crate::types::ProblemSize;
-
         let ilp_variants = graph.variants_for("ILP");
-        let input_size = ProblemSize::new(vec![]);
-        let mut best_path = None;
+        let input_size = crate::rules::ReductionGraph::compute_source_size(name, instance);
+        let mut best_path: Option<crate::rules::ReductionPath> = None;
+        let mut best_cost = f64::INFINITY;
 
         for dv in &ilp_variants {
             if let Some(path) = graph.find_cheapest_path_mode(
@@ -260,12 +265,16 @@ impl ILPSolver {
                 dv,
                 mode,
                 &input_size,
-                &crate::rules::MinimizeSteps,
+                &crate::rules::MinimizeStepsThenOverhead,
             ) {
-                let is_better = best_path
-                    .as_ref()
-                    .is_none_or(|current: &crate::rules::ReductionPath| path.len() < current.len());
-                if is_better {
+                // Use composed final output size for cross-variant comparison,
+                // since this determines the actual ILP problem size.
+                let final_size = graph
+                    .evaluate_path_overhead(&path, &input_size)
+                    .unwrap_or_default();
+                let cost = final_size.total() as f64;
+                if cost < best_cost {
+                    best_cost = cost;
                     best_path = Some(path);
                 }
             }
@@ -290,10 +299,11 @@ impl ILPSolver {
 
         let graph = crate::rules::ReductionGraph::new();
 
-        let Some(path) = self.best_path_to_ilp(&graph, name, variant, ReductionMode::Witness)
+        let Some(path) =
+            self.best_path_to_ilp(&graph, name, variant, ReductionMode::Witness, instance)
         else {
             if self
-                .best_path_to_ilp(&graph, name, variant, ReductionMode::Aggregate)
+                .best_path_to_ilp(&graph, name, variant, ReductionMode::Aggregate, instance)
                 .is_some()
             {
                 return Err(SolveViaReductionError::WitnessPathRequired {

src/types.rs

@@ -504,7 +504,7 @@ impl<V: fmt::Display> fmt::Display for Extremum<V> {
 }
 
 /// Problem size metadata (varies by problem type).
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ProblemSize {
     /// Named size components.
     pub components: Vec<(String, usize)>,
@@ -528,6 +528,11 @@ impl ProblemSize {
             .find(|(k, _)| k == name)
             .map(|(_, v)| *v)
     }
+
+    /// Sum of all component values.
+    pub fn total(&self) -> usize {
+        self.components.iter().map(|(_, v)| *v).sum()
+    }
 }
 
 impl fmt::Display for ProblemSize {

src/unit_tests/rules/cost.rs

@@ -48,3 +48,39 @@ fn test_minimize_missing_field() {
 
     assert_eq!(cost_fn.edge_cost(&overhead, &size), 0.0);
 }
+
+#[test]
+fn test_minimize_output_size() {
+    let cost_fn = MinimizeOutputSize;
+    let size = ProblemSize::new(vec![("n", 10), ("m", 5)]);
+    let overhead = test_overhead();
+
+    // output n = 20, output m = 5 → total = 25
+    assert_eq!(cost_fn.edge_cost(&overhead, &size), 25.0);
+}
+
+#[test]
+fn test_minimize_steps_then_overhead() {
+    let cost_fn = MinimizeStepsThenOverhead;
+    let size = ProblemSize::new(vec![("n", 10), ("m", 5)]);
+    let overhead = test_overhead();
+
+    let cost = cost_fn.edge_cost(&overhead, &size);
+    // Should be dominated by the step weight (1e9) with small overhead tiebreaker
+    assert!(cost > 1e8, "step weight should dominate");
+    assert!(cost < 2e9, "should be roughly 1e9 + small tiebreaker");
+
+    // Two edges with different overhead should have different costs
+    let small_overhead =
+        ReductionOverhead::new(vec![("n", Expr::Const(1.0)), ("m", Expr::Const(1.0))]);
+    let cost_small = cost_fn.edge_cost(&small_overhead, &size);
+    // Both have the same step weight but different tiebreakers
+    assert!(cost > cost_small, "larger overhead should cost more");
+}
+
+#[test]
+fn test_problem_size_total() {
+    let size = ProblemSize::new(vec![("a", 3), ("b", 7), ("c", 10)]);
+    assert_eq!(size.total(), 20);
+    assert_eq!(ProblemSize::new(vec![]).total(), 0);
+}