From 04768176243d6e80d826d5e302bfef1325a4ab8e Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Tue, 28 Jul 2026 17:34:36 -0700 Subject: [PATCH 1/8] feat(libsy): stage router with handoff notes, per-tier prompts, and capability-classifier fallback Signed-off-by: Sabhatina Selvam --- Cargo.lock | 1 + crates/libsy/Cargo.toml | 3 + crates/libsy/examples/research_agent.rs | 1 + crates/libsy/examples/research_agent_core.rs | 1 + crates/libsy/src/algorithms.rs | 8 +- crates/libsy/src/algorithms/fall_through.rs | 74 ++- crates/libsy/src/algorithms/llm_class.rs | 193 +++++-- crates/libsy/src/algorithms/stage.rs | 340 ++++++++++++ crates/libsy/src/algorithms/util.rs | 5 +- .../src/algorithms/util/handoff_notes.rs | 180 ------ crates/libsy/src/algorithms/util/prompts.rs | 306 +++++++++++ .../util/{stage_router.rs => stage.rs} | 511 ++++++++++++++---- .../libsy/src/algorithms/util/tool_signals.rs | 260 ++------- crates/libsy/src/core/processor.rs | 26 +- crates/libsy/src/lib.rs | 8 +- crates/libsy/tests/observability.rs | 1 + crates/libsy/tests/prompts.rs | 209 +++++++ crates/libsy/tests/stage_router.rs | 347 ++++++++++++ .../src/dimension_collector/tool_signals.rs | 7 +- crates/switchyard-server/src/config.rs | 103 +++- crates/switchyard-server/tests/server.rs | 91 +++- 21 files changed, 2101 insertions(+), 574 deletions(-) create mode 100644 crates/libsy/src/algorithms/stage.rs delete mode 100644 crates/libsy/src/algorithms/util/handoff_notes.rs create mode 100644 crates/libsy/src/algorithms/util/prompts.rs rename crates/libsy/src/algorithms/util/{stage_router.rs => stage.rs} (52%) create mode 100644 crates/libsy/tests/prompts.rs create mode 100644 crates/libsy/tests/stage_router.rs diff --git a/Cargo.lock b/Cargo.lock index c15c0af6..bb508564 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1736,6 +1736,7 @@ dependencies = [ "serde_json", "switchyard-llm-client", "switchyard-protocol", + "switchyard-translation", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml index c9e58e18..4769723d 100644 --- a/crates/libsy/Cargo.toml +++ b/crates/libsy/Cargo.toml @@ -31,5 +31,8 @@ tracing.workspace = true # SDK + in-memory exporter to assert what the observability layer records. opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing"] } switchyard-llm-client.workspace = true +# Decodes the wire-format fixtures the tool-signal tests are written against, so +# they keep asserting on bodies a real client would send. +switchyard-translation.workspace = true tokio.workspace = true tracing-subscriber = "0.3" diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs index 885a3882..6e17dff6 100644 --- a/crates/libsy/examples/research_agent.rs +++ b/crates/libsy/examples/research_agent.rs @@ -112,6 +112,7 @@ async fn main() -> Result<()> { capability_elevated_floor: None, session_affinity: false, message_hash_fallback: false, + recent_turn_window: None, }, )?); diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs index 3c0571b1..0cc7af8b 100644 --- a/crates/libsy/examples/research_agent_core.rs +++ b/crates/libsy/examples/research_agent_core.rs @@ -120,6 +120,7 @@ async fn main() -> Result<()> { capability_elevated_floor: None, session_affinity: false, message_hash_fallback: false, + recent_turn_window: None, }, )?); diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs index b36b542f..f2bf4f31 100644 --- a/crates/libsy/src/algorithms.rs +++ b/crates/libsy/src/algorithms.rs @@ -11,13 +11,17 @@ pub mod llm_class; pub mod noop; pub mod passthrough; pub mod rand; +pub mod stage; -pub use fall_through::{FallThrough, FallThroughDecision}; +pub use fall_through::{DefaultTarget, FallThrough, FallThroughDecision}; pub use llm_class::{LlmTaskClassifier, TaskClassifierConfig}; pub use noop::{Noop, NoopDecision}; pub use passthrough::{Passthrough, PassthroughDecision}; pub use rand::{Random, RandomClassifier, RandomDecision}; -pub use util::{AffinityRouter, SubagentOverride}; +pub use stage::{LlmFallback, StageRouter, StageRouterConfig}; +pub use util::{ + append_note, AffinityRouter, SubagentOverride, SystemPromptProcessor, TargetPrompts, +}; pub mod util; diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index 958e6d19..48eebc4e 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -19,7 +19,7 @@ use async_trait::async_trait; use parking_lot::Mutex; use tokio::sync::Mutex as AsyncMutex; -use crate::core::{Classifier, Event, Processor, Score}; +use crate::core::{Classification, Classifier, Event, Processor, Score}; use crate::{ Algorithm, Context, Decision, Driver, LibsyError, LlmClientError, LlmTarget, LlmTargetSet, Request, Response, Result, RoutedLlmClient, @@ -54,7 +54,42 @@ impl Decision for FallThroughDecision { } } -/// Processor chain → classifier cascade → routed model call. See the [module docs](self). +/// Terminal classifier for a cascade whose classifiers may all abstain. +/// +/// A classifier abstains when it cannot decide, which lets the next one try. The +/// last has no next, so a cascade that could abstain all the way through needs a +/// decider that never does. Which target that is belongs to whoever assembles the +/// cascade, not to the classifiers in it. +pub struct DefaultTarget { + target: String, +} + +impl DefaultTarget { + /// Close a cascade with `target`. + pub fn new(target: impl Into) -> Self { + Self { + target: target.into(), + } + } +} + +#[async_trait] +impl Classifier for DefaultTarget { + async fn score( + &self, + _state: &mut S, + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result { + // Zero confidence: this is a fallback, not a judgement. + Ok(Classification::Scores(vec![Score { + target: self.target.clone(), + confidence: 0.0, + }])) + } +} + +/// Processor chain → classifier cascade → routed model call. See the module docs. /// /// The generic state type is shared by every processor and classifier in the composition. pub struct FallThrough { @@ -279,17 +314,24 @@ where }); driver.info(ctx.clone(), decision.clone()).await?; - // 4. Replay the decision to the processors so stateful ones can bind it. + // 4. Post-decision replay, in two passes over the same chain. Every processor sees + // the decision first, so stateful ones bind it; only then is the outbound + // request offered for rewriting, paired with the decision that routed it. The + // order matters: a processor rewriting the request in the second pass sees the + // state every processor settled in the first. for processor in &self.processors { - processor - .process( - state, - Event::Decision { - request, - decision: decision.as_ref(), - }, - ) - .await?; + let event = Event::Decision { + request, + decision: decision.as_ref(), + }; + processor.process(state, event).await?; + } + for processor in &self.processors { + let event = Event::ModelRequest { + request, + decision: decision.as_ref(), + }; + processor.process(state, event).await?; } Ok((target, decision)) @@ -700,10 +742,11 @@ mod tests { } #[tokio::test] - async fn processor_observes_request_and_decision() -> Result<()> { + async fn processor_observes_request_decision_then_model_request() -> Result<()> { use parking_lot::Mutex; - // Records which event kinds it saw, proving the request-then-decision replay. + // Records which event kinds it saw, proving the replay order: the inbound + // request, then the decision, then the request on its way to the model. struct RecordingProcessor(Arc>>); #[async_trait] @@ -712,6 +755,7 @@ mod tests { let kind = match event { Event::Request(_) => "request", Event::Decision { .. } => "decision", + Event::ModelRequest { .. } => "model_request", _ => "other", }; self.0.lock().push(kind); @@ -725,7 +769,7 @@ mod tests { .with_classifier(fixed(vec![score("strong", 1.0)])); run(router).await?; - assert_eq!(*seen.lock(), vec!["request", "decision"]); + assert_eq!(*seen.lock(), vec!["request", "decision", "model_request"]); Ok(()) } diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 255a0c43..e2cdeaed 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -3,7 +3,7 @@ //! Task-level capability routing with a judge-backed classifier. //! -//! The algorithm owns a [`FallThrough`](super::FallThrough) cascade. Its classifier judges the +//! The algorithm owns a [`FallThrough`] cascade. Its classifier judges the //! full inbound request and selects one decisive target. Invalid, abstained, or unavailable judge //! output always selects the capable target. @@ -15,7 +15,7 @@ use serde_json::Value; use switchyard_protocol::{LlmRequest, Message, OutputParams, Role}; use super::util::{AffinityRouter, Judge, JudgeClassifier, JudgeConfig, JudgePolicy}; -use super::FallThrough; +use super::{DefaultTarget, FallThrough}; use crate::{ Algorithm, Classification, Classifier, Context, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response, Result, RoutedLlmClient, Score, State, @@ -71,8 +71,31 @@ impl TaskClassifierVerdict { /// The judge is responsible for any kind of llm judge based calls /// Example: A judge can be a capability classifier, a escalation classifier etc /// Builds capability-specific judge requests from shared classifier configuration. +/// Keeps client instructions, the opening task, and the last `recent_turn_window` +/// turns after it. A window of `0` keeps the instructions and the task alone. +/// +/// Selects by reference and clones only what survives — a coding-agent +/// conversation carries every tool result, so cloning it whole to keep a window +/// would copy the transcript on each judged turn. +fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec { + let is_instruction = |message: &Message| matches!(message.role, Role::System | Role::Developer); + let mut kept: Vec<&Message> = messages.iter().filter(|m| is_instruction(m)).collect(); + let Some(task) = messages.iter().position(|m| m.role == Role::User) else { + return kept.into_iter().cloned().collect(); + }; + kept.push(&messages[task]); + + let tail: Vec<&Message> = messages[task + 1..] + .iter() + .filter(|m| !is_instruction(m)) + .collect(); + kept.extend(&tail[tail.len().saturating_sub(recent_turn_window)..]); + kept.into_iter().cloned().collect() +} + struct CapabilityJudge { config: JudgeConfig, + recent_turn_window: Option, } impl Judge for CapabilityJudge { @@ -81,17 +104,20 @@ impl Judge for CapabilityJudge { /// For different judges, the request building logic can be different /// To have a single interface for all judges, we may make a common request building logic here. fn build_request(&self, _state: &State, request: &Request) -> Request { - // For task-based routing, classify only the newest user message with the judge prompt. - // For any turn window size setting, use TaskClassifierConfig to define the window size. - let mut messages = request - .llm_request - .messages - .iter() - .rev() - .find(|message| message.role == Role::User) - .cloned() - .into_iter() - .collect::>(); + // Task-based routing judges the newest user message alone. A configured + // window widens that to the surrounding conversation. + let mut messages = match self.recent_turn_window { + Some(window) => trim_messages(&request.llm_request.messages, window), + None => request + .llm_request + .messages + .iter() + .rev() + .find(|message| message.role == Role::User) + .cloned() + .into_iter() + .collect::>(), + }; messages.insert( 0, Message::text(Role::System, self.config.system_prompt.clone()), @@ -147,18 +173,25 @@ impl JudgePolicy for TaskClassifierPolicy { type Verdict = TaskClassifierVerdict; fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification { - // Judge output is untrusted. Anything incomplete, invalid, abstained, or below its - // configured confidence or capability threshold routes to the capable target. - let target = match verdict { - Some(verdict) - if verdict.is_valid() - && !verdict.abstain - && verdict.confidence >= self.config.min_confidence - && verdict.p_solve >= self.threshold(verdict) => - { - &self.efficient_target - } - _ => &self.capable_target, + // Judge output is untrusted, so only a complete, valid, non-abstained verdict that + // clears the configured confidence decides. Anything else is "I could not tell" — + // reported as ambiguous so the composition around this classifier chooses the + // fallback, rather than this policy silently imposing one. The capable target rides + // along as the safe suggestion for a caller that reads ambiguous scores. + let Some(verdict) = verdict + .filter(|v| v.is_valid() && !v.abstain && v.confidence >= self.config.min_confidence) + else { + return Classification::Ambiguous(vec![Score { + target: self.capable_target.clone(), + confidence: 0.0, + }]); + }; + // A usable verdict below the capability threshold is still a decision: the judge + // does not trust the efficient tier with this task. + let target = if verdict.p_solve >= self.threshold(verdict) { + &self.efficient_target + } else { + &self.capable_target }; Classification::Scores(vec![Score { target: target.clone(), @@ -184,6 +217,14 @@ pub struct TaskClassifierConfig { /// Uses the first user message as the SessionKey for sticky routing when session metadata is unavailable. #[serde(default)] pub message_hash_fallback: bool, + /// Trailing conversation turns the judge sees on top of the client + /// instructions and the opening task. + /// + /// `None` (the default) judges the newest user message alone — the task, with + /// no history. `Some(n)` widens that to the client instructions, the opening + /// task, and the last `n` turns after it. + #[serde(default)] + pub recent_turn_window: Option, } impl TaskClassifierConfig { @@ -259,6 +300,7 @@ impl LlmTaskClassifier { classifier: JudgeClassifier::new( CapabilityJudge { config: judge_config, + recent_turn_window: config.recent_turn_window, }, judge_target, TaskClassifierPolicy::new( @@ -273,6 +315,14 @@ impl LlmTaskClassifier { // The cascade is an internal detail: callers drive the algorithm, not its parts. // Affinity comes first so a retained assignment short-circuits the judge call. + // + // TODO: bind the assignment on the post-decision hook instead, the way the + // per-target system prompt does. Affinity is a fact about the decision, not a + // classification, so recording it there would let a pinned session skip the + // cascade outright on the next turn. It would also make the pin survive + // composition: a cascade that uses this classifier as one member (the stage + // router does) currently never runs the affinity wired in here, because only + // the inner classifier is consulted. let mut route = FallThrough::::new_with_state(targets).with_name(ALGORITHM_NAME); if session_affinity { let affinity = if message_hash_fallback { @@ -286,8 +336,13 @@ impl LlmTaskClassifier { .with_processor(affinity.clone()) .with_classifier(affinity); } + // The judge abstains when it cannot tell; the capable tier catches those turns + // rather than letting the cascade come back empty-handed. + let capable_fallback = DefaultTarget::new(classifier.capable_target.clone()); Ok(Self { - route: route.with_classifier(classifier.clone()), + route: route + .with_classifier(classifier.clone()) + .with_classifier(Arc::new(capable_fallback)), classifier, }) } @@ -398,6 +453,7 @@ mod tests { capability_elevated_floor: None, session_affinity: false, message_hash_fallback: false, + recent_turn_window: None, } } @@ -608,6 +664,7 @@ mod tests { TaskClassifierConfig { session_affinity: true, message_hash_fallback: true, + recent_turn_window: None, ..test_config(TEST_THRESHOLD) }, )?); @@ -670,6 +727,7 @@ mod tests { capability_elevated_floor: None, session_affinity: false, message_hash_fallback: false, + recent_turn_window: None, }, TaskClassifierConfig { base_threshold: 0.5, @@ -677,6 +735,7 @@ mod tests { capability_elevated_floor: Some(0.5), session_affinity: false, message_hash_fallback: false, + recent_turn_window: None, }, TaskClassifierConfig { base_threshold: 0.5, @@ -684,6 +743,7 @@ mod tests { capability_elevated_floor: None, session_affinity: false, message_hash_fallback: true, + recent_turn_window: None, }, ] { assert!( @@ -696,18 +756,31 @@ mod tests { } #[test] - fn invalid_or_abstained_verdict_routes_capable() -> Result<()> { + fn an_unusable_verdict_abstains() -> Result<()> { + // Invalid, abstained, unintelligible, or absent: the judge could not tell, + // so it declines to decide and leaves the fallback to whoever composed the + // cascade. let policy = policy(); - let invalid_probability = verdict(1.1, 1.0, false); - let abstained = verdict(1.0, 1.0, true); let invalid_boundary = TaskClassifierVerdict { capability_boundary: "unknown".to_string(), ..verdict(1.0, 1.0, false) }; - assert_eq!(selected(&policy, Some(&invalid_probability))?, "capable"); - assert_eq!(selected(&policy, Some(&abstained))?, "capable"); - assert_eq!(selected(&policy, Some(&invalid_boundary))?, "capable"); - assert_eq!(selected(&policy, None)?, "capable"); + let unusable = [ + Some(verdict(1.1, 1.0, false)), + Some(verdict(1.0, 1.0, true)), + Some(invalid_boundary), + None, + ]; + for verdict in unusable { + let classification = policy.to_classification(verdict.as_ref()); + assert!(matches!(classification, Classification::Ambiguous(_))); + assert!(classification.argmax(false)?.is_none()); + // The capable tier rides along for a caller that reads ambiguous scores. + assert_eq!( + classification.argmax(true)?.map(|score| score.target), + Some("capable".to_string()) + ); + } Ok(()) } @@ -737,10 +810,63 @@ mod tests { Ok(()) } + /// The text of each message a judge with `recent_turn_window` would be sent. + /// The no-window case is covered by `capability_judge_builds_a_structured_request`. + fn judged_contents(recent_turn_window: usize) -> Result> { + let judge = CapabilityJudge { + config: LlmTaskClassifier::load_judge_config()?, + recent_turn_window: Some(recent_turn_window), + }; + let request = Request { + llm_request: LlmRequest { + messages: vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + Message::text(Role::Assistant, "old response"), + Message::text(Role::User, "old follow-up"), + Message::text(Role::Assistant, "recent 1"), + Message::text(Role::User, "recent 2"), + ], + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + }; + Ok(judge + .build_request(&State::default(), &request) + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("\n")) + .collect()) + } + + #[test] + fn a_window_widens_the_judge_to_the_surrounding_conversation() -> Result<()> { + // Client instructions and the opening task, plus the last two turns. + let contents = judged_contents(2)?; + assert!(contents.contains(&"client instructions".to_string())); + assert!(contents.contains(&"initial task".to_string())); + assert!(contents.contains(&"recent 1".to_string())); + assert!(contents.contains(&"recent 2".to_string())); + assert!(!contents.contains(&"old response".to_string())); + Ok(()) + } + + #[test] + fn a_zero_window_keeps_only_the_instructions_and_the_task() -> Result<()> { + let contents = judged_contents(0)?; + assert!(contents.contains(&"client instructions".to_string())); + assert!(contents.contains(&"initial task".to_string())); + assert!(!contents.contains(&"recent 2".to_string())); + Ok(()) + } + #[test] fn capability_judge_builds_a_structured_request() -> Result<()> { let judge = CapabilityJudge { config: LlmTaskClassifier::load_judge_config()?, + recent_turn_window: None, }; let request = Request { llm_request: LlmRequest { @@ -828,6 +954,7 @@ mod tests { let reply = schema_shaped_verdict(schema)?; let judge = CapabilityJudge { config: config.clone(), + recent_turn_window: None, }; let verdict = judge.parse(&text_response(None, reply))?; diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs new file mode 100644 index 00000000..ac0652b1 --- /dev/null +++ b/crates/libsy/src/algorithms/stage.rs @@ -0,0 +1,340 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Signal-driven stage routing for coding agents. +//! +//! [`StageRouter`] is the assembled algorithm: a [`FallThrough`] pre-wired with +//! the tool-signal processor that reads each turn's tool results and the +//! [`StageClassifier`] that scores them onto the capable/efficient tiers. The cascade +//! is an internal detail — callers drive the algorithm, not its parts. +//! +//! Signals do not decide every turn. An under-threshold turn abstains and falls +//! through to the optional [`LlmTaskClassifier`] — the capability route's judge, +//! joined in unchanged — and then to the picker's default tier. The judge is +//! asked per turn and its verdict is never pinned to the session. +//! +//! Callers needing a different composition can assemble the parts themselves. + +use std::sync::Arc; + +use async_trait::async_trait; + +use super::llm_class::TaskClassifierConfig; +use super::util::prompts::{SystemPromptProcessor, TargetPrompts}; +use super::util::stage::{ + record_decision_source, DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, + StageTargets, +}; +use super::util::tool_signals::ToolSignalProcessor; +use super::{DefaultTarget, FallThrough, LlmTaskClassifier}; +use crate::{ + Algorithm, Classification, Classifier, Context, Driver, LibsyError, LlmTarget, LlmTargetSet, + Request, Response, Result, RoutedLlmClient, State, DEFAULT_RECENT_WINDOW, +}; + +/// Telemetry name for a router this module assembles. +const STAGE_ROUTER: &str = "stage_router"; + +/// Attributes a turn to the classifier it wraps, when that classifier decides it. +/// +/// The classifiers themselves are composition-agnostic and write no state; only +/// this router knows where each sits in its cascade. +struct SourceStamp { + inner: Arc>, + source: DecisionSource, +} + +#[async_trait] +impl Classifier for SourceStamp { + fn routing_tier(&self, selected_model: &str) -> Option<&'static str> { + self.inner.routing_tier(selected_model) + } + + async fn score( + &self, + state: &mut State, + request: &mut Request, + driver: Option<&Driver>, + ) -> Result { + let classification = self.inner.score(state, request, driver).await?; + // An abstaining classifier passes the turn on, so it is not its to claim. + if matches!(&classification, Classification::Scores(scores) if !scores.is_empty()) { + record_decision_source(state, self.source); + } + Ok(classification) + } +} + +/// The capability judge a stage router falls through to. +pub struct LlmFallback { + /// Target the judge model is called through. It is not a routing + /// destination, so it does not belong in the router's target set. + pub judge_target: LlmTarget, + /// How the judge routes, exactly as the standalone capability route takes it. + /// Its `recent_turn_window` is worth setting to this router's + /// `recent_window`, so the judge reads the same span of the conversation the + /// signal scorer scored. + pub config: TaskClassifierConfig, +} + +/// How a stage router scores turns, and what it hands the model it picks. +pub struct StageRouterConfig { + /// Tier a turn falls open to when the scorer is not confident. + pub mode: PickerMode, + /// How much corroboration a decisive pick needs, in `[0.0, 1.0]`. + pub confidence_threshold: f64, + /// Trailing tool results the signals are computed over. `None` uses + /// [`DEFAULT_RECENT_WINDOW`]. + pub recent_window: Option, + /// Note handed to the model on a signal-driven escalation, and on a + /// hand-back to the efficient tier when a de-escalation note is configured. + pub handoff_notes: Option, + /// System prompts keyed by target, handed over on every turn that target + /// serves. Empty by default. + pub tier_prompts: TargetPrompts, + /// Capability judge consulted on turns the signals leave undecided — the + /// judge's own target, plus the same configuration the standalone capability + /// route takes. + pub llm_fallback: Option, +} + +impl StageRouterConfig { + /// The signal-only configuration: no notes, no per-tier prompts, no judge. + /// Set the optional fields to add them. + pub fn new(mode: PickerMode, confidence_threshold: f64) -> Self { + Self { + mode, + confidence_threshold, + recent_window: None, + handoff_notes: None, + tier_prompts: TargetPrompts::default(), + llm_fallback: None, + } + } +} + +/// Routes coding-agent turns between a capable and an efficient tier: tool signals +/// decide first, an optional capability judge takes the turns they cannot, and +/// the picker's default tier closes the cascade so a turn is never left unrouted. +pub struct StageRouter { + route: FallThrough, +} + +impl StageRouter { + /// Routes between the `capable` and `efficient` targets. The + /// judge, when configured, is called through its own target and is not a + /// routing destination. + /// + /// Errors if either threshold in `config` is outside `[0.0, 1.0]`. + pub fn new( + capable: LlmTarget, + efficient: LlmTarget, + config: StageRouterConfig, + ) -> Result { + Ok(Self { + route: build_route(capable, efficient, config)?, + }) + } +} + +#[async_trait] +impl Algorithm for StageRouter { + fn name(&self) -> &str { + STAGE_ROUTER + } + + fn count_tokens_client(&self) -> Option> { + self.route.count_tokens_client() + } + + async fn create_run_task( + self: Arc, + ctx: Context, + driver: Driver, + request: Request, + ) -> Result { + self.route.execute(ctx, driver, request).await + } +} + +/// Wires the cascade the wrapper drives. +fn build_route( + capable: LlmTarget, + efficient: LlmTarget, + config: StageRouterConfig, +) -> Result> { + if !(0.0..=1.0).contains(&config.confidence_threshold) { + return Err(LibsyError::AlgorithmError { + message: format!( + "confidence_threshold must be between 0 and 1, got {}", + config.confidence_threshold + ), + }); + } + // The tiers are a fixed pair; their targets are whatever the deployment calls + // them, and the classifier scores onto those names. + let targets = StageTargets::new( + capable.semantic_name.clone(), + efficient.semantic_name.clone(), + ); + // The picker's mode fixes the fallback tier up front, so the terminal + // classifier is a constant rather than a per-turn lookup. + let fall_open = targets.name(config.mode.default_tier()).to_string(); + + let mut classifier = StageClassifier::new(targets, config.mode, config.confidence_threshold); + if let Some(notes) = config.handoff_notes { + classifier = classifier.with_handoff_notes(notes); + } + let signals = ToolSignalProcessor { + recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW), + }; + + let target_set = LlmTargetSet::new(vec![capable.clone(), efficient.clone()]); + let mut router = FallThrough::::new_with_state(target_set) + .with_name(STAGE_ROUTER) + .with_processor(Arc::new(signals)) + .with_classifier(Arc::new(classifier)); + if let Some(fallback) = config.llm_fallback { + // The capability judge takes its tiers in the same order the capability + // route passes them: efficient first, capable second. + router = router.with_classifier(Arc::new(SourceStamp { + inner: Arc::new(LlmTaskClassifier::new( + fallback.judge_target, + efficient, + capable, + fallback.config, + )?), + source: DecisionSource::LlmClassifier, + })); + } + // Nothing behind this, so the turn lands on the picker's default tier — + // including when the judge could not tell. + router = router.with_classifier(Arc::new(SourceStamp { + inner: Arc::new(DefaultTarget::new(fall_open)), + source: DecisionSource::FallOpen, + })); + // Runs on the post-decision hook, so it applies to the target the cascade + // settled on, whichever classifier picked it. With no prompts configured it + // is a no-op, so there is nothing to branch on. + router = router.with_processor(Arc::new(SystemPromptProcessor::new(config.tier_prompts))); + Ok(router) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Algorithm, LlmTarget, StateValue}; + + fn tier_target(name: &str) -> LlmTarget { + LlmTarget { + semantic_name: name.to_string(), + llm_client: None, + } + } + + /// A classifier that always picks `target`, standing in for a cascade member. + struct Fixed(&'static str); + + #[async_trait] + impl Classifier for Fixed { + async fn score( + &self, + _state: &mut State, + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result { + Ok(Classification::Scores(vec![crate::Score { + target: self.0.to_string(), + confidence: 1.0, + }])) + } + } + + /// A classifier that never decides. + struct Abstains; + + #[async_trait] + impl Classifier for Abstains { + async fn score( + &self, + _state: &mut State, + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result { + Ok(Classification::Ambiguous(vec![])) + } + } + + async fn stamped(inner: Arc>) -> Result> { + let stamp = SourceStamp { + inner, + source: DecisionSource::LlmClassifier, + }; + let mut state = State::default(); + stamp + .score(&mut state, &mut Request::default(), None) + .await?; + Ok( + match state.extra.get(crate::stage_router::DECISION_SOURCE_KEY) { + Some(StateValue::String(source)) => Some(source.clone()), + _ => None, + }, + ) + } + + #[tokio::test] + async fn a_deciding_classifier_is_credited_with_the_turn() -> Result<()> { + assert_eq!( + stamped(Arc::new(Fixed("strong"))).await?.as_deref(), + Some("llm-classifier") + ); + Ok(()) + } + + #[tokio::test] + async fn an_abstaining_classifier_claims_nothing() -> Result<()> { + // It passed the turn on, so the next classifier is the one that decided. + assert_eq!(stamped(Arc::new(Abstains)).await?, None); + Ok(()) + } + + fn config() -> StageRouterConfig { + StageRouterConfig::new(PickerMode::EfficientFirst, 0.5) + } + + #[test] + fn rejects_an_out_of_range_confidence_threshold() { + let mut config = config(); + config.confidence_threshold = 1.5; + assert!(matches!( + StageRouter::new(tier_target("strong"), tier_target("weak"), config), + Err(LibsyError::AlgorithmError { .. }) + )); + } + + #[test] + fn rejects_an_out_of_range_judge_threshold() { + let mut config = config(); + config.llm_fallback = Some(LlmFallback { + judge_target: LlmTarget { + semantic_name: "judge".to_string(), + llm_client: None, + }, + config: TaskClassifierConfig { + base_threshold: -0.1, + ..Default::default() + }, + }); + assert!(matches!( + StageRouter::new(tier_target("strong"), tier_target("weak"), config), + Err(LibsyError::AlgorithmError { .. }) + )); + } + + #[test] + fn builds_over_both_tiers() -> Result<()> { + let router = StageRouter::new(tier_target("strong"), tier_target("weak"), config())?; + assert_eq!(router.name(), STAGE_ROUTER); + Ok(()) + } +} diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 827c233c..f86e23f9 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -2,12 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 mod affinity; -pub(crate) mod handoff_notes; mod llm_judge; -pub(crate) mod stage_router; +pub(crate) mod prompts; +pub(crate) mod stage; mod subagent; pub(crate) mod tool_signals; pub use affinity::AffinityRouter; pub(crate) use llm_judge::{Judge, JudgeClassifier, JudgeConfig, JudgePolicy}; +pub use prompts::{append_note, SystemPromptProcessor, TargetPrompts}; pub use subagent::SubagentOverride; diff --git a/crates/libsy/src/algorithms/util/handoff_notes.rs b/crates/libsy/src/algorithms/util/handoff_notes.rs deleted file mode 100644 index bbf9ede0..00000000 --- a/crates/libsy/src/algorithms/util/handoff_notes.rs +++ /dev/null @@ -1,180 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Stage-router handoff notes. -//! -//! When a turn escalates to the strong tier (or hands back to the weak tier), -//! a short **deterministic** note can be handed to the model taking over so it -//! knows *why* — without re-diagnosing (weak→strong) or re-architecting settled -//! work (strong→weak). This is not a model call; the note text comes from config. -//! -//! [`HandoffNoteProcessor`] reacts to [`Event::Decision`] (replayed after the -//! tier is chosen, before the model call), derives escalate vs de-escalate from -//! the chosen tier plus the `decision_source` the picker stashed, and records -//! the matching note on the [`State`] under [`HANDOFF_NOTE_KEY`]. A later -//! request-injection step splices it into the outbound request — that step is -//! intentionally not wired here. - -use async_trait::async_trait; - -use crate::{Event, Processor, Result, State, StateValue}; - -/// `State.extra` key under which the computed handoff note is recorded for a -/// later request-injection step to consume. -pub const HANDOFF_NOTE_KEY: &str = "handoff_note"; - -/// Records a handoff note on the decision, based on the chosen tier and the -/// picker's `decision_source`. Deterministic — no model call. -pub struct HandoffNoteProcessor { - escalation_note: String, - deescalation_note: Option, - only_on_wrong_signal_escalation: bool, -} - -impl HandoffNoteProcessor { - /// Configure the notes: the `escalation_note` handed to the strong tier, an - /// optional `deescalation_note` handed back to the weak tier, and whether - /// the escalation note fires only on a signal-driven escalation - /// (`override` / `dimensions`) rather than a `fall_open` default. - pub fn new( - escalation_note: impl Into, - deescalation_note: Option, - only_on_wrong_signal_escalation: bool, - ) -> Self { - Self { - escalation_note: escalation_note.into(), - deescalation_note, - only_on_wrong_signal_escalation, - } - } - - /// The note to record for a decision routed to `tier` with picker `source`, - /// or `None` when no note applies. - fn note_for(&self, tier: &str, source: Option<&str>) -> Option { - match tier { - // Escalation to the strong tier. When gated, only a signal-driven - // escalation qualifies — never a `fall_open` default. - "strong" => { - let signal_driven = matches!(source, Some("override") | Some("dimensions")); - (!self.only_on_wrong_signal_escalation || signal_driven) - .then(|| self.escalation_note.clone()) - } - // Hand-back to the weak tier, when a de-escalation note is configured. - "weak" => self.deescalation_note.clone(), - _ => None, - } - } -} - -#[async_trait] -impl Processor for HandoffNoteProcessor { - async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> { - let Event::Decision { decision, .. } = event else { - return Ok(()); - }; - let tier = decision.selected_model().to_string(); - // Read (and release the borrow on) the source the picker stashed. - let source = match state.extra.get("decision_source") { - Some(StateValue::String(source)) => Some(source.clone()), - _ => None, - }; - if let Some(note) = self.note_for(&tier, source.as_deref()) { - state - .extra - .insert(HANDOFF_NOTE_KEY.to_string(), StateValue::String(note)); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{Decision, Request}; - - struct FakeDecision(&'static str); - impl Decision for FakeDecision { - fn selected_model(&self) -> &str { - self.0 - } - fn reasoning(&self) -> Option<&str> { - None - } - fn as_any(&self) -> &dyn std::any::Any { - self - } - } - - fn state_with_source(source: &str) -> State { - let mut state = State::default(); - state.extra.insert( - "decision_source".to_string(), - StateValue::String(source.to_string()), - ); - state - } - - async fn run(processor: &HandoffNoteProcessor, state: &mut State, tier: &'static str) { - let request = Request::default(); - let decision = FakeDecision(tier); - processor - .process( - state, - Event::Decision { - request: &request, - decision: &decision, - }, - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn escalation_note_recorded_on_signal_driven_strong() { - let processor = HandoffNoteProcessor::new("recovering from an error", None, true); - for source in ["override", "dimensions"] { - let mut state = state_with_source(source); - run(&processor, &mut state, "strong").await; - assert!(matches!( - state.extra.get(HANDOFF_NOTE_KEY), - Some(StateValue::String(note)) if note == "recovering from an error" - )); - } - } - - #[tokio::test] - async fn no_escalation_note_on_fall_open_default_when_gated() { - let processor = HandoffNoteProcessor::new("recovering from an error", None, true); - let mut state = state_with_source("fall_open"); - run(&processor, &mut state, "strong").await; - assert!(!state.extra.contains_key(HANDOFF_NOTE_KEY)); - } - - #[tokio::test] - async fn escalation_note_on_fall_open_when_not_gated() { - let processor = HandoffNoteProcessor::new("recovering from an error", None, false); - let mut state = state_with_source("fall_open"); - run(&processor, &mut state, "strong").await; - assert!(state.extra.contains_key(HANDOFF_NOTE_KEY)); - } - - #[tokio::test] - async fn deescalation_note_recorded_on_weak_when_configured() { - let processor = - HandoffNoteProcessor::new("esc", Some("settled — carry on".to_string()), true); - let mut state = state_with_source("tests_passed"); - run(&processor, &mut state, "weak").await; - assert!(matches!( - state.extra.get(HANDOFF_NOTE_KEY), - Some(StateValue::String(note)) if note == "settled — carry on" - )); - } - - #[tokio::test] - async fn no_deescalation_note_when_unconfigured() { - let processor = HandoffNoteProcessor::new("esc", None, true); - let mut state = state_with_source("tests_passed"); - run(&processor, &mut state, "weak").await; - assert!(!state.extra.contains_key(HANDOFF_NOTE_KEY)); - } -} diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs new file mode 100644 index 00000000..4096d0e4 --- /dev/null +++ b/crates/libsy/src/algorithms/util/prompts.rs @@ -0,0 +1,306 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Adding text to a request on its way to the model it was routed to. +//! +//! Two shapes, both target-agnostic — any algorithm routing between named +//! targets can use them, and neither writes anything back into the caller's +//! conversation: +//! +//! * [`append_note`] — a one-off note in the conversation itself, for telling +//! the model something about *this* turn. +//! * [`SystemPromptProcessor`] — standing instructions per target, applied on +//! every turn that target serves. +//! +//! Which text, and when, is the caller's policy; this module only knows how to +//! place it so the provider accepts it and the prompt cache survives. + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role}; + +use crate::{Event, Processor, Result}; + +/// Appends `note` to the request as conversation text. +/// +/// It joins the trailing user message when there is one, rather than opening a +/// turn of its own: Anthropic rejects two consecutive user messages, and a +/// `tool_result` must stay first within its message. A text block after it +/// satisfies both and keeps the addition a cache-safe suffix. Any other trailing +/// role — an empty conversation, or one ending on an assistant turn — takes a +/// fresh user message. +pub fn append_note(request: &mut Request, note: &str) { + match request.llm_request.messages.last_mut() { + Some(last) if last.role == Role::User => last.content.push(ContentBlock::Text { + text: note.to_string(), + }), + _ => request + .llm_request + .messages + .push(Message::text(Role::User, note)), + } +} + +/// System prompts keyed by routing target. A target left unset is routed +/// untouched. +#[derive(Clone, Debug, Default)] +pub struct TargetPrompts { + by_target: BTreeMap, +} + +impl TargetPrompts { + /// Hand `target` this prompt on every turn it serves. + pub fn with(mut self, target: impl Into, prompt: impl Into) -> Self { + self.by_target.insert(target.into(), prompt.into()); + self + } + + /// The prompt configured for `target`, if any. + pub fn get(&self, target: &str) -> Option<&str> { + self.by_target.get(target).map(String::as_str) + } + + /// Whether any target has a prompt, so a caller can skip wiring the + /// processor when none does. + pub fn is_empty(&self) -> bool { + self.by_target.is_empty() + } +} + +/// Prepends the routed target's system prompt to the outbound request. +pub struct SystemPromptProcessor { + prompts: TargetPrompts, +} + +impl SystemPromptProcessor { + /// Hand each target the prompt configured for it. + pub fn new(prompts: TargetPrompts) -> Self { + Self { prompts } + } +} + +#[async_trait] +impl Processor for SystemPromptProcessor { + async fn process(&self, _state: &mut S, event: Event<'_>) -> Result<()> { + // The outbound hook carries the decision that routed the request, so the + // target is read straight off it — whichever classifier picked it, and + // with nothing kept between turns. + let Event::ModelRequest { request, decision } = event else { + return Ok(()); + }; + let Some(prompt) = self.prompts.get(decision.selected_model()) else { + return Ok(()); + }; + // Ahead of the client's own instructions, so this framing is what the + // model reads first. + request.llm_request.instructions.insert( + 0, + InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: prompt.to_string(), + }], + }, + ); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use switchyard_protocol::{text_request, LlmRequest, ToolResult}; + + const NOTE: &str = "recovering from an error"; + const STRONG_PROMPT: &str = "diagnose before you edit"; + const WEAK_PROMPT: &str = "follow the settled plan"; + + fn request_with(messages: Vec) -> Request { + Request { + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + + #[test] + fn a_note_joins_a_trailing_user_turn_after_its_tool_result() { + // The shape a coding-agent turn actually arrives in: the tool result + // leads the trailing user message, so the note has to follow it. + let tool_result = ContentBlock::ToolResult(ToolResult { + tool_call_id: "call_1".to_string(), + content: vec![ContentBlock::Text { + text: "exit 1".to_string(), + }], + is_error: Some(true), + }); + let mut request = request_with(vec![Message { + role: Role::User, + content: vec![tool_result.clone()], + }]); + + append_note(&mut request, NOTE); + + let messages = &request.llm_request.messages; + assert_eq!(messages.len(), 1, "no second consecutive user turn"); + assert_eq!( + messages[0].content, + vec![ + tool_result, + ContentBlock::Text { + text: NOTE.to_string() + } + ] + ); + } + + #[test] + fn a_note_opens_a_user_turn_after_an_assistant_turn() { + let mut request = request_with(vec![Message::text(Role::Assistant, "done")]); + + append_note(&mut request, NOTE); + + let messages = &request.llm_request.messages; + assert_eq!(messages.len(), 2); + assert_eq!(messages[1].role, Role::User); + assert_eq!(messages[1].text_content(""), Some(NOTE.to_string())); + } + + #[test] + fn a_note_leaves_the_rest_of_the_conversation_untouched() { + let mut request = Request { + llm_request: text_request(Some("auto".to_string()), "fix the build"), + raw_request: None, + metadata: None, + }; + + append_note(&mut request, NOTE); + + let trail: Vec = request + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("|")) + .collect(); + assert_eq!(trail, vec![format!("fix the build|{NOTE}")]); + } + + /// A decision routed to `target`. + struct RoutedTo(&'static str); + impl crate::Decision for RoutedTo { + fn selected_model(&self) -> &str { + self.0 + } + fn reasoning(&self) -> Option<&str> { + None + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + } + + /// The instruction text the request carries. + fn instructions(request: &Request) -> Vec { + request + .llm_request + .instructions + .iter() + .filter_map(|block| { + block.content.iter().find_map(|content| match content { + ContentBlock::Text { text } => Some(text.clone()), + _ => None, + }) + }) + .collect() + } + + /// Runs one outbound request routed to `target` through `processor`. + async fn run(processor: &SystemPromptProcessor, target: &'static str) -> Result { + let mut request = Request::default(); + processor + .process( + &mut (), + Event::ModelRequest { + request: &mut request, + decision: &RoutedTo(target), + }, + ) + .await?; + Ok(request) + } + + fn prompts() -> TargetPrompts { + TargetPrompts::default() + .with("strong", STRONG_PROMPT) + .with("weak", WEAK_PROMPT) + } + + #[tokio::test] + async fn each_target_gets_its_own_prompt() -> Result<()> { + let processor = SystemPromptProcessor::new(prompts()); + for (target, expected) in [("strong", STRONG_PROMPT), ("weak", WEAK_PROMPT)] { + assert_eq!( + instructions(&run(&processor, target).await?), + vec![expected] + ); + } + Ok(()) + } + + #[tokio::test] + async fn an_unconfigured_target_is_left_untouched() -> Result<()> { + // One target's prompt must not leak onto another, whatever ran before. + let processor = + SystemPromptProcessor::new(TargetPrompts::default().with("strong", STRONG_PROMPT)); + assert_eq!( + instructions(&run(&processor, "strong").await?), + vec![STRONG_PROMPT] + ); + assert!(instructions(&run(&processor, "weak").await?).is_empty()); + Ok(()) + } + + #[tokio::test] + async fn the_prompt_leads_the_client_instructions() -> Result<()> { + let processor = SystemPromptProcessor::new(prompts()); + let mut request = Request::default(); + request.llm_request.instructions.push(InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: "you are a coding agent".to_string(), + }], + }); + + processor + .process( + &mut (), + Event::ModelRequest { + request: &mut request, + decision: &RoutedTo("strong"), + }, + ) + .await?; + + assert_eq!( + instructions(&request), + vec![STRONG_PROMPT, "you are a coding agent"] + ); + Ok(()) + } + + #[tokio::test] + async fn the_inbound_request_is_left_alone() -> Result<()> { + // The inbound hook runs before the cascade has picked anything. + let processor = SystemPromptProcessor::new(prompts()); + let mut request = Request::default(); + processor + .process(&mut (), Event::Request(&mut request)) + .await?; + assert!(instructions(&request).is_empty()); + Ok(()) + } +} diff --git a/crates/libsy/src/algorithms/util/stage_router.rs b/crates/libsy/src/algorithms/util/stage.rs similarity index 52% rename from crates/libsy/src/algorithms/util/stage_router.rs rename to crates/libsy/src/algorithms/util/stage.rs index a9d69251..db5de92e 100644 --- a/crates/libsy/src/algorithms/util/stage_router.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -5,7 +5,7 @@ //! //! Given a [`ToolSignals`] (extracted by the dimension collector), this //! module decides whether a coding-agent turn should go to the **capable** -//! (strong) or **efficient** (weak) tier. It is the single source of truth for +//! or **efficient** tier. It is the single source of truth for //! the decision, shared by the Rust profile and the Python processor via //! bindings — only the outer shell differs in how it fetches the decision. //! @@ -19,10 +19,10 @@ //! `(-1, +1)`; `confidence` is the magnitude. The `confidence_threshold` dials //! how much corroboration a decisive escalation needs (see [`score_signal`]). -#![allow(dead_code)] - use async_trait::async_trait; +use serde::Deserialize; +use super::prompts::append_note; use crate::{ Classification, Classifier, Driver, Request, Result, Score, State, StateValue, ToolSignals, }; @@ -46,14 +46,68 @@ const SEVERITY_CRITICAL: f32 = 1.0; /// The two tiers a turn can route to. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Tier { - /// Weak / cheap tier. + /// Efficient / cheap tier. Efficient, - /// Strong / capable tier. + /// Capable / powerful tier. Capable, } +impl Tier { + /// Stable label for stats and the [`routing_tier`](Classifier::routing_tier) + /// hook, independent of what the tiers' targets are called. These are the + /// strings the capability route reports too, so a deployment running both + /// sees one tier vocabulary. + fn label(self) -> &'static str { + match self { + Self::Capable => "strong", + Self::Efficient => "weak", + } + } +} + +/// The targets a stage router's two tiers route to. +/// +/// The tiers are a fixed pair, but their targets are whatever the deployment +/// calls them, so the classifier scores onto those names and the routed call +/// reaches the right model. +#[derive(Clone, Debug)] +pub struct StageTargets { + capable: String, + efficient: String, +} + +impl StageTargets { + /// Name the targets the two tiers route to. + pub fn new(capable: impl Into, efficient: impl Into) -> Self { + Self { + capable: capable.into(), + efficient: efficient.into(), + } + } + + /// The target `tier` routes to. + pub fn name(&self, tier: Tier) -> &str { + match tier { + Tier::Capable => &self.capable, + Tier::Efficient => &self.efficient, + } + } + + /// The tier label for a routed target, or `None` for one outside the pair. + pub fn label_for(&self, target: &str) -> Option<&'static str> { + if target == self.capable { + Some(Tier::Capable.label()) + } else if target == self.efficient { + Some(Tier::Efficient.label()) + } else { + None + } + } +} + /// Which tier to default to when the scorer is not confident. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum PickerMode { /// Default to capable unless the scorer confidently picks efficient. CapableFirst, @@ -62,7 +116,8 @@ pub enum PickerMode { } impl PickerMode { - fn default_tier(self) -> Tier { + /// Tier a turn routes to when the scorer is not confident enough to pick. + pub fn default_tier(self) -> Tier { match self { Self::CapableFirst => Tier::Capable, Self::EfficientFirst => Tier::Efficient, @@ -70,7 +125,23 @@ impl PickerMode { } } +/// `State.extra` key under which the turn's [`DecisionSource`] is recorded. +pub const DECISION_SOURCE_KEY: &str = "decision_source"; + +/// Record which component decided the turn. +pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource) { + state.extra.insert( + DECISION_SOURCE_KEY.to_string(), + StateValue::String(source.as_str().to_string()), + ); +} + /// What produced a decision — for stats and explainability. +/// +/// Each is stamped by the component that knows it, so a turn's final label names +/// whoever actually decided it. [`Ambiguous`](Self::Ambiguous) is the exception: +/// the signal scorer records it on its way out, and whichever classifier resolves +/// the turn overwrites it. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum DecisionSource { /// Hard override (critical severity or context compaction). @@ -79,7 +150,11 @@ pub enum DecisionSource { TestsPassed, /// Scorer crossed `confidence_threshold`. Dimensions, - /// Scorer was not confident; the caller should consult its classifier. + /// Scorer was not confident, so the signals did not decide this turn. + Ambiguous, + /// An LLM classifier behind the signals decided the turn. + LlmClassifier, + /// Nothing resolved the turn, so it landed on the picker's default tier. FallOpen, } @@ -90,6 +165,8 @@ impl DecisionSource { Self::Override => "override", Self::TestsPassed => "tests_passed", Self::Dimensions => "dimensions", + Self::Ambiguous => "ambiguous", + Self::LlmClassifier => "llm-classifier", Self::FallOpen => "fall_open", } } @@ -189,11 +266,11 @@ pub fn score_signal(signal: &ToolSignals) -> ScoreResult { } } -/// Hard **escalate** — force the strong tier no matter what the scorer would +/// Hard **escalate** — force the capable tier no matter what the scorer would /// say. Fires on a critical error or a compacted context. fn should_escalate(signal: &ToolSignals) -> bool { // Compaction wipes the accumulated signals, so a task that had escalated - // would snap back to weak — a context big enough to overflow belongs strong. + // would snap back to efficient — a context big enough to overflow belongs capable. if signal.compacted { return true; } @@ -213,7 +290,7 @@ fn should_deescalate(signal: &ToolSignals) -> bool { /// /// The rules run in order; the first that fires wins: /// -/// 1. **Escalate** — a hard reason to go strong (critical error / compaction). +/// 1. **Escalate** — a hard reason to go capable (critical error / compaction). /// 2. **De-escalate** — a hard reason to go cheap (a settled turn). /// 3. **Scorer** — no hard reason, so weigh the two axes; if confident, follow it. /// 4. **Fall open** — not confident: hand to the classifier, else the default. @@ -226,7 +303,7 @@ fn should_deescalate(signal: &ToolSignals) -> bool { /// returns [`PickOutcome::ConsultClassifier`] instead of calling it here. The /// `no_signal` case (no tool activity yet) is handled one level up. pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f64) -> PickOutcome { - // 1. Escalate — a hard reason to go strong, ahead of everything else. + // 1. Escalate — a hard reason to go capable, ahead of everything else. if should_escalate(signal) { return resolved(Tier::Capable, DecisionSource::Override, 0.0, Some(1.0)); } @@ -237,7 +314,7 @@ pub fn pick_tier(signal: &ToolSignals, mode: PickerMode, confidence_threshold: f } // 3. Scorer — no hard reason either way, so weigh error vs production. If - // confident enough, follow the sign: positive → strong, negative → cheap. + // confident enough, follow the sign: positive → capable, negative → efficient. let scored = score_signal(signal); if scored.confidence >= confidence_threshold { let tier = if scored.score > 0.0 { @@ -285,53 +362,134 @@ fn ratio(numerator: u32, denominator: u32) -> f64 { } } -/// Signal-only stage-router classifier: scores each turn onto the strong/weak +/// The notes a stage router hands the model it routed to, and the gate deciding +/// which one a turn earns. +/// +/// Stateless: a note describes the turn's own signals, so every turn they drive +/// carries one. It rides in the forwarded request only, never in the caller's +/// conversation, so notes cannot accumulate across turns. +#[derive(Clone, Debug, Deserialize)] +pub struct HandoffNoteConfig { + /// Note handed to the capable tier on a signal-driven escalation. + escalation_note: String, + /// Optional note handed back to the efficient tier. + #[serde(default)] + deescalation_note: Option, + /// Restricts the escalation note to signal-driven escalations. + #[serde(default = "escalation_gate_default")] + only_on_wrong_signal_escalation: bool, +} + +/// Gating the escalation note is the safe default: an ungated note can tell the +/// capable model the efficient one was stalling when it wasn't. +fn escalation_gate_default() -> bool { + true +} + +impl HandoffNoteConfig { + /// Configure the notes: the `escalation_note` handed to the capable tier, an + /// optional `deescalation_note` handed back to the efficient tier, and + /// whether the escalation note fires only on a signal-driven escalation + /// (`override` / `dimensions`) rather than an ambiguous default. + pub fn new( + escalation_note: impl Into, + deescalation_note: Option, + only_on_wrong_signal_escalation: bool, + ) -> Self { + Self { + escalation_note: escalation_note.into(), + deescalation_note, + only_on_wrong_signal_escalation, + } + } + + /// The note for a turn routed to `tier` with picker `source`, or `None` when + /// no note applies. + fn note_for(&self, tier: Tier, source: DecisionSource) -> Option<&str> { + match tier { + // Escalation to the capable tier. When gated, only a signal-driven + // escalation qualifies — never an ambiguous default, which would tell + // the capable model the efficient one was stalling when it wasn't. + Tier::Capable => { + let signal_driven = matches!( + source, + DecisionSource::Override | DecisionSource::Dimensions + ); + (!self.only_on_wrong_signal_escalation || signal_driven) + .then_some(self.escalation_note.as_str()) + } + // Hand-back to the efficient tier, when a de-escalation note is configured. + Tier::Efficient => self.deescalation_note.as_deref(), + } + } +} + +/// Signal-only stage-router classifier: scores each turn onto the capable/efficient /// tiers from tool-result signals, via the configured picker mode and the /// confidence the scorer must reach before it acts on the signal alone. +/// +/// With [`with_handoff_notes`](Self::with_handoff_notes) it also splices a note +/// into the request explaining why the signals sent the turn where they did. pub struct StageClassifier { + targets: StageTargets, mode: PickerMode, confidence_threshold: f64, + handoff_notes: Option, } impl StageClassifier { - /// Build a classifier with the given default tier (`mode`) and + /// Scores onto `targets`, with the given default tier (`mode`) and /// `confidence_threshold`. - pub fn new(mode: PickerMode, confidence_threshold: f64) -> Self { + pub fn new(targets: StageTargets, mode: PickerMode, confidence_threshold: f64) -> Self { Self { + targets, mode, confidence_threshold, + handoff_notes: None, + } + } + + /// Hand the routed model a note on a signal-driven escalation, and on a + /// hand-back to the efficient tier when a de-escalation note is configured. + pub fn with_handoff_notes(mut self, config: HandoffNoteConfig) -> Self { + self.handoff_notes = Some(config); + self + } + + /// The signals could not decide, so this turn belongs to whatever the cascade + /// has behind this classifier. + fn abstain(state: &mut State) -> Classification { + record_decision_source(state, DecisionSource::Ambiguous); + Classification::Ambiguous(Vec::new()) + } + + fn apply_handoff_note(&self, request: &mut Request, tier: Tier, source: DecisionSource) { + let Some(config) = &self.handoff_notes else { + return; + }; + if let Some(note) = config.note_for(tier, source) { + append_note(request, note); } } } #[async_trait] impl Classifier for StageClassifier { + fn routing_tier(&self, selected_model: &str) -> Option<&'static str> { + self.targets.label_for(selected_model) + } + async fn score( &self, state: &mut State, - _request: &mut Request, + request: &mut Request, _driver: Option<&Driver>, ) -> Result { let tool_signals = &state.tool_signals; let Some(signal) = tool_signals else { - // No tool activity yet — nothing to score, so fall open to the - // picker's configured default tier (same as a below-threshold turn). - let target = match self.mode.default_tier() { - Tier::Capable => "strong", - Tier::Efficient => "weak", - }; - state.extra.insert( - "default_target".to_string(), - StateValue::String(target.to_string()), - ); - state.extra.insert( - "decision_source".to_string(), - StateValue::String(DecisionSource::FallOpen.as_str().to_string()), - ); - return Ok(Classification::Ambiguous(vec![Score { - target: target.to_string(), - confidence: 0.0, - }])); + // No tool activity yet — nothing to score, so the signals have no + // opinion, same as a below-threshold turn. + return Ok(Self::abstain(state)); }; let outcome = pick_tier(signal, self.mode, self.confidence_threshold); @@ -342,14 +500,12 @@ impl Classifier for StageClassifier { score, .. } => { - let target = match tier { - Tier::Capable => "strong", - Tier::Efficient => "weak", - }; - state.extra.insert( - "decision_source".to_string(), - StateValue::String(source.as_str().to_string()), - ); + let target = self.targets.name(tier); + record_decision_source(state, source); + // Only a resolved turn routes on this classifier's target, so it + // is the only branch whose tier the signals actually chose — an + // ambiguous turn is decided further down the cascade. + self.apply_handoff_note(request, tier, source); let conf = score.abs(); // TODO add the non-target to this score set? Ok(Classification::Scores(vec![Score { @@ -357,30 +513,7 @@ impl Classifier for StageClassifier { confidence: conf, }])) } - PickOutcome::ConsultClassifier { - score, - default_tier, - .. - } => { - let target = match default_tier { - Tier::Capable => "strong", - Tier::Efficient => "weak", - }; - state.extra.insert( - "default_target".to_string(), - StateValue::String(target.to_string()), - ); - state.extra.insert( - "decision_source".to_string(), - StateValue::String(DecisionSource::FallOpen.as_str().to_string()), - ); - let conf = score.abs(); - // TODO add the non-target to this score set? - Ok(Classification::Ambiguous(vec![Score { - target: target.to_string(), - confidence: conf, - }])) - } + PickOutcome::ConsultClassifier { .. } => Ok(Self::abstain(state)), } } } @@ -444,6 +577,12 @@ mod tests { ); } + #[test] + fn the_picker_mode_names_the_tier_an_undecided_turn_falls_back_to() { + assert_eq!(PickerMode::CapableFirst.default_tier(), Tier::Capable); + assert_eq!(PickerMode::EfficientFirst.default_tier(), Tier::Efficient); + } + #[test] fn quiet_signal_falls_open_to_default() { let signal = signal_from(json!([{"role": "user", "content": "hi"}])); @@ -457,6 +596,11 @@ mod tests { // ─── StageClassifier ───────────────────────────────────────────────── + /// Tiers named the way a deployment would name them. + fn tiers() -> StageTargets { + StageTargets::new("strong", "weak") + } + /// A `State` carrying `signal` as its tool signals. fn state_with(signal: ToolSignals) -> State { State { @@ -467,39 +611,29 @@ mod tests { #[tokio::test] async fn classifier_defaults_without_tool_signals() -> Result<()> { - // No tool activity yet → route to the picker's configured default tier - // (efficient_first → weak), recorded in `extra` for the caller. + // No tool activity yet — nothing to score, so the signals have no opinion + // and the turn belongs to whatever the cascade has behind them. let mut state = State::default(); - let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5) + let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), None) .await?; - match classification { - Classification::Ambiguous(scores) => { - assert_eq!(scores.len(), 1); - assert_eq!(scores[0].target, "weak"); - } - _ => panic!("expected an ambiguous default classification"), - } - assert!(matches!( - state.extra.get("default_target"), - Some(StateValue::String(target)) if target == "weak" - )); + assert!(classification.argmax(false)?.is_none()); assert!(matches!( - state.extra.get("decision_source"), - Some(StateValue::String(source)) if source == "fall_open" + state.extra.get(DECISION_SOURCE_KEY), + Some(StateValue::String(source)) if source == "ambiguous" )); Ok(()) } #[tokio::test] async fn classifier_escalates_critical_severity_to_strong() -> Result<()> { - // Critical severity is a hard override → a definite score for the strong tier. + // Critical severity is a hard override → a definite score for the capable tier. let signal = ToolSignals { severity: SEVERITY_CRITICAL, ..Default::default() }; let mut state = state_with(signal); - let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5) + let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), None) .await?; match classification { @@ -511,7 +645,7 @@ mod tests { } // The decision source travels downstream (for handoff-note gating). assert!(matches!( - state.extra.get("decision_source"), + state.extra.get(DECISION_SOURCE_KEY), Some(StateValue::String(source)) if source == "override" )); Ok(()) @@ -520,7 +654,7 @@ mod tests { #[tokio::test] async fn classifier_deescalates_settled_turn_to_weak() -> Result<()> { // Tests passed with recent production and no error → the settled-turn shortcut - // resolves straight to a definite weak-tier score. + // resolves straight to a definite efficient-tier score. let signal = ToolSignals { tests_passed: true, recent_write_count: 1, @@ -528,7 +662,7 @@ mod tests { ..Default::default() }; let mut state = state_with(signal); - let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5) + let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), None) .await?; match classification { @@ -543,49 +677,186 @@ mod tests { #[tokio::test] async fn classifier_falls_open_to_default_and_records_it() -> Result<()> { - // A quiet signal corroborates neither axis → ambiguous, defaulting to the - // efficient (weak) tier, which is also stashed in `extra` for the caller. + // A quiet signal corroborates neither axis, so the scorer abstains and + // records why. let mut state = state_with(ToolSignals::default()); - let classification = StageClassifier::new(PickerMode::EfficientFirst, 0.5) + let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), None) .await?; - match classification { - Classification::Ambiguous(scores) => { - assert_eq!(scores.len(), 1); - assert_eq!(scores[0].target, "weak"); - } - _ => panic!("expected an ambiguous classification"), - } - assert!(matches!( - state.extra.get("default_target"), - Some(StateValue::String(target)) if target == "weak" - )); + assert!(classification.argmax(false)?.is_none()); assert!(matches!( - state.extra.get("decision_source"), - Some(StateValue::String(source)) if source == "fall_open" + state.extra.get(DECISION_SOURCE_KEY), + Some(StateValue::String(source)) if source == "ambiguous" )); Ok(()) } + const ESCALATION: &str = "recovering from an error"; + const DEESCALATION: &str = "settled — carry on"; + + fn config(only_on_wrong_signal_escalation: bool) -> HandoffNoteConfig { + HandoffNoteConfig::new( + ESCALATION, + Some(DEESCALATION.to_string()), + only_on_wrong_signal_escalation, + ) + } + + #[test] + fn escalation_note_applies_to_signal_driven_capable() { + for source in [DecisionSource::Override, DecisionSource::Dimensions] { + assert_eq!( + config(true).note_for(Tier::Capable, source), + Some(ESCALATION) + ); + } + } + + #[test] + fn no_escalation_note_on_an_ambiguous_turn_when_gated() { + assert_eq!( + config(true).note_for(Tier::Capable, DecisionSource::Ambiguous), + None + ); + } + + #[test] + fn escalation_note_on_an_ambiguous_turn_when_not_gated() { + assert_eq!( + config(false).note_for(Tier::Capable, DecisionSource::Ambiguous), + Some(ESCALATION) + ); + } + + #[test] + fn deescalation_note_applies_to_efficient_when_configured() { + assert_eq!( + config(true).note_for(Tier::Efficient, DecisionSource::TestsPassed), + Some(DEESCALATION) + ); + } + + #[test] + fn no_deescalation_note_when_unconfigured() { + let config = HandoffNoteConfig::new(ESCALATION, None, true); + assert_eq!( + config.note_for(Tier::Efficient, DecisionSource::TestsPassed), + None + ); + } + + // ─── handoff notes ─────────────────────────────────────────────────── + + /// A classifier that hands the capable tier an escalation note, gated to + /// signal-driven escalations. + fn noting_classifier(mode: PickerMode) -> StageClassifier { + StageClassifier::new(tiers(), mode, 0.5).with_handoff_notes(HandoffNoteConfig::new( + ESCALATION, + Some(DEESCALATION.to_string()), + true, + )) + } + + /// A one-user-turn request, the thing a note gets spliced into. + fn request() -> Request { + Request { + llm_request: switchyard_protocol::text_request(Some("auto".to_string()), "hi"), + raw_request: None, + metadata: None, + } + } + + /// The trailing user turn's text, note included. + fn trailing_text(request: &Request) -> Option { + request + .llm_request + .messages + .last() + .and_then(|message| message.text_content("|")) + } + + /// The signal that forces an escalation on the override path. + fn critical() -> ToolSignals { + ToolSignals { + severity: SEVERITY_CRITICAL, + ..Default::default() + } + } + #[tokio::test] - async fn classifier_honors_configured_mode_on_fall_open() -> Result<()> { - // Same quiet signal, but capable_first defaults the fall-open to strong — - // proving the configured picker mode is used, not a hardcoded default. - let mut state = state_with(ToolSignals::default()); - let classification = StageClassifier::new(PickerMode::CapableFirst, 0.5) - .score(&mut state, &mut Request::default(), None) + async fn a_signal_driven_escalation_carries_the_note() -> Result<()> { + let mut state = state_with(critical()); + let mut request = request(); + + noting_classifier(PickerMode::EfficientFirst) + .score(&mut state, &mut request, None) .await?; - match classification { - Classification::Ambiguous(scores) => { - assert_eq!(scores.len(), 1); - assert_eq!(scores[0].target, "strong"); - } - _ => panic!("expected an ambiguous classification"), + + assert_eq!(trailing_text(&request), Some(format!("hi|{ESCALATION}"))); + Ok(()) + } + + #[tokio::test] + async fn every_turn_the_signals_drive_carries_the_note() -> Result<()> { + // Stateless by design: the note describes this turn's signals, so a run + // of escalated turns each carries one. Nothing tracks the previous tier. + let classifier = noting_classifier(PickerMode::EfficientFirst); + let mut state = state_with(critical()); + + for _ in 0..3 { + let mut request = request(); + classifier.score(&mut state, &mut request, None).await?; + assert_eq!(trailing_text(&request), Some(format!("hi|{ESCALATION}"))); } - assert!(matches!( - state.extra.get("default_target"), - Some(StateValue::String(target)) if target == "strong" - )); + Ok(()) + } + + #[tokio::test] + async fn a_settled_turn_carries_the_deescalation_note() -> Result<()> { + // Tests passed with recent production resolves to weak on the settled-turn + // shortcut, which is the hand-back the de-escalation note is for. + let signal = ToolSignals { + tests_passed: true, + recent_write_count: 1, + ..Default::default() + }; + let mut state = state_with(signal); + let mut request = request(); + + noting_classifier(PickerMode::EfficientFirst) + .score(&mut state, &mut request, None) + .await?; + + assert_eq!(trailing_text(&request), Some(format!("hi|{DEESCALATION}"))); + Ok(()) + } + + #[tokio::test] + async fn no_note_on_an_ambiguous_turn() -> Result<()> { + // A quiet signal falls open: the cascade, not these signals, picks the + // tier, so there is no signal-driven handover to narrate. + let mut state = state_with(ToolSignals::default()); + let mut request = request(); + + let classification = noting_classifier(PickerMode::CapableFirst) + .score(&mut state, &mut request, None) + .await?; + + assert!(matches!(classification, Classification::Ambiguous(_))); + assert_eq!(trailing_text(&request), Some("hi".to_string())); + Ok(()) + } + + #[tokio::test] + async fn no_note_when_notes_are_unconfigured() -> Result<()> { + let mut state = state_with(critical()); + let mut request = request(); + + StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) + .score(&mut state, &mut request, None) + .await?; + + assert_eq!(trailing_text(&request), Some("hi".to_string())); Ok(()) } } diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 1b0a0d9f..2f784b0d 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -15,7 +15,7 @@ use async_trait::async_trait; use serde_json::Value; -use switchyard_protocol::{Request, WireFormat}; +use switchyard_protocol::{ContentBlock, Request}; use crate::Result; @@ -331,224 +331,71 @@ fn classify_tool_call(name: &str, command: Option<&str>) -> ToolCategory { /// Returns [`ToolSignals::default()`] when the wire format or body is absent or /// the messages list is empty — callers can always read `signal.severity`. fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals { - let Some(Some(wire_format)) = request.metadata.as_ref().map(|m| m.wire_format) else { - return ToolSignals::default(); - }; - let Some(body) = request.raw_request.as_ref() else { - return ToolSignals::default(); - }; - let Some(obj) = body.as_object() else { - return ToolSignals::default(); - }; - - let (mut signal, entries) = match wire_format { - WireFormat::OpenAiChat => { - let messages = obj - .get("messages") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - ( - extract_from_messages_openai_chat(messages, recent_window), - messages, - ) - } - WireFormat::AnthropicMessages => { - let messages = obj - .get("messages") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - ( - extract_from_messages_anthropic(messages, recent_window), - messages, - ) - } - WireFormat::OpenAiResponses => { - let items = obj - .get("input") - .and_then(Value::as_array) - .map(Vec::as_slice) - .unwrap_or(&[]); - (extract_from_input_responses(items, recent_window), items) - } - }; - - // Compaction is detected anywhere in the message/item contents (the summary stays - // in the prefix on every subsequent turn, so this self-latches once it fires). - signal.compacted = entries.iter().any(|m| { - m.as_object() - .is_some_and(|o| content_has_compaction_marker(o.get("content"))) - }); - signal -} - -// ─── format-specific extractors ────────────────────────────────────────────── - -/// Distinctive preamble Claude Code injects as a user message when it compacts an -/// overflowed context. Matched case-insensitively; normal task text never contains it. -const COMPACTION_MARKER: &str = "session is being continued"; - -/// True when a user-message content (string or text blocks) carries the compaction -/// summary preamble. -fn content_has_compaction_marker(content: Option<&Value>) -> bool { - match content { - Some(Value::String(s)) => s.to_lowercase().contains(COMPACTION_MARKER), - Some(Value::Array(blocks)) => blocks.iter().any(|b| { - b.as_object() - .and_then(|o| o.get("text")) - .and_then(Value::as_str) - .is_some_and(|t| t.to_lowercase().contains(COMPACTION_MARKER)) - }), - _ => false, - } -} - -fn extract_from_messages_openai_chat(messages: &[Value], recent_window: usize) -> ToolSignals { + // Read the decoded conversation, not the raw body: every inbound format lands + // in the same shape here, so the signals do not depend on knowing which one it + // arrived as. + let messages = &request.llm_request.messages; let mut tool_texts: Vec = Vec::new(); let mut tool_calls: Vec = Vec::new(); - - for msg in messages { - let Some(obj) = msg.as_object() else { continue }; - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); - match role { - "tool" => { - if let Some(text) = content_to_text(obj.get("content")) { - tool_texts.push(text); + let mut compacted = false; + + for message in messages { + for block in &message.content { + match block { + ContentBlock::ToolCall(call) => { + tool_calls.push(ObservedToolCall { + name: call.name.clone(), + command: command_of(&call.arguments), + }); } - } - "assistant" => { - if let Some(tc_list) = obj.get("tool_calls").and_then(Value::as_array) { - for tc in tc_list { - let Some(fn_obj) = tc - .as_object() - .and_then(|t| t.get("function")) - .and_then(|f| f.as_object()) - else { - continue; - }; - let Some(name) = fn_obj.get("name").and_then(Value::as_str) else { - continue; - }; - // OpenAI Chat encodes `arguments` as a JSON string. - let command = fn_obj - .get("arguments") - .and_then(Value::as_str) - .and_then(|s| serde_json::from_str::(s).ok()) - .and_then(|v| { - v.get("command") - .and_then(Value::as_str) - .map(|s| s.to_lowercase()) - }); - tool_calls.push(ObservedToolCall { - name: name.to_string(), - command, - }); + ContentBlock::ToolResult(result) => { + let text = result + .content + .iter() + .filter_map(text_of) + .collect::>() + .join("\n"); + if !text.is_empty() { + tool_texts.push(text); } } - } - _ => {} - } - } - - build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) -} - -fn extract_from_messages_anthropic(messages: &[Value], recent_window: usize) -> ToolSignals { - let mut tool_texts: Vec = Vec::new(); - let mut tool_calls: Vec = Vec::new(); - - for msg in messages { - let Some(obj) = msg.as_object() else { continue }; - let role = obj.get("role").and_then(Value::as_str).unwrap_or(""); - let content = obj.get("content"); - - match role { - "user" => { - if let Some(Value::Array(blocks)) = content { - for block in blocks { - let Some(b) = block.as_object() else { continue }; - if b.get("type").and_then(Value::as_str) == Some("tool_result") { - if let Some(text) = content_to_text(b.get("content")) { - tool_texts.push(text); - } - } - } + // Compaction is detected anywhere in the conversation: the summary + // stays in the prefix on every later turn, so this self-latches + // once it fires. + ContentBlock::Text { text } => { + compacted |= text.to_lowercase().contains(COMPACTION_MARKER); } + _ => {} } - "assistant" => { - if let Some(Value::Array(blocks)) = content { - for block in blocks { - let Some(b) = block.as_object() else { continue }; - if b.get("type").and_then(Value::as_str) == Some("tool_use") { - let Some(name) = b.get("name").and_then(Value::as_str) else { - continue; - }; - // Anthropic delivers `input` as a parsed object. - let command = b - .get("input") - .and_then(Value::as_object) - .and_then(|i| i.get("command")) - .and_then(Value::as_str) - .map(|s| s.to_lowercase()); - tool_calls.push(ObservedToolCall { - name: name.to_string(), - command, - }); - } - } - } - } - _ => {} } } - build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window) + let mut signal = build_signal(tool_texts, tool_calls, messages.len() as u32, recent_window); + signal.compacted = compacted; + signal } -fn extract_from_input_responses(items: &[Value], recent_window: usize) -> ToolSignals { - let mut tool_texts: Vec = Vec::new(); - let mut tool_calls: Vec = Vec::new(); +/// Distinctive preamble Claude Code injects as a user message when it compacts an +/// overflowed context. Matched case-insensitively; normal task text never contains it. +const COMPACTION_MARKER: &str = "session is being continued"; - for item in items { - let Some(obj) = item.as_object() else { - continue; - }; - let item_type = obj.get("type").and_then(Value::as_str).unwrap_or(""); +/// The shell command a tool call carries, when it has one. Harnesses name the +/// field `command`; anything else is a tool whose category comes from its name. +fn command_of(arguments: &Value) -> Option { + arguments + .get("command") + .and_then(Value::as_str) + .map(str::to_lowercase) +} - match item_type { - "function_call_output" => { - if let Some(output) = obj.get("output").and_then(Value::as_str) { - tool_texts.push(output.to_string()); - } - } - "function_call" => { - let Some(name) = obj.get("name").and_then(Value::as_str) else { - continue; - }; - let command = obj - .get("arguments") - .and_then(Value::as_str) - .and_then(|s| serde_json::from_str::(s).ok()) - .and_then(|v| { - v.get("command") - .and_then(Value::as_str) - .map(|s| s.to_lowercase()) - }); - tool_calls.push(ObservedToolCall { - name: name.to_string(), - command, - }); - } - _ => {} - } +/// Text carried by a content block, ignoring the non-textual kinds. +fn text_of(block: &ContentBlock) -> Option<&str> { + match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()), + _ => None, } - - build_signal(tool_texts, tool_calls, items.len() as u32, recent_window) } -// ─── aggregation ───────────────────────────────────────────────────────────── - fn build_signal( tool_texts: Vec, tool_calls: Vec, @@ -750,17 +597,20 @@ fn has_nonzero_failure_count(lower: &str) -> bool { mod tests { use super::*; use serde_json::json; - use switchyard_protocol::Metadata; + use switchyard_protocol::{Metadata, WireFormat}; - /// Build a `Request` carrying `body` as its raw payload, tagged with `wire_format`. + /// Decodes `body` the way an endpoint would, so these tests assert on the + /// bodies a real client sends rather than on hand-built internal shapes. fn request_with(wire_format: WireFormat, body: Value) -> Request { + let llm_request = switchyard_translation::decode_request(wire_format, &body) + .unwrap_or_else(|error| panic!("fixture is not valid {wire_format}: {error}")); Request { + llm_request, raw_request: Some(body), metadata: Some(Metadata { wire_format: Some(wire_format), ..Default::default() }), - ..Default::default() } } diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs index d3bb7856..0998f5a2 100644 --- a/crates/libsy/src/core/processor.rs +++ b/crates/libsy/src/core/processor.rs @@ -22,8 +22,13 @@ pub enum Event<'a> { /// The routing decision produced for `request`. decision: &'a dyn Decision, }, - /// A request about to be sent to a model. - ModelRequest(&'a mut Request), + /// A request about to be sent to a model, with the decision that routed it. + ModelRequest { + /// The outbound request, rewritable in place. + request: &'a mut Request, + /// Where the request is headed. + decision: &'a dyn Decision, + }, /// A buffered response received back from a model. ModelResponse(&'a AggLlmResponse), } @@ -53,7 +58,7 @@ mod tests { Event::Request(_) => "requests", Event::Signal(_) => "signals", Event::Decision { .. } => "decisions", - Event::ModelRequest(_) => "model_requests", + Event::ModelRequest { .. } => "model_requests", Event::ModelResponse(_) => "model_responses", } } @@ -111,7 +116,13 @@ mod tests { .process(&mut state, Event::Request(&mut req)) .await?; processor - .process(&mut state, Event::ModelRequest(&mut req)) + .process( + &mut state, + Event::ModelRequest { + request: &mut req, + decision: &decision, + }, + ) .await?; processor .process(&mut state, Event::ModelResponse(&response)) @@ -159,8 +170,11 @@ mod tests { #[async_trait] impl Processor for RewritingProcessor { async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> { - if let Event::Request(request) | Event::ModelRequest(request) = event { - request.llm_request.model = Some("rewritten".to_string()); + match event { + Event::Request(request) | Event::ModelRequest { request, .. } => { + request.llm_request.model = Some("rewritten".to_string()); + } + _ => {} } Ok(()) } diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 9f29db91..8ddfe1be 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -98,9 +98,11 @@ pub fn initialize_metrics() { /// core (scorer, picker, and the `StageClassifier`). // TODO cleanup once switchyard-components is removed pub mod stage_router { - pub use crate::algorithms::util::handoff_notes::{HandoffNoteProcessor, HANDOFF_NOTE_KEY}; - pub use crate::algorithms::util::stage_router::{ + pub use crate::algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; + + pub use crate::algorithms::util::stage::{ dimensions_from_signal, pick_tier, score_signal, CodingAgentDimensions, DecisionSource, - PickOutcome, PickerMode, ScoreResult, StageClassifier, Tier, + HandoffNoteConfig, PickOutcome, PickerMode, ScoreResult, StageClassifier, StageTargets, + Tier, DECISION_SOURCE_KEY, }; } diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs index 813c9453..50d73897 100644 --- a/crates/libsy/tests/observability.rs +++ b/crates/libsy/tests/observability.rs @@ -754,6 +754,7 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib capability_elevated_floor: None, session_affinity: false, message_hash_fallback: false, + recent_turn_window: None, }, )?); diff --git a/crates/libsy/tests/prompts.rs b/crates/libsy/tests/prompts.rs new file mode 100644 index 00000000..b936d361 --- /dev/null +++ b/crates/libsy/tests/prompts.rs @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for the request-text helpers on a running cascade. +//! +//! The unit tests cover placement in isolation. These drive whole turns through +//! a plain [`FallThrough`] — no stage routing — to pin the two things only a +//! running composition can show: that the text survives to the model call, and +//! that it follows the target the cascade settled on rather than the classifier +//! that named it. + +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::Mutex; + +use switchyard_libsy::algorithms::{ + append_note, DefaultTarget, FallThrough, SystemPromptProcessor, TargetPrompts, +}; +use switchyard_libsy::{ + Algorithm, Classification, Classifier, Context, Decision, Driver, Event, LlmResponse, + LlmTarget, LlmTargetSet, Processor, Request, Response, Result, RoutedLlmClient, +}; +use switchyard_protocol::{text_request, text_response}; + +const CAPABLE_PROMPT: &str = "diagnose before you edit"; +const EFFICIENT_PROMPT: &str = "follow the settled plan"; +const NOTE: &str = "the previous model was stalling"; + +/// One model call as the client saw it. +#[derive(Clone, Debug, Default)] +struct Call { + target: String, + messages: Vec, + instructions: Vec, +} + +/// Records what the model was handed, so a test asserts on that rather than on +/// composition internals. +#[derive(Default)] +struct RecordingClient(Mutex>); + +#[async_trait] +impl RoutedLlmClient for RecordingClient { + async fn call( + &self, + _ctx: Context, + request: Request, + decision: Arc, + ) -> std::result::Result { + *self.0.lock() = Some(Call { + target: decision.selected_model().to_string(), + messages: request + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("|")) + .collect(), + instructions: request + .llm_request + .instructions + .iter() + .filter_map(|block| block.content.iter().find_map(text_of)) + .collect(), + }); + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, decision.selected_model())), + metadata: None, + }) + } +} + +fn text_of(block: &switchyard_protocol::ContentBlock) -> Option { + match block { + switchyard_protocol::ContentBlock::Text { text } => Some(text.clone()), + _ => None, + } +} + +/// A classifier that never decides, so the next one in the cascade gets the turn. +struct Abstains; + +#[async_trait] +impl Classifier for Abstains { + async fn score( + &self, + _state: &mut (), + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result { + Ok(Classification::Ambiguous(Vec::new())) + } +} + +/// Appends a note to every outbound request, the way a router would on a turn +/// it wants to explain. +struct Noting; + +#[async_trait] +impl Processor for Noting { + async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> { + if let Event::ModelRequest { request, .. } = event { + append_note(request, NOTE); + } + Ok(()) + } +} + +fn targets(client: &Arc, names: &[&str]) -> LlmTargetSet { + LlmTargetSet::new( + names + .iter() + .map(|name| LlmTarget { + semantic_name: (*name).to_string(), + llm_client: Some(client.clone() as Arc), + }) + .collect(), + ) +} + +fn prompts() -> TargetPrompts { + TargetPrompts::default() + .with("capable", CAPABLE_PROMPT) + .with("efficient", EFFICIENT_PROMPT) +} + +/// Runs one turn on a cascade that always routes to `target`. +async fn routed_to(client: &Arc, router: FallThrough) -> Result { + Arc::new(router) + .run( + Context::default(), + Request { + llm_request: text_request(Some("auto".to_string()), "fix the build"), + raw_request: None, + metadata: None, + }, + ) + .await?; + let call = client.0.lock().take(); + match call { + Some(call) => Ok(call), + None => panic!("the model was never called"), + } +} + +/// A cascade that routes to `target` and applies `prompts`. +fn router(client: &Arc, target: &str, prompts: TargetPrompts) -> FallThrough { + FallThrough::new(targets(client, &["capable", "efficient"])) + .with_processor(Arc::new(SystemPromptProcessor::new(prompts))) + .with_classifier(Arc::new(DefaultTarget::new(target))) +} + +#[tokio::test] +async fn each_target_gets_its_own_prompt() -> Result<()> { + for (target, expected) in [("capable", CAPABLE_PROMPT), ("efficient", EFFICIENT_PROMPT)] { + let client = Arc::new(RecordingClient::default()); + let call = routed_to(&client, router(&client, target, prompts())).await?; + assert_eq!(call.target, target); + assert_eq!(call.instructions, vec![expected.to_string()]); + } + Ok(()) +} + +#[tokio::test] +async fn a_target_with_no_prompt_is_left_untouched() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let only_capable = TargetPrompts::default().with("capable", CAPABLE_PROMPT); + + let call = routed_to(&client, router(&client, "efficient", only_capable)).await?; + + assert!( + call.instructions.is_empty(), + "one target's prompt must not leak onto another: {:?}", + call.instructions + ); + Ok(()) +} + +#[tokio::test] +async fn the_prompt_follows_the_target_whichever_classifier_picked_it() -> Result<()> { + // The first classifier abstains, so the second decides — the prompt still + // has to follow the target the cascade settled on. + let client = Arc::new(RecordingClient::default()); + let router = FallThrough::new(targets(&client, &["capable", "efficient"])) + .with_processor(Arc::new(SystemPromptProcessor::new(prompts()))) + .with_classifier(Arc::new(Abstains)) + .with_classifier(Arc::new(DefaultTarget::new("capable"))); + + let call = routed_to(&client, router).await?; + + assert_eq!(call.target, "capable"); + assert_eq!(call.instructions, vec![CAPABLE_PROMPT.to_string()]); + Ok(()) +} + +#[tokio::test] +async fn a_note_reaches_the_model_in_the_conversation() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = FallThrough::new(targets(&client, &["capable", "efficient"])) + .with_processor(Arc::new(Noting)) + .with_classifier(Arc::new(DefaultTarget::new("capable"))); + + let call = routed_to(&client, router).await?; + + // Joined onto the trailing user turn rather than opening one of its own. + assert_eq!(call.messages, vec![format!("fix the build|{NOTE}")]); + assert!(call.instructions.is_empty(), "a note is not an instruction"); + Ok(()) +} diff --git a/crates/libsy/tests/stage_router.rs b/crates/libsy/tests/stage_router.rs new file mode 100644 index 00000000..d90c09ba --- /dev/null +++ b/crates/libsy/tests/stage_router.rs @@ -0,0 +1,347 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for a running [`StageRouter`]. +//! +//! The unit tests cover scoring, note selection and prompt placement in +//! isolation. These drive whole turns through the public API to pin what only +//! the assembled cascade can show: which tier served the turn, what the *model* +//! was handed, and which step of the cascade decided. +//! +//! The per-target prompts have their own integration tests in `prompts.rs`, +//! since they are not stage-router-specific. + +use std::sync::Arc; + +use async_trait::async_trait; +use parking_lot::Mutex; +use serde_json::json; + +use switchyard_libsy::algorithms::{StageRouter, TaskClassifierConfig}; +use switchyard_libsy::stage_router::{ + HandoffNoteConfig, LlmFallback, PickerMode, StageRouterConfig, +}; +use switchyard_libsy::{ + Algorithm, Context, Decision, LlmResponse, LlmTarget, Metadata, Request, Response, Result, + RoutedLlmClient, +}; +use switchyard_protocol::{ + text_response, ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat, +}; + +const ESCALATION: &str = "the previous model was stalling; pick up the diagnosis"; +/// Semantic name of the judge target. It is called through its own target and is +/// never a routing destination. +const JUDGE: &str = "judge"; + +/// One model call as the client saw it. +#[derive(Clone, Debug)] +struct Call { + /// Target the call was routed to. + target: String, + /// Text of each message, so a test can assert on the note. + messages: Vec, +} + +/// A client that records what each target was handed, so a test can assert on +/// what reached the model rather than on router internals. +/// +/// It also plays the judge: a call routed to the judge target answers with a +/// verdict, which is how the fallback classifier gets an answer without a real +/// model. +#[derive(Default)] +struct RecordingClient { + calls: Mutex>, + /// `p_solve` the judge reports. High keeps the turn on the weak tier. + judge_p_solve: Mutex, +} + +impl RecordingClient { + /// The calls that routed to a tier, dropping the judge's own. + fn routed(&self) -> Vec { + self.calls + .lock() + .iter() + .filter(|call| call.target != JUDGE) + .cloned() + .collect() + } +} + +#[async_trait] +impl RoutedLlmClient for RecordingClient { + async fn call( + &self, + _ctx: Context, + request: Request, + decision: Arc, + ) -> std::result::Result { + let target = decision.selected_model().to_string(); + self.calls.lock().push(Call { + target: target.clone(), + messages: request + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("|")) + .collect(), + }); + let completion = if target == JUDGE { + let p_solve = *self.judge_p_solve.lock(); + format!( + r#"{{"recommended_route":"efficient","p_solve":{p_solve},"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}}"# + ) + } else { + target + }; + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, completion)), + metadata: None, + }) + } +} + +fn target(client: &Arc, name: &str) -> LlmTarget { + LlmTarget { + semantic_name: name.to_string(), + llm_client: Some(client.clone() as Arc), + } +} + +fn router(client: Arc, config: StageRouterConfig) -> Result> { + // Only the two tiers are routing destinations; the judge has its own target. + Ok(Arc::new(StageRouter::new( + target(&client, "strong"), + target(&client, "weak"), + config, + )?)) +} + +/// The signal-only configuration these tests start from. +fn config() -> StageRouterConfig { + StageRouterConfig::new(PickerMode::EfficientFirst, 0.5) +} + +/// That configuration plus handoff notes. +fn config_with_notes() -> StageRouterConfig { + let mut config = config(); + config.handoff_notes = Some(HandoffNoteConfig::new(ESCALATION, None, true)); + config +} + +/// One turn of a coding-agent conversation, as the wire delivers it: an +/// assistant tool call answered by a tool result the signal extractor reads. +/// +/// `failed` makes the tool result a critical error, the hard override that +/// escalates a turn on the signal alone. +fn turn_request(failed: bool) -> Request { + let tool_call = json!({ + "role": "assistant", + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"}, + }], + }); + let content = if failed { + "fatal runtime error: out of memory" + } else { + "ok" + }; + let raw_request = json!({ + "model": "auto", + "messages": [ + {"role": "user", "content": "fix the build"}, + tool_call, + {"role": "tool", "tool_call_id": "call_1", "content": content}, + ], + }); + // The neutral IR the router forwards, alongside the raw body the signal + // extractor parses. An OpenAI tool result is its own `tool` turn, so a note + // is appended as a fresh user message rather than folded into it. + let messages = vec![ + Message::text(Role::User, "fix the build"), + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: "call_1".to_string(), + name: "Bash".to_string(), + arguments: json!({"command": "cargo test"}), + })], + }, + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call_1".to_string(), + content: vec![ContentBlock::Text { + text: content.to_string(), + }], + is_error: Some(failed), + })], + }, + ]; + Request { + llm_request: LlmRequest { + model: Some("auto".to_string()), + messages, + ..LlmRequest::default() + }, + raw_request: Some(raw_request), + metadata: Some(Metadata { + wire_format: Some(WireFormat::OpenAiChat), + // One session, so turns of the same test share the router's state. + session_id: Some("session-1".to_string()), + ..Default::default() + }), + } +} + +#[tokio::test] +async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = router(client.clone(), config_with_notes())?; + let ctx = Context::default(); + + // Turn 1 — a clean tool result is under threshold and falls open to weak. + router.clone().run(ctx.clone(), turn_request(false)).await?; + // Turn 2 — a critical tool error escalates on the signals alone. + router.run(ctx, turn_request(true)).await?; + + let calls = client.routed(); + assert_eq!(calls[0].target, "weak"); + assert_eq!(calls[1].target, "strong"); + assert!( + !calls[0] + .messages + .iter() + .any(|text| text.contains(ESCALATION)), + "the steady-state turn should carry no note: {:?}", + calls[0].messages + ); + // The note rides on the trailing turn, after the tool result. + assert!( + calls[1] + .messages + .last() + .is_some_and(|text| text.ends_with(ESCALATION)), + "the escalating turn should carry the note last: {:?}", + calls[1].messages + ); + Ok(()) +} + +/// That configuration plus the capability judge behind the signals. +fn config_with_judge(client: &Arc, p_solve: f64) -> StageRouterConfig { + *client.judge_p_solve.lock() = p_solve; + let mut config = config(); + config.llm_fallback = Some(LlmFallback { + judge_target: target(client, JUDGE), + config: TaskClassifierConfig { + base_threshold: 0.5, + // The same span the signal scorer reads. + recent_turn_window: Some(3), + ..Default::default() + }, + }); + config +} + +#[tokio::test] +async fn the_judge_decides_a_turn_the_signals_leave_undecided() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + // A low p_solve means the judge does not trust the weak tier with the task. + let router = router(client.clone(), config_with_judge(&client, 0.1))?; + + // A clean turn is under threshold, so without a judge it would fall open to + // the configured weak default. The judge overrides that. + router.run(Context::default(), turn_request(false)).await?; + + let judged = client.calls.lock().iter().any(|call| call.target == JUDGE); + assert!(judged, "the judge should be consulted on an undecided turn"); + assert_eq!(client.routed()[0].target, "strong"); + Ok(()) +} + +#[tokio::test] +async fn a_decisive_signal_never_reaches_the_judge() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + // The judge would say weak; the signals must win before it is asked at all. + let router = router(client.clone(), config_with_judge(&client, 0.9))?; + + router.run(Context::default(), turn_request(true)).await?; + + assert!( + !client.calls.lock().iter().any(|call| call.target == JUDGE), + "a resolved turn should not pay for a judge call" + ); + assert_eq!(client.routed()[0].target, "strong"); + Ok(()) +} + +#[tokio::test] +async fn the_judges_verdict_is_not_pinned_to_the_session() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = router(client.clone(), config_with_judge(&client, 0.1))?; + let ctx = Context::default(); + + // First undecided turn: the judge sends it to the strong tier. + router.clone().run(ctx.clone(), turn_request(false)).await?; + // The judge changes its mind; a second undecided turn asks again rather than + // replaying the first verdict. + *client.judge_p_solve.lock() = 0.9; + router.run(ctx, turn_request(false)).await?; + + let routed = client.routed(); + assert_eq!(routed[0].target, "strong"); + assert_eq!(routed[1].target, "weak"); + assert_eq!( + client + .calls + .lock() + .iter() + .filter(|call| call.target == JUDGE) + .count(), + 2, + "each undecided turn is its own question" + ); + Ok(()) +} + +#[tokio::test] +async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + // An out-of-range p_solve is an unusable verdict, so the judge abstains. + let router = router(client.clone(), config_with_judge(&client, 42.0))?; + + router.run(Context::default(), turn_request(false)).await?; + + // efficient_first, so the turn falls open to weak rather than being pushed + // to strong by the judge's own fallback. + assert_eq!(client.routed()[0].target, "weak"); + Ok(()) +} + +#[tokio::test] +async fn the_judge_reads_the_window_it_was_configured_with() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + // An undecided turn, so the judge is consulted and we can see what it got. + let router = router(client.clone(), config_with_judge(&client, 0.9))?; + + router.run(Context::default(), turn_request(false)).await?; + + // With a window the judge sees the opening task, not just the newest turn. + let judged = client + .calls + .lock() + .iter() + .find(|call| call.target == JUDGE) + .map(|call| call.messages.join("|")); + let Some(judged) = judged else { + panic!("the judge was never called"); + }; + assert!( + judged.contains("fix the build"), + "the judge should see the opening task: {judged}" + ); + Ok(()) +} diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index ad819d5d..c15ab2ca 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -23,13 +23,18 @@ fn to_protocol_request(request: &ChatRequest) -> Request { ChatRequestType::Anthropic => WireFormat::AnthropicMessages, ChatRequestType::OpenAiResponses => WireFormat::OpenAiResponses, }; + // The signals are read off the decoded conversation, so decode here rather + // than handing over a raw body the extractor cannot interpret. A body that + // fails to decode yields no signals, which is what an absent body did before. + let llm_request = + switchyard_translation::decode_request(wire_format, request.body()).unwrap_or_default(); Request { + llm_request, raw_request: Some(request.body().clone()), metadata: Some(Metadata { wire_format: Some(wire_format), ..Default::default() }), - ..Default::default() } } diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 0f4557ea..d389ae2f 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -8,7 +8,11 @@ use std::fs; use std::path::Path; use std::sync::Arc; -use libsy::algorithms::{LlmTaskClassifier, Noop, Passthrough, Random, TaskClassifierConfig}; +use libsy::algorithms::{ + LlmTaskClassifier, Noop, Passthrough, Random, StageRouter, StageRouterConfig, TargetPrompts, + TaskClassifierConfig, +}; +use libsy::stage_router::{HandoffNoteConfig, LlmFallback, PickerMode}; use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient}; use serde::Deserialize; use serde_json::Value; @@ -191,15 +195,48 @@ enum RouteConfig { #[serde(flatten)] classifier_config: TaskClassifierConfig, }, + StageRouter { + id: String, + capable_target: String, + efficient_target: String, + /// Tier a turn falls back to when the signals are not confident. + picker: PickerMode, + confidence_threshold: f64, + /// Trailing tool results the signals are computed over. + #[serde(default)] + recent_turn_window: Option, + /// Note handed to the model a signal-driven switch routes to. + #[serde(default)] + handoff_notes: Option, + /// System prompt handed to each tier on every turn it serves. + #[serde(default)] + capable_system_prompt: Option, + #[serde(default)] + efficient_system_prompt: Option, + /// Capability judge consulted on turns the signals leave undecided. + #[serde(default)] + classifier: Option, + }, +} + +/// The judge a `stage_router` route falls through to, and how it routes. +#[derive(Debug, Deserialize)] +struct StageClassifierConfig { + /// Target the judge is called through. Not a routing destination. + target: String, + #[serde(flatten)] + config: TaskClassifierConfig, } impl RouteConfig { fn id(&self) -> &str { use RouteConfig::*; match self { - Noop { id } | Random { id, .. } | LlmClassifier { id, .. } | Passthrough { id, .. } => { - id - } + Noop { id } + | Random { id, .. } + | LlmClassifier { id, .. } + | Passthrough { id, .. } + | StageRouter { id, .. } => id, } } } @@ -301,7 +338,65 @@ fn build_algorithm( })?; Ok(Arc::new(algorithm)) } + RouteConfig::StageRouter { + capable_target, + efficient_target, + picker, + confidence_threshold, + recent_turn_window, + handoff_notes, + capable_system_prompt, + efficient_system_prompt, + classifier, + .. + } => { + let capable = resolve_target(route_name, capable_target, targets)?; + let efficient = resolve_target(route_name, efficient_target, targets)?; + let mut config = StageRouterConfig::new(*picker, *confidence_threshold); + config.recent_window = *recent_turn_window; + config.handoff_notes = handoff_notes.clone(); + config.tier_prompts = tier_prompts( + &capable.semantic_name, + capable_system_prompt.as_deref(), + &efficient.semantic_name, + efficient_system_prompt.as_deref(), + ); + // The judge is called through its own target, so it is not a routing + // destination and stays out of the tier pair. + config.llm_fallback = classifier + .as_ref() + .map(|classifier| { + resolve_target(route_name, &classifier.target, targets).map(|judge_target| { + LlmFallback { + judge_target, + config: classifier.config.clone(), + } + }) + }) + .transpose()?; + let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { + ServerError::new(format!("stage_router route {route_name}: {error}")) + })?; + Ok(Arc::new(algorithm)) + } + } +} + +/// Keys each configured system prompt by the target it belongs to. +fn tier_prompts( + capable: &str, + capable_prompt: Option<&str>, + efficient: &str, + efficient_prompt: Option<&str>, +) -> TargetPrompts { + let mut prompts = TargetPrompts::default(); + if let Some(prompt) = capable_prompt { + prompts = prompts.with(capable, prompt); + } + if let Some(prompt) = efficient_prompt { + prompts = prompts.with(efficient, prompt); } + prompts } fn resolve_targets<'a>( diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index b173c46d..c08834c2 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -18,7 +18,7 @@ use axum::routing::post; use axum::{Json, Router}; use http_body_util::BodyExt; use libsy::algorithms::{FallThrough, Random}; -use libsy::stage_router::{PickerMode, StageClassifier}; +use libsy::stage_router::{PickerMode, StageClassifier, StageTargets}; use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State as AlgorithmState}; use serde_json::{json, Value}; use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; @@ -405,6 +405,72 @@ fn assert_in_order(haystack: &str, needles: &[&str]) { } } +/// A critical tool error must reach the stage router's signal scorer, which reads +/// the decoded conversation. The endpoint records no inbound wire format, so a +/// scorer that parsed the raw body instead would find nothing and route every turn +/// as if the conversation had no signals at all. +#[tokio::test] +async fn stage_route_escalates_on_a_signal_in_the_conversation() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 0.5 +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [ + {"role": "user", "content": "fix the build"}, + {"role": "assistant", "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"} + }]}, + {"role": "tool", "tool_call_id": "call_1", "content": "fatal runtime error: out of memory"}, + ] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/strong"), + "a critical error should escalate on the signals alone" + ); + Ok(()) +} + #[tokio::test] async fn toml_config_constructs_and_serves_multiple_algorithms() -> TestResult { let upstream = MockUpstream::start().await?; @@ -445,6 +511,24 @@ base_threshold = 0.5 id = "switchyard/passthrough" type = "passthrough" target = "weak" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 0.5 +recent_turn_window = 3 +capable_system_prompt = "diagnose before you edit" +efficient_system_prompt = "follow the settled plan" + +[routes.stage.handoff_notes] +escalation_note = "the previous model was stalling" + +[routes.stage.classifier] +target = "classifier" +base_threshold = 0.5 "#, base_url = upstream.base_url ))?; @@ -604,8 +688,9 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult = Arc::new( - FallThrough::::new_with_state(targets) - .with_classifier(Arc::new(StageClassifier::new(mode, 0.5))), + FallThrough::::new_with_state(targets).with_classifier(Arc::new( + StageClassifier::new(StageTargets::new("strong", "weak"), mode, 0.5), + )), ); Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?) } From 7f262c03560d0c4d88b7e54a493147695f631840 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 16:08:12 -0700 Subject: [PATCH 2/8] refactor(libsy): collapse post-routing replay to a single Decision pass Event::Decision now carries a mutable request so processors can both bind state and rewrite the outbound request (e.g. tier system prompts) in one pass. Removes Event::ModelRequest. Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/fall_through.rs | 21 +++------- crates/libsy/src/algorithms/util/affinity.rs | 12 +++--- crates/libsy/src/algorithms/util/prompts.rs | 10 ++--- crates/libsy/src/core/processor.rs | 40 ++++++-------------- crates/libsy/tests/prompts.rs | 2 +- 5 files changed, 29 insertions(+), 56 deletions(-) diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index 48eebc4e..3a76c244 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -314,11 +314,8 @@ where }); driver.info(ctx.clone(), decision.clone()).await?; - // 4. Post-decision replay, in two passes over the same chain. Every processor sees - // the decision first, so stateful ones bind it; only then is the outbound - // request offered for rewriting, paired with the decision that routed it. The - // order matters: a processor rewriting the request in the second pass sees the - // state every processor settled in the first. + // 4. Post-decision replay: every processor sees the decision so stateful ones + // can bind it, and may rewrite the outbound request (e.g. add a tier prompt). for processor in &self.processors { let event = Event::Decision { request, @@ -326,13 +323,6 @@ where }; processor.process(state, event).await?; } - for processor in &self.processors { - let event = Event::ModelRequest { - request, - decision: decision.as_ref(), - }; - processor.process(state, event).await?; - } Ok((target, decision)) } @@ -742,11 +732,11 @@ mod tests { } #[tokio::test] - async fn processor_observes_request_decision_then_model_request() -> Result<()> { + async fn processor_observes_request_then_decision() -> Result<()> { use parking_lot::Mutex; // Records which event kinds it saw, proving the replay order: the inbound - // request, then the decision, then the request on its way to the model. + // request, then the routing decision (which carries the request to the model). struct RecordingProcessor(Arc>>); #[async_trait] @@ -755,7 +745,6 @@ mod tests { let kind = match event { Event::Request(_) => "request", Event::Decision { .. } => "decision", - Event::ModelRequest { .. } => "model_request", _ => "other", }; self.0.lock().push(kind); @@ -769,7 +758,7 @@ mod tests { .with_classifier(fixed(vec![score("strong", 1.0)])); run(router).await?; - assert_eq!(*seen.lock(), vec!["request", "decision", "model_request"]); + assert_eq!(*seen.lock(), vec!["request", "decision"]); Ok(()) } diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index c71b9fb0..ee6fdc22 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -554,12 +554,12 @@ mod tests { let classifier: Arc = router; let mut state = (); - let first = request(session("session-1", "agent-a")); + let mut first = request(session("session-1", "agent-a")); processor .process( &mut state, Event::Decision { - request: &first, + request: &mut first, decision: &FixedDecision("model-a"), }, ) @@ -578,13 +578,13 @@ mod tests { async fn decision_without_an_affinity_identity_is_ignored() -> Result<(), BoxErr> { let router = AffinityRouter::new(); let mut state = (); - let unkeyed = request(Metadata::default()); + let mut unkeyed = request(Metadata::default()); router .process( &mut state, Event::Decision { - request: &unkeyed, + request: &mut unkeyed, decision: &FixedDecision("model-a"), }, ) @@ -608,7 +608,7 @@ mod tests { .process( &mut state, Event::Decision { - request: &second, + request: &mut second, decision: &FixedDecision("model-b"), }, ) @@ -617,7 +617,7 @@ mod tests { .process( &mut state, Event::Decision { - request: &first, + request: &mut first, decision: &FixedDecision("model-a"), }, ) diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs index 4096d0e4..deb1c1b9 100644 --- a/crates/libsy/src/algorithms/util/prompts.rs +++ b/crates/libsy/src/algorithms/util/prompts.rs @@ -83,10 +83,10 @@ impl SystemPromptProcessor { #[async_trait] impl Processor for SystemPromptProcessor { async fn process(&self, _state: &mut S, event: Event<'_>) -> Result<()> { - // The outbound hook carries the decision that routed the request, so the - // target is read straight off it — whichever classifier picked it, and + // The decision event carries both the routing outcome and the outbound request, + // so the target is read straight off it — whichever classifier picked it, and // with nothing kept between turns. - let Event::ModelRequest { request, decision } = event else { + let Event::Decision { request, decision } = event else { return Ok(()); }; let Some(prompt) = self.prompts.get(decision.selected_model()) else { @@ -224,7 +224,7 @@ mod tests { processor .process( &mut (), - Event::ModelRequest { + Event::Decision { request: &mut request, decision: &RoutedTo(target), }, @@ -278,7 +278,7 @@ mod tests { processor .process( &mut (), - Event::ModelRequest { + Event::Decision { request: &mut request, decision: &RoutedTo("strong"), }, diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs index 0998f5a2..9ac4e567 100644 --- a/crates/libsy/src/core/processor.rs +++ b/crates/libsy/src/core/processor.rs @@ -7,26 +7,22 @@ use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals}; /// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate state. /// -/// The two request-bearing variants borrow the request mutably, so a processor may rewrite -/// it in place and pass the rewritten request down the chain (see [`Processor::process`]). -/// The observation-only variants stay immutable. +/// Request-bearing variants ([`Event::Request`], [`Event::Decision`]) borrow the request +/// mutably, so a processor may rewrite it in place and the edit propagates to the rest of +/// the chain and to the model call. The observation-only variants stay immutable. pub enum Event<'a> { /// The inbound request that begins a turn. Request(&'a mut Request), /// An out-of-band agentic-stack signal (tool results, budget updates, …). Signal(&'a Signals), /// A routing decision paired with the request that produced it. + /// + /// The request is rewritable: a processor may add instructions or notes here that + /// are bound to the routing outcome (e.g. a tier-specific system prompt). Decision { - /// The request classified by the algorithm. - request: &'a Request, - /// The routing decision produced for `request`. - decision: &'a dyn Decision, - }, - /// A request about to be sent to a model, with the decision that routed it. - ModelRequest { - /// The outbound request, rewritable in place. + /// The request, rewritable in place. request: &'a mut Request, - /// Where the request is headed. + /// The routing decision produced for `request`. decision: &'a dyn Decision, }, /// A buffered response received back from a model. @@ -38,9 +34,8 @@ pub enum Event<'a> { pub trait Processor: Send + Sync { /// Process an event, accumulating facts into `state`. /// - /// A request-bearing event ([`Event::Request`], [`Event::ModelRequest`]) may also be - /// rewritten in place; the edit propagates to the rest of the chain and to the model - /// call. Most processors only read it. + /// Process an event, accumulating facts into `state`. Request-bearing events + /// ([`Event::Request`], [`Event::Decision`]) may also be rewritten in place. async fn process(&self, state: &mut S, event: Event<'_>) -> Result<()>; } @@ -58,7 +53,6 @@ mod tests { Event::Request(_) => "requests", Event::Signal(_) => "signals", Event::Decision { .. } => "decisions", - Event::ModelRequest { .. } => "model_requests", Event::ModelResponse(_) => "model_responses", } } @@ -115,15 +109,6 @@ mod tests { processor .process(&mut state, Event::Request(&mut req)) .await?; - processor - .process( - &mut state, - Event::ModelRequest { - request: &mut req, - decision: &decision, - }, - ) - .await?; processor .process(&mut state, Event::ModelResponse(&response)) .await?; @@ -131,7 +116,7 @@ mod tests { .process( &mut state, Event::Decision { - request: &req, + request: &mut req, decision: &decision, }, ) @@ -143,7 +128,6 @@ mod tests { assert_eq!(count(&state, "requests"), 1); assert_eq!(count(&state, "signals"), 1); assert_eq!(count(&state, "decisions"), 1); - assert_eq!(count(&state, "model_requests"), 1); assert_eq!(count(&state, "model_responses"), 1); Ok(()) } @@ -171,7 +155,7 @@ mod tests { impl Processor for RewritingProcessor { async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> { match event { - Event::Request(request) | Event::ModelRequest { request, .. } => { + Event::Request(request) | Event::Decision { request, .. } => { request.llm_request.model = Some("rewritten".to_string()); } _ => {} diff --git a/crates/libsy/tests/prompts.rs b/crates/libsy/tests/prompts.rs index b936d361..4d28e705 100644 --- a/crates/libsy/tests/prompts.rs +++ b/crates/libsy/tests/prompts.rs @@ -99,7 +99,7 @@ struct Noting; #[async_trait] impl Processor for Noting { async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> { - if let Event::ModelRequest { request, .. } = event { + if let Event::Decision { request, .. } = event { append_note(request, NOTE); } Ok(()) From 18c3e33da5d34ce3b4462d3d973f0f071ec325b4 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 16:19:41 -0700 Subject: [PATCH 3/8] refactor(libsy): ambiguous classification returns empty scores Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/llm_class.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index e2cdeaed..82cc8fa6 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -176,15 +176,11 @@ impl JudgePolicy for TaskClassifierPolicy { // Judge output is untrusted, so only a complete, valid, non-abstained verdict that // clears the configured confidence decides. Anything else is "I could not tell" — // reported as ambiguous so the composition around this classifier chooses the - // fallback, rather than this policy silently imposing one. The capable target rides - // along as the safe suggestion for a caller that reads ambiguous scores. + // fallback, rather than this policy silently imposing one. let Some(verdict) = verdict .filter(|v| v.is_valid() && !v.abstain && v.confidence >= self.config.min_confidence) else { - return Classification::Ambiguous(vec![Score { - target: self.capable_target.clone(), - confidence: 0.0, - }]); + return Classification::Ambiguous(vec![]); }; // A usable verdict below the capability threshold is still a decision: the judge // does not trust the efficient tier with this task. @@ -775,11 +771,7 @@ mod tests { let classification = policy.to_classification(verdict.as_ref()); assert!(matches!(classification, Classification::Ambiguous(_))); assert!(classification.argmax(false)?.is_none()); - // The capable tier rides along for a caller that reads ambiguous scores. - assert_eq!( - classification.argmax(true)?.map(|score| score.target), - Some("capable".to_string()) - ); + assert!(classification.argmax(true)?.is_none()); } Ok(()) } From ebbb1076cf119563ed6df4818e9b26eb81744168 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 18:43:58 -0700 Subject: [PATCH 4/8] refactor(libsy): address review comments - Remove switchyard-translation dev dep; rewrite tool_signals tests to construct LlmRequest directly - Move stage_router integration tests into stage.rs #[cfg(test)] - Trim paragraph comments in fall_through.rs Signed-off-by: Sabhatina Selvam --- Cargo.lock | 1 - crates/libsy/Cargo.toml | 3 - crates/libsy/src/algorithms/stage.rs | 275 +++++++++++++- .../libsy/src/algorithms/util/tool_signals.rs | 295 ++++++--------- crates/libsy/tests/stage_router.rs | 347 ------------------ 5 files changed, 387 insertions(+), 534 deletions(-) delete mode 100644 crates/libsy/tests/stage_router.rs diff --git a/Cargo.lock b/Cargo.lock index bb508564..c15c0af6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1736,7 +1736,6 @@ dependencies = [ "serde_json", "switchyard-llm-client", "switchyard-protocol", - "switchyard-translation", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml index 4769723d..c9e58e18 100644 --- a/crates/libsy/Cargo.toml +++ b/crates/libsy/Cargo.toml @@ -31,8 +31,5 @@ tracing.workspace = true # SDK + in-memory exporter to assert what the observability layer records. opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing"] } switchyard-llm-client.workspace = true -# Decodes the wire-format fixtures the tool-signal tests are written against, so -# they keep asserting on bodies a real client would send. -switchyard-translation.workspace = true tokio.workspace = true tracing-subscriber = "0.3" diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index ac0652b1..48dee8e9 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -222,8 +222,20 @@ fn build_route( #[cfg(test)] mod tests { + use std::sync::Arc; + + use async_trait::async_trait; + use parking_lot::Mutex; + use serde_json::json; + use switchyard_protocol::{ + text_response, ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat, + }; + use super::*; - use crate::{Algorithm, LlmTarget, StateValue}; + use crate::{ + Algorithm, Context, Decision, LlmResponse, LlmTarget, Metadata, Response, RoutedLlmClient, + StateValue, + }; fn tier_target(name: &str) -> LlmTarget { LlmTarget { @@ -337,4 +349,265 @@ mod tests { assert_eq!(router.name(), STAGE_ROUTER); Ok(()) } + + // ── routing integration tests ──────────────────────────────────────────── + + const ESCALATION: &str = "the previous model was stalling; pick up the diagnosis"; + const JUDGE: &str = "judge"; + + #[derive(Clone, Debug)] + struct Call { + target: String, + messages: Vec, + } + + /// Records what each target receives. When called as the judge target it + /// replies with a structured verdict so the fallback classifier gets an answer + /// without a real model. + #[derive(Default)] + struct RecordingClient { + calls: Mutex>, + judge_p_solve: Mutex, + } + + impl RecordingClient { + fn routed(&self) -> Vec { + self.calls + .lock() + .iter() + .filter(|call| call.target != JUDGE) + .cloned() + .collect() + } + } + + #[async_trait] + impl RoutedLlmClient for RecordingClient { + async fn call( + &self, + _ctx: Context, + request: Request, + decision: Arc, + ) -> std::result::Result { + let target = decision.selected_model().to_string(); + self.calls.lock().push(Call { + target: target.clone(), + messages: request + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("|")) + .collect(), + }); + let completion = if target == JUDGE { + let p_solve = *self.judge_p_solve.lock(); + format!( + r#"{{"recommended_route":"efficient","p_solve":{p_solve},"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}}"# + ) + } else { + target + }; + Ok(Response { + llm_response: LlmResponse::Agg(text_response(None, completion)), + metadata: None, + }) + } + } + + fn recording_target(client: &Arc, name: &str) -> LlmTarget { + LlmTarget { + semantic_name: name.to_string(), + llm_client: Some(client.clone() as Arc), + } + } + + fn recording_router( + client: Arc, + config: StageRouterConfig, + ) -> Result> { + Ok(Arc::new(StageRouter::new( + recording_target(&client, "strong"), + recording_target(&client, "weak"), + config, + )?)) + } + + fn config_with_notes() -> StageRouterConfig { + let mut c = config(); + c.handoff_notes = Some(HandoffNoteConfig::new(ESCALATION, None, true)); + c + } + + fn config_with_judge(client: &Arc, p_solve: f64) -> StageRouterConfig { + *client.judge_p_solve.lock() = p_solve; + let mut c = config(); + c.llm_fallback = Some(LlmFallback { + judge_target: recording_target(client, JUDGE), + config: TaskClassifierConfig { + base_threshold: 0.5, + recent_turn_window: Some(3), + ..Default::default() + }, + }); + c + } + + fn turn_request(failed: bool) -> Request { + let content = if failed { + "fatal runtime error: out of memory" + } else { + "ok" + }; + Request { + llm_request: LlmRequest { + model: Some("auto".to_string()), + messages: vec![ + Message::text(Role::User, "fix the build"), + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: "call_1".to_string(), + name: "Bash".to_string(), + arguments: json!({"command": "cargo test"}), + })], + }, + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call_1".to_string(), + content: vec![ContentBlock::Text { + text: content.to_string(), + }], + is_error: Some(failed), + })], + }, + ], + ..LlmRequest::default() + }, + raw_request: Some(json!({ + "model": "auto", + "messages": [ + {"role": "user", "content": "fix the build"}, + {"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", + "function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"}}]}, + {"role": "tool", "tool_call_id": "call_1", "content": content}, + ], + })), + metadata: Some(Metadata { + wire_format: Some(WireFormat::OpenAiChat), + session_id: Some("session-1".to_string()), + ..Default::default() + }), + } + } + + #[tokio::test] + async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = recording_router(client.clone(), config_with_notes())?; + let ctx = Context::default(); + + router.clone().run(ctx.clone(), turn_request(false)).await?; + router.run(ctx, turn_request(true)).await?; + + let calls = client.routed(); + assert_eq!(calls[0].target, "weak"); + assert_eq!(calls[1].target, "strong"); + assert!( + !calls[0].messages.iter().any(|t| t.contains(ESCALATION)), + "steady-state turn should carry no note: {:?}", + calls[0].messages + ); + assert!( + calls[1].messages.last().is_some_and(|t| t.ends_with(ESCALATION)), + "escalating turn should carry the note last: {:?}", + calls[1].messages + ); + Ok(()) + } + + #[tokio::test] + async fn the_judge_decides_a_turn_the_signals_leave_undecided() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = recording_router(client.clone(), config_with_judge(&client, 0.1))?; + + router.run(Context::default(), turn_request(false)).await?; + + assert!( + client.calls.lock().iter().any(|c| c.target == JUDGE), + "the judge should be consulted on an undecided turn" + ); + assert_eq!(client.routed()[0].target, "strong"); + Ok(()) + } + + #[tokio::test] + async fn a_decisive_signal_never_reaches_the_judge() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = recording_router(client.clone(), config_with_judge(&client, 0.9))?; + + router.run(Context::default(), turn_request(true)).await?; + + assert!( + !client.calls.lock().iter().any(|c| c.target == JUDGE), + "a resolved turn should not pay for a judge call" + ); + assert_eq!(client.routed()[0].target, "strong"); + Ok(()) + } + + #[tokio::test] + async fn the_judges_verdict_is_not_pinned_to_the_session() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = recording_router(client.clone(), config_with_judge(&client, 0.1))?; + let ctx = Context::default(); + + router.clone().run(ctx.clone(), turn_request(false)).await?; + *client.judge_p_solve.lock() = 0.9; + router.run(ctx, turn_request(false)).await?; + + let routed = client.routed(); + assert_eq!(routed[0].target, "strong"); + assert_eq!(routed[1].target, "weak"); + assert_eq!( + client.calls.lock().iter().filter(|c| c.target == JUDGE).count(), + 2, + "each undecided turn is its own question" + ); + Ok(()) + } + + #[tokio::test] + async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = recording_router(client.clone(), config_with_judge(&client, 42.0))?; + + router.run(Context::default(), turn_request(false)).await?; + + assert_eq!(client.routed()[0].target, "weak"); + Ok(()) + } + + #[tokio::test] + async fn the_judge_reads_the_window_it_was_configured_with() -> Result<()> { + let client = Arc::new(RecordingClient::default()); + let router = recording_router(client.clone(), config_with_judge(&client, 0.9))?; + + router.run(Context::default(), turn_request(false)).await?; + + let judged = client + .calls + .lock() + .iter() + .find(|c| c.target == JUDGE) + .map(|c| c.messages.join("|")); + let Some(judged) = judged else { + panic!("the judge was never called"); + }; + assert!( + judged.contains("fix the build"), + "the judge should see the opening task: {judged}" + ); + Ok(()) + } } diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 2f784b0d..6ae54950 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -597,33 +597,50 @@ fn has_nonzero_failure_count(lower: &str) -> bool { mod tests { use super::*; use serde_json::json; - use switchyard_protocol::{Metadata, WireFormat}; + use switchyard_protocol::{ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult}; - /// Decodes `body` the way an endpoint would, so these tests assert on the - /// bodies a real client sends rather than on hand-built internal shapes. - fn request_with(wire_format: WireFormat, body: Value) -> Request { - let llm_request = switchyard_translation::decode_request(wire_format, &body) - .unwrap_or_else(|error| panic!("fixture is not valid {wire_format}: {error}")); + fn with_messages(messages: Vec) -> Request { Request { - llm_request, - raw_request: Some(body), - metadata: Some(Metadata { - wire_format: Some(wire_format), - ..Default::default() - }), + llm_request: LlmRequest { messages, ..LlmRequest::default() }, + raw_request: None, + metadata: None, } } - fn openai_chat_request(body: Value) -> Request { - request_with(WireFormat::OpenAiChat, body) + // assistant message with a single named tool call + fn tc(name: &str) -> Message { + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: String::new(), + name: name.to_string(), + arguments: json!({}), + })], + } } - fn anthropic_request(body: Value) -> Request { - request_with(WireFormat::AnthropicMessages, body) + // assistant Bash message carrying `command` + fn bash(command: &str) -> Message { + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: String::new(), + name: "Bash".to_string(), + arguments: json!({"command": command}), + })], + } } - fn openai_responses_request(body: Value) -> Request { - request_with(WireFormat::OpenAiResponses, body) + // a tool result message (goes in a user-role message, as in Anthropic's normalised form) + fn tr(text: &str) -> Message { + Message { + role: Role::User, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: String::new(), + content: vec![ContentBlock::Text { text: text.to_string() }], + is_error: None, + })], + } } #[test] @@ -705,14 +722,11 @@ mod tests { #[test] fn severity_is_windowed_over_recent_results() { // An error two results back, then two clean results. - let request = openai_chat_request(json!({ - "messages": [ - {"role": "tool", "tool_call_id": "1", - "content": "Traceback (most recent call last):\n ValueError"}, - {"role": "tool", "tool_call_id": "2", "content": "ok"}, - {"role": "tool", "tool_call_id": "3", "content": "ok"}, - ] - })); + let request = with_messages(vec![ + tr("Traceback (most recent call last):\n ValueError"), + tr("ok"), + tr("ok"), + ]); // window covers the error → severity persists (max over the window) assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD); // window of 1 sees only the last (clean) result → severity has decayed out @@ -721,13 +735,11 @@ mod tests { #[test] fn extract_openai_chat_tool_results() { - let request = openai_chat_request(json!({ - "messages": [ - {"role": "user", "content": "do something"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, - {"role": "tool", "tool_call_id": "1", "content": "Traceback (most recent call last):\n ValueError"}, - ] - })); + let request = with_messages(vec![ + Message::text(Role::User, "do something"), + tc("Edit"), + tr("Traceback (most recent call last):\n ValueError"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, HARD); assert_eq!(sig.edit_count, 1); @@ -736,27 +748,19 @@ mod tests { #[test] fn extract_anthropic_tool_results() { - let request = anthropic_request(json!({ - "messages": [ - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "1", - "content": "Traceback (most recent call last):\n ValueError"} - ]}, - ] - })); + let request = with_messages(vec![ + tr("Traceback (most recent call last):\n ValueError"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, HARD); } #[test] fn extract_responses_api_tool_results() { - let request = openai_responses_request(json!({ - "input": [ - {"type": "function_call", "name": "Write"}, - {"type": "function_call_output", "call_id": "1", - "output": "file written successfully"}, - ] - })); + let request = with_messages(vec![ + tc("Write"), + tr("file written successfully"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, 0.0); assert_eq!(sig.write_count, 1); @@ -766,22 +770,14 @@ mod tests { fn recent_window_counts_only_last_default_window_tool_calls() { // 5 writes + 1 edit at the end → the default window (3) should see // the last 3 calls: 1 edit + 2 writes (not all 6 calls). - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); + let request = with_messages(vec![ + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Edit"), tr("ok"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.write_count, 5); assert_eq!(sig.edit_count, 1); @@ -794,22 +790,14 @@ mod tests { // Same six tool calls (1 edit at the end, 5 writes before). // With recent_window=3 → recent_writes=2, recent_edits=1. // With recent_window=6 → recent_writes=5, recent_edits=1 (all calls). - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Edit"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); + let request = with_messages(vec![ + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Write"), tr("ok"), + tc("Edit"), tr("ok"), + ]); let narrow = extract_tool_signals_with_window(&request, 3); assert_eq!(narrow.recent_write_count, 2); assert_eq!(narrow.recent_edit_count, 1); @@ -822,39 +810,28 @@ mod tests { #[test] fn compaction_marker_sets_compacted() { // The compaction summary is a user message carrying Claude Code's preamble. - let request = openai_chat_request(json!({ - "messages": [ - {"role": "user", "content": "This session is being continued from a previous conversation that ran out of context."}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, - ] - })); + let request = with_messages(vec![ + Message::text(Role::User, "This session is being continued from a previous conversation that ran out of context."), + bash("ls"), + ]); assert!(ToolSignals::from_request(&request, None).compacted); } #[test] fn no_compaction_marker_stays_uncompacted() { - let request = openai_chat_request(json!({ - "messages": [ - {"role": "user", "content": "Write a script that parses the log file."}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", "arguments": "{\"command\": \"ls\"}"}}]}, - ] - })); + let request = with_messages(vec![ + Message::text(Role::User, "Write a script that parses the log file."), + bash("ls"), + ]); assert!(!ToolSignals::from_request(&request, None).compacted); } #[test] fn bash_heredoc_counts_as_write() { // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{ - "function": { - "name": "Bash", - "arguments": "{\"command\": \"cat > /tmp/test.py <<'EOF'\\nprint(1)\\nEOF\"}" - } - }]}, - ] - })); + let request = with_messages(vec![ + bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!( sig.write_count, 1, @@ -865,16 +842,9 @@ mod tests { #[test] fn bash_sed_inplace_counts_as_edit() { - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{ - "function": { - "name": "Bash", - "arguments": "{\"command\": \"sed -i 's/foo/bar/g' /app/file.py\"}" - } - }]}, - ] - })); + let request = with_messages(vec![ + bash("sed -i 's/foo/bar/g' /app/file.py"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!( sig.edit_count, 1, @@ -886,16 +856,10 @@ mod tests { #[test] fn bash_non_mutating_does_not_count() { // ls, cat, grep — should not increment either counter. - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{ - "function": {"name": "Bash", "arguments": "{\"command\": \"ls -la /app\"}"} - }]}, - {"role": "assistant", "tool_calls": [{ - "function": {"name": "Bash", "arguments": "{\"command\": \"cat /app/main.py\"}"} - }]}, - ] - })); + let request = with_messages(vec![ + bash("ls -la /app"), + bash("cat /app/main.py"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.write_count, 0); assert_eq!(sig.edit_count, 0); @@ -958,14 +922,9 @@ mod tests { #[test] fn anthropic_bash_heredoc_extracts_command() { // Anthropic format: tool_use.input is an object, not a JSON string. - let request = anthropic_request(json!({ - "messages": [ - {"role": "assistant", "content": [ - {"type": "tool_use", "name": "Bash", - "input": {"command": "cat > /tmp/foo.txt << 'EOF'\nhi\nEOF"}} - ]}, - ] - })); + let request = with_messages(vec![ + bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!( sig.write_count, 1, @@ -975,11 +934,9 @@ mod tests { #[test] fn recent_window_falls_back_to_full_history_when_short() { - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - ] - })); + let request = with_messages(vec![ + tc("Write"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.recent_write_count, 1); assert_eq!(sig.recent_edit_count, 0); @@ -987,12 +944,10 @@ mod tests { #[test] fn clean_tool_result_has_zero_severity_and_non_empty_streak() { - let request = openai_chat_request(json!({ - "messages": [ - {"role": "tool", "tool_call_id": "1", "content": "output ok"}, - {"role": "tool", "tool_call_id": "2", "content": "another ok"}, - ] - })); + let request = with_messages(vec![ + tr("output ok"), + tr("another ok"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, 0.0); assert_eq!(sig.no_error_streak, 2); @@ -1066,25 +1021,13 @@ mod tests { #[test] fn pure_bash_streak_counts_trailing_other() { // 5 trailing non-classified Bash calls → streak == 5. - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"./configure\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make install\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"./run.sh\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"./test\"}"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); + let request = with_messages(vec![ + bash("make"), tr("ok"), + bash("./configure"), tr("ok"), + bash("make install"), tr("ok"), + bash("./run.sh"), tr("ok"), + bash("./test"), tr("ok"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.pure_bash_streak, 5); assert_eq!(sig.write_count, 0); @@ -1093,15 +1036,10 @@ mod tests { #[test] fn pure_bash_streak_resets_on_write() { - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Write"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); + let request = with_messages(vec![ + bash("make"), tr("ok"), + tc("Write"), tr("ok"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.pure_bash_streak, 0); assert_eq!(sig.write_count, 1); @@ -1110,19 +1048,12 @@ mod tests { #[test] fn recent_window_tracks_todowrite_and_read() { // Final 3 tool calls: TodoWrite, Read, TodoWrite. - let request = openai_chat_request(json!({ - "messages": [ - {"role": "assistant", "tool_calls": [{"function": {"name": "Bash", - "arguments": "{\"command\": \"make\"}"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "TodoWrite"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "Read"}}]}, - {"role": "tool", "content": "ok"}, - {"role": "assistant", "tool_calls": [{"function": {"name": "TodoWrite"}}]}, - {"role": "tool", "content": "ok"}, - ] - })); + let request = with_messages(vec![ + bash("make"), tr("ok"), + tc("TodoWrite"), tr("ok"), + tc("Read"), tr("ok"), + tc("TodoWrite"), tr("ok"), + ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.todowrite_count, 2); assert_eq!(sig.recent_todowrite_count, 2); diff --git a/crates/libsy/tests/stage_router.rs b/crates/libsy/tests/stage_router.rs deleted file mode 100644 index d90c09ba..00000000 --- a/crates/libsy/tests/stage_router.rs +++ /dev/null @@ -1,347 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for a running [`StageRouter`]. -//! -//! The unit tests cover scoring, note selection and prompt placement in -//! isolation. These drive whole turns through the public API to pin what only -//! the assembled cascade can show: which tier served the turn, what the *model* -//! was handed, and which step of the cascade decided. -//! -//! The per-target prompts have their own integration tests in `prompts.rs`, -//! since they are not stage-router-specific. - -use std::sync::Arc; - -use async_trait::async_trait; -use parking_lot::Mutex; -use serde_json::json; - -use switchyard_libsy::algorithms::{StageRouter, TaskClassifierConfig}; -use switchyard_libsy::stage_router::{ - HandoffNoteConfig, LlmFallback, PickerMode, StageRouterConfig, -}; -use switchyard_libsy::{ - Algorithm, Context, Decision, LlmResponse, LlmTarget, Metadata, Request, Response, Result, - RoutedLlmClient, -}; -use switchyard_protocol::{ - text_response, ContentBlock, LlmRequest, Message, Role, ToolCall, ToolResult, WireFormat, -}; - -const ESCALATION: &str = "the previous model was stalling; pick up the diagnosis"; -/// Semantic name of the judge target. It is called through its own target and is -/// never a routing destination. -const JUDGE: &str = "judge"; - -/// One model call as the client saw it. -#[derive(Clone, Debug)] -struct Call { - /// Target the call was routed to. - target: String, - /// Text of each message, so a test can assert on the note. - messages: Vec, -} - -/// A client that records what each target was handed, so a test can assert on -/// what reached the model rather than on router internals. -/// -/// It also plays the judge: a call routed to the judge target answers with a -/// verdict, which is how the fallback classifier gets an answer without a real -/// model. -#[derive(Default)] -struct RecordingClient { - calls: Mutex>, - /// `p_solve` the judge reports. High keeps the turn on the weak tier. - judge_p_solve: Mutex, -} - -impl RecordingClient { - /// The calls that routed to a tier, dropping the judge's own. - fn routed(&self) -> Vec { - self.calls - .lock() - .iter() - .filter(|call| call.target != JUDGE) - .cloned() - .collect() - } -} - -#[async_trait] -impl RoutedLlmClient for RecordingClient { - async fn call( - &self, - _ctx: Context, - request: Request, - decision: Arc, - ) -> std::result::Result { - let target = decision.selected_model().to_string(); - self.calls.lock().push(Call { - target: target.clone(), - messages: request - .llm_request - .messages - .iter() - .filter_map(|message| message.text_content("|")) - .collect(), - }); - let completion = if target == JUDGE { - let p_solve = *self.judge_p_solve.lock(); - format!( - r#"{{"recommended_route":"efficient","p_solve":{p_solve},"confidence":0.9,"abstain":false,"capability_boundary":"supported","primary_rule":"SUP-1","crux":"bounded task"}}"# - ) - } else { - target - }; - Ok(Response { - llm_response: LlmResponse::Agg(text_response(None, completion)), - metadata: None, - }) - } -} - -fn target(client: &Arc, name: &str) -> LlmTarget { - LlmTarget { - semantic_name: name.to_string(), - llm_client: Some(client.clone() as Arc), - } -} - -fn router(client: Arc, config: StageRouterConfig) -> Result> { - // Only the two tiers are routing destinations; the judge has its own target. - Ok(Arc::new(StageRouter::new( - target(&client, "strong"), - target(&client, "weak"), - config, - )?)) -} - -/// The signal-only configuration these tests start from. -fn config() -> StageRouterConfig { - StageRouterConfig::new(PickerMode::EfficientFirst, 0.5) -} - -/// That configuration plus handoff notes. -fn config_with_notes() -> StageRouterConfig { - let mut config = config(); - config.handoff_notes = Some(HandoffNoteConfig::new(ESCALATION, None, true)); - config -} - -/// One turn of a coding-agent conversation, as the wire delivers it: an -/// assistant tool call answered by a tool result the signal extractor reads. -/// -/// `failed` makes the tool result a critical error, the hard override that -/// escalates a turn on the signal alone. -fn turn_request(failed: bool) -> Request { - let tool_call = json!({ - "role": "assistant", - "tool_calls": [{ - "id": "call_1", - "type": "function", - "function": {"name": "Bash", "arguments": "{\"command\": \"cargo test\"}"}, - }], - }); - let content = if failed { - "fatal runtime error: out of memory" - } else { - "ok" - }; - let raw_request = json!({ - "model": "auto", - "messages": [ - {"role": "user", "content": "fix the build"}, - tool_call, - {"role": "tool", "tool_call_id": "call_1", "content": content}, - ], - }); - // The neutral IR the router forwards, alongside the raw body the signal - // extractor parses. An OpenAI tool result is its own `tool` turn, so a note - // is appended as a fresh user message rather than folded into it. - let messages = vec![ - Message::text(Role::User, "fix the build"), - Message { - role: Role::Assistant, - content: vec![ContentBlock::ToolCall(ToolCall { - id: "call_1".to_string(), - name: "Bash".to_string(), - arguments: json!({"command": "cargo test"}), - })], - }, - Message { - role: Role::Tool, - content: vec![ContentBlock::ToolResult(ToolResult { - tool_call_id: "call_1".to_string(), - content: vec![ContentBlock::Text { - text: content.to_string(), - }], - is_error: Some(failed), - })], - }, - ]; - Request { - llm_request: LlmRequest { - model: Some("auto".to_string()), - messages, - ..LlmRequest::default() - }, - raw_request: Some(raw_request), - metadata: Some(Metadata { - wire_format: Some(WireFormat::OpenAiChat), - // One session, so turns of the same test share the router's state. - session_id: Some("session-1".to_string()), - ..Default::default() - }), - } -} - -#[tokio::test] -async fn a_signal_driven_escalation_hands_the_note_to_the_model() -> Result<()> { - let client = Arc::new(RecordingClient::default()); - let router = router(client.clone(), config_with_notes())?; - let ctx = Context::default(); - - // Turn 1 — a clean tool result is under threshold and falls open to weak. - router.clone().run(ctx.clone(), turn_request(false)).await?; - // Turn 2 — a critical tool error escalates on the signals alone. - router.run(ctx, turn_request(true)).await?; - - let calls = client.routed(); - assert_eq!(calls[0].target, "weak"); - assert_eq!(calls[1].target, "strong"); - assert!( - !calls[0] - .messages - .iter() - .any(|text| text.contains(ESCALATION)), - "the steady-state turn should carry no note: {:?}", - calls[0].messages - ); - // The note rides on the trailing turn, after the tool result. - assert!( - calls[1] - .messages - .last() - .is_some_and(|text| text.ends_with(ESCALATION)), - "the escalating turn should carry the note last: {:?}", - calls[1].messages - ); - Ok(()) -} - -/// That configuration plus the capability judge behind the signals. -fn config_with_judge(client: &Arc, p_solve: f64) -> StageRouterConfig { - *client.judge_p_solve.lock() = p_solve; - let mut config = config(); - config.llm_fallback = Some(LlmFallback { - judge_target: target(client, JUDGE), - config: TaskClassifierConfig { - base_threshold: 0.5, - // The same span the signal scorer reads. - recent_turn_window: Some(3), - ..Default::default() - }, - }); - config -} - -#[tokio::test] -async fn the_judge_decides_a_turn_the_signals_leave_undecided() -> Result<()> { - let client = Arc::new(RecordingClient::default()); - // A low p_solve means the judge does not trust the weak tier with the task. - let router = router(client.clone(), config_with_judge(&client, 0.1))?; - - // A clean turn is under threshold, so without a judge it would fall open to - // the configured weak default. The judge overrides that. - router.run(Context::default(), turn_request(false)).await?; - - let judged = client.calls.lock().iter().any(|call| call.target == JUDGE); - assert!(judged, "the judge should be consulted on an undecided turn"); - assert_eq!(client.routed()[0].target, "strong"); - Ok(()) -} - -#[tokio::test] -async fn a_decisive_signal_never_reaches_the_judge() -> Result<()> { - let client = Arc::new(RecordingClient::default()); - // The judge would say weak; the signals must win before it is asked at all. - let router = router(client.clone(), config_with_judge(&client, 0.9))?; - - router.run(Context::default(), turn_request(true)).await?; - - assert!( - !client.calls.lock().iter().any(|call| call.target == JUDGE), - "a resolved turn should not pay for a judge call" - ); - assert_eq!(client.routed()[0].target, "strong"); - Ok(()) -} - -#[tokio::test] -async fn the_judges_verdict_is_not_pinned_to_the_session() -> Result<()> { - let client = Arc::new(RecordingClient::default()); - let router = router(client.clone(), config_with_judge(&client, 0.1))?; - let ctx = Context::default(); - - // First undecided turn: the judge sends it to the strong tier. - router.clone().run(ctx.clone(), turn_request(false)).await?; - // The judge changes its mind; a second undecided turn asks again rather than - // replaying the first verdict. - *client.judge_p_solve.lock() = 0.9; - router.run(ctx, turn_request(false)).await?; - - let routed = client.routed(); - assert_eq!(routed[0].target, "strong"); - assert_eq!(routed[1].target, "weak"); - assert_eq!( - client - .calls - .lock() - .iter() - .filter(|call| call.target == JUDGE) - .count(), - 2, - "each undecided turn is its own question" - ); - Ok(()) -} - -#[tokio::test] -async fn a_judge_that_cannot_tell_lands_on_the_picker_default() -> Result<()> { - let client = Arc::new(RecordingClient::default()); - // An out-of-range p_solve is an unusable verdict, so the judge abstains. - let router = router(client.clone(), config_with_judge(&client, 42.0))?; - - router.run(Context::default(), turn_request(false)).await?; - - // efficient_first, so the turn falls open to weak rather than being pushed - // to strong by the judge's own fallback. - assert_eq!(client.routed()[0].target, "weak"); - Ok(()) -} - -#[tokio::test] -async fn the_judge_reads_the_window_it_was_configured_with() -> Result<()> { - let client = Arc::new(RecordingClient::default()); - // An undecided turn, so the judge is consulted and we can see what it got. - let router = router(client.clone(), config_with_judge(&client, 0.9))?; - - router.run(Context::default(), turn_request(false)).await?; - - // With a window the judge sees the opening task, not just the newest turn. - let judged = client - .calls - .lock() - .iter() - .find(|call| call.target == JUDGE) - .map(|call| call.messages.join("|")); - let Some(judged) = judged else { - panic!("the judge was never called"); - }; - assert!( - judged.contains("fix the build"), - "the judge should see the opening task: {judged}" - ); - Ok(()) -} From badf2d231be8775880287f0908d163445bae4d1c Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 19:11:35 -0700 Subject: [PATCH 5/8] =?UTF-8?q?fix(libsy):=20review=20nits=20=E2=80=94=20d?= =?UTF-8?q?uplicate=20doc=20sentence=20and=20LlmFallback=20field=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/stage.rs | 8 ++++---- crates/libsy/src/core/processor.rs | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 48dee8e9..c22f69ab 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -70,10 +70,10 @@ pub struct LlmFallback { /// Target the judge model is called through. It is not a routing /// destination, so it does not belong in the router's target set. pub judge_target: LlmTarget, - /// How the judge routes, exactly as the standalone capability route takes it. - /// Its `recent_turn_window` is worth setting to this router's - /// `recent_window`, so the judge reads the same span of the conversation the - /// signal scorer scored. + /// Judge configuration. `recent_turn_window` is worth setting to this router's + /// `recent_window` so the judge reads the same span the signal scorer scored. + /// Note: `session_affinity` and `message_hash_fallback` have no effect here — + /// the judge runs as a cascade classifier, not a standalone algorithm. pub config: TaskClassifierConfig, } diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs index 9ac4e567..74193cb1 100644 --- a/crates/libsy/src/core/processor.rs +++ b/crates/libsy/src/core/processor.rs @@ -32,8 +32,6 @@ pub enum Event<'a> { /// Collects events as the algorithm runs and mutates the composition's state. #[async_trait] pub trait Processor: Send + Sync { - /// Process an event, accumulating facts into `state`. - /// /// Process an event, accumulating facts into `state`. Request-bearing events /// ([`Event::Request`], [`Event::Decision`]) may also be rewritten in place. async fn process(&self, state: &mut S, event: Event<'_>) -> Result<()>; From b320b6f36d8aa3a9dd26ac2007c474609fd0fcb1 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 19:14:58 -0700 Subject: [PATCH 6/8] chore(libsy): remove stale TODOs and verbose comments Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/llm_class.rs | 26 +++-------------------- crates/libsy/src/algorithms/util/stage.rs | 1 - 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 82cc8fa6..4797636a 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -21,7 +21,6 @@ use crate::{ Request, Response, Result, RoutedLlmClient, Score, State, }; -// TODO: As a first implementation, keeping the prompt and schema paths hardcoded. Add a way to dynamically load and parse user passed prompt and schema. const PROMPT_TEMPLATE: &str = include_str!("../prompts/capability-classifier/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../prompts/capability-classifier/schema.json"); /// Telemetry label for this algorithm's spans, metrics, and logs. @@ -29,9 +28,6 @@ const ALGORITHM_NAME: &str = "llm_task_classifier"; #[derive(Deserialize)] #[serde(deny_unknown_fields)] -/// Parsed judge output. Only confidence, abstention, and solve probability affect v0 routing. -/// These fields are parsed from the judge output and used to route the request. -/// For supporting a new schema, we need to add a new Verdict struct and parse the new struct TaskClassifierVerdict { #[serde(rename = "recommended_route")] _recommended_route: String, @@ -57,9 +53,7 @@ impl TaskClassifierVerdict { ) } - /// Whether this verdict needs the elevated capability threshold. - /// When capability boundary is "uncertain", "unsupported", or "unmatched", we need to use the elevated capability threshold to route the request to weak model - /// This is to ensure that we are not routing too many requests to the weak model. + /// Whether the capability boundary requires the elevated routing threshold. fn is_capability_elevated(&self) -> bool { matches!( self.capability_boundary.as_str(), @@ -68,9 +62,6 @@ impl TaskClassifierVerdict { } } -/// The judge is responsible for any kind of llm judge based calls -/// Example: A judge can be a capability classifier, a escalation classifier etc -/// Builds capability-specific judge requests from shared classifier configuration. /// Keeps client instructions, the opening task, and the last `recent_turn_window` /// turns after it. A window of `0` keeps the instructions and the task alone. /// @@ -101,8 +92,6 @@ struct CapabilityJudge { impl Judge for CapabilityJudge { type Verdict = TaskClassifierVerdict; - /// For different judges, the request building logic can be different - /// To have a single interface for all judges, we may make a common request building logic here. fn build_request(&self, _state: &State, request: &Request) -> Request { // Task-based routing judges the newest user message alone. A configured // window widens that to the surrounding conversation. @@ -309,16 +298,9 @@ impl LlmTaskClassifier { capable_target: capable_target.semantic_name, }); - // The cascade is an internal detail: callers drive the algorithm, not its parts. // Affinity comes first so a retained assignment short-circuits the judge call. - // - // TODO: bind the assignment on the post-decision hook instead, the way the - // per-target system prompt does. Affinity is a fact about the decision, not a - // classification, so recording it there would let a pinned session skip the - // cascade outright on the next turn. It would also make the pin survive - // composition: a cascade that uses this classifier as one member (the stage - // router does) currently never runs the affinity wired in here, because only - // the inner classifier is consulted. + // Note: when this classifier is embedded inside another cascade (e.g. StageRouter) + // the affinity processor never fires — only the inner score() is called. let mut route = FallThrough::::new_with_state(targets).with_name(ALGORITHM_NAME); if session_affinity { let affinity = if message_hash_fallback { @@ -343,8 +325,6 @@ impl LlmTaskClassifier { }) } - /// Loads the judge configuration from the packaged prompt and schema. - /// TODO: Move towards more generic loading and parsing of config when we multiple prompts and schemas to handle for same algorithm fn load_judge_config() -> Result { // The response schema is rendered into the prompt and sent as structured-output metadata. // One asset therefore defines both the instruction contract and provider enforcement. diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index db5de92e..86519cb1 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -507,7 +507,6 @@ impl Classifier for StageClassifier { // ambiguous turn is decided further down the cascade. self.apply_handoff_note(request, tier, source); let conf = score.abs(); - // TODO add the non-target to this score set? Ok(Classification::Scores(vec![Score { target: target.to_string(), confidence: conf, From 8682fad50b1adcf22ab658412b3ba0e97612b4cc Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 19:17:25 -0700 Subject: [PATCH 7/8] docs: update stage_router doc to TOML config format and correct observability Signed-off-by: Sabhatina Selvam --- .../stage_router_routing.md | 130 ++++++++++-------- 1 file changed, 69 insertions(+), 61 deletions(-) diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 589e9cff..f21d0ad0 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -188,101 +188,109 @@ well in stage-router as it does alone. ## Route configuration -```yaml -defaults: - base_url: https://openrouter.ai/api/v1 - api_key: ${OPENROUTER_API_KEY} - format: openai - -routes: - smart-stage-router: - type: stage_router - picker: capable_first - confidence_threshold: 0.5 - signal_recent_window: 3 - fallback_target_on_evict: strong - strong: - id: strong - model: openai/gpt-4o - weak: - id: weak - model: openai/gpt-4o-mini - enable_stats: true +```toml +schema_version = 1 + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.strong] +id = "openai/gpt-4o" +llm_client = "openrouter" + +[targets.weak] +id = "openai/gpt-4o-mini" +llm_client = "openrouter" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 0.5 +recent_turn_window = 3 # optional, defaults to 3 ``` -Save the file as `routes.yaml` and start it with: +Save as `routes.toml` and start the server: ```bash -switchyard --routing-profiles routes.yaml -- serve --port 4000 +switchyard-server --config routes.toml --port 4000 ``` This is the recommended default: routing on tool signals alone, no classifier. -If you omit `confidence_threshold`, the config default of `0.5` applies; the -example sets it explicitly. -`fallback_target_on_evict` is required and must reference one of the -declared target ids. See [Context-Window Handling](../operations/context_window.md) for -exception types and error envelopes. +### Optional: handoff notes -### Optional: add an LLM classifier +Add a `[routes.stage.handoff_notes]` section to pass a contextual note to the +model the router switches to. The escalation note is sent to the capable tier on +a signal-driven escalation; the de-escalation note is sent back to the efficient +tier when a settled signal drops the turn there. -By default the router uses tool signals only. If you want a model to break the -tie on low-confidence turns, add a `classifier:` block and set +```toml +[routes.stage.handoff_notes] +escalation_note = "the previous model was stalling; pick up the diagnosis" +# deescalation_note = "..." # optional +# only_on_wrong_signal_escalation = true # default; set false to always send +``` + +### Optional: per-tier system prompts + +```toml +[routes.stage] +# ... +capable_system_prompt = "diagnose before you edit" +efficient_system_prompt = "follow the settled plan" +``` + +### Optional: LLM classifier fallback + +By default the router uses tool signals only. To break ties on low-confidence +turns with a model call, add a `[routes.stage.classifier]` block and set `confidence_threshold` above `0.0`. The classifier is consulted only for turns that fall below the threshold: -```yaml - classifier: - model: openai/gpt-4o-mini - api_key: ${OPENROUTER_API_KEY} # prefer a separate key/quota in production - base_url: https://openrouter.ai/api/v1 - timeout_secs: 30.0 - recent_turn_window: 3 +```toml +[routes.stage.classifier] +target = "strong" # target the judge is called through (not a routing destination) +base_threshold = 0.5 # p_solve floor to route efficient; below this → capable +min_confidence = 0.7 # judge confidence floor; below this → abstain +recent_turn_window = 3 # conversation span the judge sees ``` -Give the classifier its own credential or quota bucket where you can. Sharing +Give the classifier its own LLM client or quota bucket where possible. Sharing one provider bucket with the efficient tier adds a request per classified turn and can cause sustained 429s at scale. -The same route bundle works with the launchers. - ## Observability -### Per-tier token / cost stats (standard) +Each response carries two routing headers: -```bash -curl http://localhost:4000/v1/stats -``` - -Returns the stats snapshot: per-model calls, tokens, latency, and cost, bucketed -by served model, each tagged with a `capable` or `efficient` `tier`. Batch -harnesses usually capture this same snapshot to a file (see below). +| Header | Content | +|---|---| +| `x-model-router-selected-model` | The model ID the turn was routed to. | +| `x-model-router-rationale` | Human-readable routing reason (e.g. `stage_router selected weak (confidence 0.612)`). | -### Decision-source metadata (stage-router-specific) +### Decision sources -The route counts why each turn was routed the way it was under -`routing_decisions.stage_router` in the stats JSON: +The `decision_source` recorded internally for each turn explains why the routing +went the way it did. It appears in per-tier metrics tagged on the decision: | Source | When | |---|---| | `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. | -| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — dropped the turn to the efficient tier. | +| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — landed the turn on the efficient tier. | | `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. | | `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | -| `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured, so the default tier was used. | -| `no_signal` | The request arrived before any tool-result history existed. | - -To capture a snapshot for a batch run, redirect the endpoint to a file: - -```bash -curl -s http://localhost:4000/v1/stats > routing_stats_final.json -``` +| `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured; the default tier was used. | ## When *not* to use stage-router - **Single-model deployments.** Use a `model` route instead. - **Probabilistic A/B splits.** Use - [Random Routing](random_routing.md) (`type: random_routing`). + [Random Routing](random_routing.md) (`type = "random"`). The stage-router's signals are wasted on a fixed traffic ratio. - **No tool-result history.** Stage-router needs meaningful tool-call traffic to populate the tool-result signal. For pure chat-completion workloads every From e6fb712e39d9581b4f86642bc28645606ced6445 Mon Sep 17 00:00:00 2001 From: Sabhatina Selvam Date: Wed, 29 Jul 2026 19:26:47 -0700 Subject: [PATCH 8/8] style(libsy): cargo fmt Signed-off-by: Sabhatina Selvam --- crates/libsy/src/algorithms/stage.rs | 12 +- .../libsy/src/algorithms/util/tool_signals.rs | 112 +++++++++--------- 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index c22f69ab..8161c188 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -519,7 +519,10 @@ mod tests { calls[0].messages ); assert!( - calls[1].messages.last().is_some_and(|t| t.ends_with(ESCALATION)), + calls[1] + .messages + .last() + .is_some_and(|t| t.ends_with(ESCALATION)), "escalating turn should carry the note last: {:?}", calls[1].messages ); @@ -570,7 +573,12 @@ mod tests { assert_eq!(routed[0].target, "strong"); assert_eq!(routed[1].target, "weak"); assert_eq!( - client.calls.lock().iter().filter(|c| c.target == JUDGE).count(), + client + .calls + .lock() + .iter() + .filter(|c| c.target == JUDGE) + .count(), 2, "each undecided turn is its own question" ); diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs index 6ae54950..969564d5 100644 --- a/crates/libsy/src/algorithms/util/tool_signals.rs +++ b/crates/libsy/src/algorithms/util/tool_signals.rs @@ -601,7 +601,10 @@ mod tests { fn with_messages(messages: Vec) -> Request { Request { - llm_request: LlmRequest { messages, ..LlmRequest::default() }, + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, raw_request: None, metadata: None, } @@ -637,7 +640,9 @@ mod tests { role: Role::User, content: vec![ContentBlock::ToolResult(ToolResult { tool_call_id: String::new(), - content: vec![ContentBlock::Text { text: text.to_string() }], + content: vec![ContentBlock::Text { + text: text.to_string(), + }], is_error: None, })], } @@ -748,19 +753,14 @@ mod tests { #[test] fn extract_anthropic_tool_results() { - let request = with_messages(vec![ - tr("Traceback (most recent call last):\n ValueError"), - ]); + let request = with_messages(vec![tr("Traceback (most recent call last):\n ValueError")]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, HARD); } #[test] fn extract_responses_api_tool_results() { - let request = with_messages(vec![ - tc("Write"), - tr("file written successfully"), - ]); + let request = with_messages(vec![tc("Write"), tr("file written successfully")]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, 0.0); assert_eq!(sig.write_count, 1); @@ -771,12 +771,18 @@ mod tests { // 5 writes + 1 edit at the end → the default window (3) should see // the last 3 calls: 1 edit + 2 writes (not all 6 calls). let request = with_messages(vec![ - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Edit"), tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Edit"), + tr("ok"), ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.write_count, 5); @@ -791,12 +797,18 @@ mod tests { // With recent_window=3 → recent_writes=2, recent_edits=1. // With recent_window=6 → recent_writes=5, recent_edits=1 (all calls). let request = with_messages(vec![ - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Write"), tr("ok"), - tc("Edit"), tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Write"), + tr("ok"), + tc("Edit"), + tr("ok"), ]); let narrow = extract_tool_signals_with_window(&request, 3); assert_eq!(narrow.recent_write_count, 2); @@ -829,9 +841,7 @@ mod tests { #[test] fn bash_heredoc_counts_as_write() { // Claude Code's pattern on TB 2.0 — write a scratch file via heredoc. - let request = with_messages(vec![ - bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF"), - ]); + let request = with_messages(vec![bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF")]); let sig = ToolSignals::from_request(&request, None); assert_eq!( sig.write_count, 1, @@ -842,9 +852,7 @@ mod tests { #[test] fn bash_sed_inplace_counts_as_edit() { - let request = with_messages(vec![ - bash("sed -i 's/foo/bar/g' /app/file.py"), - ]); + let request = with_messages(vec![bash("sed -i 's/foo/bar/g' /app/file.py")]); let sig = ToolSignals::from_request(&request, None); assert_eq!( sig.edit_count, 1, @@ -856,10 +864,7 @@ mod tests { #[test] fn bash_non_mutating_does_not_count() { // ls, cat, grep — should not increment either counter. - let request = with_messages(vec![ - bash("ls -la /app"), - bash("cat /app/main.py"), - ]); + let request = with_messages(vec![bash("ls -la /app"), bash("cat /app/main.py")]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.write_count, 0); assert_eq!(sig.edit_count, 0); @@ -922,9 +927,7 @@ mod tests { #[test] fn anthropic_bash_heredoc_extracts_command() { // Anthropic format: tool_use.input is an object, not a JSON string. - let request = with_messages(vec![ - bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF"), - ]); + let request = with_messages(vec![bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF")]); let sig = ToolSignals::from_request(&request, None); assert_eq!( sig.write_count, 1, @@ -934,9 +937,7 @@ mod tests { #[test] fn recent_window_falls_back_to_full_history_when_short() { - let request = with_messages(vec![ - tc("Write"), - ]); + let request = with_messages(vec![tc("Write")]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.recent_write_count, 1); assert_eq!(sig.recent_edit_count, 0); @@ -944,10 +945,7 @@ mod tests { #[test] fn clean_tool_result_has_zero_severity_and_non_empty_streak() { - let request = with_messages(vec![ - tr("output ok"), - tr("another ok"), - ]); + let request = with_messages(vec![tr("output ok"), tr("another ok")]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.severity, 0.0); assert_eq!(sig.no_error_streak, 2); @@ -1022,11 +1020,16 @@ mod tests { fn pure_bash_streak_counts_trailing_other() { // 5 trailing non-classified Bash calls → streak == 5. let request = with_messages(vec![ - bash("make"), tr("ok"), - bash("./configure"), tr("ok"), - bash("make install"), tr("ok"), - bash("./run.sh"), tr("ok"), - bash("./test"), tr("ok"), + bash("make"), + tr("ok"), + bash("./configure"), + tr("ok"), + bash("make install"), + tr("ok"), + bash("./run.sh"), + tr("ok"), + bash("./test"), + tr("ok"), ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.pure_bash_streak, 5); @@ -1036,10 +1039,7 @@ mod tests { #[test] fn pure_bash_streak_resets_on_write() { - let request = with_messages(vec![ - bash("make"), tr("ok"), - tc("Write"), tr("ok"), - ]); + let request = with_messages(vec![bash("make"), tr("ok"), tc("Write"), tr("ok")]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.pure_bash_streak, 0); assert_eq!(sig.write_count, 1); @@ -1049,10 +1049,14 @@ mod tests { fn recent_window_tracks_todowrite_and_read() { // Final 3 tool calls: TodoWrite, Read, TodoWrite. let request = with_messages(vec![ - bash("make"), tr("ok"), - tc("TodoWrite"), tr("ok"), - tc("Read"), tr("ok"), - tc("TodoWrite"), tr("ok"), + bash("make"), + tr("ok"), + tc("TodoWrite"), + tr("ok"), + tc("Read"), + tr("ok"), + tc("TodoWrite"), + tr("ok"), ]); let sig = ToolSignals::from_request(&request, None); assert_eq!(sig.todowrite_count, 2);