diff --git a/AGENTS.md b/AGENTS.md index e214dc9b..ab5b7feb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -144,9 +144,10 @@ Everything flows through a fixed-shape chain enforced at construction time: request-side component* → LLMBackend → response-side component* → TranslationEngine ``` -The chain executor is `Switchyard` (`switchyard/lib/switchyard.py`). `LLMBackend` remains the -shared role class re-exported from `switchyard/lib/roles.py`; request-side and response-side -processors are plain async components with `process(...)` methods. +The chain executor is `Switchyard` (`switchyard/lib/switchyard.py`). `LLMBackend` is the shared +Python role class re-exported from `switchyard/lib/roles.py`; native implementations register with +it, and request-side and response-side processors are plain async components with `process(...)` +methods. Direct Rust bindings for migrated concrete processors/backends are exposed from `switchyard_rust.components` and implemented under `crates/switchyard-py/src/component_bindings/`. @@ -163,7 +164,7 @@ Direct Rust bindings for migrated concrete processors/backends are exposed from switchyard/ ├── __init__.py # Public API — all exports live here ├── lib/ # Core library -│ ├── roles.py # Rust-owned LLMBackend re-export and translation aliases +│ ├── roles.py # Python LLMBackend re-export and translation aliases │ ├── switchyard.py # Switchyard — chain executor │ ├── proxy_context.py # ProxyContext — per-request state carrier │ ├── profiles/ # Profile configs/runtimes for pre-built routing behavior @@ -372,7 +373,7 @@ uvicorn.run(build_switchyard_app(switchyard), port=4000) ### Ask first - Modifying `pyproject.toml` dependencies. -- Changes to the chain shape or Rust-owned role classes in `switchyard/lib/roles.py`. +- Changes to the chain shape or public role classes in `switchyard/lib/roles.py`. - Adding new HTTP endpoints. - Removing or renaming any public API currently in `switchyard/__init__.__all__`. diff --git a/Cargo.lock b/Cargo.lock index 1cc8e794..6ba0168f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,12 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - [[package]] name = "anstream" version = "1.0.0" @@ -432,12 +426,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -609,17 +597,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -854,7 +831,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown", ] [[package]] @@ -935,15 +912,6 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -1739,30 +1707,16 @@ version = "0.1.0" dependencies = [ "async-stream", "async-trait", + "futures-core", "futures-util", "parking_lot", "rand 0.8.7", "reqwest", "serde", "serde_json", - "switchyard-core", "switchyard-libsy", "switchyard-protocol", "switchyard-translation", - "tokio", - "tracing", -] - -[[package]] -name = "switchyard-core" -version = "0.1.0" -dependencies = [ - "async-trait", - "futures-core", - "lru", - "rand 0.8.7", - "serde", - "serde_json", "thiserror 2.0.18", "tokio", "tracing", @@ -1828,7 +1782,6 @@ dependencies = [ "serde", "serde_json", "switchyard-components", - "switchyard-core", "switchyard-libsy", "switchyard-protocol", "switchyard-translation", diff --git a/Cargo.toml b/Cargo.toml index 2fcbbda3..0bca0c95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,6 @@ members = [ "crates/libsy", "crates/libsy-llm-client", "crates/switchyard-components", - "crates/switchyard-core", "crates/switchyard-py", "crates/protocol", "crates/switchyard-server", @@ -32,7 +31,7 @@ rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots", "stream"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -switchyard-core = { path = "crates/switchyard-core", version = "0.1.0" } +switchyard-components = { path = "crates/switchyard-components", version = "0.1.0" } switchyard-libsy = { path = "crates/libsy", version = "0.1.0" } switchyard-llm-client = { path = "crates/libsy-llm-client", version = "0.1.0" } switchyard-protocol = { path = "crates/protocol", version = "0.1.0" } diff --git a/crates/switchyard-components/Cargo.toml b/crates/switchyard-components/Cargo.toml index ae78030d..93c3e677 100644 --- a/crates/switchyard-components/Cargo.toml +++ b/crates/switchyard-components/Cargo.toml @@ -14,17 +14,18 @@ rust-version.workspace = true [dependencies] async-stream.workspace = true async-trait.workspace = true +futures-core = "0.3" futures-util.workspace = true parking_lot.workspace = true rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true -switchyard-core.workspace = true switchyard-libsy.workspace = true switchyard-protocol.workspace = true switchyard-translation.workspace = true tokio.workspace = true +thiserror.workspace = true tracing.workspace = true [dev-dependencies] diff --git a/crates/switchyard-components/src/backends/anthropic.rs b/crates/switchyard-components/src/backends/anthropic.rs index 8611c960..bd4e7f0a 100644 --- a/crates/switchyard-components/src/backends/anthropic.rs +++ b/crates/switchyard-components/src/backends/anthropic.rs @@ -8,15 +8,15 @@ use std::env; use std::fmt; use std::sync::Arc; -use async_stream::try_stream; -use async_trait::async_trait; -use futures_util::StreamExt; -use serde_json::{json, Map, Value}; -use switchyard_core::{ +use crate::{ merge_target_extra_body, BackendFormat, BoxResponseStream, ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTarget, LlmTargetId, ProxyContext, Result, StreamEvent, SwitchyardError, }; +use async_stream::try_stream; +use async_trait::async_trait; +use futures_util::StreamExt; +use serde_json::{json, Map, Value}; use switchyard_translation::{ normalize_anthropic_tool_use_ids, TranslationEngine, TranslationPolicy, WireFormat, }; @@ -546,9 +546,9 @@ fn anthropic_sse_stream(response: reqwest::Response) -> BoxResponseStream { #[cfg(test)] mod tests { + use crate::{EndpointConfig, LlmTargetId, ModelId}; use parking_lot::Mutex; use serde_json::json; - use switchyard_core::{EndpointConfig, LlmTargetId, ModelId}; use super::*; diff --git a/crates/switchyard-components/src/backends/common.rs b/crates/switchyard-components/src/backends/common.rs index 0494c089..7623aaaf 100644 --- a/crates/switchyard-components/src/backends/common.rs +++ b/crates/switchyard-components/src/backends/common.rs @@ -6,8 +6,8 @@ use std::sync::{Arc, OnceLock}; use std::time::Duration; +use crate::{ChatRequestType, Result, SwitchyardError}; use serde_json::{Map, Value}; -use switchyard_core::{ChatRequestType, Result, SwitchyardError}; use switchyard_translation::{TranslationEngine, WireFormat}; pub(crate) enum ParsedSseFrame { diff --git a/crates/switchyard-components/src/backends/multi.rs b/crates/switchyard-components/src/backends/multi.rs index a3cf8b91..42f48ad6 100644 --- a/crates/switchyard-components/src/backends/multi.rs +++ b/crates/switchyard-components/src/backends/multi.rs @@ -12,11 +12,11 @@ use std::collections::HashSet; use std::fmt; use std::sync::Arc; -use async_trait::async_trait; -use switchyard_core::{ +use crate::{ ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTarget, LlmTargetId, ProxyContext, Result, SwitchyardError, }; +use async_trait::async_trait; use super::{BackendSelection, BackendSelectionReason}; diff --git a/crates/switchyard-components/src/backends/openai.rs b/crates/switchyard-components/src/backends/openai.rs index da1cf643..8911466b 100644 --- a/crates/switchyard-components/src/backends/openai.rs +++ b/crates/switchyard-components/src/backends/openai.rs @@ -8,15 +8,15 @@ use std::env; use std::fmt; use std::sync::Arc; -use async_stream::try_stream; -use async_trait::async_trait; -use futures_util::StreamExt; -use serde_json::{Map, Value}; -use switchyard_core::{ +use crate::{ merge_target_extra_body, BackendFormat, BoxResponseStream, ChatRequest, ChatRequestType, ChatResponse, EndpointConfig, LlmBackend, LlmTarget, LlmTargetId, ModelId, ProxyContext, Result, StreamEvent, SwitchyardError, }; +use async_stream::try_stream; +use async_trait::async_trait; +use futures_util::StreamExt; +use serde_json::{Map, Value}; use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; use super::common::{ @@ -580,9 +580,9 @@ fn openai_sse_stream(response: reqwest::Response) -> BoxResponseStream { #[cfg(test)] mod tests { + use crate::{EndpointConfig, LlmTargetId, ModelId}; use parking_lot::Mutex; use serde_json::json; - use switchyard_core::{EndpointConfig, LlmTargetId, ModelId}; use super::*; diff --git a/crates/switchyard-components/src/backends/selection.rs b/crates/switchyard-components/src/backends/selection.rs index 1693be17..1416b587 100644 --- a/crates/switchyard-components/src/backends/selection.rs +++ b/crates/switchyard-components/src/backends/selection.rs @@ -3,8 +3,8 @@ //! Backend execution metadata for observability components. +use crate::{LlmTargetId, ModelId}; use serde::{Deserialize, Serialize}; -use switchyard_core::{LlmTargetId, ModelId}; /// How a backend resolved the final upstream target/model for a request. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] diff --git a/crates/switchyard-components/src/backends/stats.rs b/crates/switchyard-components/src/backends/stats.rs index 6c589b47..80ff8f94 100644 --- a/crates/switchyard-components/src/backends/stats.rs +++ b/crates/switchyard-components/src/backends/stats.rs @@ -7,10 +7,8 @@ use std::fmt; use std::sync::Arc; use std::time::Instant; +use crate::{ChatRequest, ChatRequestType, ChatResponse, LlmBackend, ProxyContext, Result}; use async_trait::async_trait; -use switchyard_core::{ - ChatRequest, ChatRequestType, ChatResponse, LlmBackend, ProxyContext, Result, -}; use crate::stats::{ selected_stats_model, selected_stats_tier, StatsAccumulator, StatsBackendLatency, diff --git a/crates/switchyard-core/src/backend.rs b/crates/switchyard-components/src/contracts/backend.rs similarity index 98% rename from crates/switchyard-core/src/backend.rs rename to crates/switchyard-components/src/contracts/backend.rs index 5265bffc..9beef8a1 100644 --- a/crates/switchyard-core/src/backend.rs +++ b/crates/switchyard-components/src/contracts/backend.rs @@ -1,14 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! LLM target configuration shared by routing and factory code. +//! LLM target configuration shared by compatibility routing and factory code. use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::ids::{LlmTargetId, ModelId}; +use super::ids::{LlmTargetId, ModelId}; /// Wire format expected by an LLM target. #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Serialize, Deserialize)] diff --git a/crates/switchyard-core/src/context.rs b/crates/switchyard-components/src/contracts/context.rs similarity index 97% rename from crates/switchyard-core/src/context.rs rename to crates/switchyard-components/src/contracts/context.rs index b04652df..f864508b 100644 --- a/crates/switchyard-core/src/context.rs +++ b/crates/switchyard-components/src/contracts/context.rs @@ -1,14 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Per-request context and typed extension storage for chain components. +//! Per-request context and typed extension storage for compatibility components. use std::any::{Any, TypeId}; use std::collections::{HashMap, HashSet}; use std::fmt; -use crate::ids::{LlmTargetId, RequestId}; -use crate::types::ChatRequestType; +use super::ids::{LlmTargetId, RequestId}; +use super::types::ChatRequestType; /// Mutable state shared across processors and the backend for one request. #[derive(Default)] diff --git a/crates/switchyard-core/src/error.rs b/crates/switchyard-components/src/contracts/error.rs similarity index 95% rename from crates/switchyard-core/src/error.rs rename to crates/switchyard-components/src/contracts/error.rs index 0be5aa97..115e3b92 100644 --- a/crates/switchyard-core/src/error.rs +++ b/crates/switchyard-components/src/contracts/error.rs @@ -1,12 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Error types used by the Rust core contracts. +//! Errors shared by compatibility backends and Python bindings. use thiserror::Error; -use crate::ids::{InvalidId, ModelId}; -use crate::types::ChatRequestType; +use super::ids::{InvalidId, ModelId}; +use super::types::ChatRequestType; /// Result alias for core Switchyard operations. pub type Result = std::result::Result; diff --git a/crates/switchyard-core/src/ids.rs b/crates/switchyard-components/src/contracts/ids.rs similarity index 94% rename from crates/switchyard-core/src/ids.rs rename to crates/switchyard-components/src/contracts/ids.rs index c7f00483..90104c60 100644 --- a/crates/switchyard-core/src/ids.rs +++ b/crates/switchyard-components/src/contracts/ids.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Non-empty string identifiers used across core routing and component wiring. +//! Non-empty string identifiers used by compatibility component wiring. use std::fmt; use std::str::FromStr; @@ -106,10 +106,6 @@ macro_rules! string_id { }; } -string_id!(BackendId); -string_id!(ComponentId); -string_id!(EndpointId); string_id!(LlmTargetId); string_id!(ModelId); -string_id!(ProfileId); string_id!(RequestId); diff --git a/crates/switchyard-components/src/contracts/mod.rs b/crates/switchyard-components/src/contracts/mod.rs new file mode 100644 index 00000000..4ef3dde0 --- /dev/null +++ b/crates/switchyard-components/src/contracts/mod.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Contracts retained by the compatibility component and Python layers. + +mod backend; +mod context; +mod error; +mod ids; +mod roles; +mod types; + +pub use backend::*; +pub use context::*; +pub use error::*; +pub use ids::*; +pub use roles::*; +pub use types::*; diff --git a/crates/switchyard-core/src/roles.rs b/crates/switchyard-components/src/contracts/roles.rs similarity index 81% rename from crates/switchyard-core/src/roles.rs rename to crates/switchyard-components/src/contracts/roles.rs index a42951b3..8ad074ec 100644 --- a/crates/switchyard-core/src/roles.rs +++ b/crates/switchyard-components/src/contracts/roles.rs @@ -1,13 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Backend trait shared by Rust-owned LLM callers. +//! Backend trait shared by compatibility LLM callers. use async_trait::async_trait; -use crate::context::ProxyContext; -use crate::error::Result; -use crate::types::{ChatRequest, ChatRequestType, ChatResponse}; +use super::context::ProxyContext; +use super::error::Result; +use super::types::{ChatRequest, ChatRequestType, ChatResponse}; /// Backend abstraction responsible for making the LLM call. #[async_trait] diff --git a/crates/switchyard-core/src/types.rs b/crates/switchyard-components/src/contracts/types.rs similarity index 98% rename from crates/switchyard-core/src/types.rs rename to crates/switchyard-components/src/contracts/types.rs index 58e79281..d50ee683 100644 --- a/crates/switchyard-core/src/types.rs +++ b/crates/switchyard-components/src/contracts/types.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Provider-agnostic request and response wrappers used by core chains. +//! Provider-agnostic request and response wrappers used by compatibility chains. use std::fmt; use std::pin::Pin; @@ -10,7 +10,7 @@ use futures_core::Stream; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::error::{Result, SwitchyardError}; +use super::error::{Result, SwitchyardError}; /// Supported inbound request wire formats. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] diff --git a/crates/switchyard-components/src/dimension_collector/response/checks.rs b/crates/switchyard-components/src/dimension_collector/response/checks.rs index ee52a801..fac3b0c1 100644 --- a/crates/switchyard-components/src/dimension_collector/response/checks.rs +++ b/crates/switchyard-components/src/dimension_collector/response/checks.rs @@ -14,8 +14,8 @@ //! formats are different enough that uniform JSONPath would either be //! lossy or 4x more code than direct shape access. +use crate::ChatResponse; use serde_json::Value; -use switchyard_core::ChatResponse; /// Returns `true` if any `tool_calls[].function.arguments` string is not /// valid JSON. Operates on OpenAI-Chat and Anthropic responses. diff --git a/crates/switchyard-components/src/dimension_collector/response/mod.rs b/crates/switchyard-components/src/dimension_collector/response/mod.rs index 756a3323..685b185e 100644 --- a/crates/switchyard-components/src/dimension_collector/response/mod.rs +++ b/crates/switchyard-components/src/dimension_collector/response/mod.rs @@ -15,8 +15,8 @@ pub mod checks; +use crate::ChatResponse; use serde::{Deserialize, Serialize}; -use switchyard_core::ChatResponse; /// Aggregate of response-side quality flags emitted by [`extract_response_signals`]. /// diff --git a/crates/switchyard-components/src/dimension_collector/tool_signals.rs b/crates/switchyard-components/src/dimension_collector/tool_signals.rs index f0440be5..ad819d5d 100644 --- a/crates/switchyard-components/src/dimension_collector/tool_signals.rs +++ b/crates/switchyard-components/src/dimension_collector/tool_signals.rs @@ -8,7 +8,7 @@ //! [`switchyard_protocol::Request`] (raw body + wire-format metadata) so the two //! request models share one implementation. -use switchyard_core::{ChatRequest, ChatRequestType}; +use crate::{ChatRequest, ChatRequestType}; use switchyard_protocol::{Metadata, Request, WireFormat}; /// The tool-signal output type. Re-exported from libsy so downstream consumers diff --git a/crates/switchyard-components/src/intake/client.rs b/crates/switchyard-components/src/intake/client.rs index 4cc18ec6..a4a58574 100644 --- a/crates/switchyard-components/src/intake/client.rs +++ b/crates/switchyard-components/src/intake/client.rs @@ -8,10 +8,10 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; +use crate::{Result, SwitchyardError}; use async_trait::async_trait; use parking_lot::Mutex; use serde_json::Value; -use switchyard_core::{Result, SwitchyardError}; use tokio::sync::mpsc; use tokio::task::JoinHandle; @@ -326,7 +326,7 @@ fn nibble_to_hex(value: u8) -> char { #[cfg(test)] mod tests { - use switchyard_core::Result; + use crate::Result; use super::*; diff --git a/crates/switchyard-components/src/intake/context.rs b/crates/switchyard-components/src/intake/context.rs index fb063ec7..34a4a5e7 100644 --- a/crates/switchyard-components/src/intake/context.rs +++ b/crates/switchyard-components/src/intake/context.rs @@ -5,8 +5,8 @@ use std::collections::BTreeMap; +use crate::{ChatRequest, ChatRequestType}; use serde::{Deserialize, Serialize}; -use switchyard_core::{ChatRequest, ChatRequestType}; /// Header carrying the client session ID used by intake. pub const PROXY_SESSION_ID_HEADER: &str = "proxy_x_session_id"; diff --git a/crates/switchyard-components/src/intake/payload.rs b/crates/switchyard-components/src/intake/payload.rs index f6bac053..d56dfd48 100644 --- a/crates/switchyard-components/src/intake/payload.rs +++ b/crates/switchyard-components/src/intake/payload.rs @@ -7,11 +7,11 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -use serde_json::{json, Map, Value}; -use switchyard_core::{ +use crate::{ ChatRequest, ChatRequestType, ChatResponse, ChatResponseType, ProxyContext, Result, StreamEvent, SwitchyardError, }; +use serde_json::{json, Map, Value}; use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; use crate::backends::BackendSelection; @@ -1632,7 +1632,7 @@ impl AnthropicContentBlockState { #[cfg(test)] mod tests { use super::*; - use switchyard_core::Result; + use crate::Result; // Timestamp formatting should match the Python intake payloads exactly. #[test] diff --git a/crates/switchyard-components/src/lib.rs b/crates/switchyard-components/src/lib.rs index 094f58e5..958c4331 100644 --- a/crates/switchyard-components/src/lib.rs +++ b/crates/switchyard-components/src/lib.rs @@ -1,13 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Concrete Switchyard implementations built on `switchyard-core`. +//! Compatibility backends and processors for the Python Switchyard runtime. //! -//! `switchyard-core` owns traits and wire wrappers. This crate owns built-in -//! compatibility implementations: backends, request processors, and response -//! processors. New Rust orchestration belongs in libsy algorithms and clients. +//! New Rust orchestration belongs in libsy algorithms and clients. The contracts +//! here remain only while the Python compatibility runtime uses these components. pub mod backends; +mod contracts; pub mod dimension_collector; pub mod intake; pub mod request_processors; @@ -20,6 +20,7 @@ pub use backends::{ AnthropicNativeBackend, BackendSelection, BackendSelectionReason, LlmTargetBackend, MultiLlmBackend, OpenAiNativeBackend, OpenAiPassthroughBackend, StatsLlmBackend, }; +pub use contracts::*; pub use dimension_collector::{ extract_tool_signals, ResponseFlag, ResponseSignals, ToolResultSignal, }; diff --git a/crates/switchyard-components/src/request_processors/dimension_collector.rs b/crates/switchyard-components/src/request_processors/dimension_collector.rs index 617ae1a7..f5437597 100644 --- a/crates/switchyard-components/src/request_processors/dimension_collector.rs +++ b/crates/switchyard-components/src/request_processors/dimension_collector.rs @@ -7,7 +7,7 @@ //! It walks the request's tool-call history and stamps the resulting //! [`ToolResultSignal`] into `ProxyContext` for the stage_router picker to read. -use switchyard_core::{ChatRequest, ProxyContext, Result}; +use crate::{ChatRequest, ProxyContext, Result}; use crate::dimension_collector::{ extract_tool_signals_with_window, ToolResultSignal, DEFAULT_RECENT_WINDOW, diff --git a/crates/switchyard-components/src/request_processors/intake.rs b/crates/switchyard-components/src/request_processors/intake.rs index f76a350b..319c88a7 100644 --- a/crates/switchyard-components/src/request_processors/intake.rs +++ b/crates/switchyard-components/src/request_processors/intake.rs @@ -3,8 +3,8 @@ //! Request-side intake processor. +use crate::{ChatRequest, ProxyContext, Result}; use serde_json::Value; -use switchyard_core::{ChatRequest, ProxyContext, Result}; use crate::intake::context::{IntakeRequestState, RequestMetadata}; use crate::intake::payload::now_millis; diff --git a/crates/switchyard-components/src/request_processors/random_routing.rs b/crates/switchyard-components/src/request_processors/random_routing.rs index 2f6cac59..097f9b03 100644 --- a/crates/switchyard-components/src/request_processors/random_routing.rs +++ b/crates/switchyard-components/src/request_processors/random_routing.rs @@ -5,11 +5,11 @@ use std::fmt; +use crate::{LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError}; use parking_lot::Mutex; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use serde::{Deserialize, Serialize}; -use switchyard_core::{LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError}; const DEFAULT_STRONG_PROBABILITY: f64 = 0.5; diff --git a/crates/switchyard-components/src/request_processors/stats.rs b/crates/switchyard-components/src/request_processors/stats.rs index 9c8998d6..9b3188ec 100644 --- a/crates/switchyard-components/src/request_processors/stats.rs +++ b/crates/switchyard-components/src/request_processors/stats.rs @@ -3,7 +3,7 @@ //! Request-side stats processor. -use switchyard_core::{ChatRequest, ProxyContext, Result}; +use crate::{ChatRequest, ProxyContext, Result}; use crate::stats::{prefix_probe, StatsRequestStart}; diff --git a/crates/switchyard-components/src/response_processors/intake.rs b/crates/switchyard-components/src/response_processors/intake.rs index 8a6c09cd..8aa22a8b 100644 --- a/crates/switchyard-components/src/response_processors/intake.rs +++ b/crates/switchyard-components/src/response_processors/intake.rs @@ -5,9 +5,9 @@ use std::sync::Arc; +use crate::{BoxResponseStream, ChatResponse, ProxyContext, Result}; use async_stream::try_stream; use futures_util::StreamExt; -use switchyard_core::{BoxResponseStream, ChatResponse, ProxyContext, Result}; use crate::intake::{ now_millis, HttpIntakeSink, IntakePayloadBuilder, IntakePayloadContext, IntakeRequestState, diff --git a/crates/switchyard-components/src/response_processors/response_signals.rs b/crates/switchyard-components/src/response_processors/response_signals.rs index d6e3002c..02405885 100644 --- a/crates/switchyard-components/src/response_processors/response_signals.rs +++ b/crates/switchyard-components/src/response_processors/response_signals.rs @@ -9,7 +9,7 @@ //! stage_router / escalation logic can read structured response-quality //! flags without re-parsing the wire body. -use switchyard_core::{ChatResponse, ProxyContext, Result}; +use crate::{ChatResponse, ProxyContext, Result}; use crate::dimension_collector::response::{extract_response_signals, ResponseSignals}; @@ -47,7 +47,7 @@ mod tests { use crate::dimension_collector::response::ResponseFlag; use serde_json::json; - use switchyard_core::Result; + use crate::Result; #[tokio::test] async fn stamps_response_signals_for_buffered_response() -> Result<()> { diff --git a/crates/switchyard-components/src/response_processors/stats.rs b/crates/switchyard-components/src/response_processors/stats.rs index c69d9b8f..4a10de9d 100644 --- a/crates/switchyard-components/src/response_processors/stats.rs +++ b/crates/switchyard-components/src/response_processors/stats.rs @@ -3,9 +3,9 @@ //! Response-side stats processor. +use crate::{BoxResponseStream, ChatResponse, ProxyContext, Result}; use async_stream::try_stream; use futures_util::StreamExt; -use switchyard_core::{BoxResponseStream, ChatResponse, ProxyContext, Result}; use crate::stats::{ openai_chat_usage_from_stream_event, openai_responses_usage_from_stream_event, diff --git a/crates/switchyard-components/src/stats/accumulator.rs b/crates/switchyard-components/src/stats/accumulator.rs index 36a583ce..4f523eaf 100644 --- a/crates/switchyard-components/src/stats/accumulator.rs +++ b/crates/switchyard-components/src/stats/accumulator.rs @@ -7,9 +7,9 @@ use std::cmp::{Ordering, Reverse}; use std::collections::{btree_map::Entry, BTreeMap, BinaryHeap, HashSet}; use std::sync::Arc; +use crate::Result; use parking_lot::{Mutex, MutexGuard}; use serde::{Deserialize, Serialize}; -use switchyard_core::Result; use super::cost::{estimate_cost, CostEstimate}; use super::{PrefixProbe, TokenUsage}; diff --git a/crates/switchyard-components/src/stats/context.rs b/crates/switchyard-components/src/stats/context.rs index 1a8e7008..6d397eb9 100644 --- a/crates/switchyard-components/src/stats/context.rs +++ b/crates/switchyard-components/src/stats/context.rs @@ -5,8 +5,8 @@ use std::time::{Duration, Instant}; +use crate::ProxyContext; use serde::{Deserialize, Serialize}; -use switchyard_core::ProxyContext; use crate::backends::BackendSelection; use crate::request_processors::RandomRoutingDecision; diff --git a/crates/switchyard-components/src/stats/usage.rs b/crates/switchyard-components/src/stats/usage.rs index 884b80c9..10b939a3 100644 --- a/crates/switchyard-components/src/stats/usage.rs +++ b/crates/switchyard-components/src/stats/usage.rs @@ -3,9 +3,9 @@ //! Provider usage extraction for buffered and streaming responses. +use crate::StreamEvent; use serde::{Deserialize, Serialize}; use serde_json::Value; -use switchyard_core::StreamEvent; /// Normalized token usage counters. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] diff --git a/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs b/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs index 875359c4..a00f9b5d 100644 --- a/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs +++ b/crates/switchyard-components/tests/adversarial_multi_llm_backend.rs @@ -9,12 +9,12 @@ use async_trait::async_trait; use parking_lot::Mutex; use serde_json::{json, Value}; use switchyard_components::{ - BackendSelection, BackendSelectionReason, LlmTargetBackend, MultiLlmBackend, -}; -use switchyard_core::{ BackendFormat, ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTarget, LlmTargetId, ModelId, ProxyContext, Result, SwitchyardError, }; +use switchyard_components::{ + BackendSelection, BackendSelectionReason, LlmTargetBackend, MultiLlmBackend, +}; /// One backend call observed by the recording backend. #[derive(Clone, Debug, PartialEq)] diff --git a/crates/switchyard-components/tests/adversarial_native_backends.rs b/crates/switchyard-components/tests/adversarial_native_backends.rs index f9bc85a8..21c75006 100644 --- a/crates/switchyard-components/tests/adversarial_native_backends.rs +++ b/crates/switchyard-components/tests/adversarial_native_backends.rs @@ -10,7 +10,7 @@ use serde_json::{json, Value}; use switchyard_components::{ AnthropicNativeBackend, BackendSelection, OpenAiNativeBackend, OpenAiPassthroughBackend, }; -use switchyard_core::{ +use switchyard_components::{ BackendFormat, ChatRequest, ChatRequestType, ChatResponse, ChatResponseType, EndpointConfig, LlmBackend, LlmTarget, LlmTargetId, ModelId, ProxyContext, Result, StreamEvent, SwitchyardError, diff --git a/crates/switchyard-components/tests/adversarial_random_routing.rs b/crates/switchyard-components/tests/adversarial_random_routing.rs index c67ff702..746d7090 100644 --- a/crates/switchyard-components/tests/adversarial_random_routing.rs +++ b/crates/switchyard-components/tests/adversarial_random_routing.rs @@ -4,11 +4,11 @@ //! Adversarial tests for the random routing engine. use serde_json::json; -use switchyard_components::{RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier}; -use switchyard_core::{ +use switchyard_components::{ BackendFormat, ChatRequest, EndpointConfig, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError, }; +use switchyard_components::{RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier}; // Builds a deterministic strong/weak config for routing tests. fn config(strong_probability: f64, rng_seed: u64) -> Result { diff --git a/crates/switchyard-core/tests/adversarial_types.rs b/crates/switchyard-components/tests/contracts.rs similarity index 90% rename from crates/switchyard-core/tests/adversarial_types.rs rename to crates/switchyard-components/tests/contracts.rs index 75643cdb..39c80186 100644 --- a/crates/switchyard-core/tests/adversarial_types.rs +++ b/crates/switchyard-components/tests/contracts.rs @@ -1,16 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Adversarial tests for core identifiers, context storage, and wire wrappers. +//! Adversarial tests for compatibility identifiers, context storage, and wire wrappers. use std::pin::Pin; use std::task::{Context, Poll}; use futures_core::Stream; use serde_json::json; -use switchyard_core::{ - BackendFormat, ChatRequest, ChatRequestType, ChatResponse, ChatResponseType, ComponentId, - LlmTarget, LlmTargetId, ModelId, ProxyContext, StreamEvent, +use switchyard_components::{ + BackendFormat, ChatRequest, ChatRequestType, ChatResponse, ChatResponseType, LlmTarget, + LlmTargetId, ModelId, ProxyContext, StreamEvent, }; type TestResult = std::result::Result<(), Box>; @@ -22,7 +22,7 @@ struct ContextMarker(&'static str); struct EmptyStream; impl Stream for EmptyStream { - type Item = switchyard_core::Result; + type Item = switchyard_components::Result; fn poll_next(self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { Poll::Ready(None) @@ -34,17 +34,11 @@ impl Stream for EmptyStream { fn ids_reject_empty_and_whitespace_values_from_constructors_and_serde() -> TestResult { assert!(ModelId::new("").is_err()); assert!(ModelId::new(" ").is_err()); - assert!(ComponentId::new("").is_err()); - assert!(ComponentId::new(" ").is_err()); assert!(serde_json::from_value::(json!("")).is_err()); assert!(serde_json::from_value::(json!(" ")).is_err()); - assert!(serde_json::from_value::(json!("")).is_err()); - assert!(serde_json::from_value::(json!(" ")).is_err()); let parsed = serde_json::from_value::(json!("real-model"))?; assert_eq!(parsed.as_str(), "real-model"); - let parsed = serde_json::from_value::(json!("route"))?; - assert_eq!(parsed.as_str(), "route"); Ok(()) } diff --git a/crates/switchyard-components/tests/flat_target_live.rs b/crates/switchyard-components/tests/flat_target_live.rs index 7e129cc6..75f19b69 100644 --- a/crates/switchyard-components/tests/flat_target_live.rs +++ b/crates/switchyard-components/tests/flat_target_live.rs @@ -14,10 +14,10 @@ use std::time::{SystemTime, UNIX_EPOCH}; use serde_json::{json, Value}; use switchyard_components::intake::to_flat_document; +use switchyard_components::Result; use switchyard_components::{ HttpIntakeSink, IntakeFormat, IntakeSink, IntakeSinkConfig, IntakeTarget, }; -use switchyard_core::Result; // A representative chat-completions intake payload, the input the production // builder produces before flattening. diff --git a/crates/switchyard-components/tests/intake_http_sink.rs b/crates/switchyard-components/tests/intake_http_sink.rs index de08bb3f..32e4afe1 100644 --- a/crates/switchyard-components/tests/intake_http_sink.rs +++ b/crates/switchyard-components/tests/intake_http_sink.rs @@ -7,7 +7,7 @@ use serde_json::json; use switchyard_components::{ HttpIntakeSink, IntakeFormat, IntakeSink, IntakeSinkConfig, IntakeTarget, }; -use switchyard_core::{Result, SwitchyardError}; +use switchyard_components::{Result, SwitchyardError}; use support::{OneShotServer, SequenceServer}; diff --git a/crates/switchyard-components/tests/intake_payload.rs b/crates/switchyard-components/tests/intake_payload.rs index 460cd3a0..59ea655e 100644 --- a/crates/switchyard-components/tests/intake_payload.rs +++ b/crates/switchyard-components/tests/intake_payload.rs @@ -6,12 +6,14 @@ mod support; use serde_json::{json, Value}; +use switchyard_components::{ + ChatRequest, ChatRequestType, LlmTargetId, ModelId, ProxyContext, Result, +}; use switchyard_components::{ IntakeFormat, IntakePayloadBuilder, IntakeRequestMetadata, IntakeRequestState, IntakeSinkConfig, IntakeTarget, RandomRoutingDecision, RandomRoutingTier, RequestMetadata, StatsRouteLabel, SubModelCall, }; -use switchyard_core::{ChatRequest, ChatRequestType, LlmTargetId, ModelId, ProxyContext, Result}; use support::intake::{ completion, completion_with_usage, openai_chat_request, record_backend_selection, diff --git a/crates/switchyard-components/tests/intake_request_processor.rs b/crates/switchyard-components/tests/intake_request_processor.rs index e5d2f969..88a873af 100644 --- a/crates/switchyard-components/tests/intake_request_processor.rs +++ b/crates/switchyard-components/tests/intake_request_processor.rs @@ -6,10 +6,10 @@ mod support; use std::collections::BTreeMap; use serde_json::json; +use switchyard_components::{ChatRequest, ChatRequestType, ProxyContext, Result, SwitchyardError}; use switchyard_components::{ IntakeRequestMetadata, IntakeRequestProcessor, IntakeRequestState, RequestMetadata, }; -use switchyard_core::{ChatRequest, ChatRequestType, ProxyContext, Result, SwitchyardError}; use support::intake::request; diff --git a/crates/switchyard-components/tests/intake_response_processor.rs b/crates/switchyard-components/tests/intake_response_processor.rs index 56ecd955..9938bbf4 100644 --- a/crates/switchyard-components/tests/intake_response_processor.rs +++ b/crates/switchyard-components/tests/intake_response_processor.rs @@ -9,11 +9,11 @@ use std::sync::Arc; use serde_json::json; use switchyard_components::{ - IntakeRequestState, IntakeResponseProcessor, IntakeSinkConfig, SubModelCall, SubModelCalls, -}; -use switchyard_core::{ ChatRequestType, ChatResponse, ModelId, ProxyContext, Result, StreamEvent, SwitchyardError, }; +use switchyard_components::{ + IntakeRequestState, IntakeResponseProcessor, IntakeSinkConfig, SubModelCall, SubModelCalls, +}; use support::intake::{ completion, drain_stream, opted_in_context, record_backend_selection, RecordingSink, diff --git a/crates/switchyard-components/tests/stats_accumulator.rs b/crates/switchyard-components/tests/stats_accumulator.rs index 1f2e5134..47c04856 100644 --- a/crates/switchyard-components/tests/stats_accumulator.rs +++ b/crates/switchyard-components/tests/stats_accumulator.rs @@ -7,7 +7,7 @@ use serde_json::json; use switchyard_components::{ prefix_probe, ModelStatsSnapshot, StatsAccumulator, StatsSnapshot, TokenUsage, }; -use switchyard_core::{Result, SwitchyardError}; +use switchyard_components::{Result, SwitchyardError}; #[test] fn accumulator_snapshot_starts_zero_and_reset_clears_all_state() -> Result<()> { diff --git a/crates/switchyard-components/tests/stats_processors.rs b/crates/switchyard-components/tests/stats_processors.rs index 5ab649d2..860d885f 100644 --- a/crates/switchyard-components/tests/stats_processors.rs +++ b/crates/switchyard-components/tests/stats_processors.rs @@ -16,7 +16,7 @@ use switchyard_components::{ StatsAccumulator, StatsBackendLatency, StatsLlmBackend, StatsRequestProcessor, StatsRequestStart, StatsResponseProcessor, StatsRouteLabel, }; -use switchyard_core::{ +use switchyard_components::{ ChatRequest, ChatRequestType, ChatResponse, LlmBackend, LlmTargetId, ModelId, ProxyContext, Result, StreamEvent, SwitchyardError, }; diff --git a/crates/switchyard-components/tests/support/config.rs b/crates/switchyard-components/tests/support/config.rs index 50161bf1..404873d3 100644 --- a/crates/switchyard-components/tests/support/config.rs +++ b/crates/switchyard-components/tests/support/config.rs @@ -5,7 +5,7 @@ use std::collections::BTreeMap; -use switchyard_core::{ +use switchyard_components::{ BackendFormat, EndpointConfig, LlmTarget, LlmTargetId, ModelId, Result, SwitchyardError, }; diff --git a/crates/switchyard-components/tests/support/intake.rs b/crates/switchyard-components/tests/support/intake.rs index 9713aa95..07fd9683 100644 --- a/crates/switchyard-components/tests/support/intake.rs +++ b/crates/switchyard-components/tests/support/intake.rs @@ -13,7 +13,7 @@ use switchyard_components::{ BackendSelection, BackendSelectionReason, IntakeRequestMetadata, IntakeRequestState, IntakeSink, RequestMetadata, }; -use switchyard_core::{ +use switchyard_components::{ ChatRequest, ChatRequestType, ChatResponse, ModelId, ProxyContext, Result, StreamEvent, SwitchyardError, }; diff --git a/crates/switchyard-components/tests/support/mod.rs b/crates/switchyard-components/tests/support/mod.rs index b149188b..97ed90f0 100644 --- a/crates/switchyard-components/tests/support/mod.rs +++ b/crates/switchyard-components/tests/support/mod.rs @@ -12,7 +12,7 @@ use std::thread::{self, JoinHandle}; use std::time::Duration; use serde_json::Value; -use switchyard_core::{Result, SwitchyardError}; +use switchyard_components::{Result, SwitchyardError}; pub mod config; pub mod intake; diff --git a/crates/switchyard-core/Cargo.toml b/crates/switchyard-core/Cargo.toml deleted file mode 100644 index 907f0030..00000000 --- a/crates/switchyard-core/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -[package] -name = "switchyard-core" -version = "0.1.0" -description = "Core contracts for Switchyard's Rust implementation" -authors.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true - -[dependencies] -async-trait.workspace = true -futures-core = "0.3" -lru = "0.12" -rand.workspace = true -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -tracing.workspace = true - -[dev-dependencies] -tokio.workspace = true diff --git a/crates/switchyard-core/src/lib.rs b/crates/switchyard-core/src/lib.rs deleted file mode 100644 index c2d06984..00000000 --- a/crates/switchyard-core/src/lib.rs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Core contracts for Switchyard's Rust implementation. -//! -//! This crate owns provider-agnostic values, IDs, errors, context storage, and -//! the backend trait used by concrete LLM callers. Request/response processor -//! logic now lives on concrete components and profile runtimes instead of core -//! role traits. - -pub mod backend; -pub mod context; -pub mod error; -pub mod ids; -pub mod roles; -pub mod session; -pub mod types; - -pub use backend::*; -pub use context::*; -pub use error::*; -pub use ids::*; -pub use roles::*; -pub use session::*; -pub use types::*; diff --git a/crates/switchyard-core/src/session.rs b/crates/switchyard-core/src/session.rs deleted file mode 100644 index 0e80b8d8..00000000 --- a/crates/switchyard-core/src/session.rs +++ /dev/null @@ -1,510 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Session-affinity primitives: a stable per-conversation key derived from a -//! request body and a bounded, access-ordered LRU cache keyed by that string. - -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; -use std::num::NonZeroUsize; - -use lru::LruCache; -use serde_json::Value; - -/// Derive a stable per-conversation key from a request body. -/// -/// Hashes only the prefix a harness never rewrites — system prompt + first user -/// message — so every turn of a conversation shares a key while distinct -/// conversations differ. Returns a 16-char lowercase hex string. -pub fn session_key_from_body(body: &Value) -> String { - hash_conversation_prefix(body, 0).0 -} - -/// Deep variant of [`session_key_from_body`]: extend the hashed prefix with -/// the first `depth` messages after the first user message, so repeated runs -/// of an identical task diverge via early model responses (repeated-trial -/// benchmarking). -/// -/// Returns `None` until the conversation actually contains a first user -/// message plus `depth` later messages — a shorter prefix would hash to a key -/// that later turns of the same conversation no longer match. `depth == 0` is -/// exactly the base key and always yields `Some`. -pub fn session_key_from_body_with_depth(body: &Value, depth: usize) -> Option { - let (key, complete) = hash_conversation_prefix(body, depth); - complete.then_some(key) -} - -/// Shared hashing core for both key variants: anchors (top-level system, -/// in-list system/developer messages, first user message) plus the first -/// `depth` non-system messages after the first user. Returns the key and -/// whether the requested prefix was complete (always true at depth 0). -fn hash_conversation_prefix(body: &Value, depth: usize) -> (String, bool) { - let mut hasher = DefaultHasher::new(); - - // Anthropic carries the system prompt at the top level. - flatten_text(body.get("system")).hash(&mut hasher); - - // OpenAI uses "messages"; the Responses API uses "input". A messages list - // with no user message falls through to "input". - let mut anchored = false; - let mut tail_taken = 0usize; - for seq_key in ["messages", "input"] { - if let Some(Value::Array(items)) = body.get(seq_key) { - for item in items { - match item.get("role").and_then(Value::as_str) { - Some("system") | Some("developer") => { - flatten_text(item.get("content")).hash(&mut hasher); - } - Some("user") if !anchored => { - flatten_text(item.get("content")).hash(&mut hasher); - anchored = true; - } - // Post-anchor tail (any non-system item, roleless - // Responses items included): hash the full message — - // tool calls too, so assistant turns that differ only - // in tool calls still diverge the deep key. - _ if anchored && tail_taken < depth => { - flatten_message_text(item).hash(&mut hasher); - tail_taken += 1; - } - _ => {} - } - if anchored && tail_taken >= depth { - break; - } - } - } - if anchored { - break; - } - } - - let complete = depth == 0 || (anchored && tail_taken >= depth); - (format!("{:016x}", hasher.finish()), complete) -} - -/// Flatten a whole message for deep-key hashing: `content` text plus tool-call -/// payloads (OpenAI chat `tool_calls`, Responses `function_call` items) and -/// tool `output`. Anchors keep hashing `content` only — this richer form is -/// applied to post-anchor tail messages, where the divergence signal between -/// repeated trials often lives entirely in the tool calls. -fn flatten_message_text(item: &Value) -> String { - let mut parts: Vec = Vec::new(); - let content = flatten_text(item.get("content")); - if !content.is_empty() { - parts.push(content); - } - // Anthropic tool_use blocks carry their payload in name/input rather than - // text/content, so the plain content flatten yields nothing for them. - if let Some(Value::Array(blocks)) = item.get("content") { - for block in blocks { - if block.get("type").and_then(Value::as_str) == Some("tool_use") { - let name = block.get("name").and_then(Value::as_str).unwrap_or(""); - let input = block.get("input").map(Value::to_string).unwrap_or_default(); - parts.push(format!("tool_call {name}({input})")); - } - } - } - if let Some(Value::Array(calls)) = item.get("tool_calls") { - for call in calls { - if let Some(function) = call.get("function").and_then(Value::as_object) { - let name = function.get("name").and_then(Value::as_str).unwrap_or(""); - let arguments = function - .get("arguments") - .and_then(Value::as_str) - .unwrap_or(""); - parts.push(format!("tool_call {name}({arguments})")); - } - } - } - if item.get("type").and_then(Value::as_str) == Some("function_call") { - let name = item.get("name").and_then(Value::as_str).unwrap_or(""); - let arguments = item.get("arguments").and_then(Value::as_str).unwrap_or(""); - parts.push(format!("tool_call {name}({arguments})")); - } - let output = flatten_text(item.get("output")); - if !output.is_empty() { - parts.push(output); - } - parts.join(" ") -} - -/// Flatten a message-content value into a single string for hashing: strings -/// pass through, content-block arrays concatenate their first non-empty -/// `text`/`content` field (or the raw block for non-objects), null/absent yield -/// empty, and other scalars stringify. Text-only by design (block metadata such -/// as `cache_control` is excluded so it can't perturb the key). -fn flatten_text(content: Option<&Value>) -> String { - match content { - Some(Value::String(s)) => s.clone(), - Some(Value::Array(blocks)) => { - let mut out = String::new(); - for block in blocks { - if let Value::Object(map) = block { - let text = map - .get("text") - .and_then(Value::as_str) - .filter(|s| !s.is_empty()) - .or_else(|| { - map.get("content") - .and_then(Value::as_str) - .filter(|s| !s.is_empty()) - }); - if let Some(text) = text { - out.push_str(text); - } - } else { - out.push_str(&block.to_string()); - } - } - out - } - None | Some(Value::Null) => String::new(), - Some(other) => other.to_string(), - } -} - -/// A bounded, access-ordered LRU cache keyed by `String`. -/// -/// Recency refreshes on both [`SessionCache::get`] and [`SessionCache::put`]; -/// the least-recently-used entry is evicted when capacity is exceeded. A -/// capacity of 0 retains nothing. -/// -/// NOT thread-safe — intended for single-event-loop use. -pub struct SessionCache { - cache: Option>, -} - -impl SessionCache { - /// Create a cache holding at most `max_sessions` entries (0 retains nothing). - pub fn new(max_sessions: usize) -> Self { - Self { - cache: NonZeroUsize::new(max_sessions).map(LruCache::new), - } - } - - /// Look up `key`, refreshing its recency to most-recently-used. - pub fn get(&mut self, key: &str) -> Option<&V> { - self.cache.as_mut().and_then(|c| c.get(key)) - } - - /// Insert `value` as most-recently-used, evicting the LRU entry if over - /// capacity. No-op when capacity is 0. - pub fn put(&mut self, key: String, value: V) { - if let Some(c) = self.cache.as_mut() { - c.put(key, value); - } - } - - /// Number of entries currently retained. - pub fn len(&self) -> usize { - self.cache.as_ref().map_or(0, LruCache::len) - } - - /// Whether the cache holds no entries. - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// The configured maximum number of sessions. - pub fn max_sessions(&self) -> usize { - self.cache.as_ref().map_or(0, |c| c.cap().get()) - } - - /// Iterate over the retained values (order unspecified). - pub fn values(&self) -> impl Iterator { - self.cache.iter().flat_map(|c| c.iter().map(|(_, v)| v)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn get_refreshes_recency() { - let mut cache: SessionCache = SessionCache::new(2); - cache.put("a".to_string(), 1); - cache.put("b".to_string(), 2); - // Touch "a" so "b" becomes the LRU entry. - assert!(cache.get("a").is_some()); - cache.put("c".to_string(), 3); - assert!(cache.get("a").is_some()); - assert!(cache.get("b").is_none()); - assert!(cache.get("c").is_some()); - } - - #[test] - fn eviction_bounds_len() { - let mut cache: SessionCache = SessionCache::new(2); - cache.put("a".to_string(), 1); - cache.put("b".to_string(), 2); - cache.put("c".to_string(), 3); - assert_eq!(cache.len(), 2); - } - - #[test] - fn zero_capacity_retains_nothing() { - let mut cache: SessionCache<&str> = SessionCache::new(0); - cache.put("a".to_string(), "x"); - assert!(cache.get("a").is_none()); - assert_eq!(cache.len(), 0); - assert!(cache.is_empty()); - assert_eq!(cache.max_sessions(), 0); - } - - #[test] - fn session_key_stable_across_appended_turns() { - let base = json!({ - "system": "you are helpful", - "messages": [ - {"role": "user", "content": "first question"}, - ], - }); - let extended = json!({ - "system": "you are helpful", - "messages": [ - {"role": "user", "content": "first question"}, - {"role": "assistant", "content": "an answer"}, - {"role": "user", "content": "a follow up"}, - ], - }); - assert_eq!( - session_key_from_body(&base), - session_key_from_body(&extended) - ); - } - - #[test] - fn session_key_distinct_on_system_or_first_user() { - let base = json!({ - "system": "you are helpful", - "messages": [{"role": "user", "content": "first question"}], - }); - let diff_system = json!({ - "system": "you are terse", - "messages": [{"role": "user", "content": "first question"}], - }); - let diff_user = json!({ - "system": "you are helpful", - "messages": [{"role": "user", "content": "another question"}], - }); - assert_ne!( - session_key_from_body(&base), - session_key_from_body(&diff_system) - ); - assert_ne!( - session_key_from_body(&base), - session_key_from_body(&diff_user) - ); - } - - #[test] - fn session_key_blocks_equal_plain() { - let blocks = json!({ - "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], - }); - let plain = json!({ - "messages": [{"role": "user", "content": "hi"}], - }); - assert_eq!( - session_key_from_body(&blocks), - session_key_from_body(&plain) - ); - } - - #[test] - fn key_is_16_char_hex() { - let key = session_key_from_body(&json!({"messages": [{"role": "user", "content": "x"}]})); - assert_eq!(key.len(), 16); - assert!(key.chars().all(|c| c.is_ascii_hexdigit())); - } - - #[test] - fn deep_key_depth_zero_matches_base() { - let body = json!({ - "system": "you are helpful", - "messages": [{"role": "user", "content": "first question"}], - }); - assert_eq!( - session_key_from_body_with_depth(&body, 0), - Some(session_key_from_body(&body)) - ); - } - - #[test] - fn deep_key_diverges_on_early_responses() { - // Identical anchors, different first assistant response: the deep key - // separates the trials while the base key does not. - let trial = |first_response: &str| { - json!({ - "messages": [ - {"role": "user", "content": "identical task text"}, - {"role": "assistant", "content": first_response}, - ], - }) - }; - let a = trial("read the tests first"); - let b = trial("look at the repo layout"); - assert_eq!(session_key_from_body(&a), session_key_from_body(&b)); - assert_ne!( - session_key_from_body_with_depth(&a, 1), - session_key_from_body_with_depth(&b, 1) - ); - } - - #[test] - fn deep_key_stable_as_conversation_grows() { - // The deep key hashes a fixed prefix, so it survives appended turns. - let base = json!({ - "messages": [ - {"role": "user", "content": "task"}, - {"role": "assistant", "content": "first response"}, - ], - }); - let grown = json!({ - "messages": [ - {"role": "user", "content": "task"}, - {"role": "assistant", "content": "first response"}, - {"role": "user", "content": "tool result"}, - {"role": "assistant", "content": "much later turn"}, - ], - }); - assert_eq!( - session_key_from_body_with_depth(&base, 1), - session_key_from_body_with_depth(&grown, 1) - ); - } - - #[test] - fn deep_key_none_until_prefix_complete() { - // One post-user message can't satisfy depth 2; no user at all can't - // satisfy any positive depth. - let short = json!({ - "messages": [ - {"role": "user", "content": "task"}, - {"role": "assistant", "content": "first response"}, - ], - }); - assert_eq!(session_key_from_body_with_depth(&short, 2), None); - let unanchored = json!({"messages": [{"role": "system", "content": "sys"}]}); - assert_eq!(session_key_from_body_with_depth(&unanchored, 1), None); - } - - #[test] - fn deep_key_sees_tool_calls() { - // Assistant turns whose divergence lives entirely in tool calls - // (empty content) must still separate the trials. - let trial = |arguments: &str| { - json!({ - "messages": [ - {"role": "user", "content": "task"}, - {"role": "assistant", "content": null, "tool_calls": [ - {"function": {"name": "read_file", "arguments": arguments}}, - ]}, - ], - }) - }; - assert_ne!( - session_key_from_body_with_depth(&trial("{\"path\": \"a.rs\"}"), 1), - session_key_from_body_with_depth(&trial("{\"path\": \"b.rs\"}"), 1) - ); - } - - #[test] - fn deep_key_sees_anthropic_tool_use() { - // Anthropic assistant turns whose divergence lives in tool_use blocks - // (no text content) must still separate the trials. - let trial = |command: &str| { - json!({ - "messages": [ - {"role": "user", "content": "task"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "name": "bash", "input": {"command": command}}, - ]}, - ], - }) - }; - assert_ne!( - session_key_from_body_with_depth(&trial("ls tests/"), 1), - session_key_from_body_with_depth(&trial("cat README.md"), 1) - ); - } - - #[test] - fn session_key_uses_input_when_messages_has_no_user() { - // A messages list with no user message falls through to `input`; the - // first user there anchors the key (proven by it affecting the hash). - let alpha = json!({ - "messages": [{"role": "system", "content": "sys"}], - "input": [{"role": "user", "content": "alpha"}], - }); - let beta = json!({ - "messages": [{"role": "system", "content": "sys"}], - "input": [{"role": "user", "content": "beta"}], - }); - assert_ne!(session_key_from_body(&alpha), session_key_from_body(&beta)); - } - - #[test] - fn session_key_supports_responses_input_only() { - // Responses API bodies carry turns under `input` with no `messages`. - let a = json!({"input": [{"role": "user", "content": "x"}]}); - let b = json!({"input": [{"role": "user", "content": "y"}]}); - assert_ne!(session_key_from_body(&a), session_key_from_body(&b)); - } - - #[test] - fn session_key_includes_system_message_in_messages() { - // A system/developer message inside `messages` (OpenAI shape) contributes. - let a = json!({ - "messages": [ - {"role": "system", "content": "sys A"}, - {"role": "user", "content": "q"}, - ], - }); - let b = json!({ - "messages": [ - {"role": "system", "content": "sys B"}, - {"role": "user", "content": "q"}, - ], - }); - assert_ne!(session_key_from_body(&a), session_key_from_body(&b)); - } - - #[test] - fn get_missing_returns_none() { - let mut cache: SessionCache = SessionCache::new(2); - assert!(cache.get("nope").is_none()); - cache.put("a".to_string(), 1); - assert_eq!(cache.get("a"), Some(&1)); - assert!(cache.get("b").is_none()); - } - - #[test] - fn put_overwrites_existing_value() { - // Re-pinning a session updates its value (mirrors pin-on-success). - let mut cache: SessionCache = SessionCache::new(2); - cache.put("a".to_string(), 1); - cache.put("a".to_string(), 2); - assert_eq!(cache.get("a"), Some(&2)); - assert_eq!(cache.len(), 1); - } - - #[test] - fn max_sessions_reports_capacity() { - let cache: SessionCache = SessionCache::new(5); - assert_eq!(cache.max_sessions(), 5); - } - - #[test] - fn values_yields_all_retained() { - let mut cache: SessionCache = SessionCache::new(3); - cache.put("a".to_string(), 1); - cache.put("b".to_string(), 2); - let mut vals: Vec = cache.values().copied().collect(); - vals.sort(); - assert_eq!(vals, vec![1, 2]); - } -} diff --git a/crates/switchyard-py/Cargo.toml b/crates/switchyard-py/Cargo.toml index 6f5bc973..90d2e76f 100644 --- a/crates/switchyard-py/Cargo.toml +++ b/crates/switchyard-py/Cargo.toml @@ -25,9 +25,8 @@ pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] } pythonize = "0.28.0" serde.workspace = true serde_json.workspace = true -switchyard-core.workspace = true switchyard-protocol.workspace = true -switchyard-components = { path = "../switchyard-components" } +switchyard-components.workspace = true switchyard-translation.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/switchyard-py/src/component_bindings/backends.rs b/crates/switchyard-py/src/component_bindings/backends.rs index f299a2b9..6551042d 100644 --- a/crates/switchyard-py/src/component_bindings/backends.rs +++ b/crates/switchyard-py/src/component_bindings/backends.rs @@ -12,13 +12,13 @@ use switchyard_components::{ AnthropicNativeBackend, LlmTargetBackend, MultiLlmBackend, OpenAiNativeBackend, OpenAiPassthroughBackend, StatsLlmBackend, }; -use switchyard_core::{ChatRequestType, EndpointConfig, LlmBackend, LlmTargetId}; +use switchyard_components::{ChatRequestType, EndpointConfig, LlmBackend, LlmTargetId}; use super::config::{endpoint_config_from_python, PyEndpointConfig, PyLlmTarget}; use super::stats::PyStatsAccumulator; -use crate::core_bindings::request::request_type_from_python; -use crate::core_bindings::roles::PyLlmBackend; use crate::errors::py_core_error; +use crate::interop::request::request_type_from_python; +use crate::interop::roles::PyLlmBackend; #[pyclass(name = "LlmTargetBackend", frozen, skip_from_py_object)] #[derive(Clone, Debug)] diff --git a/crates/switchyard-py/src/component_bindings/config.rs b/crates/switchyard-py/src/component_bindings/config.rs index dd576aed..f37eaafd 100644 --- a/crates/switchyard-py/src/component_bindings/config.rs +++ b/crates/switchyard-py/src/component_bindings/config.rs @@ -8,11 +8,11 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyBool; use serde::Serialize; +use switchyard_components::{BackendFormat, EndpointConfig, LlmTarget, LlmTargetId, ModelId}; use switchyard_components::{ IntakeFormat, IntakeQueueFullPolicy, IntakeSinkConfig, IntakeTarget, RandomRoutingProcessorConfig, }; -use switchyard_core::{BackendFormat, EndpointConfig, LlmTarget, LlmTargetId, ModelId}; use crate::errors::py_core_error; use crate::py_serde::{value_from_python, value_to_python}; diff --git a/crates/switchyard-py/src/component_bindings/dimension_collector.rs b/crates/switchyard-py/src/component_bindings/dimension_collector.rs index 8e161ddb..5ba7a763 100644 --- a/crates/switchyard-py/src/component_bindings/dimension_collector.rs +++ b/crates/switchyard-py/src/component_bindings/dimension_collector.rs @@ -17,6 +17,7 @@ use pyo3::prelude::*; use pyo3::types::PyList; +use switchyard_components::ChatResponse; use switchyard_components::{ dimension_collector::{ extract_response_signals as core_extract_response_signals, ResponseFlag, ResponseSignals, @@ -24,13 +25,12 @@ use switchyard_components::{ }, DimensionCollector, ResponseSignalCollector, }; -use switchyard_core::ChatResponse; use crate::py_serde::value_from_python; -use crate::core_bindings::context::PyProxyContext; -use crate::core_bindings::request::PyChatRequest; -use crate::core_bindings::response::PyChatResponse; +use crate::interop::context::{get_cloned_from_python, lease_from_python}; +use crate::interop::request::{request_from_python, request_to_python}; +use crate::interop::response::{response_from_python, response_to_python}; /// Request-side component that runs the dimension collector. #[pyclass(name = "DimensionCollector", skip_from_py_object)] @@ -61,20 +61,18 @@ impl PyDimensionCollector { fn process<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - request: PyRef<'_, PyChatRequest>, + ctx: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, ) -> PyResult> { let processor = self.inner.clone(); - let mut lease = ctx.lease()?; - let request = request.clone_core(); + let mut lease = lease_from_python(ctx)?; + let request = request_from_python(request)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = processor.process(lease.context_mut()?, request).await; let restore_result = lease.restore(); let request = result.map_err(crate::errors::py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatRequest::from_core(request)).map(|request| request.into_any()) - }) + Python::attach(|py| request_to_python(py, request)) }) } @@ -196,11 +194,11 @@ impl PyResponseSignalCollector { fn process<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - mut response: PyRefMut<'_, PyChatResponse>, + ctx: &Bound<'_, PyAny>, + response: &Bound<'_, PyAny>, ) -> PyResult> { - let mut lease = ctx.lease()?; - let response = response.take_core(py)?; + let mut lease = lease_from_python(ctx)?; + let response = response_from_python(response)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = ResponseSignalCollector .process(lease.context_mut()?, response) @@ -208,10 +206,7 @@ impl PyResponseSignalCollector { let restore_result = lease.restore(); let response = result.map_err(crate::errors::py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatResponse::from_core(py, response)?) - .map(|response| response.into_any()) - }) + Python::attach(|py| response_to_python(py, response)) }) } @@ -226,10 +221,8 @@ impl PyResponseSignalCollector { /// response was a streaming response (which the buffered-body checks /// can't introspect). #[pyfunction] -fn get_response_signals(ctx: PyRef<'_, PyProxyContext>) -> PyResult> { - Ok(ctx - .get_cloned::()? - .map(PyResponseSignals::from_core)) +fn get_response_signals(ctx: &Bound<'_, PyAny>) -> PyResult> { + Ok(get_cloned_from_python::(ctx)?.map(PyResponseSignals::from_core)) } /// Runs the response-side checks against an inline response body dict. @@ -388,10 +381,8 @@ impl PyToolResultSignal { /// /// Returns ``None`` when the collector has not run on this context yet. #[pyfunction] -fn get_tool_result_signal(ctx: PyRef<'_, PyProxyContext>) -> PyResult> { - Ok(ctx - .get_cloned::()? - .map(PyToolResultSignal::from_core)) +fn get_tool_result_signal(ctx: &Bound<'_, PyAny>) -> PyResult> { + Ok(get_cloned_from_python::(ctx)?.map(PyToolResultSignal::from_core)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/switchyard-py/src/component_bindings/intake.rs b/crates/switchyard-py/src/component_bindings/intake.rs index 811f3dad..6cbe3634 100644 --- a/crates/switchyard-py/src/component_bindings/intake.rs +++ b/crates/switchyard-py/src/component_bindings/intake.rs @@ -12,7 +12,7 @@ use pyo3::types::{PyBool, PyDict, PyType}; use serde::Serialize; use switchyard_components::{IntakeRequestMetadata, RequestMetadata}; -use crate::core_bindings::context::PyProxyContext; +use crate::interop::context::insert_into_python; use crate::py_serde::value_to_python; #[pyclass(name = "IntakeRequestMetadata", frozen, skip_from_py_object)] @@ -146,8 +146,8 @@ impl PyRequestMetadata { to_python(py, &self.inner) } - fn apply_to_context(&self, ctx: PyRef<'_, PyProxyContext>) -> PyResult<()> { - ctx.insert_value(self.clone_core()) + fn apply_to_context(&self, ctx: &Bound<'_, PyAny>) -> PyResult<()> { + insert_into_python(ctx, self.clone_core()) } fn __richcmp__( diff --git a/crates/switchyard-py/src/component_bindings/request_processors.rs b/crates/switchyard-py/src/component_bindings/request_processors.rs index 8c68725a..50a4fa58 100644 --- a/crates/switchyard-py/src/component_bindings/request_processors.rs +++ b/crates/switchyard-py/src/component_bindings/request_processors.rs @@ -8,9 +8,9 @@ use switchyard_components::{ tracking_enabled_from_env, IntakeRequestProcessor, StatsRequestProcessor, }; -use crate::core_bindings::context::PyProxyContext; -use crate::core_bindings::request::PyChatRequest; use crate::errors::py_core_error; +use crate::interop::context::lease_from_python; +use crate::interop::request::{request_from_python, request_to_python}; #[pyclass(name = "StatsRequestProcessor", skip_from_py_object)] #[derive(Clone, Copy, Debug)] @@ -38,20 +38,18 @@ impl PyStatsRequestProcessor { fn process<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - request: PyRef<'_, PyChatRequest>, + ctx: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, ) -> PyResult> { let processor = self.inner; - let mut lease = ctx.lease()?; - let request = request.clone_core(); + let mut lease = lease_from_python(ctx)?; + let request = request_from_python(request)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = processor.process(lease.context_mut()?, request).await; let restore_result = lease.restore(); let request = result.map_err(py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatRequest::from_core(request)).map(|request| request.into_any()) - }) + Python::attach(|py| request_to_python(py, request)) }) } @@ -82,11 +80,11 @@ impl PyIntakeRequestProcessor { fn process<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - request: PyRef<'_, PyChatRequest>, + ctx: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, ) -> PyResult> { - let mut lease = ctx.lease()?; - let request = request.clone_core(); + let mut lease = lease_from_python(ctx)?; + let request = request_from_python(request)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = IntakeRequestProcessor .process(lease.context_mut()?, request) @@ -94,9 +92,7 @@ impl PyIntakeRequestProcessor { let restore_result = lease.restore(); let request = result.map_err(py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatRequest::from_core(request)).map(|request| request.into_any()) - }) + Python::attach(|py| request_to_python(py, request)) }) } diff --git a/crates/switchyard-py/src/component_bindings/response_processors.rs b/crates/switchyard-py/src/component_bindings/response_processors.rs index 41fd8765..f8072ac4 100644 --- a/crates/switchyard-py/src/component_bindings/response_processors.rs +++ b/crates/switchyard-py/src/component_bindings/response_processors.rs @@ -8,9 +8,9 @@ use switchyard_components::{IntakeResponseProcessor, StatsResponseProcessor}; use super::config::PyIntakeSinkConfig; use super::stats::PyStatsAccumulator; -use crate::core_bindings::context::PyProxyContext; -use crate::core_bindings::response::PyChatResponse; use crate::errors::py_core_error; +use crate::interop::context::lease_from_python; +use crate::interop::response::{response_from_python, response_to_python}; #[pyclass(name = "StatsResponseProcessor", skip_from_py_object)] #[derive(Clone, Debug)] @@ -46,21 +46,18 @@ impl PyStatsResponseProcessor { fn process<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - mut response: PyRefMut<'_, PyChatResponse>, + ctx: &Bound<'_, PyAny>, + response: &Bound<'_, PyAny>, ) -> PyResult> { let processor = self.inner.clone(); - let mut lease = ctx.lease()?; - let response = response.take_core(py)?; + let mut lease = lease_from_python(ctx)?; + let response = response_from_python(response)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = processor.process(lease.context_mut()?, response).await; let restore_result = lease.restore(); let response = result.map_err(py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatResponse::from_core(py, response)?) - .map(|response| response.into_any()) - }) + Python::attach(|py| response_to_python(py, response)) }) } @@ -113,21 +110,18 @@ impl PyIntakeResponseProcessor { fn process<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - mut response: PyRefMut<'_, PyChatResponse>, + ctx: &Bound<'_, PyAny>, + response: &Bound<'_, PyAny>, ) -> PyResult> { let processor = self.inner.clone(); - let mut lease = ctx.lease()?; - let response = response.take_core(py)?; + let mut lease = lease_from_python(ctx)?; + let response = response_from_python(response)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = processor.process(lease.context_mut()?, response).await; let restore_result = lease.restore(); let response = result.map_err(py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatResponse::from_core(py, response)?) - .map(|response| response.into_any()) - }) + Python::attach(|py| response_to_python(py, response)) }) } diff --git a/crates/switchyard-py/src/component_bindings/stats.rs b/crates/switchyard-py/src/component_bindings/stats.rs index c0b09263..08a7023f 100644 --- a/crates/switchyard-py/src/component_bindings/stats.rs +++ b/crates/switchyard-py/src/component_bindings/stats.rs @@ -8,8 +8,8 @@ use pyo3::prelude::*; use serde::Serialize; use switchyard_components::{StatsAccumulator, StatsRouteLabel, TokenUsage}; -use crate::core_bindings::context::PyProxyContext; use crate::errors::py_core_error; +use crate::interop::context::insert_into_python; use crate::py_serde::value_to_python; #[pyclass(name = "StatsAccumulator", skip_from_py_object)] @@ -226,12 +226,12 @@ fn to_python(py: Python<'_>, value: &impl Serialize) -> PyResult> { } #[pyfunction] -fn set_stats_route_label(ctx: PyRef<'_, PyProxyContext>, label: &str) -> PyResult<()> { +fn set_stats_route_label(ctx: &Bound<'_, PyAny>, label: &str) -> PyResult<()> { let label = label.trim(); if label.is_empty() { return Err(PyValueError::new_err("stats route label must not be empty")); } - ctx.insert_value(StatsRouteLabel::new(label)) + insert_into_python(ctx, StatsRouteLabel::new(label)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/switchyard-py/src/core_bindings/request.rs b/crates/switchyard-py/src/core_bindings/request.rs deleted file mode 100644 index 171e7399..00000000 --- a/crates/switchyard-py/src/core_bindings/request.rs +++ /dev/null @@ -1,202 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for Rust-owned chat request values. - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyType; -use switchyard_core::{ChatRequest, ChatRequestType}; - -use crate::errors::py_core_error; -use crate::py_serde::{value_from_python, value_to_python}; - -#[pyclass(name = "ChatRequestType", frozen, eq, skip_from_py_object)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct PyChatRequestType { - inner: ChatRequestType, -} - -impl PyChatRequestType { - const fn new(inner: ChatRequestType) -> Self { - Self { inner } - } -} - -#[pymethods] -impl PyChatRequestType { - #[classattr] - const OPENAI_CHAT: Self = Self::new(ChatRequestType::OpenAiChat); - - #[classattr] - const OPENAI_RESPONSES: Self = Self::new(ChatRequestType::OpenAiResponses); - - #[classattr] - const ANTHROPIC: Self = Self::new(ChatRequestType::Anthropic); - - #[getter] - fn value(&self) -> &'static str { - request_type_name(self.inner) - } - - fn __repr__(&self) -> String { - format!("ChatRequestType.{}", request_type_variant_name(self.inner)) - } - - fn __str__(&self) -> &'static str { - request_type_name(self.inner) - } - - fn __hash__(&self) -> isize { - match self.inner { - ChatRequestType::OpenAiChat => 1, - ChatRequestType::OpenAiResponses => 2, - ChatRequestType::Anthropic => 3, - } - } -} - -#[pyclass(name = "ChatRequest")] -#[derive(Debug, PartialEq)] -pub(crate) struct PyChatRequest { - inner: ChatRequest, -} - -impl PyChatRequest { - pub(crate) fn from_core(inner: ChatRequest) -> Self { - Self { inner } - } - - pub(crate) fn clone_core(&self) -> ChatRequest { - self.inner.clone() - } -} - -#[pymethods] -impl PyChatRequest { - #[classmethod] - fn openai_chat(_cls: &Bound<'_, PyType>, body: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - inner: ChatRequest::openai_chat(value_from_python(body)?), - }) - } - - #[classmethod] - fn openai_responses(_cls: &Bound<'_, PyType>, body: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - inner: ChatRequest::openai_responses(value_from_python(body)?), - }) - } - - #[classmethod] - fn anthropic(_cls: &Bound<'_, PyType>, body: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - inner: ChatRequest::anthropic(value_from_python(body)?), - }) - } - - /// Validates the request at the inbound trust boundary. - /// - /// Kept separate from construction so internal re-wraps (e.g. the - /// translation engine rebuilding a request after format conversion) are - /// not subject to inbound-only checks. Endpoints call this explicitly on - /// the client-supplied request; raises ``SwitchyardInvalidRequestError`` - /// for a present-but-empty ``messages`` array. - fn validate(&self) -> PyResult<()> { - self.inner.validate().map_err(py_core_error) - } - - #[getter] - fn request_type(&self, py: Python<'_>) -> PyResult> { - request_type_object(py, self.inner.request_type()) - } - - #[getter] - fn body(&self, py: Python<'_>) -> PyResult> { - self.to_body(py) - } - - #[getter] - fn model(&self) -> Option { - self.inner.model().map(str::to_owned) - } - - fn set_model(&mut self, model: &str) { - self.inner.set_model(model); - } - - fn replace_body(&mut self, body: &Bound<'_, PyAny>) -> PyResult<()> { - let body = value_from_python(body)?; - self.inner = match self.inner.request_type() { - ChatRequestType::OpenAiChat => ChatRequest::openai_chat(body), - ChatRequestType::OpenAiResponses => ChatRequest::openai_responses(body), - ChatRequestType::Anthropic => ChatRequest::anthropic(body), - }; - Ok(()) - } - - fn to_body(&self, py: Python<'_>) -> PyResult> { - value_to_python(py, self.inner.body()) - } - - fn __repr__(&self) -> String { - match self.inner.model() { - Some(model) => format!( - "ChatRequest(request_type='{}', model='{}')", - request_type_name(self.inner.request_type()), - model - ), - None => format!( - "ChatRequest(request_type='{}')", - request_type_name(self.inner.request_type()) - ), - } - } -} - -pub(crate) fn request_type_from_python(value: &Bound<'_, PyAny>) -> PyResult { - let raw = if let Ok(value_attr) = value.getattr("value") { - value_attr.extract::()? - } else { - value.extract::()? - }; - match raw.as_str() { - "openai_chat" => Ok(ChatRequestType::OpenAiChat), - "openai_responses" => Ok(ChatRequestType::OpenAiResponses), - "anthropic" | "anthropic_messages" => Ok(ChatRequestType::Anthropic), - _ => Err(PyValueError::new_err(format!( - "Unknown request type: {raw:?}" - ))), - } -} - -pub(crate) fn request_type_name(request_type: ChatRequestType) -> &'static str { - match request_type { - ChatRequestType::OpenAiChat => "openai_chat", - ChatRequestType::OpenAiResponses => "openai_responses", - ChatRequestType::Anthropic => "anthropic", - } -} - -pub(crate) fn request_type_variant_name(request_type: ChatRequestType) -> &'static str { - match request_type { - ChatRequestType::OpenAiChat => "OPENAI_CHAT", - ChatRequestType::OpenAiResponses => "OPENAI_RESPONSES", - ChatRequestType::Anthropic => "ANTHROPIC", - } -} - -pub(crate) fn request_type_object( - py: Python<'_>, - request_type: ChatRequestType, -) -> PyResult> { - py.get_type::() - .getattr(request_type_variant_name(request_type)) - .map(Bound::unbind) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - Ok(()) -} diff --git a/crates/switchyard-py/src/core_bindings/session.rs b/crates/switchyard-py/src/core_bindings/session.rs deleted file mode 100644 index f5114ee3..00000000 --- a/crates/switchyard-py/src/core_bindings/session.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Python bindings for Rust-owned session-affinity primitives. - -use pyo3::prelude::*; - -use crate::py_serde::value_from_python; - -/// Derives a stable session key from a request body mapping. -/// -/// Accepts any Python mapping (e.g. a request body dict) and returns the key -/// the core derivation produces for it, used to pin requests of one session to -/// a previously selected model. `depth == 0` (default) always yields a key; -/// `depth > 0` extends the hashed prefix with the first `depth` post-first-user -/// messages and returns `None` until that prefix exists. -#[pyfunction] -#[pyo3(signature = (body, depth = 0))] -pub(crate) fn session_key_from_body( - body: &Bound<'_, PyAny>, - depth: usize, -) -> PyResult> { - Ok(switchyard_core::session_key_from_body_with_depth( - &value_from_python(body)?, - depth, - )) -} - -/// LRU-bounded cache mapping session keys to arbitrary Python values. -/// -/// Stores `Py` so any router can reuse it; the Python consumer keeps -/// `str` model-ids today. `get` refreshes recency, so it requires `&mut self`. -#[pyclass(name = "SessionCache")] -pub(crate) struct PySessionCache { - /// Core LRU cache holding GIL-bound Python references. - inner: switchyard_core::SessionCache>, -} - -#[pymethods] -impl PySessionCache { - /// Creates a cache bounded to at most `max_sessions` entries. - #[new] - fn new(max_sessions: usize) -> Self { - Self { - inner: switchyard_core::SessionCache::new(max_sessions), - } - } - - /// Returns the value for `key`, refreshing its recency, or `None`. - fn get(&mut self, py: Python<'_>, key: &str) -> Option> { - self.inner.get(key).map(|value| value.clone_ref(py)) - } - - /// Inserts or updates the value for `key`, evicting the oldest if full. - fn put(&mut self, key: String, value: Py) { - self.inner.put(key, value); - } - - /// Returns all cached values (order unspecified). - fn values(&self, py: Python<'_>) -> Vec> { - self.inner - .values() - .map(|value| value.clone_ref(py)) - .collect() - } - - /// Returns the configured maximum number of sessions. - #[getter] - fn max_sessions(&self) -> usize { - self.inner.max_sessions() - } - - /// Returns the current number of cached sessions. - fn __len__(&self) -> usize { - self.inner.len() - } -} - -/// Registers session-affinity bindings into the Python module. -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(session_key_from_body, module)?)?; - Ok(()) -} diff --git a/crates/switchyard-py/src/errors.rs b/crates/switchyard-py/src/errors.rs index 5eccc784..99ab5ed8 100644 --- a/crates/switchyard-py/src/errors.rs +++ b/crates/switchyard-py/src/errors.rs @@ -6,7 +6,7 @@ use pyo3::create_exception; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use switchyard_core::SwitchyardError; +use switchyard_components::SwitchyardError; use switchyard_translation::TranslationError; create_exception!(_switchyard_rust, SwitchyardRuntimeError, PyRuntimeError); diff --git a/crates/switchyard-py/src/core_bindings.rs b/crates/switchyard-py/src/interop.rs similarity index 79% rename from crates/switchyard-py/src/core_bindings.rs rename to crates/switchyard-py/src/interop.rs index fece0fe6..2088ed48 100644 --- a/crates/switchyard-py/src/core_bindings.rs +++ b/crates/switchyard-py/src/interop.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! PyO3 bindings for `switchyard-core`. +//! Private adapters between Python compatibility objects and native components. use pyo3::prelude::*; @@ -9,15 +9,12 @@ pub(crate) mod context; pub(crate) mod request; pub(crate) mod response; pub(crate) mod roles; -pub(crate) mod session; pub(crate) mod subagent; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { context::register(module)?; - request::register(module)?; response::register(module)?; roles::register(module)?; - session::register(module)?; subagent::register(module)?; Ok(()) } diff --git a/crates/switchyard-py/src/core_bindings/context.rs b/crates/switchyard-py/src/interop/context.rs similarity index 62% rename from crates/switchyard-py/src/core_bindings/context.rs rename to crates/switchyard-py/src/interop/context.rs index 1b361f81..7fa9eeae 100644 --- a/crates/switchyard-py/src/core_bindings/context.rs +++ b/crates/switchyard-py/src/interop/context.rs @@ -1,291 +1,32 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Python bindings for Rust-owned request context values. +//! Private adapter for Rust-owned request context values. -use std::collections::BTreeMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use parking_lot::{Mutex, MutexGuard}; -use pyo3::class::basic::CompareOp; -use pyo3::exceptions::{PyKeyError, PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; -use pyo3::types::{PyBool, PyDict}; +use pyo3::types::PyDict; use std::time::Duration; use switchyard_components::{ BackendSelection, BackendSelectionReason, StatsBackendLatency, SubModelCall, SubModelCalls, }; -use switchyard_core::{EvictedTargets, LlmTargetId, ModelId, ProxyContext, RequestId}; +use switchyard_components::{EvictedTargets, LlmTargetId, ModelId, ProxyContext, RequestId}; use crate::component_bindings::intake::request_metadata_from_mapping; use super::request::{request_type_from_python, request_type_object}; -/// Python-owned metadata values kept beside the Rust proxy context. -#[derive(Default)] -struct MetadataStore { - /// Arbitrary metadata values keyed by Python string. - values: BTreeMap>, -} - -/// Cloneable handle to the shared Python metadata store. -#[derive(Clone)] -pub(crate) struct PyProxyMetadataStore { - /// Shared metadata storage used by context clones. - inner: Arc>, -} - -/// Dict-like Python object used for `ProxyContext.metadata`. -#[pyclass(name = "ProxyMetadata")] -struct PyProxyMetadata { - /// Shared metadata storage protected across Python/Rust calls. - inner: Arc>, -} - -impl PyProxyMetadata { - /// Creates an empty metadata object. - fn new() -> Self { - Self::from_store(PyProxyMetadataStore::default()) - } - - /// Creates a metadata object from an existing shared store. - fn from_store(store: PyProxyMetadataStore) -> Self { - Self { inner: store.inner } - } - - /// Returns the cloneable store handle for sharing with `ProxyContext`. - fn metadata_store(&self) -> PyProxyMetadataStore { - PyProxyMetadataStore { - inner: Arc::clone(&self.inner), - } - } - - /// Builds metadata from any Python mapping accepted by `dict.update`. - fn from_mapping(py: Python<'_>, metadata: &Bound<'_, PyAny>) -> PyResult { - let store = Self::new(); - store.update_from_mapping(py, metadata)?; - Ok(store) - } - - /// Locks the metadata store. - fn lock(&self) -> MutexGuard<'_, MetadataStore> { - self.inner.lock() - } - - /// Merges entries from a Python mapping into the metadata store. - fn update_from_mapping(&self, py: Python<'_>, metadata: &Bound<'_, PyAny>) -> PyResult<()> { - let entries = metadata_entries(py, metadata)?; - let mut store = self.lock(); - store.values.extend(entries); - Ok(()) - } - - /// Materializes the metadata store as a Python dict. - fn to_dict(&self, py: Python<'_>) -> PyResult> { - let entries = { - let store = self.lock(); - store - .values - .iter() - .map(|(key, value)| (key.clone(), value.clone_ref(py))) - .collect::>() - }; - let dict = PyDict::new(py); - for (key, value) in entries { - dict.set_item(key, value)?; - } - Ok(dict.unbind()) - } -} - -impl Default for PyProxyMetadataStore { - fn default() -> Self { - Self { - inner: Arc::new(Mutex::new(MetadataStore::default())), - } - } -} - -#[pymethods] -impl PyProxyMetadata { - /// Creates metadata from an optional mapping. - #[new] - #[pyo3(signature = (metadata=None))] - fn py_new(py: Python<'_>, metadata: Option<&Bound<'_, PyAny>>) -> PyResult { - let store = Self::new(); - if let Some(metadata) = metadata { - if !metadata.is_none() { - store.update_from_mapping(py, metadata)?; - } - } - Ok(store) - } - - /// Implements `dict.get`. - #[pyo3(signature = (key, default=None))] - fn get( - &self, - py: Python<'_>, - key: &str, - default: Option<&Bound<'_, PyAny>>, - ) -> PyResult> { - let value = self.lock().values.get(key).map(|value| value.clone_ref(py)); - match value { - Some(value) => Ok(value), - None => Ok(default - .map(|value| value.clone().unbind()) - .unwrap_or_else(|| py.None())), - } - } - - /// Implements `dict.setdefault`. - #[pyo3(signature = (key, default=None))] - fn setdefault( - &self, - py: Python<'_>, - key: String, - default: Option<&Bound<'_, PyAny>>, - ) -> PyResult> { - let default = default - .map(|value| value.clone().unbind()) - .unwrap_or_else(|| py.None()); - let mut store = self.lock(); - if let Some(value) = store.values.get(&key) { - return Ok(value.clone_ref(py)); - } - store.values.insert(key, default.clone_ref(py)); - Ok(default) - } - - /// Implements `dict.update`. - fn update(&self, py: Python<'_>, metadata: &Bound<'_, PyAny>) -> PyResult<()> { - self.update_from_mapping(py, metadata) - } - - /// Returns a shallow Python dict copy. - fn copy(&self, py: Python<'_>) -> PyResult> { - self.to_dict(py) - } - - /// Returns a Python keys view. - fn keys(&self, py: Python<'_>) -> PyResult> { - self.to_dict(py)? - .bind(py) - .call_method0("keys") - .map(Bound::unbind) - } - - /// Returns a Python values view. - fn values(&self, py: Python<'_>) -> PyResult> { - self.to_dict(py)? - .bind(py) - .call_method0("values") - .map(Bound::unbind) - } - - /// Returns a Python items view. - fn items(&self, py: Python<'_>) -> PyResult> { - self.to_dict(py)? - .bind(py) - .call_method0("items") - .map(Bound::unbind) - } - - /// Implements metadata indexing. - fn __getitem__(&self, py: Python<'_>, key: &str) -> PyResult> { - self.lock() - .values - .get(key) - .map(|value| value.clone_ref(py)) - .ok_or_else(|| PyKeyError::new_err(key.to_string())) - } - - /// Implements metadata assignment. - fn __setitem__(&self, key: String, value: &Bound<'_, PyAny>) -> PyResult<()> { - self.lock().values.insert(key, value.clone().unbind()); - Ok(()) - } - - /// Implements metadata deletion. - fn __delitem__(&self, key: &str) -> PyResult<()> { - self.lock() - .values - .remove(key) - .map(|_| ()) - .ok_or_else(|| PyKeyError::new_err(key.to_string())) - } - - /// Implements `key in metadata`. - fn __contains__(&self, key: &str) -> PyResult { - Ok(self.lock().values.contains_key(key)) - } - - /// Implements `len(metadata)`. - fn __len__(&self) -> PyResult { - Ok(self.lock().values.len()) - } - - /// Iterates over metadata keys like a Python dict. - fn __iter__(&self, py: Python<'_>) -> PyResult> { - self.keys(py)? - .bind(py) - .call_method0("__iter__") - .map(Bound::unbind) - } - - /// Compares metadata by Python dict equality semantics. - fn __richcmp__( - &self, - py: Python<'_>, - other: &Bound<'_, PyAny>, - op: CompareOp, - ) -> PyResult> { - match op { - CompareOp::Eq | CompareOp::Ne => { - let equals = self - .to_dict(py)? - .bind(py) - .rich_compare(other, CompareOp::Eq)? - .extract::()?; - let result = if matches!(op, CompareOp::Eq) { - equals - } else { - !equals - }; - Ok(PyBool::new(py, result).to_owned().unbind().into_any()) - } - _ => Ok(py.NotImplemented()), - } - } - - /// Returns the Python dict representation. - fn __repr__(&self, py: Python<'_>) -> PyResult { - self.to_dict(py)?.bind(py).repr()?.extract() - } -} - -/// Extracts string-keyed entries from any mapping accepted by `dict.update`. -fn metadata_entries( - py: Python<'_>, - metadata: &Bound<'_, PyAny>, -) -> PyResult)>> { - let dict = PyDict::new(py); - dict.call_method1("update", (metadata,))?; - dict.iter() - .map(|(key, value)| Ok((key.extract::()?, value.clone().unbind()))) - .collect() -} - /// Python-facing proxy context backed by the Rust `ProxyContext`. -#[pyclass(name = "ProxyContext", skip_from_py_object)] +#[pyclass(name = "_NativeProxyContext", skip_from_py_object)] pub(crate) struct PyProxyContext { /// Shared Rust context guarded across Python and async Rust calls. inner: Arc>, /// Borrow flag that prevents Python mutation while Rust owns the context. in_use: Arc, - /// Dict-like compatibility metadata store. - metadata: Py, } impl PyProxyContext { @@ -338,6 +79,36 @@ impl PyProxyContext { } } +/// Leases the native context carried by a Python `ProxyContext`. +pub(crate) fn lease_from_python(value: &Bound<'_, PyAny>) -> PyResult { + value + .getattr("_native")? + .extract::>()? + .lease() +} + +/// Inserts a typed value into a Python context's native state. +pub(crate) fn insert_into_python(value: &Bound<'_, PyAny>, item: T) -> PyResult<()> +where + T: Send + Sync + 'static, +{ + value + .getattr("_native")? + .extract::>()? + .insert_value(item) +} + +/// Reads a typed value from a Python context's native state. +pub(crate) fn get_cloned_from_python(value: &Bound<'_, PyAny>) -> PyResult> +where + T: Clone + Send + Sync + 'static, +{ + value + .getattr("_native")? + .extract::>()? + .get_cloned::() +} + /// Temporary ownership lease for passing `ProxyContext` into async Rust roles. pub(crate) struct PyProxyContextLease { /// Shared storage that receives the context when the lease is restored. @@ -391,13 +162,6 @@ impl PyProxyContext { Some(metadata) if !metadata.is_none() => request_metadata_from_mapping(py, metadata)?, _ => None, }; - let metadata = match metadata { - Some(metadata) if !metadata.is_none() => { - Py::new(py, PyProxyMetadata::from_mapping(py, metadata)?)? - } - _ => Py::new(py, PyProxyMetadata::new())?, - }; - let request_id = request_id .map(RequestId::new) .transpose() @@ -407,8 +171,6 @@ impl PyProxyContext { let mut inner = ProxyContext::default(); inner.request_id = request_id; - let metadata_store = metadata.borrow(py).metadata_store(); - inner.insert(metadata_store); if let Some(request_metadata) = request_metadata { inner.insert(request_metadata); } @@ -416,16 +178,9 @@ impl PyProxyContext { Ok(Self { inner: Arc::new(Mutex::new(inner)), in_use: Arc::new(AtomicBool::new(false)), - metadata, }) } - /// Returns the dict-like metadata object. - #[getter] - fn metadata(&self, py: Python<'_>) -> Py { - self.metadata.clone_ref(py) - } - /// Records one routing-strategy sub-model call for intake capture. /// /// Routers that call a model to pick a route (the LLM classifier, the @@ -677,7 +432,6 @@ impl PyProxyContext { /// Registers proxy context bindings into the Python module. pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; module.add_class::()?; Ok(()) } diff --git a/crates/switchyard-py/src/interop/request.rs b/crates/switchyard-py/src/interop/request.rs new file mode 100644 index 00000000..376d3272 --- /dev/null +++ b/crates/switchyard-py/src/interop/request.rs @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Private adapter for Rust-owned chat request values. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use switchyard_components::{ChatRequest, ChatRequestType}; + +use crate::py_serde::{value_from_python, value_to_python}; + +/// Converts a public Python request into the native request value. +pub(crate) fn request_from_python(value: &Bound<'_, PyAny>) -> PyResult { + let request_type = request_type_from_python(&value.getattr("request_type")?)?; + let body = value_from_python(&value.getattr("_body")?)?; + Ok(match request_type { + ChatRequestType::OpenAiChat => ChatRequest::openai_chat(body), + ChatRequestType::OpenAiResponses => ChatRequest::openai_responses(body), + ChatRequestType::Anthropic => ChatRequest::anthropic(body), + }) +} + +/// Converts a native request into the public Python request value. +pub(crate) fn request_to_python(py: Python<'_>, request: ChatRequest) -> PyResult> { + let request_type = request.request_type(); + let body = value_to_python(py, request.body())?; + let factory = match request_type { + ChatRequestType::OpenAiChat => "openai_chat", + ChatRequestType::OpenAiResponses => "openai_responses", + ChatRequestType::Anthropic => "anthropic", + }; + py.import("switchyard_rust.core")? + .getattr("ChatRequest")? + .call_method1(factory, (body,)) + .map(Bound::unbind) +} + +pub(crate) fn request_type_from_python(value: &Bound<'_, PyAny>) -> PyResult { + let raw = if let Ok(value_attr) = value.getattr("value") { + value_attr.extract::()? + } else { + value.extract::()? + }; + match raw.as_str() { + "openai_chat" => Ok(ChatRequestType::OpenAiChat), + "openai_responses" => Ok(ChatRequestType::OpenAiResponses), + "anthropic" | "anthropic_messages" => Ok(ChatRequestType::Anthropic), + _ => Err(PyValueError::new_err(format!( + "Unknown request type: {raw:?}" + ))), + } +} + +pub(crate) fn request_type_variant_name(request_type: ChatRequestType) -> &'static str { + match request_type { + ChatRequestType::OpenAiChat => "OPENAI_CHAT", + ChatRequestType::OpenAiResponses => "OPENAI_RESPONSES", + ChatRequestType::Anthropic => "ANTHROPIC", + } +} + +pub(crate) fn request_type_object( + py: Python<'_>, + request_type: ChatRequestType, +) -> PyResult> { + py.import("switchyard_rust.core")? + .getattr("ChatRequestType")? + .getattr(request_type_variant_name(request_type)) + .map(Bound::unbind) +} diff --git a/crates/switchyard-py/src/core_bindings/response.rs b/crates/switchyard-py/src/interop/response.rs similarity index 61% rename from crates/switchyard-py/src/core_bindings/response.rs rename to crates/switchyard-py/src/interop/response.rs index e9a02ede..f8d31fbc 100644 --- a/crates/switchyard-py/src/core_bindings/response.rs +++ b/crates/switchyard-py/src/interop/response.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Python bindings for Rust-owned chat response values. +//! Private adapter for Rust-owned chat response values. use std::pin::Pin; use std::sync::atomic::{AtomicBool, Ordering}; @@ -10,311 +10,103 @@ use std::task::{Context, Poll}; use futures_util::{stream, Stream, StreamExt}; use parking_lot::Mutex; -use pyo3::exceptions::{PyAttributeError, PyRuntimeError, PyStopAsyncIteration, PyValueError}; +use pyo3::exceptions::{PyRuntimeError, PyStopAsyncIteration, PyValueError}; use pyo3::prelude::*; -use pyo3::types::PyType; -use serde_json::Value; -use switchyard_core::{BoxResponseStream, ChatResponse, ChatResponseType, StreamEvent}; +use switchyard_components::{BoxResponseStream, ChatResponse, ChatResponseType, StreamEvent}; use crate::errors::py_core_error; use crate::py_serde::{value_from_python, value_to_python}; -#[pyclass(name = "ChatResponseType", frozen, eq, skip_from_py_object)] -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -struct PyChatResponseType { - inner: ChatResponseType, -} - -impl PyChatResponseType { - const fn new(inner: ChatResponseType) -> Self { - Self { inner } - } -} - -#[pymethods] -impl PyChatResponseType { - #[classattr] - const OPENAI_COMPLETION: Self = Self::new(ChatResponseType::OpenAiCompletion); - - #[classattr] - const OPENAI_STREAM: Self = Self::new(ChatResponseType::OpenAiStream); - - #[classattr] - const OPENAI_RESPONSES_COMPLETION: Self = - Self::new(ChatResponseType::OpenAiResponsesCompletion); - - #[classattr] - const OPENAI_RESPONSES_STREAM: Self = Self::new(ChatResponseType::OpenAiResponsesStream); - - #[classattr] - const ANTHROPIC_COMPLETION: Self = Self::new(ChatResponseType::AnthropicCompletion); - - #[classattr] - const ANTHROPIC_STREAM: Self = Self::new(ChatResponseType::AnthropicStream); - - #[getter] - fn value(&self) -> &'static str { - response_type_name(self.inner) - } - - fn __repr__(&self) -> String { - format!( - "ChatResponseType.{}", - response_type_variant_name(self.inner) - ) - } - - fn __str__(&self) -> &'static str { - response_type_name(self.inner) - } - - fn __hash__(&self) -> isize { - match self.inner { - ChatResponseType::OpenAiCompletion => 1, - ChatResponseType::OpenAiStream => 2, - ChatResponseType::OpenAiResponsesCompletion => 3, - ChatResponseType::OpenAiResponsesStream => 4, - ChatResponseType::AnthropicCompletion => 5, - ChatResponseType::AnthropicStream => 6, +/// Converts a public Python response into the native response value. +pub(crate) fn response_from_python(value: &Bound<'_, PyAny>) -> PyResult { + let response_type = response_type_from_python(&value.getattr("response_type")?)?; + Ok(match response_type { + ChatResponseType::OpenAiCompletion => { + ChatResponse::openai_completion(value_from_python(&value.getattr("_body")?)?) } - } -} - -enum PyChatResponseInner { - Buffered { - response_type: ChatResponseType, - body: Value, - }, - Stream { - response_type: ChatResponseType, - stream: Py, - }, -} - -#[pyclass(name = "ChatResponse")] -pub(crate) struct PyChatResponse { - inner: PyChatResponseInner, -} - -#[pymethods] -impl PyChatResponse { - #[classmethod] - fn openai_completion(_cls: &Bound<'_, PyType>, body: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - inner: PyChatResponseInner::Buffered { - response_type: ChatResponseType::OpenAiCompletion, - body: value_from_python(body)?, - }, - }) - } - - #[classmethod] - fn openai_stream(_cls: &Bound<'_, PyType>, stream: &Bound<'_, PyAny>) -> PyResult { - Self::stream_response(stream.py(), ChatResponseType::OpenAiStream, stream) - } - - #[classmethod] - fn openai_responses_completion( - _cls: &Bound<'_, PyType>, - body: &Bound<'_, PyAny>, - ) -> PyResult { - Ok(Self { - inner: PyChatResponseInner::Buffered { - response_type: ChatResponseType::OpenAiResponsesCompletion, - body: value_from_python(body)?, - }, - }) - } - - #[classmethod] - fn openai_responses_stream( - _cls: &Bound<'_, PyType>, - stream: &Bound<'_, PyAny>, - ) -> PyResult { - Self::stream_response(stream.py(), ChatResponseType::OpenAiResponsesStream, stream) - } - - #[classmethod] - fn anthropic_completion(_cls: &Bound<'_, PyType>, body: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - inner: PyChatResponseInner::Buffered { - response_type: ChatResponseType::AnthropicCompletion, - body: value_from_python(body)?, - }, - }) - } - - #[classmethod] - fn anthropic_stream(_cls: &Bound<'_, PyType>, stream: &Bound<'_, PyAny>) -> PyResult { - Self::stream_response(stream.py(), ChatResponseType::AnthropicStream, stream) - } - - #[getter] - fn response_type(&self, py: Python<'_>) -> PyResult> { - response_type_object(py, self.response_type_inner()) - } - - #[getter] - fn body(&self, py: Python<'_>) -> PyResult> { - self.to_body(py) - } - - #[getter] - fn stream(&self, py: Python<'_>) -> PyResult> { - match &self.inner { - PyChatResponseInner::Stream { stream, .. } => Ok(stream.clone_ref(py).into_any()), - PyChatResponseInner::Buffered { .. } => Err(PyAttributeError::new_err( - "buffered ChatResponse values do not have a stream", - )), + ChatResponseType::OpenAiResponsesCompletion => { + ChatResponse::openai_responses_completion(value_from_python(&value.getattr("_body")?)?) } - } - - fn replace_body(&mut self, body: &Bound<'_, PyAny>) -> PyResult<()> { - self.inner = match self.response_type_inner() { - ChatResponseType::OpenAiCompletion => { - let body = value_from_python(body)?; - PyChatResponseInner::Buffered { - response_type: ChatResponseType::OpenAiCompletion, - body, - } - } - ChatResponseType::OpenAiResponsesCompletion => { - let body = value_from_python(body)?; - PyChatResponseInner::Buffered { - response_type: ChatResponseType::OpenAiResponsesCompletion, - body, - } - } - ChatResponseType::AnthropicCompletion => { - let body = value_from_python(body)?; - PyChatResponseInner::Buffered { - response_type: ChatResponseType::AnthropicCompletion, - body, - } - } - ChatResponseType::OpenAiStream - | ChatResponseType::OpenAiResponsesStream - | ChatResponseType::AnthropicStream => { - return Err(PyValueError::new_err( - "streaming ChatResponse values do not have a replaceable body", - )); - } - }; - Ok(()) - } - - fn to_body(&self, py: Python<'_>) -> PyResult> { - match &self.inner { - PyChatResponseInner::Buffered { body, .. } => value_to_python(py, body), - PyChatResponseInner::Stream { .. } => Err(PyAttributeError::new_err( - "streaming ChatResponse values do not have a buffered body", - )), + ChatResponseType::AnthropicCompletion => { + ChatResponse::anthropic_completion(value_from_python(&value.getattr("_body")?)?) } - } - - fn __repr__(&self) -> String { - format!( - "ChatResponse(response_type='{}')", - response_type_name(self.response_type_inner()) - ) - } + ChatResponseType::OpenAiStream => ChatResponse::OpenAiStream(stream_from_python(value)?), + ChatResponseType::OpenAiResponsesStream => { + ChatResponse::OpenAiResponsesStream(stream_from_python(value)?) + } + ChatResponseType::AnthropicStream => { + ChatResponse::AnthropicStream(stream_from_python(value)?) + } + }) } -impl PyChatResponse { - pub(crate) fn from_core(py: Python<'_>, response: ChatResponse) -> PyResult { - let inner = match response { - ChatResponse::OpenAiCompletion(response) => PyChatResponseInner::Buffered { - response_type: ChatResponseType::OpenAiCompletion, - body: response.into_body(), - }, - ChatResponse::OpenAiStream(stream) => PyChatResponseInner::Stream { - response_type: ChatResponseType::OpenAiStream, - stream: Py::new(py, PyResponseStream::from_core_stream(stream))?, - }, - ChatResponse::OpenAiResponsesCompletion(response) => PyChatResponseInner::Buffered { - response_type: ChatResponseType::OpenAiResponsesCompletion, - body: response.into_body(), - }, - ChatResponse::OpenAiResponsesStream(stream) => PyChatResponseInner::Stream { - response_type: ChatResponseType::OpenAiResponsesStream, - stream: Py::new(py, PyResponseStream::from_core_stream(stream))?, - }, - ChatResponse::AnthropicCompletion(response) => PyChatResponseInner::Buffered { - response_type: ChatResponseType::AnthropicCompletion, - body: response.into_body(), - }, - ChatResponse::AnthropicStream(stream) => PyChatResponseInner::Stream { - response_type: ChatResponseType::AnthropicStream, - stream: Py::new(py, PyResponseStream::from_core_stream(stream))?, - }, - }; - Ok(Self { inner }) - } - - pub(crate) fn take_core(&mut self, py: Python<'_>) -> PyResult { - match &self.inner { - PyChatResponseInner::Buffered { - response_type, - body, - } => Ok(match response_type { - ChatResponseType::OpenAiCompletion => ChatResponse::openai_completion(body.clone()), - ChatResponseType::OpenAiResponsesCompletion => { - ChatResponse::openai_responses_completion(body.clone()) - } - ChatResponseType::AnthropicCompletion => { - ChatResponse::anthropic_completion(body.clone()) - } - ChatResponseType::OpenAiStream - | ChatResponseType::OpenAiResponsesStream - | ChatResponseType::AnthropicStream => { - return Err(PyRuntimeError::new_err( - "streaming response type stored without a stream", - )); - } - }), - PyChatResponseInner::Stream { - response_type, - stream, - } => { - let stream = stream.bind(py).borrow().take_core_stream()?; - Ok(match response_type { - ChatResponseType::OpenAiStream => ChatResponse::OpenAiStream(stream), - ChatResponseType::OpenAiResponsesStream => { - ChatResponse::OpenAiResponsesStream(stream) - } - ChatResponseType::AnthropicStream => ChatResponse::AnthropicStream(stream), - ChatResponseType::OpenAiCompletion - | ChatResponseType::OpenAiResponsesCompletion - | ChatResponseType::AnthropicCompletion => { - return Err(PyRuntimeError::new_err( - "buffered response type stored with a stream", - )); - } - }) - } +/// Converts a native response into the public Python response value. +pub(crate) fn response_to_python(py: Python<'_>, response: ChatResponse) -> PyResult> { + let core = py.import("switchyard_rust.core")?; + let (factory, payload) = match response { + ChatResponse::OpenAiCompletion(response) => ( + "openai_completion", + value_to_python(py, &response.into_body())?, + ), + ChatResponse::OpenAiResponsesCompletion(response) => ( + "openai_responses_completion", + value_to_python(py, &response.into_body())?, + ), + ChatResponse::AnthropicCompletion(response) => ( + "anthropic_completion", + value_to_python(py, &response.into_body())?, + ), + ChatResponse::OpenAiStream(stream) => { + ("openai_stream", stream_to_python(py, &core, stream)?) } - } - - fn stream_response( - py: Python<'_>, - response_type: ChatResponseType, - stream: &Bound<'_, PyAny>, - ) -> PyResult { - let stream = Py::new(py, PyResponseStream::new(stream.clone().unbind()))?; - Ok(Self { - inner: PyChatResponseInner::Stream { - response_type, - stream, - }, - }) - } - - fn response_type_inner(&self) -> ChatResponseType { - match &self.inner { - PyChatResponseInner::Buffered { response_type, .. } => *response_type, - PyChatResponseInner::Stream { response_type, .. } => *response_type, + ChatResponse::OpenAiResponsesStream(stream) => ( + "openai_responses_stream", + stream_to_python(py, &core, stream)?, + ), + ChatResponse::AnthropicStream(stream) => { + ("anthropic_stream", stream_to_python(py, &core, stream)?) } - } + }; + core.getattr("ChatResponse")? + .call_method1(factory, (payload,)) + .map(Bound::unbind) +} + +fn response_type_from_python(value: &Bound<'_, PyAny>) -> PyResult { + let raw = if let Ok(value_attr) = value.getattr("value") { + value_attr.extract::()? + } else { + value.extract::()? + }; + match raw.as_str() { + "openai_completion" => Ok(ChatResponseType::OpenAiCompletion), + "openai_stream" => Ok(ChatResponseType::OpenAiStream), + "openai_responses_completion" => Ok(ChatResponseType::OpenAiResponsesCompletion), + "openai_responses_stream" => Ok(ChatResponseType::OpenAiResponsesStream), + "anthropic_completion" => Ok(ChatResponseType::AnthropicCompletion), + "anthropic_stream" => Ok(ChatResponseType::AnthropicStream), + _ => Err(PyValueError::new_err(format!( + "Unknown response type: {raw:?}" + ))), + } +} + +fn stream_from_python(value: &Bound<'_, PyAny>) -> PyResult { + let native = value.getattr("stream")?.getattr("_native")?; + native + .extract::>()? + .take_core_stream() +} + +fn stream_to_python( + py: Python<'_>, + core: &Bound<'_, PyModule>, + stream: BoxResponseStream, +) -> PyResult> { + let native = Py::new(py, PyResponseStream::from_core_stream(stream))?; + core.getattr("ChatResponseStream")? + .call_method1("_from_native", (native,)) + .map(Bound::unbind) } struct PyResponseStreamSource { @@ -325,7 +117,7 @@ struct PyResponseStreamSource { type BoxPyResponseStream = std::pin::Pin>> + Send>>; -#[pyclass(name = "ChatResponseStream")] +#[pyclass(name = "_NativeChatResponseStream")] pub(crate) struct PyResponseStream { stream: Arc>>, taps: Arc>>>, @@ -400,7 +192,7 @@ impl PyResponseStream { }; ( event.map_err(|error| { - switchyard_core::SwitchyardError::Processor(error.to_string()) + switchyard_components::SwitchyardError::Processor(error.to_string()) }), (stream, taps, maps, on_complete, completed), ) @@ -637,7 +429,7 @@ impl SourceClosingStream { } impl Stream for SourceClosingStream { - type Item = switchyard_core::Result; + type Item = switchyard_components::Result; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { // All fields are ``Unpin``, so projecting through ``get_mut`` is sound. @@ -815,37 +607,7 @@ fn is_stop_async_iteration(error: &PyErr) -> bool { Python::attach(|py| error.is_instance_of::(py)) } -fn response_type_name(response_type: ChatResponseType) -> &'static str { - match response_type { - ChatResponseType::OpenAiCompletion => "openai_completion", - ChatResponseType::OpenAiStream => "openai_stream", - ChatResponseType::OpenAiResponsesCompletion => "openai_responses_completion", - ChatResponseType::OpenAiResponsesStream => "openai_responses_stream", - ChatResponseType::AnthropicCompletion => "anthropic_completion", - ChatResponseType::AnthropicStream => "anthropic_stream", - } -} - -fn response_type_variant_name(response_type: ChatResponseType) -> &'static str { - match response_type { - ChatResponseType::OpenAiCompletion => "OPENAI_COMPLETION", - ChatResponseType::OpenAiStream => "OPENAI_STREAM", - ChatResponseType::OpenAiResponsesCompletion => "OPENAI_RESPONSES_COMPLETION", - ChatResponseType::OpenAiResponsesStream => "OPENAI_RESPONSES_STREAM", - ChatResponseType::AnthropicCompletion => "ANTHROPIC_COMPLETION", - ChatResponseType::AnthropicStream => "ANTHROPIC_STREAM", - } -} - -fn response_type_object(py: Python<'_>, response_type: ChatResponseType) -> PyResult> { - py.get_type::() - .getattr(response_type_variant_name(response_type)) - .map(Bound::unbind) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; module.add_class::()?; Ok(()) } diff --git a/crates/switchyard-py/src/core_bindings/roles.rs b/crates/switchyard-py/src/interop/roles.rs similarity index 84% rename from crates/switchyard-py/src/core_bindings/roles.rs rename to crates/switchyard-py/src/interop/roles.rs index 6e3b3b8e..e2ede8bc 100644 --- a/crates/switchyard-py/src/core_bindings/roles.rs +++ b/crates/switchyard-py/src/interop/roles.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Python bindings for Rust-owned backend role abstractions. +//! Private adapter for Rust-owned backend role abstractions. use std::sync::Arc; @@ -9,14 +9,14 @@ use pyo3::exceptions::{PyNotImplementedError, PyTypeError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple, PyType}; use pyo3::PyTypeInfo; -use switchyard_core::LlmBackend; +use switchyard_components::LlmBackend; -use super::context::PyProxyContext; -use super::request::{request_type_object, PyChatRequest}; -use super::response::PyChatResponse; +use super::context::lease_from_python; +use super::request::{request_from_python, request_type_object}; +use super::response::response_to_python; use crate::errors::py_core_error; -#[pyclass(name = "LLMBackend", subclass)] +#[pyclass(name = "_NativeLlmBackend", subclass)] pub(crate) struct PyLlmBackend { inner: Option>, } @@ -78,24 +78,21 @@ impl PyLlmBackend { fn call<'py>( &self, py: Python<'py>, - ctx: PyRef<'_, PyProxyContext>, - request: PyRef<'_, PyChatRequest>, + ctx: &Bound<'_, PyAny>, + request: &Bound<'_, PyAny>, ) -> PyResult> { let backend = self .inner .clone() .ok_or_else(|| PyNotImplementedError::new_err("LLMBackend.call must be implemented"))?; - let mut lease = ctx.lease()?; - let request = request.clone_core(); + let mut lease = lease_from_python(ctx)?; + let request = request_from_python(request)?; pyo3_async_runtimes::tokio::future_into_py(py, async move { let result = backend.call(lease.context_mut()?, &request).await; let restore_result = lease.restore(); let response = result.map_err(py_core_error)?; restore_result?; - Python::attach(|py| { - Py::new(py, PyChatResponse::from_core(py, response)?) - .map(|response| response.into_any()) - }) + Python::attach(|py| response_to_python(py, response)) }) } diff --git a/crates/switchyard-py/src/core_bindings/subagent.rs b/crates/switchyard-py/src/interop/subagent.rs similarity index 100% rename from crates/switchyard-py/src/core_bindings/subagent.rs rename to crates/switchyard-py/src/interop/subagent.rs diff --git a/crates/switchyard-py/src/lib.rs b/crates/switchyard-py/src/lib.rs index 266aa858..f9862c69 100644 --- a/crates/switchyard-py/src/lib.rs +++ b/crates/switchyard-py/src/lib.rs @@ -4,8 +4,8 @@ use pyo3::prelude::*; mod component_bindings; -mod core_bindings; mod errors; +mod interop; mod libsy_bindings; mod py_serde; mod translation; @@ -14,7 +14,7 @@ mod translation; fn _switchyard_rust(module: &Bound<'_, PyModule>) -> PyResult<()> { errors::register(module)?; translation::register(module)?; - core_bindings::register(module)?; + interop::register(module)?; component_bindings::register(module)?; libsy_bindings::register(module)?; Ok(()) diff --git a/switchyard/lib/affinity_pin_store.py b/switchyard/lib/affinity_pin_store.py index 39b76689..a0a07691 100644 --- a/switchyard/lib/affinity_pin_store.py +++ b/switchyard/lib/affinity_pin_store.py @@ -3,7 +3,7 @@ """Pluggable async L2 store for session-affinity pins. -The in-process Rust ``SessionCache`` stays the L1 inside +The in-process ``SessionCache`` stays the L1 inside :class:`switchyard.lib.session_affinity.SessionAffinity`. An optional object implementing this protocol is the L2: a shared, out-of-process store (e.g. Redis) that lets pins be read by every Switchyard worker/pod and survive pod diff --git a/switchyard/lib/backends/multi_llm_backend.py b/switchyard/lib/backends/multi_llm_backend.py index 6a7deeed..546458b9 100644 --- a/switchyard/lib/backends/multi_llm_backend.py +++ b/switchyard/lib/backends/multi_llm_backend.py @@ -7,6 +7,7 @@ import logging from collections.abc import Iterable, Mapping +from typing import cast from switchyard.lib.backends.backend_format_resolver import BackendFormatResolver from switchyard.lib.backends.llm_target import ( @@ -71,9 +72,12 @@ def build_multi_llm_backend( ] else: target_values = targets - return MultiLlmBackend( - [build_target_backend(target) for target in target_values], - default_target_id=default_target_id, + return cast( # type: ignore[redundant-cast] + MultiLlmBackend, + MultiLlmBackend( + [build_target_backend(target) for target in target_values], + default_target_id=default_target_id, + ), ) diff --git a/switchyard/lib/conversation_turn.py b/switchyard/lib/conversation_turn.py index 9ebd4120..b075dc4c 100644 --- a/switchyard/lib/conversation_turn.py +++ b/switchyard/lib/conversation_turn.py @@ -25,14 +25,11 @@ def conversation_turn_number(request: ChatRequest) -> int: if not isinstance(body, dict): return 1 - request_type = request.request_type - if request_type is ChatRequestType.OPENAI_CHAT: - return _count_assistant_messages(body.get("messages")) + 1 - if request_type is ChatRequestType.ANTHROPIC: - return _count_assistant_messages(body.get("messages")) + 1 - if request_type is ChatRequestType.OPENAI_RESPONSES: - return _count_responses_turn(body) - return 1 + match request.request_type: + case ChatRequestType.OPENAI_CHAT | ChatRequestType.ANTHROPIC: + return _count_assistant_messages(body.get("messages")) + 1 + case ChatRequestType.OPENAI_RESPONSES: + return _count_responses_turn(body) def _count_assistant_messages(messages: Any) -> int: diff --git a/switchyard/lib/profiles/chain.py b/switchyard/lib/profiles/chain.py index 0bb1f084..db9c8e29 100644 --- a/switchyard/lib/profiles/chain.py +++ b/switchyard/lib/profiles/chain.py @@ -89,8 +89,10 @@ def __init__( self._fallback_target_on_evict = fallback_target_on_evict # Maps session key → frozenset of target IDs that overflowed for that session. # Only allocated when eviction is configured; None otherwise. - self._session_evictions: SessionCache | None = ( - SessionCache(self._SESSION_CACHE_MAX) if fallback_target_on_evict is not None else None + self._session_evictions: SessionCache[frozenset[str]] | None = ( + SessionCache[frozenset[str]](self._SESSION_CACHE_MAX) + if fallback_target_on_evict is not None + else None ) def iter_components(self) -> list[object]: diff --git a/switchyard/lib/profiles/translate_profile_config.py b/switchyard/lib/profiles/translate_profile_config.py index 2d442c3f..f926035a 100644 --- a/switchyard/lib/profiles/translate_profile_config.py +++ b/switchyard/lib/profiles/translate_profile_config.py @@ -68,8 +68,9 @@ def supported_request_types(self) -> list[ChatRequestType]: ChatRequestType.ANTHROPIC, ] - async def call(self, _ctx: ProxyContext, _request: ChatRequest) -> ChatResponse: + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: """Reject complete execution; external hosts own the backend call.""" + del ctx, request raise SwitchyardBackendError( "TranslateProfileConfig is hook-only; call process(), invoke the host " "backend, then call rprocess()" diff --git a/switchyard/lib/proxy_context.py b/switchyard/lib/proxy_context.py index 01c9b78a..9b715a7f 100644 --- a/switchyard/lib/proxy_context.py +++ b/switchyard/lib/proxy_context.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""ProxyContext constants and Rust-owned context binding.""" +"""ProxyContext re-export and shared metadata keys.""" from switchyard_rust.core import ProxyContext diff --git a/switchyard/lib/request_metadata.py b/switchyard/lib/request_metadata.py index 5bc28959..2f56f4d5 100644 --- a/switchyard/lib/request_metadata.py +++ b/switchyard/lib/request_metadata.py @@ -46,7 +46,7 @@ def attach_request_metadata( metadata: RequestMetadata, headers: Mapping[str, str] | None = None, ) -> None: - """Attach request metadata to both Python and Rust-owned context storage.""" + """Attach request metadata to both Python and native typed context storage.""" ctx.metadata[CTX_REQUEST_METADATA] = metadata if headers is not None: # Redact credential headers before retaining the map: the caller key is diff --git a/switchyard/lib/roles.py b/switchyard/lib/roles.py index 97becb58..397b9f2e 100644 --- a/switchyard/lib/roles.py +++ b/switchyard/lib/roles.py @@ -1,11 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Rust-owned backend role class and translated-response aliases. +"""Python backend role class and translated-response aliases. -Request-side and response-side components are now plain objects with async +Request-side and response-side components are plain objects with async ``process(...)`` methods. The backend remains nominal because it owns upstream -transport behavior and request-format support. +transport behavior and request-format support; native implementations register +with the same Python role class. """ from __future__ import annotations diff --git a/switchyard/lib/session_affinity.py b/switchyard/lib/session_affinity.py index 1d5f48d7..93693f97 100644 --- a/switchyard/lib/session_affinity.py +++ b/switchyard/lib/session_affinity.py @@ -108,7 +108,7 @@ def __init__( raise ValueError("l2_breaker_cooldown_s must be > 0") self._enabled = enabled self._warmup_turns = warmup_turns - self._pins: SessionCache = SessionCache(max_sessions) + self._pins = SessionCache[str](max_sessions) #: Optional shared/persistent L2 store; ``None`` keeps behavior L1-only. self._l2 = l2 # L2 observability: hits = pins rescued from the shared store after an diff --git a/switchyard/lib/session_cache.py b/switchyard/lib/session_cache.py index 6e3dc0a5..76fb35c1 100644 --- a/switchyard/lib/session_cache.py +++ b/switchyard/lib/session_cache.py @@ -1,13 +1,54 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Rust-backed bounded, access-ordered LRU cache keyed by a session key. - -Pair with :func:`switchyard.lib.session_key.session_key_from_body` for the key. -""" +"""Bounded, access-ordered LRU cache keyed by a session key.""" from __future__ import annotations -from switchyard_rust.core import SessionCache +from collections import OrderedDict +from typing import Generic, TypeVar, cast + +_V = TypeVar("_V") +_MISSING = object() + + +class SessionCache(Generic[_V]): + """Retain the most recently accessed session values up to a fixed capacity.""" + + def __init__(self, max_sessions: int) -> None: + if max_sessions < 0: + raise ValueError("max_sessions must be non-negative") + self._max_sessions = max_sessions + self._values: OrderedDict[str, _V] = OrderedDict() + + @property + def max_sessions(self) -> int: + """Return the configured cache capacity.""" + return self._max_sessions + + def get(self, key: str) -> _V | None: + """Return a cached value and mark it as most recently used.""" + value = self._values.get(key, _MISSING) + if value is _MISSING: + return None + self._values.move_to_end(key) + return cast(_V, value) + + def put(self, key: str, value: _V) -> None: + """Insert a value and evict the least recently used entry if needed.""" + if self._max_sessions == 0: + return + self._values[key] = value + self._values.move_to_end(key) + if len(self._values) > self._max_sessions: + self._values.popitem(last=False) + + def values(self) -> list[_V]: + """Return the retained values in least-to-most-recent order.""" + return list(self._values.values()) + + def __len__(self) -> int: + return len(self._values) + __all__ = ["SessionCache"] diff --git a/switchyard/lib/session_key.py b/switchyard/lib/session_key.py index 2026a0ab..30a99113 100644 --- a/switchyard/lib/session_key.py +++ b/switchyard/lib/session_key.py @@ -1,18 +1,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Rust-backed stable per-conversation key. - -Derived from the prefix an agent harness never rewrites — the system prompt -plus the first user message — so every turn of one conversation hashes alike. -""" +"""Stable per-conversation key derived from the request prefix.""" from __future__ import annotations +import hashlib +import json from typing import Any, Literal, overload -from switchyard_rust.core import session_key_from_body as _native_session_key_from_body - @overload def session_key_from_body(body: Any, depth: Literal[0] = 0) -> str: ... @@ -23,15 +19,123 @@ def session_key_from_body(body: Any, depth: int) -> str | None: ... def session_key_from_body(body: Any, depth: int = 0) -> str | None: - """Derive the per-conversation key from a request body. - - ``depth == 0`` (default) hashes the stable anchors only and always returns - a key. ``depth > 0`` extends the hashed prefix with the first ``depth`` - post-first-user messages — so repeated trials of an identical task diverge - via early model responses — and returns ``None`` until the conversation is - long enough for that prefix to exist. - """ - return _native_session_key_from_body(body, depth) + """Hash stable conversation anchors and an optional early-response prefix.""" + if depth < 0: + raise ValueError("depth must be non-negative") + + parts: list[str] = [_flatten_text(_field(body, "system"))] + anchored = False + tail_taken = 0 + + for sequence_name in ("messages", "input"): + items = _field(body, sequence_name) + if not isinstance(items, list): + continue + for item in items: + role = _field(item, "role") + if role in {"system", "developer"}: + parts.append(_flatten_text(_field(item, "content"))) + elif role == "user" and not anchored: + parts.append(_flatten_text(_field(item, "content"))) + anchored = True + elif anchored and tail_taken < depth: + parts.append(_flatten_message(item)) + tail_taken += 1 + if anchored and tail_taken >= depth: + break + if anchored: + break + + if depth > 0 and (not anchored or tail_taken < depth): + return None + + digest = hashlib.blake2b(digest_size=8) + for part in parts: + encoded = part.encode() + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + +def _field(value: Any, name: str) -> Any: + if isinstance(value, dict): + return value.get(name) + return getattr(value, name, None) + + +def _flatten_message(item: Any) -> str: + parts: list[str] = [] + content = _flatten_text(_field(item, "content")) + if content: + parts.append(content) + + blocks = _field(item, "content") + if isinstance(blocks, list): + for block in blocks: + if _field(block, "type") == "tool_use": + name = _field(block, "name") or "" + raw_input = _field(block, "input") + payload = _json_text(raw_input) if raw_input is not None else "" + parts.append(f"tool_call {name}({payload})") + + calls = _field(item, "tool_calls") + if isinstance(calls, list): + for call in calls: + function = _field(call, "function") + if function is not None: + raw_name = _field(function, "name") + raw_arguments = _field(function, "arguments") + name = raw_name if isinstance(raw_name, str) else "" + arguments = ( + raw_arguments + if isinstance(raw_arguments, str) + else _json_text(raw_arguments) + if raw_arguments is not None + else "" + ) + parts.append(f"tool_call {name}({arguments})") + + if _field(item, "type") == "function_call": + raw_name = _field(item, "name") + raw_arguments = _field(item, "arguments") + name = raw_name if isinstance(raw_name, str) else "" + arguments = ( + raw_arguments + if isinstance(raw_arguments, str) + else _json_text(raw_arguments) + if raw_arguments is not None + else "" + ) + parts.append(f"tool_call {name}({arguments})") + + output = _flatten_text(_field(item, "output")) + if output: + parts.append(output) + return " ".join(parts) + + +def _flatten_text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, dict): + text = block.get("text") + if not isinstance(text, str) or not text: + text = block.get("content") + if isinstance(text, str) and text: + parts.append(text) + else: + parts.append(_json_text(block)) + return "".join(parts) + if content is None: + return "" + return _json_text(content) + + +def _json_text(value: Any) -> str: + return json.dumps(value, separators=(",", ":"), sort_keys=True) __all__ = ["session_key_from_body"] diff --git a/switchyard/server/switchyard_app.py b/switchyard/server/switchyard_app.py index b8f6b04a..404b77b9 100644 --- a/switchyard/server/switchyard_app.py +++ b/switchyard/server/switchyard_app.py @@ -140,11 +140,11 @@ async def _request_validation_error_handler( async def _invalid_request_handler( _request: Request, exc: SwitchyardInvalidRequestError ) -> Response: - """Map Rust-side request validation failures to the 400 envelope. + """Map request validation failures to the 400 envelope. ``ChatRequest.validate()`` (called by the inbound endpoints) raises - this from the Rust core when a body is structurally valid but - semantically invalid. The only such check today is a present-but-empty + this when a body is structurally valid but semantically invalid. The + only such check today is a present-but-empty ``messages`` array, so the envelope uses ``code="empty_messages"``; revisit if more validations start sharing this error. """ diff --git a/switchyard_rust/components.py b/switchyard_rust/components.py index f7578950..a451f6f0 100644 --- a/switchyard_rust/components.py +++ b/switchyard_rust/components.py @@ -44,6 +44,15 @@ "stage_score_signal", } ) +_BACKEND_EXPORTS = frozenset( + { + "AnthropicNativeBackend", + "MultiLlmBackend", + "OpenAiNativeBackend", + "OpenAiPassthroughBackend", + "StatsLlmBackend", + } +) if TYPE_CHECKING: AnthropicNativeBackend: type[Any] @@ -81,7 +90,12 @@ def __getattr__(name: str) -> object: if name in _COMPONENT_EXPORTS: - return getattr(_load_native(), name) + value = getattr(_load_native(), name) + if name in _BACKEND_EXPORTS: + from switchyard_rust.core import LLMBackend + + LLMBackend.register(value) + return value raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/switchyard_rust/core.py b/switchyard_rust/core.py index 551fa3aa..1b10f663 100644 --- a/switchyard_rust/core.py +++ b/switchyard_rust/core.py @@ -1,135 +1,318 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Switchyard core bindings plus Python-only compatibility chain wrappers.""" +"""Python core values and compatibility chain wrappers.""" from __future__ import annotations import importlib import importlib.metadata +import json import os -from collections.abc import ItemsView, Iterable, Iterator, KeysView, Mapping, ValuesView +from abc import ABCMeta +from collections.abc import Iterable, Mapping +from enum import Enum from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, cast JsonScalar: TypeAlias = bool | int | float | str | None JsonValue: TypeAlias = JsonScalar | list["JsonValue"] | dict[str, "JsonValue"] -class _ChatRequestType(Protocol): - """Rust-owned request format tag exposed through PyO3.""" +class ChatRequestType(Enum): + """Wire format carried by a :class:`ChatRequest`.""" - OPENAI_CHAT: ClassVar[_ChatRequestType] - OPENAI_RESPONSES: ClassVar[_ChatRequestType] - ANTHROPIC: ClassVar[_ChatRequestType] + OPENAI_CHAT = "openai_chat" + OPENAI_RESPONSES = "openai_responses" + ANTHROPIC = "anthropic" - value: str + def __str__(self) -> str: + return self.value -class _ChatRequest(Protocol): - """Rust-owned chat request value exposed through PyO3.""" +class ChatResponseType(Enum): + """Wire format and delivery mode carried by a :class:`ChatResponse`.""" - request_type: _ChatRequestType - body: Any - model: str | None + OPENAI_COMPLETION = "openai_completion" + OPENAI_STREAM = "openai_stream" + OPENAI_RESPONSES_COMPLETION = "openai_responses_completion" + OPENAI_RESPONSES_STREAM = "openai_responses_stream" + ANTHROPIC_COMPLETION = "anthropic_completion" + ANTHROPIC_STREAM = "anthropic_stream" + + def __str__(self) -> str: + return self.value + + +def _json_copy(value: object) -> JsonValue: + """Normalize an object to an owned JSON value.""" + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + value = model_dump(mode="json", exclude_none=True) + else: + to_dict = getattr(value, "to_dict", None) + if callable(to_dict): + value = to_dict() + try: + return cast(JsonValue, json.loads(json.dumps(value, allow_nan=False))) + except (TypeError, ValueError) as error: + raise ValueError(str(error)) from error + + +class ChatRequest: + """Owned provider request body with an explicit wire format.""" + + def __init__(self, request_type: ChatRequestType, body: object) -> None: + self.request_type = request_type + self._body = _json_copy(body) @classmethod - def openai_chat(cls, body: Mapping[str, Any] | JsonValue) -> _ChatRequest: ... + def openai_chat(cls, body: object) -> ChatRequest: + """Build an OpenAI Chat Completions request.""" + return cls(ChatRequestType.OPENAI_CHAT, body) @classmethod - def openai_responses(cls, body: Mapping[str, Any] | JsonValue) -> _ChatRequest: ... + def openai_responses(cls, body: object) -> ChatRequest: + """Build an OpenAI Responses request.""" + return cls(ChatRequestType.OPENAI_RESPONSES, body) @classmethod - def anthropic(cls, body: Mapping[str, Any] | JsonValue) -> _ChatRequest: ... + def anthropic(cls, body: object) -> ChatRequest: + """Build an Anthropic Messages request.""" + return cls(ChatRequestType.ANTHROPIC, body) + + @property + def body(self) -> Any: + """Return an owned copy of the request body.""" + return _json_copy(self._body) + + @property + def model(self) -> str | None: + """Return the request model when it is a string.""" + if isinstance(self._body, dict): + model = self._body.get("model") + return model if isinstance(model, str) else None + return None + + def validate(self) -> None: + """Reject semantically invalid message-based requests.""" + if self.request_type in {ChatRequestType.OPENAI_CHAT, ChatRequestType.ANTHROPIC}: + if isinstance(self._body, dict) and self._body.get("messages") == []: + raise _load_native().SwitchyardInvalidRequestError( + "messages must be a non-empty array" + ) - def set_model(self, model: str) -> None: ... - def replace_body(self, body: Mapping[str, Any] | JsonValue) -> None: ... - def to_body(self) -> JsonValue: ... + def set_model(self, model: str) -> None: + """Set the request model, replacing a malformed non-object body.""" + if not isinstance(self._body, dict): + self._body = {} + self._body["model"] = model + + def replace_body(self, body: object) -> None: + """Replace the body without changing its wire format.""" + self._body = _json_copy(body) + + def to_body(self) -> JsonValue: + """Return an owned copy of the request body.""" + return _json_copy(self._body) + + def __repr__(self) -> str: + model = f", model={self.model!r}" if self.model is not None else "" + return f"ChatRequest(request_type={self.request_type.value!r}{model})" + + +class ChatResponseStream: + """Single-consumer async response stream with callback transforms.""" + + def __init__(self, source: object) -> None: + self._native = _load_native()._NativeChatResponseStream(source) + + @classmethod + def _from_native(cls, stream: Any) -> ChatResponseStream: + instance = cls.__new__(cls) + instance._native = stream + return instance + def tap(self, callback: object) -> ChatResponseStream: + """Observe each event before mapping.""" + self._native.tap(callback) + return self -class _ChatResponseType(Protocol): - """Rust-owned response format/delivery tag exposed through PyO3.""" + def map(self, callback: object) -> ChatResponseStream: + """Transform each event in registration order.""" + self._native.map(callback) + return self - OPENAI_COMPLETION: ClassVar[_ChatResponseType] - OPENAI_STREAM: ClassVar[_ChatResponseType] - OPENAI_RESPONSES_COMPLETION: ClassVar[_ChatResponseType] - OPENAI_RESPONSES_STREAM: ClassVar[_ChatResponseType] - ANTHROPIC_COMPLETION: ClassVar[_ChatResponseType] - ANTHROPIC_STREAM: ClassVar[_ChatResponseType] + def on_complete(self, callback: object) -> ChatResponseStream: + """Run a callback once after normal stream exhaustion.""" + self._native.on_complete(callback) + return self - value: str + def __aiter__(self) -> ChatResponseStream: + self._native.__aiter__() + return self + async def __anext__(self) -> Any: + return await self._native.__anext__() -class _ChatResponse(Protocol): - """Rust-owned chat response value exposed through PyO3.""" + async def aclose(self) -> None: + """Close the upstream source and release its connection.""" + await self._native.aclose() - response_type: _ChatResponseType - body: Any - stream: Any + def __repr__(self) -> str: + return "ChatResponseStream()" + + +class ChatResponse: + """Owned buffered response or live response stream.""" + + def __init__( + self, + response_type: ChatResponseType, + *, + body: object | None = None, + stream: object | None = None, + ) -> None: + self.response_type = response_type + self._body: JsonValue | None + self._stream: ChatResponseStream | None + if response_type.value.endswith("_stream"): + if stream is None: + raise TypeError("streaming ChatResponse requires a stream") + self._stream = ( + stream if isinstance(stream, ChatResponseStream) else ChatResponseStream(stream) + ) + self._body = None + else: + self._body = _json_copy(body) + self._stream = None @classmethod - def openai_completion(cls, body: Mapping[str, Any] | JsonValue | object) -> _ChatResponse: ... + def openai_completion(cls, body: object) -> ChatResponse: + """Build a buffered OpenAI Chat Completions response.""" + return cls(ChatResponseType.OPENAI_COMPLETION, body=body) @classmethod - def openai_stream(cls, stream: object) -> _ChatResponse: ... + def openai_stream(cls, stream: object) -> ChatResponse: + """Build a streaming OpenAI Chat Completions response.""" + return cls(ChatResponseType.OPENAI_STREAM, stream=stream) @classmethod - def openai_responses_completion( - cls, body: Mapping[str, Any] | JsonValue | object, - ) -> _ChatResponse: ... + def openai_responses_completion(cls, body: object) -> ChatResponse: + """Build a buffered OpenAI Responses response.""" + return cls(ChatResponseType.OPENAI_RESPONSES_COMPLETION, body=body) @classmethod - def openai_responses_stream(cls, stream: object) -> _ChatResponse: ... + def openai_responses_stream(cls, stream: object) -> ChatResponse: + """Build a streaming OpenAI Responses response.""" + return cls(ChatResponseType.OPENAI_RESPONSES_STREAM, stream=stream) @classmethod - def anthropic_completion(cls, body: Mapping[str, Any] | JsonValue | object) -> _ChatResponse: ... + def anthropic_completion(cls, body: object) -> ChatResponse: + """Build a buffered Anthropic response.""" + return cls(ChatResponseType.ANTHROPIC_COMPLETION, body=body) @classmethod - def anthropic_stream(cls, stream: object) -> _ChatResponse: ... + def anthropic_stream(cls, stream: object) -> ChatResponse: + """Build a streaming Anthropic response.""" + return cls(ChatResponseType.ANTHROPIC_STREAM, stream=stream) + + @property + def body(self) -> Any: + """Return an owned buffered body.""" + if self._stream is not None: + raise AttributeError("streaming ChatResponse values do not have a buffered body") + return _json_copy(self._body) + + @property + def stream(self) -> ChatResponseStream: + """Return the live stream.""" + if self._stream is None: + raise AttributeError("buffered ChatResponse values do not have a stream") + return self._stream + + def replace_body(self, body: object) -> None: + """Replace a buffered body without changing its wire shape.""" + if self._stream is not None: + raise ValueError("streaming ChatResponse values do not have a replaceable body") + self._body = _json_copy(body) + + def to_body(self) -> JsonValue: + """Return an owned buffered body.""" + if self._stream is not None: + raise AttributeError("streaming ChatResponse values do not have a buffered body") + return _json_copy(self._body) + + def __repr__(self) -> str: + return f"ChatResponse(response_type={self.response_type.value!r})" + + +class ProxyMetadata(dict[str, Any]): + """Mutable per-request metadata owned by Python.""" - def replace_body(self, body: Mapping[str, Any] | JsonValue | object) -> None: ... - def to_body(self) -> JsonValue: ... +class ProxyContext: + """Per-request state shared by Python and native components.""" -class _ChatResponseStream(Protocol): - """Rust-owned async stream adapter exposed through PyO3.""" + def __init__( + self, + metadata: Mapping[str, Any] | None = None, + request_id: str | None = None, + ) -> None: + self._native = _load_native()._NativeProxyContext(metadata, request_id) + self.metadata = ProxyMetadata(metadata or {}) + + @property + def request_id(self) -> str | None: + return cast(str | None, self._native.request_id) + + @request_id.setter + def request_id(self, value: str | None) -> None: + self._native.request_id = value + + @property + def inbound_format(self) -> ChatRequestType | None: + value = self._native.inbound_format + return request_type_enum(value) if value is not None else None + + @inbound_format.setter + def inbound_format(self, value: object | None) -> None: + self._native.inbound_format = value + + @property + def selected_model(self) -> str | None: + return cast(str | None, self._native.selected_model) - def __init__(self, source: object) -> None: ... - def tap(self, callback: object) -> _ChatResponseStream: ... - def map(self, callback: object) -> _ChatResponseStream: ... - def on_complete(self, callback: object) -> _ChatResponseStream: ... - def __aiter__(self) -> _ChatResponseStream: ... - async def __anext__(self) -> Any: ... + @selected_model.setter + def selected_model(self, value: str | None) -> None: + self._native.selected_model = value + @property + def selected_target(self) -> str | None: + return cast(str | None, self._native.selected_target) -class _ProxyMetadata(Protocol): - """Rust-owned metadata map exposed through PyO3.""" + @selected_target.setter + def selected_target(self, value: str | None) -> None: + self._native.selected_target = value - def __getitem__(self, key: str) -> Any: ... - def __setitem__(self, key: str, value: Any) -> None: ... - def __delitem__(self, key: str) -> None: ... - def __contains__(self, key: object) -> bool: ... - def __len__(self) -> int: ... - def __iter__(self) -> Iterator[str]: ... - def get(self, key: str, default: Any = None) -> Any: ... - def setdefault(self, key: str, default: Any = None) -> Any: ... - def update(self, metadata: Mapping[str, Any] | Iterable[tuple[str, Any]], /) -> None: ... - def copy(self) -> dict[str, Any]: ... - def keys(self) -> KeysView[str]: ... - def values(self) -> ValuesView[Any]: ... - def items(self) -> ItemsView[str, Any]: ... + @property + def evicted_targets(self) -> list[str] | None: + return cast(list[str] | None, self._native.evicted_targets) + @evicted_targets.setter + def evicted_targets(self, value: list[str] | None) -> None: + self._native.evicted_targets = value -class _ProxyContext(Protocol): - """Rust-owned per-request context exposed through PyO3.""" + @property + def backend_call_latency_ms(self) -> float | None: + return cast(float | None, self._native.backend_call_latency_ms) - metadata: _ProxyMetadata - request_id: str | None - inbound_format: _ChatRequestType | None - selected_model: str | None - selected_target: str | None - backend_call_latency_ms: float | None - evicted_targets: list[str] | None - submodel_calls: list[dict[str, Any]] + @backend_call_latency_ms.setter + def backend_call_latency_ms(self, value: float | None) -> None: + self._native.backend_call_latency_ms = value + + @property + def submodel_calls(self) -> list[dict[str, Any]]: + return cast(list[dict[str, Any]], self._native.submodel_calls) def record_submodel_call( self, @@ -139,52 +322,51 @@ def record_submodel_call( cached_tokens: int, router_type: str, routed_to: str, - ) -> None: ... + ) -> None: + """Record one routing model call for intake and stats.""" + self._native.record_submodel_call( + model, + prompt_tokens, + completion_tokens, + cached_tokens, + router_type, + routed_to, + ) + def __repr__(self) -> str: + return repr(self._native) -class _LLMBackend(Protocol): - """Rust-owned backend role exposed through PyO3.""" - supported_request_types: list[_ChatRequestType] +class LLMBackend(metaclass=ABCMeta): # noqa: B024 + """Base class for Python and registered native LLM backends.""" - async def call(self, ctx: _ProxyContext, request: _ChatRequest) -> _ChatResponse: ... - def startup(self) -> Any: ... - def shutdown(self) -> Any: ... + def __new__(cls, *args: object, **kwargs: object) -> LLMBackend: + if cls is LLMBackend: + raise TypeError("can't instantiate abstract role LLMBackend") + return super().__new__(cls) + @property + def supported_request_types(self) -> list[ChatRequestType]: + """Return request formats accepted by this backend.""" + raise NotImplementedError("LLMBackend.supported_request_types must be implemented") -class _Switchyard(Protocol): - """Python compatibility Switchyard chain.""" + async def startup(self) -> None: + """Start backend resources.""" + return None - state_key: ClassVar[str] - _backend: _LLMBackend - _translator: Any + async def shutdown(self) -> None: + """Stop backend resources.""" + return None - def __init__( - self, - *, - request_processors: Iterable[Any] | None = None, - backend: _LLMBackend, - response_processors: Iterable[Any] | None = None, - translator: Any, - ) -> None: ... - def iter_components(self) -> list[Any]: ... - async def call( - self, - request: _ChatRequest, - *, - ctx: _ProxyContext | None = None, - ) -> Any: ... + async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: + """Call the upstream model.""" + raise NotImplementedError("LLMBackend.call must be implemented") class _NativeModule(Protocol): - ChatRequest: type[_ChatRequest] - ChatRequestType: type[_ChatRequestType] - ChatResponse: type[_ChatResponse] - ChatResponseStream: type[_ChatResponseStream] - ChatResponseType: type[_ChatResponseType] - LLMBackend: type[_LLMBackend] - ProxyMetadata: type[_ProxyMetadata] - ProxyContext: type[_ProxyContext] + _NativeChatResponseStream: Any + _NativeProxyContext: Any + is_subagent_request: Any SwitchyardRuntimeError: type[RuntimeError] SwitchyardConfigError: type[RuntimeError] SwitchyardInvalidIdError: type[RuntimeError] @@ -197,9 +379,6 @@ class _NativeModule(Protocol): SwitchyardUpstreamError: type[RuntimeError] SwitchyardContextWindowExceededError: type[RuntimeError] SwitchyardContextPoolExhaustedError: type[RuntimeError] - # Session-affinity primitives. - SessionCache: type[Any] - session_key_from_body: Any def _ensure_switchyard_version_env() -> None: @@ -221,11 +400,70 @@ def _load_native() -> _NativeModule: return cast(_NativeModule, importlib.import_module("switchyard_rust._switchyard_rust")) except ImportError as exc: # pragma: no cover - broken install guard raise RuntimeError( - "Rust core bindings are required. Run `uv run maturin develop` " + "The Switchyard native extension is required. Run `uv run maturin develop` " "or install a built switchyard wheel." ) from exc +if TYPE_CHECKING: + class SwitchyardRuntimeError(RuntimeError): + """Base class for native Switchyard runtime errors.""" + + class SwitchyardConfigError(SwitchyardRuntimeError): + """Raised for invalid Switchyard configuration.""" + + class SwitchyardInvalidIdError(SwitchyardRuntimeError): + """Raised when a Switchyard identifier is invalid.""" + + class SwitchyardDuplicateRegistrationError(SwitchyardRuntimeError): + """Raised when a registry receives a duplicate ID.""" + + class SwitchyardModelNotFoundError(SwitchyardRuntimeError): + """Raised when a route table cannot find a model.""" + + class SwitchyardUnsupportedRequestTypeError(SwitchyardRuntimeError): + """Raised when a component rejects a request format.""" + + class SwitchyardInvalidRequestError(SwitchyardRuntimeError): + """Raised when a request body fails semantic validation.""" + + class SwitchyardProcessorError(SwitchyardRuntimeError): + """Raised when a processor fails.""" + + class SwitchyardBackendError(SwitchyardRuntimeError): + """Raised when a backend fails.""" + + class SwitchyardUpstreamError(SwitchyardRuntimeError): + """Raised when an upstream call fails.""" + + status_code: int + body: str + + class SwitchyardContextWindowExceededError(SwitchyardBackendError): + """Raised when an upstream request exceeds the context window.""" + + class SwitchyardContextPoolExhaustedError(SwitchyardBackendError): + """Raised when every attempted routing target was evicted.""" +else: + SwitchyardRuntimeError = _load_native().SwitchyardRuntimeError + SwitchyardConfigError = _load_native().SwitchyardConfigError + SwitchyardInvalidIdError = _load_native().SwitchyardInvalidIdError + SwitchyardDuplicateRegistrationError = _load_native().SwitchyardDuplicateRegistrationError + SwitchyardModelNotFoundError = _load_native().SwitchyardModelNotFoundError + SwitchyardUnsupportedRequestTypeError = _load_native().SwitchyardUnsupportedRequestTypeError + SwitchyardInvalidRequestError = _load_native().SwitchyardInvalidRequestError + SwitchyardProcessorError = _load_native().SwitchyardProcessorError + SwitchyardBackendError = _load_native().SwitchyardBackendError + SwitchyardUpstreamError = _load_native().SwitchyardUpstreamError + SwitchyardContextWindowExceededError = _load_native().SwitchyardContextWindowExceededError + SwitchyardContextPoolExhaustedError = _load_native().SwitchyardContextPoolExhaustedError + + +def is_subagent_request(headers: Mapping[str, str]) -> bool: + """Return whether canonical protocol metadata marks delegated agent work.""" + return bool(_load_native().is_subagent_request(headers)) + + def _role_type_name(value: object) -> str: """Return a stable display name for role validation errors.""" return type(value).__name__ @@ -261,7 +499,7 @@ def __init__( translator: Any, fallback_target_on_evict: str | None = None, ) -> None: - if not isinstance(backend, _load_native().LLMBackend): + if not isinstance(backend, LLMBackend): actual = _role_type_name(backend) raise TypeError(f"Switchyard backend must be LLMBackend, got {actual}") self._request_components = tuple(request_processors or ()) @@ -286,7 +524,7 @@ async def call( ctx: Any | None = None, ) -> Any: """Run the compatibility chain and translate the final response.""" - context = ctx if ctx is not None else _load_native().ProxyContext() + context = ctx if ctx is not None else ProxyContext() processed_request = await self._process_request_components(context, request) try: response = await self._call_backend_stage(context, processed_request) @@ -308,12 +546,12 @@ async def _call_backend_stage( """Call the backend and then response processors.""" native = _load_native() try: - response = await self._backend.call(ctx, request) + response: object = await cast(Any, self._backend).call(ctx, request) except native.SwitchyardContextWindowExceededError: raise except Exception as error: raise _backend_error(error) from error - if not isinstance(response, native.ChatResponse): + if not isinstance(response, ChatResponse): actual = _role_type_name(response) raise native.SwitchyardBackendError( f"Switchyard backend returned {actual}, expected ChatResponse", @@ -335,7 +573,7 @@ async def _process_request_components(self, ctx: Any, request: Any) -> Any: current = await process(ctx, current) except Exception as error: raise _processor_error(error) from error - if not isinstance(current, native.ChatRequest): + if not isinstance(current, ChatRequest): actual = _role_type_name(current) raise native.SwitchyardProcessorError( f"Request component returned {actual}, expected ChatRequest", @@ -357,7 +595,7 @@ async def _process_response_components(self, ctx: Any, response: Any) -> Any: current = await process(ctx, current) except Exception as error: raise _processor_error(error) from error - if not isinstance(current, native.ChatResponse): + if not isinstance(current, ChatResponse): actual = _role_type_name(current) raise native.SwitchyardProcessorError( f"Response component returned {actual}, expected ChatResponse", @@ -410,224 +648,8 @@ def _rewrite_evicted_pick(self, ctx: Any) -> None: ctx.selected_target = self._fallback_target_on_evict -if TYPE_CHECKING: - class ChatRequestType: - """Static view of the Rust-owned ChatRequestType class.""" - - OPENAI_CHAT: ClassVar[ChatRequestType] - OPENAI_RESPONSES: ClassVar[ChatRequestType] - ANTHROPIC: ClassVar[ChatRequestType] - value: str - - class ChatRequest: - """Static view of the Rust-owned ChatRequest class.""" - - request_type: ChatRequestType - body: Any - model: str | None - - @classmethod - def openai_chat(cls, body: Mapping[str, Any] | JsonValue) -> ChatRequest: ... - @classmethod - def openai_responses(cls, body: Mapping[str, Any] | JsonValue) -> ChatRequest: ... - @classmethod - def anthropic(cls, body: Mapping[str, Any] | JsonValue) -> ChatRequest: ... - def validate(self) -> None: ... - def set_model(self, model: str) -> None: ... - def replace_body(self, body: Mapping[str, Any] | JsonValue) -> None: ... - def to_body(self) -> JsonValue: ... - - class ChatResponseType: - """Static view of the Rust-owned ChatResponseType class.""" - - OPENAI_COMPLETION: ClassVar[ChatResponseType] - OPENAI_STREAM: ClassVar[ChatResponseType] - OPENAI_RESPONSES_COMPLETION: ClassVar[ChatResponseType] - OPENAI_RESPONSES_STREAM: ClassVar[ChatResponseType] - ANTHROPIC_COMPLETION: ClassVar[ChatResponseType] - ANTHROPIC_STREAM: ClassVar[ChatResponseType] - value: str - - class ChatResponse: - """Static view of the Rust-owned ChatResponse class.""" - - response_type: ChatResponseType - body: Any - stream: Any - - @classmethod - def openai_completion( - cls, - body: Mapping[str, Any] | JsonValue | object, - ) -> ChatResponse: ... - @classmethod - def openai_stream(cls, stream: object) -> ChatResponse: ... - @classmethod - def openai_responses_completion( - cls, - body: Mapping[str, Any] | JsonValue | object, - ) -> ChatResponse: ... - @classmethod - def openai_responses_stream(cls, stream: object) -> ChatResponse: ... - @classmethod - def anthropic_completion( - cls, - body: Mapping[str, Any] | JsonValue | object, - ) -> ChatResponse: ... - @classmethod - def anthropic_stream(cls, stream: object) -> ChatResponse: ... - def replace_body(self, body: Mapping[str, Any] | JsonValue | object) -> None: ... - def to_body(self) -> JsonValue: ... - - class ChatResponseStream: - """Static view of the Rust-owned ChatResponseStream class.""" - - def __init__(self, source: object) -> None: ... - def tap(self, callback: object) -> ChatResponseStream: ... - def map(self, callback: object) -> ChatResponseStream: ... - def on_complete(self, callback: object) -> ChatResponseStream: ... - def __aiter__(self) -> ChatResponseStream: ... - async def __anext__(self) -> Any: ... - - class ProxyMetadata(_ProxyMetadata, Protocol): - """Static view of the Rust-owned ProxyMetadata class.""" - - class ProxyContext: - """Static view of the Rust-owned ProxyContext class.""" - - metadata: _ProxyMetadata - request_id: str | None - inbound_format: _ChatRequestType | None - selected_model: str | None - selected_target: str | None - backend_call_latency_ms: float | None - evicted_targets: list[str] | None - submodel_calls: list[dict[str, Any]] - - def __init__( - self, - metadata: Mapping[str, Any] | None = None, - request_id: str | None = None, - ) -> None: ... - - def record_submodel_call( - self, - model: str, - prompt_tokens: int, - completion_tokens: int, - cached_tokens: int, - router_type: str, - routed_to: str, - ) -> None: ... - - class LLMBackend: - """Static view of the Rust-owned LLMBackend role class.""" - - @property - def supported_request_types(self) -> list[ChatRequestType]: ... - async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse: ... - def startup(self) -> Any: ... - def shutdown(self) -> Any: ... - - class SwitchyardRuntimeError(RuntimeError): - """Base class for Rust-owned Switchyard runtime errors.""" - - class SwitchyardConfigError(SwitchyardRuntimeError): - """Raised for invalid Rust Switchyard configuration.""" - - class SwitchyardInvalidIdError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard identifier is invalid.""" - - class SwitchyardDuplicateRegistrationError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard registry receives a duplicate ID.""" - - class SwitchyardModelNotFoundError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard route table cannot find a model.""" - - class SwitchyardUnsupportedRequestTypeError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard component rejects a request format.""" - - class SwitchyardInvalidRequestError(SwitchyardRuntimeError): - """Raised when a request body fails semantic validation (e.g. empty messages).""" - - class SwitchyardProcessorError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard processor fails.""" - - class SwitchyardBackendError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard backend fails.""" - - class SwitchyardUpstreamError(SwitchyardRuntimeError): - """Raised when a Rust Switchyard upstream call fails.""" - status_code: int - body: str - - class SwitchyardContextWindowExceededError(SwitchyardBackendError): - """Raised by an LLMBackend on an upstream context-window overflow.""" - - class SwitchyardContextPoolExhaustedError(SwitchyardBackendError): - """Raised when every attempted routing target was evicted.""" - - class SessionCache: - """Static view of the Rust-owned bounded-LRU session cache.""" - - def __init__(self, max_sessions: int) -> None: ... - def get(self, key: str) -> Any | None: ... - def put(self, key: str, value: Any) -> None: ... - def values(self) -> list[Any]: ... - @property - def max_sessions(self) -> int: ... - def __len__(self) -> int: ... - - def session_key_from_body( - body: Mapping[str, Any] | JsonValue, depth: int = 0 - ) -> str | None: ... - - def is_subagent_request(headers: Mapping[str, str]) -> bool: ... - - -def __getattr__(name: str) -> object: - if name == "is_subagent_request": - return _load_native().is_subagent_request - if name == "SessionCache": - return _load_native().SessionCache - if name == "session_key_from_body": - return _load_native().session_key_from_body - if name == "ChatRequest": - return _load_native().ChatRequest - if name == "ChatRequestType": - return _load_native().ChatRequestType - if name == "ChatResponse": - return _load_native().ChatResponse - if name == "ChatResponseStream": - return _load_native().ChatResponseStream - if name == "ChatResponseType": - return _load_native().ChatResponseType - if name == "LLMBackend": - return _load_native().LLMBackend - if name == "ProxyMetadata": - return _load_native().ProxyMetadata - if name == "ProxyContext": - return _load_native().ProxyContext - if name in { - "SwitchyardRuntimeError", - "SwitchyardConfigError", - "SwitchyardInvalidIdError", - "SwitchyardDuplicateRegistrationError", - "SwitchyardModelNotFoundError", - "SwitchyardUnsupportedRequestTypeError", - "SwitchyardInvalidRequestError", - "SwitchyardProcessorError", - "SwitchyardBackendError", - "SwitchyardUpstreamError", - "SwitchyardContextWindowExceededError", - "SwitchyardContextPoolExhaustedError", - }: - return getattr(_load_native(), name) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - def request_type_value(value: object) -> str: - """Normalize Rust request-format tags and wire strings.""" + """Normalize request-format tags and wire strings.""" raw = value.value if hasattr(value, "value") else value if not isinstance(raw, str): raise TypeError(f"Request type must be a string-like value, got {type(raw).__name__}") @@ -638,39 +660,30 @@ def request_type_value(value: object) -> str: raise ValueError(f"Unknown request type: {value!r}") -def request_type_enum(value: object) -> _ChatRequestType: - """Normalize a request-format tag to the Rust-bound enum object.""" - normalized = request_type_value(value) - request_type = _load_native().ChatRequestType - if normalized == "openai_chat": - return request_type.OPENAI_CHAT - if normalized == "openai_responses": - return request_type.OPENAI_RESPONSES - if normalized == "anthropic": - return request_type.ANTHROPIC - raise ValueError(f"Unknown request type: {value!r}") +def request_type_enum(value: object) -> ChatRequestType: + """Normalize a request-format tag to the Python enum.""" + return ChatRequestType(request_type_value(value)) def request_type_matches(request: object, request_type: object) -> bool: - """Return whether a Rust-backed request has the requested wire format.""" + """Return whether a request has the requested wire format.""" return request_type_value(cast(Any, request).request_type) == request_type_value(request_type) def request_with_type(request_type: object, body: Mapping[str, Any] | JsonValue) -> ChatRequest: - """Build a Rust-backed request with the given wire format.""" + """Build a request with the given wire format.""" normalized = request_type_value(request_type) - chat_request = _load_native().ChatRequest if normalized == "openai_chat": - return cast("ChatRequest", chat_request.openai_chat(body)) + return ChatRequest.openai_chat(body) if normalized == "openai_responses": - return cast("ChatRequest", chat_request.openai_responses(body)) + return ChatRequest.openai_responses(body) if normalized == "anthropic": - return cast("ChatRequest", chat_request.anthropic(body)) + return ChatRequest.anthropic(body) raise ValueError(f"Unknown request type: {request_type!r}") def response_type_value(value: object) -> str: - """Normalize Rust response-format tags and wire strings.""" + """Normalize response-format tags and wire strings.""" raw = value.value if hasattr(value, "value") else value if not isinstance(raw, str): raise TypeError(f"Response type must be a string-like value, got {type(raw).__name__}") @@ -693,23 +706,9 @@ def response_type_value(value: object) -> str: raise ValueError(f"Unknown response type: {value!r}") -def response_type_enum(value: object) -> _ChatResponseType: - """Normalize a response-format tag to the Rust-bound enum object.""" - normalized = response_type_value(value) - response_type = _load_native().ChatResponseType - if normalized == "openai_completion": - return response_type.OPENAI_COMPLETION - if normalized == "openai_stream": - return response_type.OPENAI_STREAM - if normalized == "openai_responses_completion": - return response_type.OPENAI_RESPONSES_COMPLETION - if normalized == "openai_responses_stream": - return response_type.OPENAI_RESPONSES_STREAM - if normalized == "anthropic_completion": - return response_type.ANTHROPIC_COMPLETION - if normalized == "anthropic_stream": - return response_type.ANTHROPIC_STREAM - raise ValueError(f"Unknown response type: {value!r}") +def response_type_enum(value: object) -> ChatResponseType: + """Normalize a response-format tag to the Python enum.""" + return ChatResponseType(response_type_value(value)) def response_type_matches(response: object, response_type: object) -> bool: @@ -721,21 +720,20 @@ def response_with_type( response_type: object, body_or_stream: Mapping[str, Any] | JsonValue | object, ) -> ChatResponse: - """Build a Rust-backed response with the given wire shape.""" + """Build a response with the given wire shape.""" normalized = response_type_value(response_type) - chat_response = _load_native().ChatResponse if normalized == "openai_completion": - return cast("ChatResponse", chat_response.openai_completion(body_or_stream)) + return ChatResponse.openai_completion(body_or_stream) if normalized == "openai_stream": - return cast("ChatResponse", chat_response.openai_stream(body_or_stream)) + return ChatResponse.openai_stream(body_or_stream) if normalized == "openai_responses_completion": - return cast("ChatResponse", chat_response.openai_responses_completion(body_or_stream)) + return ChatResponse.openai_responses_completion(body_or_stream) if normalized == "openai_responses_stream": - return cast("ChatResponse", chat_response.openai_responses_stream(body_or_stream)) + return ChatResponse.openai_responses_stream(body_or_stream) if normalized == "anthropic_completion": - return cast("ChatResponse", chat_response.anthropic_completion(body_or_stream)) + return ChatResponse.anthropic_completion(body_or_stream) if normalized == "anthropic_stream": - return cast("ChatResponse", chat_response.anthropic_stream(body_or_stream)) + return ChatResponse.anthropic_stream(body_or_stream) raise ValueError(f"Unknown response type: {response_type!r}") @@ -743,7 +741,7 @@ def response_type_for_request_type( request_type: object, *, stream: bool, -) -> _ChatResponseType: +) -> ChatResponseType: """Return the response shape that corresponds to a request format.""" normalized = request_type_value(request_type) if normalized == "openai_chat": @@ -790,9 +788,7 @@ def response_is_streaming(response: object) -> bool: "LLMBackend", "ProxyMetadata", "ProxyContext", - "SessionCache", "Switchyard", - "session_key_from_body", "SwitchyardBackendError", "SwitchyardConfigError", "SwitchyardContextPoolExhaustedError", diff --git a/tests/test_switchyard_rust_core_bindings.py b/tests/test_switchyard_rust_core_bindings.py index 7720ab40..3ad6f1a7 100644 --- a/tests/test_switchyard_rust_core_bindings.py +++ b/tests/test_switchyard_rust_core_bindings.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Tests for Rust-owned switchyard_rust core bindings.""" +"""Tests for Python core values and their native component interop.""" from __future__ import annotations @@ -56,7 +56,7 @@ def test_request_constructors_preserve_wire_format() -> None: assert anthropic.model == "claude-sonnet-4.5" -def test_set_model_mutates_rust_owned_body() -> None: +def test_set_model_mutates_owned_body() -> None: request = ChatRequest.openai_chat({"model": "old", "messages": []}) request.set_model("new") @@ -113,6 +113,13 @@ def test_openai_completion_response_owns_body() -> None: assert response.body["model"] == "gpt-4o" +def test_buffered_response_preserves_json_null_body() -> None: + response = ChatResponse.openai_completion(None) + + assert response.body is None + assert response.to_body() is None + + def test_response_constructors_preserve_wire_shape() -> None: responses = ChatResponse.openai_responses_completion({ "id": "resp-test", @@ -131,7 +138,7 @@ def test_response_constructors_preserve_wire_shape() -> None: assert anthropic.response_type.value == "anthropic_completion" -async def test_stream_response_uses_rust_owned_async_stream_and_rejects_body_access() -> None: +async def test_stream_response_uses_owned_async_stream_and_rejects_body_access() -> None: async def source(): yield {"delta": "hello"} @@ -185,7 +192,7 @@ async def on_complete() -> None: _ = [event async for event in stream] -def test_proxy_context_is_rust_owned_with_rust_metadata_mapping() -> None: +def test_proxy_context_uses_python_wrappers_with_shared_native_state() -> None: metadata = {"request_id": "client-visible", "nested": {"value": 1}} ctx = ProxyContext(metadata=metadata, request_id="rust-request") @@ -240,7 +247,7 @@ def test_provider_stream_adapters_are_rust_chat_response_stream_aliases() -> Non assert AnthropicResponseStream is ChatResponseStream -def test_backend_role_class_is_rust_owned_public_export() -> None: +def test_backend_role_class_is_the_public_python_export() -> None: from switchyard.lib.roles import LLMBackend as PublicLLMBackend assert PublicLLMBackend is LLMBackend