From 24494dc6e6e63dba24c46b38c317bdefd0aeea00 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 24 Jun 2026 13:42:57 +0530 Subject: [PATCH 01/26] add structured telemetry event definitions --- Cargo.toml | 1 + mofa-observability/Cargo.toml | 10 + mofa-observability/src/events.rs | 553 +++++++++++++++++++++++++++++++ mofa-observability/src/lib.rs | 1 + 4 files changed, 565 insertions(+) create mode 100644 mofa-observability/Cargo.toml create mode 100644 mofa-observability/src/events.rs create mode 100644 mofa-observability/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 6e7adfc..49103f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "mofa-engine-core", "mofa-engine-sdk", "mofa-engine-app", + "mofa-observability", ] [workspace.package] diff --git a/mofa-observability/Cargo.toml b/mofa-observability/Cargo.toml new file mode 100644 index 0000000..b1402f5 --- /dev/null +++ b/mofa-observability/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mofa-observability" +version.workspace = true +edition.workspace = true +license.workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } diff --git a/mofa-observability/src/events.rs b/mofa-observability/src/events.rs new file mode 100644 index 0000000..094e134 --- /dev/null +++ b/mofa-observability/src/events.rs @@ -0,0 +1,553 @@ +//! # MoFA Engine — Event Type Definitions +//! +//! Structured event types for the observability subsystem. +//! Every interesting thing the engine does is captured as one of these events. +//! +//! Design principles: +//! - Zero side effects: events are data, not actions. +//! - Privacy: no prompt text, file contents, API keys, or user-identifying info. Ever. +//! - Bounded enums: capability, reason, source, status use enums, not free-form strings. +//! +//! Reference: observability_plan.md §3 + +use serde::{Deserialize, Serialize}; +use std::time::{SystemTime, UNIX_EPOCH}; + +// ─── Shared Enums ──────────────────────────────────────────────────────────── + +/// What the caller asked for. Bounded enum — no free-form strings. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Capability { + Chat, + Tts, + Asr, + Vlm, + ImageGen, + VideoGen, + Embedding, +} + +impl std::fmt::Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Capability::Chat => write!(f, "chat"), + Capability::Tts => write!(f, "tts"), + Capability::Asr => write!(f, "asr"), + Capability::Vlm => write!(f, "vlm"), + Capability::ImageGen => write!(f, "image_gen"), + Capability::VideoGen => write!(f, "video_gen"), + Capability::Embedding => write!(f, "embedding"), + } + } +} + +/// Why a model was unloaded. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum UnloadReason { + /// Ollama's keep_alive timer expired. + IdleTimeout, + /// Memory pressure forced eviction. + Eviction, + /// Caller explicitly requested unload. + Explicit, +} + +impl std::fmt::Display for UnloadReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + UnloadReason::IdleTimeout => write!(f, "idle_timeout"), + UnloadReason::Eviction => write!(f, "eviction"), + UnloadReason::Explicit => write!(f, "explicit"), + } + } +} + +/// Where the Preflight prediction came from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalSource { + /// App explicitly said what comes next. Confidence = 1.0. + Hint, + /// Learned from past session transitions. Confidence < 1.0. + History, +} + +impl std::fmt::Display for SignalSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SignalSource::Hint => write!(f, "hint"), + SignalSource::History => write!(f, "history"), + } + } +} + +// ─── Event Envelope ────────────────────────────────────────────────────────── + +/// Every event carries this standard envelope. +/// Reference: observability_plan.md §3.1 +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventEnvelope { + /// Unix timestamp in milliseconds. + pub timestamp_ms: u64, + /// Correlation ID — ties events from the same request together. + pub request_id: Option, + /// App or user session (if provided by caller). + pub session_id: Option, + /// The event payload. + pub event: EngineEvent, +} + +impl EventEnvelope { + /// Create an envelope with the current timestamp. + pub fn now(event: EngineEvent) -> Self { + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + + Self { + timestamp_ms, + request_id: None, + session_id: None, + event, + } + } + + /// Attach a request ID. + pub fn with_request_id(mut self, id: impl Into) -> Self { + self.request_id = Some(id.into()); + self + } + + /// Attach a session ID. + pub fn with_session_id(mut self, id: impl Into) -> Self { + self.session_id = Some(id.into()); + self + } +} + +// ─── Engine Event (discriminated union) ────────────────────────────────────── + +/// All possible engine events. The collector pattern-matches on this. +/// Reference: observability_plan.md §3.2 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EngineEvent { + // ── Request Events ─────────────────────────────────────────────────── + RequestReceived(RequestReceived), + RoutingDecision(RoutingDecision), + RequestCompleted(RequestCompleted), + + // ── Model Lifecycle Events ─────────────────────────────────────────── + ModelLoaded(ModelLoaded), + ModelUnloaded(ModelUnloaded), + + // ── Memory Events ──────────────────────────────────────────────────── + EvictionTriggered(EvictionTriggered), + + // ── Preflight Events ───────────────────────────────────────────────── + PreflightSignal(PreflightSignal), + PreflightHit(PreflightHit), + PreflightMiss(PreflightMiss), + + // ── Infrastructure Events ──────────────────────────────────────────── + ProviderDiscovered(ProviderDiscovered), + FailoverTriggered(FailoverTriggered), +} + +// ─── Request Events ────────────────────────────────────────────────────────── + +/// Emitted when the engine accepts a new inference request. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestReceived { + /// What the caller asked for. + pub capability: Capability, + /// Specific model name if the caller used named routing. + pub model: Option, + /// The `hint_next` value if provided. + pub hint: Option, +} + +/// Emitted after the router selects a model. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RoutingDecision { + /// Requested capability. + pub capability: Capability, + /// How many models were eligible. + pub candidates_count: u32, + /// The model chosen. + pub selected_model: String, + /// The backend hosting that model. + pub selected_backend: String, + /// Whether this was a fallback selection. + pub is_fallback: bool, + /// Why this model was chosen (e.g., "local_first", "only_candidate", "fallback_after_failure"). + pub reason: String, +} + +/// Emitted when inference finishes (success or failure). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequestCompleted { + /// Which model ran. + pub model_id: String, + /// Which backend. + pub backend: String, + /// What capability was served. + pub capability: Capability, + /// Total request time in milliseconds. + pub duration_ms: u64, + /// Time to first token in milliseconds (for streaming). + pub ttft_ms: Option, + /// Input token count (text models). + pub tokens_in: Option, + /// Output token count (text models). + pub tokens_out: Option, + /// Was the model already loaded when the request arrived? + /// Populated by the engine. Enables warm vs cold latency split. + pub model_was_hot: Option, + /// Whether it succeeded. + pub success: bool, + /// Error code if failed. + pub error_code: Option, +} + +// ─── Model Lifecycle Events ────────────────────────────────────────────────── + +/// A model finished loading into memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelLoaded { + /// Which model. + pub model_id: String, + /// Which backend. + pub backend: String, + /// What capability. + pub capability: Capability, + /// How long the load took in milliseconds. + pub load_duration_ms: u64, + /// How much memory it occupies in bytes. + pub memory_bytes: u64, +} + +/// A model was removed from memory. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelUnloaded { + /// Which model. + pub model_id: String, + /// Why it was unloaded. + pub reason: UnloadReason, + /// How much memory was released in bytes. + pub memory_freed_bytes: u64, +} + +// ─── Memory Events ─────────────────────────────────────────────────────────── + +/// Memory pressure forced a model eviction. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvictionTriggered { + /// Which model was evicted. + pub evicted_model: String, + /// Used memory before eviction in bytes. + pub memory_before_bytes: u64, + /// Used memory after eviction in bytes. + pub memory_after_bytes: u64, + /// Total memory budget in bytes. + pub budget_bytes: u64, +} + +// ─── Preflight Events ──────────────────────────────────────────────────────── + +/// Preflight made a prediction. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PreflightSignal { + /// What Preflight thinks comes next. + pub predicted_capability: Capability, + /// 0.0–1.0 (hints are always 1.0). + pub confidence: f64, + /// Where the prediction came from. + pub source: SignalSource, +} + +/// The prediction was correct — the predicted model was used next. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PreflightHit { + /// What was predicted. + pub predicted_capability: Capability, + /// How much load time was saved in milliseconds. + /// Computed by engine/scheduler, not Preflight. + pub cold_start_avoided_ms: u64, +} + +/// The prediction was wrong — a different capability was requested. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PreflightMiss { + /// What was predicted. + pub predicted_capability: Capability, + /// What actually happened. + pub actual_capability: Capability, +} + +// ─── Infrastructure Events ─────────────────────────────────────────────────── + +/// A backend was found during discovery. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProviderDiscovered { + /// Backend name. + pub provider_name: String, + /// How many models were discovered. + pub models_found: u32, + /// What capabilities are available. + pub capabilities: Vec, +} + +/// Primary model failed, switching to fallback. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FailoverTriggered { + /// The model that failed. + pub failed_model: String, + /// Its backend. + pub failed_backend: String, + /// The replacement model. + pub fallback_model: String, + /// Its backend. + pub fallback_backend: String, +} + +// ─── Privacy Contract ──────────────────────────────────────────────────────── +// +// These fields are NEVER present in any event: +// - Prompt text or generated text +// - File contents or file paths +// - API keys, tokens, or credentials +// - User-identifying information +// +// All categorical fields use bounded enums, not free-form strings: +// - capability → Capability enum (7 variants) +// - reason → UnloadReason enum (3 variants) +// - source → SignalSource enum (2 variants) +// - status → bool (success/failure) +// +// Reference: observability_plan.md §3.3 + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_capability_display() { + assert_eq!(Capability::Chat.to_string(), "chat"); + assert_eq!(Capability::Tts.to_string(), "tts"); + assert_eq!(Capability::ImageGen.to_string(), "image_gen"); + } + + #[test] + fn test_unload_reason_display() { + assert_eq!(UnloadReason::IdleTimeout.to_string(), "idle_timeout"); + assert_eq!(UnloadReason::Eviction.to_string(), "eviction"); + assert_eq!(UnloadReason::Explicit.to_string(), "explicit"); + } + + #[test] + fn test_signal_source_display() { + assert_eq!(SignalSource::Hint.to_string(), "hint"); + assert_eq!(SignalSource::History.to_string(), "history"); + } + + #[test] + fn test_envelope_creation() { + let event = EngineEvent::RequestReceived(RequestReceived { + capability: Capability::Chat, + model: None, + hint: Some("tts".into()), + }); + + let envelope = EventEnvelope::now(event) + .with_request_id("req-001") + .with_session_id("sess-001"); + + assert!(envelope.timestamp_ms > 0); + assert_eq!(envelope.request_id.as_deref(), Some("req-001")); + assert_eq!(envelope.session_id.as_deref(), Some("sess-001")); + } + + #[test] + fn test_request_completed_with_model_was_hot() { + let event = RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: Some(115), + tokens_in: Some(50), + tokens_out: Some(277), + model_was_hot: Some(true), + success: true, + error_code: None, + }; + + // Warm TTFT should be ~0.1s (Phase 0 measured 0.115s) + assert_eq!(event.model_was_hot, Some(true)); + assert!(event.ttft_ms.unwrap() < 200); // warm is fast + } + + #[test] + fn test_preflight_signal_hint() { + let signal = PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 1.0, + source: SignalSource::Hint, + }; + + // Phase 0 proved: hints have accuracy 1.0 + assert_eq!(signal.confidence, 1.0); + assert_eq!(signal.source, SignalSource::Hint); + } + + #[test] + fn test_preflight_hit() { + let hit = PreflightHit { + predicted_capability: Capability::Tts, + // Phase 0 measured: Kokoro TTS saving = 2404ms on Mac M4 + cold_start_avoided_ms: 2404, + }; + + assert_eq!(hit.predicted_capability, Capability::Tts); + assert!(hit.cold_start_avoided_ms > 0); + } + + #[test] + fn test_preflight_miss() { + let miss = PreflightMiss { + predicted_capability: Capability::Tts, + actual_capability: Capability::Asr, + }; + + assert_ne!(miss.predicted_capability, miss.actual_capability); + } + + #[test] + fn test_model_loaded_event() { + let loaded = ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + // Phase 0 measured: 1532ms cold load on Mac M4 + load_duration_ms: 1532, + // Phase 0 measured: 4.7 GB + memory_bytes: 4_700_000_000, + }; + + assert!(loaded.load_duration_ms > 0); + assert!(loaded.memory_bytes > 0); + } + + #[test] + fn test_eviction_event() { + let eviction = EvictionTriggered { + evicted_model: "gemma3:4b".into(), + memory_before_bytes: 8_000_000_000, + memory_after_bytes: 4_700_000_000, + budget_bytes: 11_000_000_000, + }; + + assert!(eviction.memory_after_bytes < eviction.memory_before_bytes); + assert!(eviction.memory_after_bytes < eviction.budget_bytes); + } + + #[test] + fn test_event_serialization_roundtrip() { + let original = EventEnvelope::now(EngineEvent::PreflightSignal(PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 1.0, + source: SignalSource::Hint, + })) + .with_request_id("req-042"); + + let json = serde_json::to_string(&original).expect("serialize"); + let restored: EventEnvelope = serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(original.timestamp_ms, restored.timestamp_ms); + assert_eq!(original.request_id, restored.request_id); + } + + #[test] + fn test_all_event_variants_serialize() { + // Every variant must serialize without panic. + let events: Vec = vec![ + EngineEvent::RequestReceived(RequestReceived { + capability: Capability::Chat, + model: None, + hint: None, + }), + EngineEvent::RoutingDecision(RoutingDecision { + capability: Capability::Chat, + candidates_count: 2, + selected_model: "qwen2.5:7b".into(), + selected_backend: "ollama".into(), + is_fallback: false, + reason: "local_first".into(), + }), + EngineEvent::RequestCompleted(RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: Some(115), + tokens_in: Some(50), + tokens_out: Some(277), + model_was_hot: Some(true), + success: true, + error_code: None, + }), + EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }), + EngineEvent::ModelUnloaded(ModelUnloaded { + model_id: "gemma3:4b".into(), + reason: UnloadReason::IdleTimeout, + memory_freed_bytes: 3_300_000_000, + }), + EngineEvent::EvictionTriggered(EvictionTriggered { + evicted_model: "gemma3:4b".into(), + memory_before_bytes: 8_000_000_000, + memory_after_bytes: 4_700_000_000, + budget_bytes: 11_000_000_000, + }), + EngineEvent::PreflightSignal(PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 1.0, + source: SignalSource::Hint, + }), + EngineEvent::PreflightHit(PreflightHit { + predicted_capability: Capability::Tts, + cold_start_avoided_ms: 2404, + }), + EngineEvent::PreflightMiss(PreflightMiss { + predicted_capability: Capability::Tts, + actual_capability: Capability::Chat, + }), + EngineEvent::ProviderDiscovered(ProviderDiscovered { + provider_name: "ollama".into(), + models_found: 3, + capabilities: vec![Capability::Chat, Capability::Vlm], + }), + EngineEvent::FailoverTriggered(FailoverTriggered { + failed_model: "qwen2.5:7b".into(), + failed_backend: "ollama".into(), + fallback_model: "gemma3:4b".into(), + fallback_backend: "ollama".into(), + }), + ]; + + for event in events { + let envelope = EventEnvelope::now(event); + let json = serde_json::to_string(&envelope); + assert!(json.is_ok(), "Failed to serialize: {:?}", envelope); + } + } +} diff --git a/mofa-observability/src/lib.rs b/mofa-observability/src/lib.rs new file mode 100644 index 0000000..a9970c2 --- /dev/null +++ b/mofa-observability/src/lib.rs @@ -0,0 +1 @@ +pub mod events; From 6d3e667ccf0f9febb48e8114e82eb0e0d438ac8d Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 24 Jun 2026 13:46:30 +0530 Subject: [PATCH 02/26] implement async metrics collector --- mofa-observability/src/collector.rs | 1016 +++++++++++++++++++++++++++ mofa-observability/src/lib.rs | 1 + 2 files changed, 1017 insertions(+) create mode 100644 mofa-observability/src/collector.rs diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs new file mode 100644 index 0000000..1bc8e72 --- /dev/null +++ b/mofa-observability/src/collector.rs @@ -0,0 +1,1016 @@ +//! # MoFA Engine — Metrics Collector +//! +//! Receives engine events from a channel and updates in-memory metric state. +//! The Prometheus renderer reads this state when scraped. +//! +//! Design: +//! - Single writer (collector task), multiple readers (Prometheus scrapes). +//! - No blocking I/O. No disk. Just in-memory atomics behind a RwLock. +//! - Bounded event channel with drop-oldest backpressure. +//! +//! Reference: observability_plan.md §4 + +use crate::events::*; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{mpsc, RwLock}; + +// ─── Labels ────────────────────────────────────────────────────────────────── + +/// Metric labels. Sorted by key for consistent ordering. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Labels(Vec<(String, String)>); + +impl Labels { + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn add(mut self, key: impl Into, value: impl Into) -> Self { + self.0.push((key.into(), value.into())); + self.0.sort_by(|a, b| a.0.cmp(&b.0)); + self + } + + pub fn pairs(&self) -> &[(String, String)] { + &self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl Default for Labels { + fn default() -> Self { + Self::new() + } +} + +// ─── Metric Types ──────────────────────────────────────────────────────────── + +/// A single histogram's accumulated state. +#[derive(Debug, Clone)] +pub struct HistogramValue { + /// (upper_bound, cumulative_count) — sorted ascending. + pub buckets: Vec<(f64, u64)>, + pub sum: f64, + pub count: u64, +} + +impl HistogramValue { + fn new(boundaries: &[f64]) -> Self { + let buckets = boundaries.iter().map(|&b| (b, 0u64)).collect(); + Self { + buckets, + sum: 0.0, + count: 0, + } + } + + fn observe(&mut self, value: f64) { + self.sum += value; + self.count += 1; + for bucket in &mut self.buckets { + if value <= bucket.0 { + bucket.1 += 1; + } + } + } +} + +/// A family of labeled counters (e.g., mofa_requests_total with labels). +#[derive(Debug, Clone)] +pub struct CounterFamily { + pub name: &'static str, + pub help: &'static str, + pub values: HashMap, +} + +impl CounterFamily { + fn new(name: &'static str, help: &'static str) -> Self { + Self { + name, + help, + values: HashMap::new(), + } + } + + fn inc(&mut self, labels: Labels) { + *self.values.entry(labels).or_insert(0) += 1; + } + + fn inc_by(&mut self, labels: Labels, n: u64) { + *self.values.entry(labels).or_insert(0) += n; + } +} + +/// A family of labeled histograms. +#[derive(Debug, Clone)] +pub struct HistogramFamily { + pub name: &'static str, + pub help: &'static str, + pub bucket_boundaries: Vec, + pub values: HashMap, +} + +impl HistogramFamily { + fn new(name: &'static str, help: &'static str, boundaries: Vec) -> Self { + Self { + name, + help, + bucket_boundaries: boundaries, + values: HashMap::new(), + } + } + + fn observe(&mut self, labels: Labels, value: f64) { + self.values + .entry(labels) + .or_insert_with(|| HistogramValue::new(&self.bucket_boundaries)) + .observe(value); + } +} + +/// A family of labeled gauges. +#[derive(Debug, Clone)] +pub struct GaugeFamily { + pub name: &'static str, + pub help: &'static str, + pub values: HashMap, +} + +impl GaugeFamily { + fn new(name: &'static str, help: &'static str) -> Self { + Self { + name, + help, + values: HashMap::new(), + } + } + + fn set(&mut self, labels: Labels, value: f64) { + self.values.insert(labels, value); + } + + fn inc(&mut self, labels: Labels) { + *self.values.entry(labels).or_insert(0.0) += 1.0; + } + + fn dec(&mut self, labels: Labels) { + *self.values.entry(labels).or_insert(0.0) -= 1.0; + } + + fn add(&mut self, labels: Labels, n: f64) { + *self.values.entry(labels).or_insert(0.0) += n; + } + + fn sub(&mut self, labels: Labels, n: f64) { + *self.values.entry(labels).or_insert(0.0) -= n; + } +} + +// ─── Metrics State ─────────────────────────────────────────────────────────── + +/// All metric state. One instance per engine. +/// The collector writes, the Prometheus renderer reads. +/// +/// Reference: observability_plan.md §4.2–4.4 +#[derive(Debug, Clone)] +pub struct MetricsState { + // ── Counters (§4.2) ────────────────────────────────────────────────── + pub requests_total: CounterFamily, + pub model_loads_total: CounterFamily, + pub model_unloads_total: CounterFamily, + pub failovers_total: CounterFamily, + pub evictions_total: CounterFamily, + pub preflight_predictions_total: CounterFamily, + pub preflight_hits_total: CounterFamily, + pub preflight_misses_total: CounterFamily, + pub tokens_input_total: CounterFamily, + pub tokens_output_total: CounterFamily, + pub events_dropped_total: CounterFamily, + + // ── Histograms (§4.3) ──────────────────────────────────────────────── + pub request_duration_seconds: HistogramFamily, + pub model_load_seconds: HistogramFamily, + pub ttft_seconds: HistogramFamily, + + // ── Gauges (§4.4) ──────────────────────────────────────────────────── + pub memory_used_bytes: GaugeFamily, + pub memory_budget_bytes: GaugeFamily, + pub models_loaded: GaugeFamily, + pub active_requests: GaugeFamily, +} + +impl MetricsState { + /// Create a fresh state with all metrics initialized. + /// Bucket boundaries are calibrated to Phase 0 measurements. + pub fn new() -> Self { + Self { + // Counters + requests_total: CounterFamily::new( + "mofa_requests_total", + "Total inference requests", + ), + model_loads_total: CounterFamily::new( + "mofa_model_loads_total", + "Total model load operations", + ), + model_unloads_total: CounterFamily::new( + "mofa_model_unloads_total", + "Total model unload operations", + ), + failovers_total: CounterFamily::new( + "mofa_failovers_total", + "Total failover events", + ), + evictions_total: CounterFamily::new( + "mofa_evictions_total", + "Total memory pressure evictions", + ), + preflight_predictions_total: CounterFamily::new( + "mofa_preflight_predictions_total", + "Total Preflight predictions", + ), + preflight_hits_total: CounterFamily::new( + "mofa_preflight_hits_total", + "Total correct Preflight predictions", + ), + preflight_misses_total: CounterFamily::new( + "mofa_preflight_misses_total", + "Total incorrect Preflight predictions", + ), + tokens_input_total: CounterFamily::new( + "mofa_tokens_input_total", + "Total input tokens processed", + ), + tokens_output_total: CounterFamily::new( + "mofa_tokens_output_total", + "Total output tokens generated", + ), + events_dropped_total: CounterFamily::new( + "mofa_events_dropped_total", + "Total events dropped due to channel backpressure", + ), + + // Histograms — bucket boundaries from Phase 0 measurements. + // + // mofa_request_duration_seconds: + // Phase 0: article gen = 13–18s, warm inference = 0.1–0.6s. + request_duration_seconds: HistogramFamily::new( + "mofa_request_duration_seconds", + "Request duration in seconds", + vec![0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 60.0], + ), + // mofa_model_load_seconds: + // Phase 0: 1.5s (Qwen 7B Mac) to 97s (Qwen 7B FP16 CUDA). + // Most common range: 2–5s. + model_load_seconds: HistogramFamily::new( + "mofa_model_load_seconds", + "Time to load a model from cold", + vec![0.5, 1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0], + ), + // mofa_ttft_seconds: + // Phase 0: warm TTFT = 0.1–0.2s (Mac), 0.4–0.6s (CUDA). + // Cold TTFT = 1.5–5.5s. Fine granularity in 0.1–0.2s range + // because that's where Preflight's value shows up. + ttft_seconds: HistogramFamily::new( + "mofa_ttft_seconds", + "Time to first token", + vec![0.05, 0.1, 0.15, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], + ), + + // Gauges + memory_used_bytes: GaugeFamily::new( + "mofa_memory_used_bytes", + "Current engine memory usage in bytes", + ), + memory_budget_bytes: GaugeFamily::new( + "mofa_memory_budget_bytes", + "Total engine memory budget in bytes", + ), + models_loaded: GaugeFamily::new( + "mofa_models_loaded", + "Number of models currently loaded", + ), + active_requests: GaugeFamily::new( + "mofa_active_requests", + "Number of requests currently in flight", + ), + } + } + + /// Process a single engine event, updating all relevant metrics. + pub fn process_event(&mut self, envelope: &EventEnvelope) { + match &envelope.event { + EngineEvent::RequestReceived(e) => { + self.active_requests.inc(Labels::new()); + // RequestReceived doesn't increment requests_total — + // that happens on RequestCompleted with success/error status. + let _ = e; // capability is used for routing, not counted here. + } + + EngineEvent::RoutingDecision(_) => { + // Routing metadata is captured for future decision-record phases. + // Phase A: no derived metrics from routing decisions. + } + + EngineEvent::RequestCompleted(e) => { + // Counter: requests by capability × status + let status = if e.success { "success" } else { "error" }; + self.requests_total.inc( + Labels::new() + .add("capability", e.capability.to_string()) + .add("status", status), + ); + + // Histogram: request duration + self.request_duration_seconds.observe( + Labels::new().add("capability", e.capability.to_string()), + e.duration_ms as f64 / 1000.0, + ); + + // Histogram: TTFT (if present) + if let Some(ttft) = e.ttft_ms { + self.ttft_seconds.observe( + Labels::new() + .add("model", &e.model_id) + .add("backend", &e.backend), + ttft as f64 / 1000.0, + ); + } + + // Counter: tokens + if let Some(tokens_in) = e.tokens_in { + self.tokens_input_total.inc_by( + Labels::new().add("model", &e.model_id), + tokens_in, + ); + } + if let Some(tokens_out) = e.tokens_out { + self.tokens_output_total.inc_by( + Labels::new().add("model", &e.model_id), + tokens_out, + ); + } + + // Gauge: active requests down + self.active_requests.dec(Labels::new()); + } + + EngineEvent::ModelLoaded(e) => { + // Counter + self.model_loads_total.inc( + Labels::new() + .add("model", &e.model_id) + .add("backend", &e.backend), + ); + + // Histogram: load duration + self.model_load_seconds.observe( + Labels::new() + .add("model", &e.model_id) + .add("backend", &e.backend), + e.load_duration_ms as f64 / 1000.0, + ); + + // Gauge: memory up, model count up + self.memory_used_bytes + .add(Labels::new(), e.memory_bytes as f64); + self.models_loaded.inc(Labels::new()); + } + + EngineEvent::ModelUnloaded(e) => { + // Counter + self.model_unloads_total.inc( + Labels::new() + .add("model", &e.model_id) + .add("reason", e.reason.to_string()), + ); + + // Gauge: memory down, model count down + self.memory_used_bytes + .sub(Labels::new(), e.memory_freed_bytes as f64); + self.models_loaded.dec(Labels::new()); + } + + EngineEvent::EvictionTriggered(_) => { + self.evictions_total.inc(Labels::new()); + } + + EngineEvent::PreflightSignal(e) => { + self.preflight_predictions_total.inc( + Labels::new().add("source", e.source.to_string()), + ); + } + + EngineEvent::PreflightHit(_) => { + self.preflight_hits_total.inc(Labels::new()); + } + + EngineEvent::PreflightMiss(_) => { + self.preflight_misses_total.inc(Labels::new()); + } + + EngineEvent::ProviderDiscovered(_) => { + // Logged, not metricked. Provider discovery is a startup event. + } + + EngineEvent::FailoverTriggered(e) => { + // We don't have capability on FailoverTriggered, so label-free. + let _ = e; + self.failovers_total.inc(Labels::new()); + } + } + } +} + +impl Default for MetricsState { + fn default() -> Self { + Self::new() + } +} + +// ─── Event Channel ─────────────────────────────────────────────────────────── + +/// Sender half of the event channel. Clone this for each event source. +#[derive(Clone)] +pub struct EventSender { + tx: mpsc::Sender, + dropped: Arc, +} + +impl EventSender { + /// Send an event. If the channel is full, the event is dropped + /// and the dropped counter is incremented. + pub fn send(&self, event: EventEnvelope) { + if self.tx.try_send(event).is_err() { + self.dropped + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + } + + /// How many events have been dropped due to backpressure. + pub fn dropped_count(&self) -> u64 { + self.dropped.load(std::sync::atomic::Ordering::Relaxed) + } +} + +/// Receiver half of the event channel. Owned by the collector. +pub struct EventReceiver { + rx: mpsc::Receiver, + dropped: Arc, +} + +/// Create a bounded event channel. +/// +/// `capacity`: max events buffered before backpressure kicks in. +/// Returns (sender, receiver). Clone the sender for multiple producers. +pub fn create_event_channel(capacity: usize) -> (EventSender, EventReceiver) { + let (tx, rx) = mpsc::channel(capacity); + let dropped = Arc::new(std::sync::atomic::AtomicU64::new(0)); + + let sender = EventSender { + tx, + dropped: Arc::clone(&dropped), + }; + let receiver = EventReceiver { rx, dropped }; + + (sender, receiver) +} + +// ─── Metrics Collector ─────────────────────────────────────────────────────── + +/// The collector. Owns the shared metrics state and runs the processing loop. +/// +/// Usage: +/// ```ignore +/// let (sender, receiver) = create_event_channel(1024); +/// let collector = MetricsCollector::new(receiver); +/// let state = collector.state(); // share with Prometheus renderer +/// tokio::spawn(collector.run()); +/// ``` +pub struct MetricsCollector { + state: Arc>, + receiver: EventReceiver, +} + +impl MetricsCollector { + pub fn new(receiver: EventReceiver) -> Self { + Self { + state: Arc::new(RwLock::new(MetricsState::new())), + receiver, + } + } + + /// Get a handle to the shared metrics state. + /// Pass this to the Prometheus renderer. + pub fn state(&self) -> Arc> { + Arc::clone(&self.state) + } + + /// Run the collector loop. Processes events until the channel closes. + pub async fn run(mut self) { + while let Some(envelope) = self.receiver.rx.recv().await { + let mut state = self.state.write().await; + + // Sync the dropped counter into metrics state. + let dropped = self + .receiver + .dropped + .swap(0, std::sync::atomic::Ordering::Relaxed); + if dropped > 0 { + state + .events_dropped_total + .inc_by(Labels::new(), dropped); + } + + state.process_event(&envelope); + } + } +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn make_state() -> MetricsState { + MetricsState::new() + } + + // ── Counter tests ──────────────────────────────────────────────────── + + #[test] + fn test_request_completed_increments_counter() { + let mut state = make_state(); + let event = EventEnvelope::now(EngineEvent::RequestCompleted(RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: Some(115), + tokens_in: Some(50), + tokens_out: Some(277), + model_was_hot: Some(true), + success: true, + error_code: None, + })); + + state.process_event(&event); + + let labels = Labels::new() + .add("capability", "chat") + .add("status", "success"); + assert_eq!(state.requests_total.values[&labels], 1); + } + + #[test] + fn test_five_requests_counted() { + let mut state = make_state(); + + for _ in 0..5 { + let event = EventEnvelope::now(EngineEvent::RequestCompleted(RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: true, + error_code: None, + })); + state.process_event(&event); + } + + let labels = Labels::new() + .add("capability", "chat") + .add("status", "success"); + assert_eq!(state.requests_total.values[&labels], 5); + } + + #[test] + fn test_error_requests_separate_from_success() { + let mut state = make_state(); + + // 3 successes + for _ in 0..3 { + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: true, + error_code: None, + }, + ))); + } + + // 1 error + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 500, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: false, + error_code: Some("model_not_found".into()), + }, + ))); + + let success = Labels::new() + .add("capability", "chat") + .add("status", "success"); + let error = Labels::new() + .add("capability", "chat") + .add("status", "error"); + + assert_eq!(state.requests_total.values[&success], 3); + assert_eq!(state.requests_total.values[&error], 1); + } + + // ── Histogram tests ────────────────────────────────────────────────── + + #[test] + fn test_model_load_histogram_bucketing() { + let mut state = make_state(); + + // Phase 0: Qwen 7B cold load on Mac = 1.532s + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + + let labels = Labels::new() + .add("backend", "ollama") + .add("model", "qwen2.5:7b"); + let hist = &state.model_load_seconds.values[&labels]; + + // 1.532s should be in the le=2.0 bucket but NOT le=1.0 + assert_eq!(hist.buckets[1].1, 0); // le=1.0: no + assert_eq!(hist.buckets[2].1, 1); // le=2.0: yes + assert_eq!(hist.count, 1); + assert!((hist.sum - 1.532).abs() < 0.001); + } + + #[test] + fn test_ttft_histogram_warm_vs_cold() { + let mut state = make_state(); + + // Phase 0: warm TTFT = 0.115s + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: Some(115), + tokens_in: None, + tokens_out: None, + model_was_hot: Some(true), + success: true, + error_code: None, + }, + ))); + + let labels = Labels::new() + .add("backend", "ollama") + .add("model", "qwen2.5:7b"); + let hist = &state.ttft_seconds.values[&labels]; + + // 0.115s should be in le=0.15 but NOT le=0.1 + assert_eq!(hist.buckets[1].1, 0); // le=0.1: no + assert_eq!(hist.buckets[2].1, 1); // le=0.15: yes + } + + // ── Gauge tests ────────────────────────────────────────────────────── + + #[test] + fn test_models_loaded_gauge_up_down() { + let mut state = make_state(); + + // Load a model + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + assert_eq!(state.models_loaded.values[&Labels::new()], 1.0); + + // Load another + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "gemma3:4b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 2428, + memory_bytes: 3_300_000_000, + }))); + assert_eq!(state.models_loaded.values[&Labels::new()], 2.0); + + // Unload one + state.process_event(&EventEnvelope::now(EngineEvent::ModelUnloaded( + ModelUnloaded { + model_id: "gemma3:4b".into(), + reason: UnloadReason::IdleTimeout, + memory_freed_bytes: 3_300_000_000, + }, + ))); + assert_eq!(state.models_loaded.values[&Labels::new()], 1.0); + } + + #[test] + fn test_memory_gauge_tracks_bytes() { + let mut state = make_state(); + + // Load Qwen 7B (4.7 GB) + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + assert_eq!( + state.memory_used_bytes.values[&Labels::new()], + 4_700_000_000.0 + ); + + // Load Gemma 4B (3.3 GB) + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "gemma3:4b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 2428, + memory_bytes: 3_300_000_000, + }))); + assert_eq!( + state.memory_used_bytes.values[&Labels::new()], + 8_000_000_000.0 + ); + + // Unload Gemma + state.process_event(&EventEnvelope::now(EngineEvent::ModelUnloaded( + ModelUnloaded { + model_id: "gemma3:4b".into(), + reason: UnloadReason::Eviction, + memory_freed_bytes: 3_300_000_000, + }, + ))); + assert_eq!( + state.memory_used_bytes.values[&Labels::new()], + 4_700_000_000.0 + ); + } + + #[test] + fn test_active_requests_gauge() { + let mut state = make_state(); + + // Request arrives + state.process_event(&EventEnvelope::now(EngineEvent::RequestReceived( + RequestReceived { + capability: Capability::Chat, + model: None, + hint: Some("tts".into()), + }, + ))); + assert_eq!(state.active_requests.values[&Labels::new()], 1.0); + + // Another arrives + state.process_event(&EventEnvelope::now(EngineEvent::RequestReceived( + RequestReceived { + capability: Capability::Tts, + model: None, + hint: None, + }, + ))); + assert_eq!(state.active_requests.values[&Labels::new()], 2.0); + + // First completes + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: true, + error_code: None, + }, + ))); + assert_eq!(state.active_requests.values[&Labels::new()], 1.0); + } + + // ── Preflight tests ────────────────────────────────────────────────── + + #[test] + fn test_preflight_accuracy_from_counters() { + let mut state = make_state(); + + // 3 hits + 1 miss = 75% accuracy + for _ in 0..3 { + state.process_event(&EventEnvelope::now(EngineEvent::PreflightHit( + PreflightHit { + predicted_capability: Capability::Tts, + cold_start_avoided_ms: 2404, + }, + ))); + } + state.process_event(&EventEnvelope::now(EngineEvent::PreflightMiss( + PreflightMiss { + predicted_capability: Capability::Tts, + actual_capability: Capability::Chat, + }, + ))); + + let hits = state.preflight_hits_total.values[&Labels::new()]; + let misses = state.preflight_misses_total.values[&Labels::new()]; + let accuracy = hits as f64 / (hits + misses) as f64; + + assert_eq!(hits, 3); + assert_eq!(misses, 1); + assert!((accuracy - 0.75).abs() < 0.001); + } + + #[test] + fn test_preflight_predictions_by_source() { + let mut state = make_state(); + + // 2 hint predictions + 1 history prediction + for _ in 0..2 { + state.process_event(&EventEnvelope::now(EngineEvent::PreflightSignal( + PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 1.0, + source: SignalSource::Hint, + }, + ))); + } + state.process_event(&EventEnvelope::now(EngineEvent::PreflightSignal( + PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 0.9, + source: SignalSource::History, + }, + ))); + + let hint_labels = Labels::new().add("source", "hint"); + let history_labels = Labels::new().add("source", "history"); + + assert_eq!(state.preflight_predictions_total.values[&hint_labels], 2); + assert_eq!( + state.preflight_predictions_total.values[&history_labels], + 1 + ); + } + + // ── Token counting tests ───────────────────────────────────────────── + + #[test] + fn test_token_counting() { + let mut state = make_state(); + + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: None, + tokens_in: Some(50), + tokens_out: Some(277), + model_was_hot: None, + success: true, + error_code: None, + }, + ))); + + let labels = Labels::new().add("model", "qwen2.5:7b"); + assert_eq!(state.tokens_input_total.values[&labels], 50); + assert_eq!(state.tokens_output_total.values[&labels], 277); + } + + // ── Channel + collector integration test ───────────────────────────── + + #[tokio::test] + async fn test_collector_processes_events_from_channel() { + let (sender, receiver) = create_event_channel(64); + let collector = MetricsCollector::new(receiver); + let state = collector.state(); + + // Start collector in background + let handle = tokio::spawn(collector.run()); + + // Send events + sender.send(EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + + sender.send(EventEnvelope::now(EngineEvent::PreflightSignal( + PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 1.0, + source: SignalSource::Hint, + }, + ))); + + // Drop sender to close channel — collector loop will exit + drop(sender); + handle.await.unwrap(); + + // Verify state + let state = state.read().await; + assert_eq!(state.models_loaded.values[&Labels::new()], 1.0); + + let hint_labels = Labels::new().add("source", "hint"); + assert_eq!(state.preflight_predictions_total.values[&hint_labels], 1); + } + + // ── Backpressure test ──────────────────────────────────────────────── + + #[test] + fn test_backpressure_counts_drops() { + // Channel of size 2 + let (sender, _receiver) = create_event_channel(2); + + // Fill channel + for _ in 0..5 { + sender.send(EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "test".into(), + memory_before_bytes: 100, + memory_after_bytes: 50, + budget_bytes: 100, + }, + ))); + } + + // At least some should be dropped (channel size = 2, sent 5) + assert!(sender.dropped_count() >= 3); + } + + // ── Label cardinality test ─────────────────────────────────────────── + + #[test] + fn test_label_cardinality_bounded() { + let mut state = make_state(); + + // Simulate 100 events across all 7 capabilities × 2 statuses = 14 series max + let capabilities = [ + Capability::Chat, + Capability::Tts, + Capability::Asr, + Capability::Vlm, + Capability::ImageGen, + Capability::VideoGen, + Capability::Embedding, + ]; + + for cap in &capabilities { + for success in [true, false] { + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "test".into(), + backend: "ollama".into(), + capability: *cap, + duration_ms: 1000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success, + error_code: None, + }, + ))); + } + } + + // 7 capabilities × 2 statuses = 14 series. Within §4.5 bounds. + assert_eq!(state.requests_total.values.len(), 14); + } +} diff --git a/mofa-observability/src/lib.rs b/mofa-observability/src/lib.rs index a9970c2..bf168e8 100644 --- a/mofa-observability/src/lib.rs +++ b/mofa-observability/src/lib.rs @@ -1 +1,2 @@ pub mod events; +pub mod collector; From 43ae6e52b6033cc65c8ec582e684a358d3afce67 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 24 Jun 2026 13:50:50 +0530 Subject: [PATCH 03/26] add OpenTelemetry span fields for distributed tracing to envelope --- mofa-observability/src/events.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mofa-observability/src/events.rs b/mofa-observability/src/events.rs index 094e134..e5682f5 100644 --- a/mofa-observability/src/events.rs +++ b/mofa-observability/src/events.rs @@ -95,6 +95,12 @@ pub struct EventEnvelope { pub request_id: Option, /// App or user session (if provided by caller). pub session_id: Option, + /// OpenTelemetry trace ID (128-bit hex string). + pub trace_id: Option, + /// OpenTelemetry span ID (64-bit hex string). + pub span_id: Option, + /// OpenTelemetry span end timestamp (Unix ms). + pub span_end_timestamp: Option, /// The event payload. pub event: EngineEvent, } @@ -111,6 +117,9 @@ impl EventEnvelope { timestamp_ms, request_id: None, session_id: None, + trace_id: None, + span_id: None, + span_end_timestamp: None, event, } } @@ -126,6 +135,19 @@ impl EventEnvelope { self.session_id = Some(id.into()); self } + + /// Attach OpenTelemetry tracing context. + pub fn with_trace(mut self, trace_id: impl Into, span_id: impl Into) -> Self { + self.trace_id = Some(trace_id.into()); + self.span_id = Some(span_id.into()); + self + } + + /// Set span end timestamp. + pub fn with_span_end(mut self, timestamp_ms: u64) -> Self { + self.span_end_timestamp = Some(timestamp_ms); + self + } } // ─── Engine Event (discriminated union) ────────────────────────────────────── From 5680e769368f47bcdef17450a08af36d66137f57 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 24 Jun 2026 13:53:40 +0530 Subject: [PATCH 04/26] chore: update Cargo.lock for mofa-observability --- Cargo.lock | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 022e0d5..1e4489e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1093,6 +1093,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "mofa-observability" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "tokio", +] + [[package]] name = "native-tls" version = "0.2.18" From 1a9d07a4941d2c87beed7a20eb3ac2fd7c4b7b74 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Sat, 27 Jun 2026 19:29:18 +0530 Subject: [PATCH 05/26] add prometheus text exposition renderer --- mofa-observability/src/lib.rs | 1 + mofa-observability/src/prometheus.rs | 506 +++++++++++++++++++++++++++ 2 files changed, 507 insertions(+) create mode 100644 mofa-observability/src/prometheus.rs diff --git a/mofa-observability/src/lib.rs b/mofa-observability/src/lib.rs index bf168e8..9105b77 100644 --- a/mofa-observability/src/lib.rs +++ b/mofa-observability/src/lib.rs @@ -1,2 +1,3 @@ pub mod events; pub mod collector; +pub mod prometheus; diff --git a/mofa-observability/src/prometheus.rs b/mofa-observability/src/prometheus.rs new file mode 100644 index 0000000..b62c487 --- /dev/null +++ b/mofa-observability/src/prometheus.rs @@ -0,0 +1,506 @@ +//! # Prometheus Text Exposition Renderer +//! +//! Converts the in-memory `MetricsState` into valid Prometheus text format. +//! +//! Reference: https://prometheus.io/docs/instrumenting/exposition_formats/ +//! +//! The renderer is a pure function: `MetricsState` in, `String` out. +//! No I/O. Minimal allocations for sorting and formatting. + +use crate::collector::{CounterFamily, GaugeFamily, HistogramFamily, Labels, MetricsState}; +use std::fmt::Write; + +// ─── Label Formatting ──────────────────────────────────────────────────────── + +/// Format labels into Prometheus `{key="value",key2="value2"}` syntax. +/// Returns empty string if there are no labels. +fn format_labels(labels: &Labels) -> String { + if labels.is_empty() { + return String::new(); + } + let mut out = String::from("{"); + for (i, (key, value)) in labels.pairs().iter().enumerate() { + if i > 0 { + out.push(','); + } + // Escape backslash, double-quote, and newline in label values. + let escaped = value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n"); + write!(out, "{}=\"{}\"", key, escaped).unwrap(); + } + out.push('}'); + out +} + +/// Format labels with an extra label appended (used for histogram `le` buckets). +/// The extra label is added after existing labels. +fn format_labels_with_extra(labels: &Labels, extra_key: &str, extra_value: &str) -> String { + let mut out = String::from("{"); + for (key, value) in labels.pairs() { + let escaped = value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n"); + write!(out, "{}=\"{}\",", key, escaped).unwrap(); + } + write!(out, "{}=\"{}\"", extra_key, extra_value).unwrap(); + out.push('}'); + out +} + +// ─── Metric Rendering ──────────────────────────────────────────────────────── + +/// Render a counter family into Prometheus text format. +fn render_counter(buf: &mut String, family: &CounterFamily) { + if family.values.is_empty() { + return; + } + writeln!(buf, "# HELP {} {}", family.name, family.help).unwrap(); + writeln!(buf, "# TYPE {} counter", family.name).unwrap(); + + // Sort label sets for deterministic output without allocations. + let mut entries: Vec<_> = family.values.iter().collect(); + entries.sort_by(|a, b| a.0.pairs().cmp(b.0.pairs())); + + for (labels, value) in entries { + writeln!(buf, "{}{} {}", family.name, format_labels(labels), value).unwrap(); + } + buf.push('\n'); +} + +/// Render a gauge family into Prometheus text format. +fn render_gauge(buf: &mut String, family: &GaugeFamily) { + if family.values.is_empty() { + return; + } + writeln!(buf, "# HELP {} {}", family.name, family.help).unwrap(); + writeln!(buf, "# TYPE {} gauge", family.name).unwrap(); + + let mut entries: Vec<_> = family.values.iter().collect(); + entries.sort_by(|a, b| a.0.pairs().cmp(b.0.pairs())); + + for (labels, value) in entries { + // Render integer values without decimal point for cleanliness. + if *value == value.floor() && value.is_finite() { + writeln!( + buf, + "{}{} {}", + family.name, + format_labels(labels), + *value as i64 + ) + .unwrap(); + } else { + writeln!(buf, "{}{} {}", family.name, format_labels(labels), format_float(*value)).unwrap(); + } + } + buf.push('\n'); +} + +/// Render a histogram family into Prometheus text format. +/// +/// Each histogram produces: +/// - One line per bucket: `name_bucket{...,le="bound"} count` +/// - A `+Inf` bucket line +/// - A `name_sum{...} value` line +/// - A `name_count{...} value` line +fn render_histogram(buf: &mut String, family: &HistogramFamily) { + if family.values.is_empty() { + return; + } + writeln!(buf, "# HELP {} {}", family.name, family.help).unwrap(); + writeln!(buf, "# TYPE {} histogram", family.name).unwrap(); + + let mut entries: Vec<_> = family.values.iter().collect(); + entries.sort_by(|a, b| a.0.pairs().cmp(b.0.pairs())); + + for (labels, hist) in entries { + // Render each bucket. + for (bound, count) in &hist.buckets { + // Format the bound: remove trailing zeros for cleanliness. + let bound_str = format_float(*bound); + writeln!( + buf, + "{}_bucket{} {}", + family.name, + format_labels_with_extra(labels, "le", &bound_str), + count + ) + .unwrap(); + } + + // +Inf bucket (always equals total count). + writeln!( + buf, + "{}_bucket{} {}", + family.name, + format_labels_with_extra(labels, "le", "+Inf"), + hist.count + ) + .unwrap(); + + // Sum and count. + writeln!( + buf, + "{}_sum{} {}", + family.name, + format_labels(labels), + format_float(hist.sum) + ) + .unwrap(); + writeln!( + buf, + "{}_count{} {}", + family.name, + format_labels(labels), + hist.count + ) + .unwrap(); + } + buf.push('\n'); +} + +/// Format a float for Prometheus output. +/// Handles special values (+Inf, -Inf, NaN) strictly per the Prometheus spec. +fn format_float(v: f64) -> String { + if v.is_nan() { + "NaN".to_string() + } else if v == f64::INFINITY { + "+Inf".to_string() + } else if v == f64::NEG_INFINITY { + "-Inf".to_string() + } else if v == v.floor() && v.is_finite() { + // Whole number — render with one decimal place (e.g., "1.0", "10.0"). + format!("{:.1}", v) + } else { + // Has fractional part — render naturally. + format!("{}", v) + } +} + +// ─── Public API ────────────────────────────────────────────────────────────── + +/// Render the full metrics state into Prometheus text exposition format. +/// +/// The output is suitable for serving at a `/metrics` HTTP endpoint. +/// Prometheus can scrape this directly. +/// +/// # Example +/// +/// ```ignore +/// let state = collector.state().read().await; +/// let output = prometheus::render(&state); +/// // Return `output` as text/plain from your HTTP handler. +/// ``` +pub fn render(state: &MetricsState) -> String { + let mut buf = String::with_capacity(4096); + + // ── Counters ───────────────────────────────────────────────────────── + render_counter(&mut buf, &state.requests_total); + render_counter(&mut buf, &state.model_loads_total); + render_counter(&mut buf, &state.model_unloads_total); + render_counter(&mut buf, &state.failovers_total); + render_counter(&mut buf, &state.evictions_total); + render_counter(&mut buf, &state.preflight_predictions_total); + render_counter(&mut buf, &state.preflight_hits_total); + render_counter(&mut buf, &state.preflight_misses_total); + render_counter(&mut buf, &state.tokens_input_total); + render_counter(&mut buf, &state.tokens_output_total); + render_counter(&mut buf, &state.events_dropped_total); + + // ── Histograms ─────────────────────────────────────────────────────── + render_histogram(&mut buf, &state.request_duration_seconds); + render_histogram(&mut buf, &state.model_load_seconds); + render_histogram(&mut buf, &state.ttft_seconds); + + // ── Gauges ─────────────────────────────────────────────────────────── + render_gauge(&mut buf, &state.memory_used_bytes); + render_gauge(&mut buf, &state.memory_budget_bytes); + render_gauge(&mut buf, &state.models_loaded); + render_gauge(&mut buf, &state.active_requests); + + buf +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::collector::MetricsState; + use crate::events::*; + + #[test] + fn test_empty_state_renders_empty() { + let state = MetricsState::new(); + let output = render(&state); + // Empty state has no values, so no output. + assert!(output.is_empty(), "Expected empty output, got:\n{}", output); + } + + #[test] + fn test_counter_format() { + let mut state = MetricsState::new(); + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: true, + error_code: None, + }, + ))); + + let output = render(&state); + + // Must contain HELP and TYPE headers. + assert!(output.contains("# HELP mofa_requests_total Total inference requests")); + assert!(output.contains("# TYPE mofa_requests_total counter")); + // Must contain the labeled counter value. + assert!(output.contains("mofa_requests_total{capability=\"chat\",status=\"success\"} 1")); + } + + #[test] + fn test_gauge_format() { + let mut state = MetricsState::new(); + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + + let output = render(&state); + + assert!(output.contains("# HELP mofa_models_loaded Number of models currently loaded")); + assert!(output.contains("# TYPE mofa_models_loaded gauge")); + assert!(output.contains("mofa_models_loaded 1")); + assert!(output.contains("mofa_memory_used_bytes 4700000000")); + } + + #[test] + fn test_histogram_format() { + let mut state = MetricsState::new(); + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + + let output = render(&state); + + // Must have TYPE histogram. + assert!(output.contains("# TYPE mofa_model_load_seconds histogram")); + // Must have bucket lines with le labels. + assert!(output.contains("mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"0.5\"} 0")); + assert!(output.contains("mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"2.0\"} 1")); + // Must have +Inf bucket. + assert!(output.contains("mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"+Inf\"} 1")); + // Must have sum and count. + assert!(output.contains("mofa_model_load_seconds_sum{backend=\"ollama\",model=\"qwen2.5:7b\"} 1.532")); + assert!(output.contains("mofa_model_load_seconds_count{backend=\"ollama\",model=\"qwen2.5:7b\"} 1")); + } + + #[test] + fn test_labels_sorted_alphabetically() { + let mut state = MetricsState::new(); + // RequestCompleted generates labels: capability, status. + // These should appear in alphabetical order: capability before status. + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "test".into(), + backend: "ollama".into(), + capability: Capability::Tts, + duration_ms: 1000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: true, + error_code: None, + }, + ))); + + let output = render(&state); + // Labels must be sorted: capability comes before status. + assert!(output.contains("{capability=\"tts\",status=\"success\"}")); + } + + #[test] + fn test_label_value_escaping() { + let mut state = MetricsState::new(); + // Simulate a model ID with a quote character to test escaping. + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "model\"with\"quotes".into(), + backend: "test\\backend".into(), + capability: Capability::Chat, + load_duration_ms: 1000, + memory_bytes: 1000, + }))); + + let output = render(&state); + // Quotes and backslashes must be escaped. + assert!(output.contains("model\\\"with\\\"quotes")); + assert!(output.contains("test\\\\backend")); + } + + #[test] + fn test_multiple_label_combinations() { + let mut state = MetricsState::new(); + + // Chat success + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 1000, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: true, + error_code: None, + }, + ))); + + // TTS error + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "kokoro".into(), + backend: "ollama".into(), + capability: Capability::Tts, + duration_ms: 500, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success: false, + error_code: Some("timeout".into()), + }, + ))); + + let output = render(&state); + + // Both label combinations must appear. + assert!(output.contains("{capability=\"chat\",status=\"success\"} 1")); + assert!(output.contains("{capability=\"tts\",status=\"error\"} 1")); + } + + #[test] + fn test_full_render_is_valid_prometheus() { + let mut state = MetricsState::new(); + + // Generate a mix of events. + state.process_event(&EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }))); + state.process_event(&EventEnvelope::now(EngineEvent::RequestReceived( + RequestReceived { + capability: Capability::Chat, + model: None, + hint: None, + }, + ))); + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 15000, + ttft_ms: Some(115), + tokens_in: Some(50), + tokens_out: Some(277), + model_was_hot: Some(true), + success: true, + error_code: None, + }, + ))); + state.process_event(&EventEnvelope::now(EngineEvent::PreflightHit( + PreflightHit { + predicted_capability: Capability::Tts, + cold_start_avoided_ms: 2404, + }, + ))); + + let output = render(&state); + + // Basic validity checks: + // 1. Every HELP line is followed by a TYPE line. + let lines: Vec<&str> = output.lines().collect(); + for (i, line) in lines.iter().enumerate() { + if line.starts_with("# HELP") { + assert!( + i + 1 < lines.len() && lines[i + 1].starts_with("# TYPE"), + "HELP line not followed by TYPE: {}", + line + ); + } + } + + // 2. No empty metric names. + for line in &lines { + if !line.starts_with('#') && !line.is_empty() { + assert!( + !line.starts_with('{') && !line.starts_with(' '), + "Metric line has no name: {}", + line + ); + } + } + + // 3. Contains expected sections. + assert!(output.contains("mofa_requests_total")); + assert!(output.contains("mofa_model_load_seconds")); + assert!(output.contains("mofa_memory_used_bytes")); + assert!(output.contains("mofa_preflight_hits_total")); + } + + #[test] + fn test_no_labels_metric_has_no_braces() { + let mut state = MetricsState::new(); + state.process_event(&EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "test".into(), + memory_before_bytes: 100, + memory_after_bytes: 50, + budget_bytes: 100, + }, + ))); + + let output = render(&state); + // The evictions_total counter has no labels. + // It should render as "mofa_evictions_total 1" without {}. + assert!( + output.contains("mofa_evictions_total 1"), + "Expected no-label metric, got:\n{}", + output + ); + // Make sure there's no empty braces. + assert!(!output.contains("mofa_evictions_total{}")); + } + + #[test] + fn test_float_formatting_spec_compliance() { + let mut state = MetricsState::new(); + state.memory_used_bytes.values.insert(Labels::new(), f64::INFINITY); + state.active_requests.values.insert(Labels::new(), f64::NAN); + let output = render(&state); + assert!(output.contains("mofa_memory_used_bytes +Inf"), "Failed to render +Inf: {}", output); + assert!(output.contains("mofa_active_requests NaN"), "Failed to render NaN: {}", output); + } +} From d09b64daf582790b1b1011b4ea0206b460d9e1aa Mon Sep 17 00:00:00 2001 From: ashum9 Date: Sun, 28 Jun 2026 00:44:40 +0530 Subject: [PATCH 06/26] add privacy contract test, fix memory gauge drift via eviction reconciliation --- mofa-observability/src/collector.rs | 22 +++--- mofa-observability/src/events.rs | 108 ++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 14 deletions(-) diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs index 1bc8e72..524d5b8 100644 --- a/mofa-observability/src/collector.rs +++ b/mofa-observability/src/collector.rs @@ -7,8 +7,8 @@ //! - Single writer (collector task), multiple readers (Prometheus scrapes). //! - No blocking I/O. No disk. Just in-memory atomics behind a RwLock. //! - Bounded event channel with drop-oldest backpressure. -//! -//! Reference: observability_plan.md §4 +//! - Memory gauge reconciles against absolute values from eviction events +//! to prevent cumulative drift under event loss. use crate::events::*; use std::collections::HashMap; @@ -174,11 +174,9 @@ impl GaugeFamily { /// All metric state. One instance per engine. /// The collector writes, the Prometheus renderer reads. -/// -/// Reference: observability_plan.md §4.2–4.4 #[derive(Debug, Clone)] pub struct MetricsState { - // ── Counters (§4.2) ────────────────────────────────────────────────── + // ── Counters ───────────────────────────────────────────────────────── pub requests_total: CounterFamily, pub model_loads_total: CounterFamily, pub model_unloads_total: CounterFamily, @@ -191,12 +189,12 @@ pub struct MetricsState { pub tokens_output_total: CounterFamily, pub events_dropped_total: CounterFamily, - // ── Histograms (§4.3) ──────────────────────────────────────────────── + // ── Histograms ─────────────────────────────────────────────────────── pub request_duration_seconds: HistogramFamily, pub model_load_seconds: HistogramFamily, pub ttft_seconds: HistogramFamily, - // ── Gauges (§4.4) ──────────────────────────────────────────────────── + // ── Gauges ─────────────────────────────────────────────────────────── pub memory_used_bytes: GaugeFamily, pub memory_budget_bytes: GaugeFamily, pub models_loaded: GaugeFamily, @@ -395,8 +393,14 @@ impl MetricsState { self.models_loaded.dec(Labels::new()); } - EngineEvent::EvictionTriggered(_) => { + EngineEvent::EvictionTriggered(e) => { self.evictions_total.inc(Labels::new()); + + // Reconcile memory gauge against the absolute post-eviction value. + // This corrects any cumulative drift caused by dropped ModelUnloaded + // events under channel backpressure. The running sum is replaced with + // the authoritative value from the eviction subsystem. + self.memory_used_bytes.set(Labels::new(), e.memory_after_bytes as f64); } EngineEvent::PreflightSignal(e) => { @@ -1010,7 +1014,7 @@ mod tests { } } - // 7 capabilities × 2 statuses = 14 series. Within §4.5 bounds. + // 7 capabilities × 2 statuses = 14 series. Within defined cardinality bounds. assert_eq!(state.requests_total.values.len(), 14); } } diff --git a/mofa-observability/src/events.rs b/mofa-observability/src/events.rs index e5682f5..1541c74 100644 --- a/mofa-observability/src/events.rs +++ b/mofa-observability/src/events.rs @@ -7,8 +7,6 @@ //! - Zero side effects: events are data, not actions. //! - Privacy: no prompt text, file contents, API keys, or user-identifying info. Ever. //! - Bounded enums: capability, reason, source, status use enums, not free-form strings. -//! -//! Reference: observability_plan.md §3 use serde::{Deserialize, Serialize}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -86,7 +84,6 @@ impl std::fmt::Display for SignalSource { // ─── Event Envelope ────────────────────────────────────────────────────────── /// Every event carries this standard envelope. -/// Reference: observability_plan.md §3.1 #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EventEnvelope { /// Unix timestamp in milliseconds. @@ -153,7 +150,6 @@ impl EventEnvelope { // ─── Engine Event (discriminated union) ────────────────────────────────────── /// All possible engine events. The collector pattern-matches on this. -/// Reference: observability_plan.md §3.2 #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum EngineEvent { @@ -350,7 +346,7 @@ pub struct FailoverTriggered { // - source → SignalSource enum (2 variants) // - status → bool (success/failure) // -// Reference: observability_plan.md §3.3 +// Reference: see privacy contract in module-level doc comment above. #[cfg(test)] mod tests { @@ -572,4 +568,106 @@ mod tests { assert!(json.is_ok(), "Failed to serialize: {:?}", envelope); } } + + /// Privacy contract: serialized events must never contain prompt text, + /// file contents, API keys, credentials, or user-identifying information. + /// This test enforces the contract by serializing every event variant and + /// asserting that no forbidden field names appear in the JSON output. + #[test] + fn test_privacy_contract_no_forbidden_fields() { + let forbidden_fields = [ + "prompt", "text", "generated_text", "response_text", + "file_path", "file_content", "file_contents", + "api_key", "api_secret", "token", "auth_token", "access_token", + "password", "credential", "credentials", + "user_id", "username", "email", "ip_address", + ]; + + let events = vec![ + EngineEvent::RequestReceived(RequestReceived { + capability: Capability::Chat, + model: Some("test-model".into()), + hint: Some("test-hint".into()), + }), + EngineEvent::RoutingDecision(RoutingDecision { + capability: Capability::Chat, + candidates_count: 3, + selected_model: "qwen2.5:7b".into(), + selected_backend: "ollama".into(), + is_fallback: false, + reason: "local_first".into(), + }), + EngineEvent::RequestCompleted(RequestCompleted { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 1500, + ttft_ms: Some(115), + tokens_in: Some(50), + tokens_out: Some(277), + model_was_hot: Some(true), + success: true, + error_code: None, + }), + EngineEvent::ModelLoaded(ModelLoaded { + model_id: "qwen2.5:7b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + load_duration_ms: 1532, + memory_bytes: 4_700_000_000, + }), + EngineEvent::ModelUnloaded(ModelUnloaded { + model_id: "gemma3:4b".into(), + reason: UnloadReason::IdleTimeout, + memory_freed_bytes: 3_300_000_000, + }), + EngineEvent::EvictionTriggered(EvictionTriggered { + evicted_model: "gemma3:4b".into(), + memory_before_bytes: 8_000_000_000, + memory_after_bytes: 4_700_000_000, + budget_bytes: 11_000_000_000, + }), + EngineEvent::PreflightSignal(PreflightSignal { + predicted_capability: Capability::Tts, + confidence: 0.95, + source: SignalSource::History, + }), + EngineEvent::PreflightHit(PreflightHit { + predicted_capability: Capability::Tts, + cold_start_avoided_ms: 2404, + }), + EngineEvent::PreflightMiss(PreflightMiss { + predicted_capability: Capability::Tts, + actual_capability: Capability::Chat, + }), + EngineEvent::ProviderDiscovered(ProviderDiscovered { + provider_name: "ollama".into(), + models_found: 3, + capabilities: vec![Capability::Chat, Capability::Vlm], + }), + EngineEvent::FailoverTriggered(FailoverTriggered { + failed_model: "qwen2.5:7b".into(), + failed_backend: "ollama".into(), + fallback_model: "gemma3:4b".into(), + fallback_backend: "ollama".into(), + }), + ]; + + for event in events { + let envelope = EventEnvelope::now(event); + let json = serde_json::to_string(&envelope).expect("serialize failed"); + let json_lower = json.to_lowercase(); + + for field in &forbidden_fields { + // Check for the field as a JSON key: "field_name": + let pattern = format!("\"{}\":", field); + assert!( + !json_lower.contains(&pattern), + "Privacy violation: forbidden field '{}' found in serialized event: {}", + field, + json + ); + } + } + } } From 2c7258559fe3ba5b19b44cf0ba97471a2e39f143 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Sun, 28 Jun 2026 00:44:53 +0530 Subject: [PATCH 07/26] add Grafana dashboard definitions for engine overview, memory lifecycle, and preflight routing --- .../dashboards/engine_overview.json | 99 +++++++++++++++++++ .../dashboards/memory_lifecycle.json | 94 ++++++++++++++++++ .../dashboards/preflight_routing.json | 81 +++++++++++++++ 3 files changed, 274 insertions(+) create mode 100644 mofa-observability/dashboards/engine_overview.json create mode 100644 mofa-observability/dashboards/memory_lifecycle.json create mode 100644 mofa-observability/dashboards/preflight_routing.json diff --git a/mofa-observability/dashboards/engine_overview.json b/mofa-observability/dashboards/engine_overview.json new file mode 100644 index 0000000..a1e4c50 --- /dev/null +++ b/mofa-observability/dashboards/engine_overview.json @@ -0,0 +1,99 @@ +{ + "title": "MoFA Engine Overview", + "refresh": "10s", + "schemaVersion": 39, + "timezone": "browser", + "panels": [ + { + "type": "stat", + "title": "Request Rate (1m)", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { + "expr": "sum(rate(mofa_requests_total[1m]))", + "legendFormat": "Requests / sec" + } + ] + }, + { + "type": "stat", + "title": "Error Rate", + "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { + "expr": "sum(rate(mofa_requests_total{status=\"error\"}[5m])) / sum(rate(mofa_requests_total[5m])) * 100", + "legendFormat": "Error %" + } + ] + }, + { + "type": "stat", + "title": "Models Loaded", + "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { + "expr": "mofa_models_loaded", + "legendFormat": "Count" + } + ] + }, + { + "type": "stat", + "title": "Memory Usage %", + "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, + "targets": [ + { + "expr": "mofa_memory_used_bytes / mofa_memory_budget_bytes * 100", + "legendFormat": "%" + } + ] + }, + { + "type": "timeseries", + "title": "P95 Request Latency by Capability", + "gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(mofa_request_duration_seconds_bucket[5m])) by (le, capability))", + "legendFormat": "{{capability}} P95" + } + ], + "fieldConfig": { "defaults": { "unit": "s" } } + }, + { + "type": "timeseries", + "title": "P95 Time to First Token", + "gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(mofa_ttft_seconds_bucket[5m])) by (le, model))", + "legendFormat": "{{model}} TTFT P95" + } + ], + "fieldConfig": { "defaults": { "unit": "s" } } + }, + { + "type": "timeseries", + "title": "Requests per Second (by Capability)", + "gridPos": { "x": 0, "y": 12, "w": 12, "h": 8 }, + "targets": [ + { + "expr": "sum(rate(mofa_requests_total[1m])) by (capability)", + "legendFormat": "{{capability}}" + } + ], + "fieldConfig": { "defaults": { "custom": { "fillOpacity": 70 } } } + }, + { + "type": "timeseries", + "title": "Output Tokens per Second (by Model)", + "gridPos": { "x": 12, "y": 12, "w": 12, "h": 8 }, + "targets": [ + { + "expr": "sum(rate(mofa_tokens_output_total[1m])) by (model)", + "legendFormat": "{{model}}" + } + ] + } + ] +} diff --git a/mofa-observability/dashboards/memory_lifecycle.json b/mofa-observability/dashboards/memory_lifecycle.json new file mode 100644 index 0000000..65a03f0 --- /dev/null +++ b/mofa-observability/dashboards/memory_lifecycle.json @@ -0,0 +1,94 @@ +{ + "title": "MoFA Memory & Lifecycle", + "refresh": "10s", + "schemaVersion": 39, + "timezone": "browser", + "panels": [ + { + "type": "gauge", + "title": "Memory Utilization", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 6 }, + "targets": [ + { + "expr": "mofa_memory_used_bytes / mofa_memory_budget_bytes * 100", + "legendFormat": "% Used" + } + ], + "fieldConfig": { + "defaults": { + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 70 }, + { "color": "red", "value": 90 } + ] + } + } + } + }, + { + "type": "timeseries", + "title": "Memory Usage vs Budget", + "gridPos": { "x": 6, "y": 0, "w": 18, "h": 6 }, + "targets": [ + { + "expr": "mofa_memory_used_bytes", + "legendFormat": "Used" + }, + { + "expr": "mofa_memory_budget_bytes", + "legendFormat": "Budget" + } + ], + "fieldConfig": { "defaults": { "unit": "bytes" } } + }, + { + "type": "barchart", + "title": "Model Loads (Last 5m)", + "gridPos": { "x": 0, "y": 6, "w": 8, "h": 8 }, + "targets": [ + { + "expr": "sum(increase(mofa_model_loads_total[5m])) by (model)", + "legendFormat": "{{model}}" + } + ] + }, + { + "type": "barchart", + "title": "Model Unloads by Reason", + "gridPos": { "x": 8, "y": 6, "w": 8, "h": 8 }, + "targets": [ + { + "expr": "sum(increase(mofa_model_unloads_total[5m])) by (reason)", + "legendFormat": "{{reason}}" + } + ] + }, + { + "type": "timeseries", + "title": "Eviction Rate", + "gridPos": { "x": 16, "y": 6, "w": 8, "h": 8 }, + "targets": [ + { + "expr": "sum(rate(mofa_evictions_total[1m]))", + "legendFormat": "Evictions / sec" + } + ] + }, + { + "type": "heatmap", + "title": "Cold Load Duration Heatmap", + "gridPos": { "x": 0, "y": 14, "w": 24, "h": 8 }, + "targets": [ + { + "expr": "sum(rate(mofa_model_load_seconds_bucket[5m])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}" + } + ] + } + ] +} diff --git a/mofa-observability/dashboards/preflight_routing.json b/mofa-observability/dashboards/preflight_routing.json new file mode 100644 index 0000000..7442ac8 --- /dev/null +++ b/mofa-observability/dashboards/preflight_routing.json @@ -0,0 +1,81 @@ +{ + "title": "MoFA Preflight & Routing", + "refresh": "10s", + "schemaVersion": 39, + "timezone": "browser", + "panels": [ + { + "type": "gauge", + "title": "Prediction Accuracy", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 6 }, + "targets": [ + { + "expr": "sum(rate(mofa_preflight_hits_total[1h])) / (sum(rate(mofa_preflight_hits_total[1h])) + sum(rate(mofa_preflight_misses_total[1h]))) * 100", + "legendFormat": "Accuracy %" + } + ], + "fieldConfig": { + "defaults": { + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 70 }, + { "color": "green", "value": 90 } + ] + } + } + } + }, + { + "type": "timeseries", + "title": "Predictions by Source", + "gridPos": { "x": 6, "y": 0, "w": 10, "h": 6 }, + "targets": [ + { + "expr": "sum(rate(mofa_preflight_predictions_total[5m])) by (source)", + "legendFormat": "{{source}}" + } + ] + }, + { + "type": "piechart", + "title": "Hit vs Miss", + "gridPos": { "x": 16, "y": 0, "w": 8, "h": 6 }, + "targets": [ + { + "expr": "sum(mofa_preflight_hits_total)", + "legendFormat": "Hits" + }, + { + "expr": "sum(mofa_preflight_misses_total)", + "legendFormat": "Misses" + } + ] + }, + { + "type": "timeseries", + "title": "Failover Rate", + "gridPos": { "x": 0, "y": 6, "w": 12, "h": 6 }, + "targets": [ + { + "expr": "sum(rate(mofa_failovers_total[5m]))", + "legendFormat": "Failovers / sec" + } + ] + }, + { + "type": "piechart", + "title": "Requests by Backend", + "gridPos": { "x": 12, "y": 6, "w": 12, "h": 6 }, + "targets": [ + { + "expr": "sum(rate(mofa_requests_total[5m])) by (backend)", + "legendFormat": "{{backend}}" + } + ] + } + ] +} From 006ccc1ff684632a3c1791d48c60a2c6592dcf78 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Sun, 28 Jun 2026 00:45:23 +0530 Subject: [PATCH 08/26] add mock event harness and Docker compose stack for local observability demo --- mofa-observability/Cargo.toml | 4 + mofa-observability/docker/Dockerfile | 15 ++ mofa-observability/docker/docker-compose.yml | 34 +++ .../provisioning/dashboards/dashboards.yml | 11 + .../provisioning/datasources/prometheus.yml | 9 + .../docker/prometheus/prometheus.yml | 7 + mofa-observability/examples/mock_harness.rs | 251 ++++++++++++++++++ 7 files changed, 331 insertions(+) create mode 100644 mofa-observability/docker/Dockerfile create mode 100644 mofa-observability/docker/docker-compose.yml create mode 100644 mofa-observability/docker/grafana/provisioning/dashboards/dashboards.yml create mode 100644 mofa-observability/docker/grafana/provisioning/datasources/prometheus.yml create mode 100644 mofa-observability/docker/prometheus/prometheus.yml create mode 100644 mofa-observability/examples/mock_harness.rs diff --git a/mofa-observability/Cargo.toml b/mofa-observability/Cargo.toml index b1402f5..589570a 100644 --- a/mofa-observability/Cargo.toml +++ b/mofa-observability/Cargo.toml @@ -8,3 +8,7 @@ license.workspace = true serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } + +[dev-dependencies] +axum = { workspace = true } +rand = "0.8" diff --git a/mofa-observability/docker/Dockerfile b/mofa-observability/docker/Dockerfile new file mode 100644 index 0000000..d04751d --- /dev/null +++ b/mofa-observability/docker/Dockerfile @@ -0,0 +1,15 @@ +FROM rust:1.89-slim as builder + +WORKDIR /usr/src/mofa +# Copy entire workspace since mofa-observability depends on workspace Cargo.toml +COPY . . + +WORKDIR /usr/src/mofa/mofa-observability +RUN cargo build --example mock_harness --release + +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* +COPY --from=builder /usr/src/mofa/target/release/examples/mock_harness /usr/local/bin/mock_harness + +EXPOSE 9090 +CMD ["mock_harness"] diff --git a/mofa-observability/docker/docker-compose.yml b/mofa-observability/docker/docker-compose.yml new file mode 100644 index 0000000..1f0fde8 --- /dev/null +++ b/mofa-observability/docker/docker-compose.yml @@ -0,0 +1,34 @@ +version: '3.8' + +services: + mock-harness: + build: + context: ../../ + dockerfile: mofa-observability/docker/Dockerfile + ports: + - "9090:9090" + + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9091:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + depends_on: + - mock-harness + + grafana: + image: grafana/grafana:latest + ports: + - "3000:3000" + volumes: + - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources + - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards + - ../dashboards:/var/lib/grafana/dashboards + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + - GF_USERS_ALLOW_SIGN_UP=false + depends_on: + - prometheus diff --git a/mofa-observability/docker/grafana/provisioning/dashboards/dashboards.yml b/mofa-observability/docker/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..885f802 --- /dev/null +++ b/mofa-observability/docker/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: 'MoFA Dashboards' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards diff --git a/mofa-observability/docker/grafana/provisioning/datasources/prometheus.yml b/mofa-observability/docker/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..bb009bb --- /dev/null +++ b/mofa-observability/docker/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/mofa-observability/docker/prometheus/prometheus.yml b/mofa-observability/docker/prometheus/prometheus.yml new file mode 100644 index 0000000..454bc13 --- /dev/null +++ b/mofa-observability/docker/prometheus/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 5s + +scrape_configs: + - job_name: 'mofa_mock_harness' + static_configs: + - targets: ['mock-harness:9090'] diff --git a/mofa-observability/examples/mock_harness.rs b/mofa-observability/examples/mock_harness.rs new file mode 100644 index 0000000..d7d55dc --- /dev/null +++ b/mofa-observability/examples/mock_harness.rs @@ -0,0 +1,251 @@ +//! Mock Event Harness for MoFA Engine Observability +//! +//! Generates synthetic engine events and serves them over an HTTP `/metrics` endpoint. +//! This allows testing the Prometheus renderer and populating Grafana dashboards +//! before the real engine is fully integrated. +//! +//! Run with: `cargo run --example mock_harness` + +use axum::{extract::State, routing::get, Router}; +use mofa_observability::{ + collector::{create_event_channel, EventSender, MetricsCollector, MetricsState}, + events::*, + prometheus, +}; +use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; +use tokio::sync::RwLock; + +/// Starts the HTTP server serving `/metrics`. +async fn metrics_handler(State(state): State>>) -> String { + let state_reader = state.read().await; + prometheus::render(&state_reader) +} + +/// Generates a continuous stream of synthetic engine events. +async fn simulate_engine_traffic(sender: EventSender) { + let mut rng = StdRng::from_entropy(); + + // Available models for the mock + let models = [ + ("qwen2.5:7b", "ollama", Capability::Chat), + ("gemma3:4b", "ollama", Capability::Chat), + ("whisper-1", "openai", Capability::Asr), + ("kokoro", "local", Capability::Tts), + ]; + + // Initial setup: Load a couple of models to have baseline memory. + let m1 = models[0]; + sender.send(EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: m1.0.into(), + backend: m1.1.into(), + capability: m1.2, + load_duration_ms: rng.gen_range(1500..3500), + memory_bytes: 4_700_000_000, + }))); + + let m2 = models[3]; + sender.send(EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: m2.0.into(), + backend: m2.1.into(), + capability: m2.2, + load_duration_ms: rng.gen_range(500..1200), + memory_bytes: 800_000_000, + }))); + + // Continuous event loop + loop { + // Sleep between 50ms and 500ms to generate roughly 2-20 requests/sec. + tokio::time::sleep(Duration::from_millis(rng.gen_range(50..500))).await; + + let action = rng.gen_range(0..100); + let selected_model = models.choose(&mut rng).unwrap(); + + match action { + // 5% chance: Load a new model + 0..=4 => { + sender.send(EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { + model_id: selected_model.0.into(), + backend: selected_model.1.into(), + capability: selected_model.2, + load_duration_ms: rng.gen_range(1000..5000), + memory_bytes: rng.gen_range(500_000_000..8_000_000_000), + }))); + } + + // 5% chance: Unload a model + 5..=9 => { + let reason = match rng.gen_range(0..3) { + 0 => UnloadReason::IdleTimeout, + 1 => UnloadReason::Eviction, + _ => UnloadReason::Explicit, + }; + sender.send(EventEnvelope::now(EngineEvent::ModelUnloaded( + ModelUnloaded { + model_id: selected_model.0.into(), + reason, + memory_freed_bytes: rng.gen_range(500_000_000..8_000_000_000), + }, + ))); + + // If eviction, also trigger eviction event + if reason == UnloadReason::Eviction { + sender.send(EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: selected_model.0.into(), + memory_before_bytes: 15_000_000_000, + memory_after_bytes: 12_000_000_000, + budget_bytes: 16_000_000_000, + }, + ))); + } + } + + // 10% chance: Preflight Hit/Miss + 10..=19 => { + let is_hit = rng.gen_bool(0.85); // 85% accuracy + let source = if rng.gen_bool(0.7) { + SignalSource::History + } else { + SignalSource::Hint + }; + + sender.send(EventEnvelope::now(EngineEvent::PreflightSignal( + PreflightSignal { + predicted_capability: selected_model.2, + confidence: rng.gen_range(0.6..1.0), + source, + }, + ))); + + if is_hit { + sender.send(EventEnvelope::now(EngineEvent::PreflightHit( + PreflightHit { + predicted_capability: selected_model.2, + cold_start_avoided_ms: rng.gen_range(1000..5000), + }, + ))); + } else { + let mut wrong_cap = Capability::Chat; + if selected_model.2 == Capability::Chat { + wrong_cap = Capability::Tts; + } + sender.send(EventEnvelope::now(EngineEvent::PreflightMiss( + PreflightMiss { + predicted_capability: selected_model.2, + actual_capability: wrong_cap, + }, + ))); + } + } + + // 1% chance: Failover + 20..=20 => { + sender.send(EventEnvelope::now(EngineEvent::FailoverTriggered( + FailoverTriggered { + failed_model: selected_model.0.into(), + failed_backend: selected_model.1.into(), + fallback_model: "fallback-model".into(), + fallback_backend: "cloud".into(), + }, + ))); + } + + // 79% chance: Normal Inference Request + _ => { + // 1. Request Received + sender.send(EventEnvelope::now(EngineEvent::RequestReceived( + RequestReceived { + capability: selected_model.2, + model: None, + hint: None, + }, + ))); + + // Sleep slightly to simulate inference time + let duration_ms = match selected_model.2 { + Capability::Chat => rng.gen_range(200..15000), // Text generation is variable + _ => rng.gen_range(100..2000), // API calls faster + }; + tokio::time::sleep(Duration::from_millis(duration_ms.min(100))).await; + + let success = rng.gen_bool(0.98); // 98% success rate + + let ttft_ms = if selected_model.2 == Capability::Chat && success { + Some(rng.gen_range(50..800)) + } else { + None + }; + + let tokens_in = if selected_model.2 == Capability::Chat { + Some(rng.gen_range(10..1000)) + } else { + None + }; + + let tokens_out = if selected_model.2 == Capability::Chat && success { + Some(rng.gen_range(5..800)) + } else { + None + }; + + // 2. Request Completed + sender.send(EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: selected_model.0.into(), + backend: selected_model.1.into(), + capability: selected_model.2, + duration_ms, + ttft_ms, + tokens_in, + tokens_out, + model_was_hot: Some(rng.gen_bool(0.9)), // Usually hot + success, + error_code: if success { + None + } else { + Some("internal_error".into()) + }, + }, + ))); + } + } + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + println!("Starting MoFA Engine Observability Mock Harness..."); + + // 1. Initialize Event Channel (capacity 1024) + let (sender, receiver) = create_event_channel(1024); + + // 2. Initialize Metrics Collector + let collector = MetricsCollector::new(receiver); + let state = collector.state(); + + // 3. Spawn Collector Task + tokio::spawn(collector.run()); + + // 4. Spawn Event Generator Task + tokio::spawn(simulate_engine_traffic(sender)); + + // 5. Setup HTTP Server for Prometheus scraping + let app = Router::new() + .route("/metrics", get(metrics_handler)) + .with_state(state); + + let port = 9090; + let addr = SocketAddr::from(([0, 0, 0, 0], port)); + let listener = tokio::net::TcpListener::bind(addr).await?; + + println!("======================================================="); + println!("🚀 Mock Harness Running!"); + println!("📊 Prometheus Endpoint: http://localhost:{}/metrics", port); + println!(" Try running: curl http://localhost:{}/metrics", port); + println!("======================================================="); + + axum::serve(listener, app).await?; + + Ok(()) +} From 6c32e245d1e0d23c6aa8f670a6f3f1bf5511d28c Mon Sep 17 00:00:00 2001 From: ashum9 Date: Sun, 28 Jun 2026 00:46:20 +0530 Subject: [PATCH 09/26] update cargo.lock for mofa-obseravbility --- Cargo.lock | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1e4489e..cb24070 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,6 +1097,8 @@ dependencies = [ name = "mofa-observability" version = "0.1.0" dependencies = [ + "axum", + "rand 0.8.6", "serde", "serde_json", "tokio", @@ -1315,14 +1317,35 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1332,7 +1355,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", ] [[package]] @@ -2072,7 +2104,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.9.4", "sha1", "thiserror", ] From f947d37e41ef3fdf3a4135e2510e046e13e60671 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Tue, 30 Jun 2026 21:25:17 +0530 Subject: [PATCH 10/26] prevent memory leak by evicting stale metric labels --- mofa-observability/src/collector.rs | 233 ++++++++++++++++++++++++++-- 1 file changed, 218 insertions(+), 15 deletions(-) diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs index 524d5b8..a6c8190 100644 --- a/mofa-observability/src/collector.rs +++ b/mofa-observability/src/collector.rs @@ -13,6 +13,7 @@ use crate::events::*; use std::collections::HashMap; use std::sync::Arc; +use std::time::{Duration, Instant}; use tokio::sync::{mpsc, RwLock}; // ─── Labels ────────────────────────────────────────────────────────────────── @@ -85,6 +86,7 @@ pub struct CounterFamily { pub name: &'static str, pub help: &'static str, pub values: HashMap, + pub last_seen: HashMap, } impl CounterFamily { @@ -93,16 +95,31 @@ impl CounterFamily { name, help, values: HashMap::new(), + last_seen: HashMap::new(), } } fn inc(&mut self, labels: Labels) { + self.last_seen.insert(labels.clone(), Instant::now()); *self.values.entry(labels).or_insert(0) += 1; } fn inc_by(&mut self, labels: Labels, n: u64) { + self.last_seen.insert(labels.clone(), Instant::now()); *self.values.entry(labels).or_insert(0) += n; } + + fn evict_stale(&mut self, now: Instant, max_idle: Duration) { + let last_seen = &mut self.last_seen; + self.values.retain(|k, _| { + if let Some(&time) = last_seen.get(k) { + now.saturating_duration_since(time) < max_idle + } else { + true + } + }); + self.last_seen.retain(|_, v| now.saturating_duration_since(*v) < max_idle); + } } /// A family of labeled histograms. @@ -112,6 +129,7 @@ pub struct HistogramFamily { pub help: &'static str, pub bucket_boundaries: Vec, pub values: HashMap, + pub last_seen: HashMap, } impl HistogramFamily { @@ -121,15 +139,29 @@ impl HistogramFamily { help, bucket_boundaries: boundaries, values: HashMap::new(), + last_seen: HashMap::new(), } } fn observe(&mut self, labels: Labels, value: f64) { + self.last_seen.insert(labels.clone(), Instant::now()); self.values .entry(labels) .or_insert_with(|| HistogramValue::new(&self.bucket_boundaries)) .observe(value); } + + fn evict_stale(&mut self, now: Instant, max_idle: Duration) { + let last_seen = &mut self.last_seen; + self.values.retain(|k, _| { + if let Some(&time) = last_seen.get(k) { + now.saturating_duration_since(time) < max_idle + } else { + true + } + }); + self.last_seen.retain(|_, v| now.saturating_duration_since(*v) < max_idle); + } } /// A family of labeled gauges. @@ -138,6 +170,7 @@ pub struct GaugeFamily { pub name: &'static str, pub help: &'static str, pub values: HashMap, + pub last_seen: HashMap, } impl GaugeFamily { @@ -146,28 +179,46 @@ impl GaugeFamily { name, help, values: HashMap::new(), + last_seen: HashMap::new(), } } fn set(&mut self, labels: Labels, value: f64) { + self.last_seen.insert(labels.clone(), Instant::now()); self.values.insert(labels, value); } fn inc(&mut self, labels: Labels) { + self.last_seen.insert(labels.clone(), Instant::now()); *self.values.entry(labels).or_insert(0.0) += 1.0; } fn dec(&mut self, labels: Labels) { + self.last_seen.insert(labels.clone(), Instant::now()); *self.values.entry(labels).or_insert(0.0) -= 1.0; } fn add(&mut self, labels: Labels, n: f64) { + self.last_seen.insert(labels.clone(), Instant::now()); *self.values.entry(labels).or_insert(0.0) += n; } fn sub(&mut self, labels: Labels, n: f64) { + self.last_seen.insert(labels.clone(), Instant::now()); *self.values.entry(labels).or_insert(0.0) -= n; } + + fn evict_stale(&mut self, now: Instant, max_idle: Duration) { + let last_seen = &mut self.last_seen; + self.values.retain(|k, _| { + if let Some(&time) = last_seen.get(k) { + now.saturating_duration_since(time) < max_idle + } else { + true + } + }); + self.last_seen.retain(|_, v| now.saturating_duration_since(*v) < max_idle); + } } // ─── Metrics State ─────────────────────────────────────────────────────────── @@ -315,6 +366,15 @@ impl MetricsState { } EngineEvent::RequestCompleted(e) => { + tracing::info!( + trace_id = ?envelope.trace_id, + request_id = ?envelope.request_id, + model = %e.model_id, + duration_ms = e.duration_ms, + success = e.success, + "Request completed" + ); + // Counter: requests by capability × status let status = if e.success { "success" } else { "error" }; self.requests_total.inc( @@ -358,6 +418,13 @@ impl MetricsState { } EngineEvent::ModelLoaded(e) => { + tracing::info!( + model = %e.model_id, + backend = %e.backend, + load_duration_ms = e.load_duration_ms, + "Model loaded" + ); + // Counter self.model_loads_total.inc( Labels::new() @@ -394,6 +461,13 @@ impl MetricsState { } EngineEvent::EvictionTriggered(e) => { + tracing::warn!( + evicted_model = %e.evicted_model, + memory_before = e.memory_before_bytes, + memory_after = e.memory_after_bytes, + "Eviction triggered" + ); + self.evictions_total.inc(Labels::new()); // Reconcile memory gauge against the absolute post-eviction value. @@ -422,12 +496,47 @@ impl MetricsState { } EngineEvent::FailoverTriggered(e) => { + tracing::error!( + failed_model = %e.failed_model, + fallback_model = %e.fallback_model, + "Failover triggered" + ); + // We don't have capability on FailoverTriggered, so label-free. let _ = e; self.failovers_total.inc(Labels::new()); } } } + + /// Evict metric labels that haven't been updated in `max_idle` duration. + pub fn evict_stale_labels(&mut self, max_idle: Duration) { + let now = Instant::now(); + + // Counters + self.requests_total.evict_stale(now, max_idle); + self.model_loads_total.evict_stale(now, max_idle); + self.model_unloads_total.evict_stale(now, max_idle); + self.failovers_total.evict_stale(now, max_idle); + self.evictions_total.evict_stale(now, max_idle); + self.preflight_predictions_total.evict_stale(now, max_idle); + self.preflight_hits_total.evict_stale(now, max_idle); + self.preflight_misses_total.evict_stale(now, max_idle); + self.tokens_input_total.evict_stale(now, max_idle); + self.tokens_output_total.evict_stale(now, max_idle); + self.events_dropped_total.evict_stale(now, max_idle); + + // Histograms + self.request_duration_seconds.evict_stale(now, max_idle); + self.model_load_seconds.evict_stale(now, max_idle); + self.ttft_seconds.evict_stale(now, max_idle); + + // Gauges + self.memory_used_bytes.evict_stale(now, max_idle); + self.memory_budget_bytes.evict_stale(now, max_idle); + self.models_loaded.evict_stale(now, max_idle); + self.active_requests.evict_stale(now, max_idle); + } } impl Default for MetricsState { @@ -447,7 +556,7 @@ pub struct EventSender { impl EventSender { /// Send an event. If the channel is full, the event is dropped - /// and the dropped counter is incremented. + /// and the dropped counter is incremented. Use for high-volume telemetry. pub fn send(&self, event: EventEnvelope) { if self.tx.try_send(event).is_err() { self.dropped @@ -455,6 +564,19 @@ impl EventSender { } } + /// Send a critical event asynchronously. If the channel is full, this will await + /// and apply backpressure to the caller instead of dropping the event. + /// Use for rare, critical events like `FailoverTriggered` and `EvictionTriggered`. + pub async fn send_critical(&self, event: EventEnvelope) -> Result<(), mpsc::error::SendError> { + self.tx.send(event).await + } + + /// Send a critical event synchronously. If the channel is full, this will block + /// the current thread. Must not be called from within an async runtime context. + pub fn send_blocking(&self, event: EventEnvelope) -> Result<(), mpsc::error::SendError> { + self.tx.blocking_send(event) + } + /// How many events have been dropped due to backpressure. pub fn dropped_count(&self) -> u64 { self.dropped.load(std::sync::atomic::Ordering::Relaxed) @@ -515,22 +637,39 @@ impl MetricsCollector { } /// Run the collector loop. Processes events until the channel closes. + /// Also periodically evicts stale metric labels to prevent memory leaks. pub async fn run(mut self) { - while let Some(envelope) = self.receiver.rx.recv().await { - let mut state = self.state.write().await; - - // Sync the dropped counter into metrics state. - let dropped = self - .receiver - .dropped - .swap(0, std::sync::atomic::Ordering::Relaxed); - if dropped > 0 { - state - .events_dropped_total - .inc_by(Labels::new(), dropped); + let mut cleanup_interval = tokio::time::interval(Duration::from_secs(60)); + let max_idle = Duration::from_secs(600); // 10 minutes + + loop { + tokio::select! { + envelope_opt = self.receiver.rx.recv() => { + match envelope_opt { + Some(envelope) => { + let mut state = self.state.write().await; + + // Sync the dropped counter into metrics state. + let dropped = self + .receiver + .dropped + .swap(0, std::sync::atomic::Ordering::Relaxed); + if dropped > 0 { + state + .events_dropped_total + .inc_by(Labels::new(), dropped); + } + + state.process_event(&envelope); + } + None => break, // Channel closed + } + } + _ = cleanup_interval.tick() => { + let mut state = self.state.write().await; + state.evict_stale_labels(max_idle); + } } - - state.process_event(&envelope); } } } @@ -978,6 +1117,46 @@ mod tests { assert!(sender.dropped_count() >= 3); } + #[tokio::test] + async fn test_send_critical_blocks() { + // Channel of size 1 + let (sender, mut receiver) = create_event_channel(1); + + // Fill channel + sender.send(EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "test1".into(), + memory_before_bytes: 100, + memory_after_bytes: 50, + budget_bytes: 100, + }, + ))); + + // Try to send critical event. It should block because the channel is full. + // We use tokio::time::timeout to prove it blocks. + let critical_event = EventEnvelope::now(EngineEvent::FailoverTriggered( + FailoverTriggered { + failed_model: "failed".into(), + failed_backend: "backend".into(), + fallback_model: "fallback".into(), + fallback_backend: "backend".into(), + }, + )); + + let send_future = sender.send_critical(critical_event.clone()); + let result = tokio::time::timeout(Duration::from_millis(50), send_future).await; + + // It should timeout because it's blocking + assert!(result.is_err()); + + // Now drain the channel + let _ = receiver.rx.recv().await; + + // Sending should now succeed + let send_result = sender.send_critical(critical_event).await; + assert!(send_result.is_ok()); + } + // ── Label cardinality test ─────────────────────────────────────────── #[test] @@ -1017,4 +1196,28 @@ mod tests { // 7 capabilities × 2 statuses = 14 series. Within defined cardinality bounds. assert_eq!(state.requests_total.values.len(), 14); } + + #[test] + fn test_evict_stale_labels() { + let mut state = make_state(); + let labels_fresh = Labels::new().add("model", "fresh"); + let labels_stale = Labels::new().add("model", "stale"); + + // Insert labels + state.requests_total.inc(labels_fresh.clone()); + state.requests_total.inc(labels_stale.clone()); + + // Override the last_seen for the stale label to be 20 minutes ago + let twenty_mins_ago = Instant::now() - Duration::from_secs(1200); + state.requests_total.last_seen.insert(labels_stale.clone(), twenty_mins_ago); + + // Evict labels older than 10 minutes + state.evict_stale_labels(Duration::from_secs(600)); + + // The stale label should be gone, the fresh one should remain + assert!(state.requests_total.values.contains_key(&labels_fresh)); + assert!(!state.requests_total.values.contains_key(&labels_stale)); + assert!(state.requests_total.last_seen.contains_key(&labels_fresh)); + assert!(!state.requests_total.last_seen.contains_key(&labels_stale)); + } } From e5113bd1b998780d41b7948fe26f540af53dbe59 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Tue, 30 Jun 2026 21:25:48 +0530 Subject: [PATCH 11/26] introduce send_critical to prevent dropping key events under backpressure --- mofa-observability/examples/mock_harness.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mofa-observability/examples/mock_harness.rs b/mofa-observability/examples/mock_harness.rs index d7d55dc..384c98b 100644 --- a/mofa-observability/examples/mock_harness.rs +++ b/mofa-observability/examples/mock_harness.rs @@ -90,14 +90,14 @@ async fn simulate_engine_traffic(sender: EventSender) { // If eviction, also trigger eviction event if reason == UnloadReason::Eviction { - sender.send(EventEnvelope::now(EngineEvent::EvictionTriggered( + sender.send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( EvictionTriggered { evicted_model: selected_model.0.into(), memory_before_bytes: 15_000_000_000, memory_after_bytes: 12_000_000_000, budget_bytes: 16_000_000_000, }, - ))); + ))).await.expect("Failed to send critical event"); } } @@ -141,14 +141,14 @@ async fn simulate_engine_traffic(sender: EventSender) { // 1% chance: Failover 20..=20 => { - sender.send(EventEnvelope::now(EngineEvent::FailoverTriggered( + sender.send_critical(EventEnvelope::now(EngineEvent::FailoverTriggered( FailoverTriggered { failed_model: selected_model.0.into(), failed_backend: selected_model.1.into(), fallback_model: "fallback-model".into(), fallback_backend: "cloud".into(), }, - ))); + ))).await.expect("Failed to send critical event"); } // 79% chance: Normal Inference Request @@ -215,6 +215,10 @@ async fn simulate_engine_traffic(sender: EventSender) { #[tokio::main] async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env().add_directive("mofa_observability=info".parse().unwrap())) + .init(); + println!("Starting MoFA Engine Observability Mock Harness..."); // 1. Initialize Event Channel (capacity 1024) From 858c8da02a606f379df98ffa6fcbdbb3fd4f5b25 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 12/26] add ObservabilityConfig struct --- mofa-engine-core/src/config.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mofa-engine-core/src/config.rs b/mofa-engine-core/src/config.rs index a31c82c..f502ed6 100644 --- a/mofa-engine-core/src/config.rs +++ b/mofa-engine-core/src/config.rs @@ -22,6 +22,20 @@ pub struct EngineConfig { /// Provider definitions #[serde(default)] pub providers: Vec, + /// Observability settings + #[serde(default)] + pub observability: ObservabilityConfig, +} + +/// Observability configuration. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ObservabilityConfig { + /// Whether observability is enabled. + #[serde(default)] + pub enabled: bool, + /// The OpenTelemetry OTLP endpoint (e.g., http://localhost:4317) + #[serde(default)] + pub otlp_endpoint: Option, } /// Network listen configuration. @@ -399,6 +413,7 @@ impl EngineConfig { listen: ListenConfig::default(), memory: MemoryConfig::default(), providers, + observability: ObservabilityConfig::default(), } } } @@ -470,6 +485,7 @@ mod tests { #[test] fn toml_roundtrip() { let cfg = EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig::default(), providers: vec![ProviderConfig { @@ -492,6 +508,7 @@ mod tests { #[test] fn validate_rejects_unknown_provider_kind() { let cfg = EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig::default(), providers: vec![ProviderConfig { From 12f83251fbb2deb26642ed12a39d6cc15dfe0373 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 13/26] add OpenTelemetry workspace dependencies --- Cargo.lock | 417 ++++++++++++++++++++++++++++++++-- Cargo.toml | 4 + mofa-engine-app/Cargo.toml | 5 + mofa-engine-sdk/Cargo.toml | 1 + mofa-observability/Cargo.toml | 2 + 5 files changed, 410 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb24070..99121ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,28 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -99,13 +121,40 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core", + "axum-core 0.5.6", "base64", "bytes", "form_urlencoded", @@ -116,7 +165,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -129,12 +178,32 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-tungstenite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.6" @@ -160,8 +229,8 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9963ff19f40c6102c76756ef0a46004c0d58957d87259fc9208ff8441c12ab96" dependencies = [ - "axum", - "axum-core", + "axum 0.8.9", + "axum-core 0.5.6", "bytes", "futures-util", "headers", @@ -399,6 +468,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "encoding_rs" version = "0.8.35" @@ -479,6 +554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -487,6 +563,34 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -506,8 +610,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-io", + "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] @@ -558,6 +665,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "h2" version = "0.4.14" @@ -570,13 +683,19 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -710,6 +829,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -743,7 +875,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -884,6 +1016,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -908,6 +1050,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -989,6 +1140,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -1036,8 +1193,13 @@ dependencies = [ "clap", "mofa-engine-core", "mofa-engine-sdk", + "mofa-observability", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "tokio", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] @@ -1067,17 +1229,18 @@ dependencies = [ name = "mofa-engine-sdk" version = "0.1.0" dependencies = [ - "axum", + "axum 0.8.9", "axum-extra", "chrono", "mofa-engine-core", "mofa-kernel", + "mofa-observability", "serde", "serde_json", "thiserror", "tokio", "tokio-stream", - "tower", + "tower 0.5.3", "tower-http", "tracing", ] @@ -1097,11 +1260,13 @@ dependencies = [ name = "mofa-observability" version = "0.1.0" dependencies = [ - "axum", + "axum 0.8.9", "rand 0.8.6", "serde", "serde_json", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1212,6 +1377,88 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "236e667b670a5cdf90c258f5a55794ec5ac5027e960c224bff8367a59e1e6426" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", + "tracing", +] + +[[package]] +name = "opentelemetry-http" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8863faf2910030d139fb48715ad5ff2f35029fc5f244f6d5f689ddcf4d26253" +dependencies = [ + "async-trait", + "bytes", + "http", + "opentelemetry", + "reqwest", + "tracing", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bef114c6d41bea83d6dc60eb41720eedd0261a67af57b66dd2b84ac46c01d91" +dependencies = [ + "async-trait", + "futures-core", + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f8870d3024727e99212eb3bb1762ec16e255e3e6f58eeb3dc8db1aa226746d" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "opentelemetry", + "percent-encoding", + "rand 0.8.6", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -1247,6 +1494,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1296,6 +1563,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quote" version = "1.0.45" @@ -1422,6 +1712,7 @@ dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", @@ -1446,7 +1737,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tower", + "tower 0.5.3", "tower-http", "tower-service", "url", @@ -1694,6 +1985,16 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.4" @@ -1851,7 +2152,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.4", "tokio-macros", "windows-sys 0.61.2", ] @@ -1951,7 +2252,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap", + "indexmap 2.14.0", "serde", "serde_spanned", "toml_datetime", @@ -1965,6 +2266,56 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.9", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.6", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -1993,7 +2344,7 @@ dependencies = [ "http", "http-body", "pin-project-lite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", @@ -2056,6 +2407,24 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721f2d2569dce9f3dfbbddee5906941e953bfcdf736a62da3377f5751650cc36" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" @@ -2297,7 +2666,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -2310,7 +2679,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "semver", ] @@ -2324,6 +2693,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2594,7 +2973,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap", + "indexmap 2.14.0", "prettyplease", "syn", "wasm-metadata", @@ -2625,7 +3004,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -2644,7 +3023,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.14.0", "log", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index 49103f3..a7a9867 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,10 @@ anyhow = "1" # Logging tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +opentelemetry = "0.28.0" +opentelemetry_sdk = { version = "0.28.0", features = ["rt-tokio"] } +tracing-opentelemetry = "0.29.0" +opentelemetry-otlp = { version = "0.28.0", features = ["grpc-tonic", "trace"] } # Utilities uuid = { version = "1", features = ["v4"] } diff --git a/mofa-engine-app/Cargo.toml b/mofa-engine-app/Cargo.toml index 6e7bc16..9bd22b6 100644 --- a/mofa-engine-app/Cargo.toml +++ b/mofa-engine-app/Cargo.toml @@ -8,8 +8,13 @@ description = "MoFA Engine binary entry point" [dependencies] mofa-engine-core = { workspace = true } mofa-engine-sdk = { workspace = true } +mofa-observability = { path = "../mofa-observability" } tokio = { workspace = true } clap = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +tracing-opentelemetry = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } +opentelemetry-otlp = { workspace = true } anyhow = { workspace = true } diff --git a/mofa-engine-sdk/Cargo.toml b/mofa-engine-sdk/Cargo.toml index eb785c4..9c20c37 100644 --- a/mofa-engine-sdk/Cargo.toml +++ b/mofa-engine-sdk/Cargo.toml @@ -19,3 +19,4 @@ thiserror = { workspace = true } tracing = { workspace = true } chrono = { workspace = true } tokio-stream = { workspace = true } +mofa-observability = { path = "../mofa-observability" } diff --git a/mofa-observability/Cargo.toml b/mofa-observability/Cargo.toml index 589570a..9c36c5c 100644 --- a/mofa-observability/Cargo.toml +++ b/mofa-observability/Cargo.toml @@ -8,7 +8,9 @@ license.workspace = true serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } [dev-dependencies] axum = { workspace = true } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } rand = "0.8" From 56e2611e231a49a98ebb09bde4e2c631eb96e100 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 14/26] wire observability pipeline and OTLP tracer --- mofa-engine-app/src/main.rs | 85 ++++++++++++++++++++++++++++++++----- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/mofa-engine-app/src/main.rs b/mofa-engine-app/src/main.rs index 2f163c8..859797e 100644 --- a/mofa-engine-app/src/main.rs +++ b/mofa-engine-app/src/main.rs @@ -7,7 +7,30 @@ use clap::Parser; use mofa_engine_core::{Engine, EngineConfig}; use mofa_engine_sdk::start_server; use std::path::PathBuf; -use tracing_subscriber::{EnvFilter, fmt}; +use std::sync::Arc; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, fmt}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_sdk::Resource; + +fn init_tracer(endpoint: &str) -> Result { + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build()?; + + let provider = SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_resource( + Resource::builder().with_service_name("mofa-engine").build() + ) + .build(); + + opentelemetry::global::set_tracer_provider(provider.clone()); + + use opentelemetry::trace::TracerProvider; + Ok(provider.tracer("mofa-engine")) +} /// MoFA Engine — multimodal AI model orchestration #[derive(Parser, Debug)] @@ -24,14 +47,6 @@ struct Cli { #[tokio::main] async fn main() -> anyhow::Result<()> { - // Initialise tracing - fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), - ) - .with_target(false) - .init(); - let cli = Cli::parse(); // Load and validate configuration. @@ -45,13 +60,63 @@ async fn main() -> anyhow::Result<()> { let host = config.listen.host.clone(); let port = config.listen.port; + #[allow(unused_variables)] + let observability_enabled = config.observability.enabled; + #[allow(unused_variables)] + let otlp_endpoint = config.observability.otlp_endpoint.clone(); + + // Initialize Tracing Subscriber + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let fmt_layer = fmt::layer().with_target(false); + + let otel_layer = if observability_enabled { + if let Some(endpoint) = &otlp_endpoint { + match init_tracer(endpoint) { + Ok(tracer) => Some(tracing_opentelemetry::layer().with_tracer(tracer)), + Err(e) => { + eprintln!("Failed to initialize OTLP tracer: {e}"); + None + } + } + } else { + None + } + } else { + None + }; + + tracing_subscriber::registry() + .with(env_filter) + .with(fmt_layer) + .with(otel_layer) + .init(); + + let mut metrics_state = None; + let mut obs_sender = None; + + if observability_enabled { + tracing::info!("Initializing observability pipeline..."); + let (metrics, sender) = mofa_observability::collector::init_pipeline(otlp_endpoint.as_deref()); + metrics_state = Some(metrics); + obs_sender = Some(sender); + } + tracing::info!("MoFA Engine v{} starting", env!("CARGO_PKG_VERSION")); // Create engine let engine = Engine::try_new(config).await?; + // Start Observability Bridge + if let Some(sender) = obs_sender { + let rx = engine.subscribe_events(); + let engine_clone = Arc::clone(&engine); + tokio::spawn(async move { + mofa_engine_sdk::observability_bridge::run(rx, sender, engine_clone).await; + }); + } + // Start server - start_server(engine, &host, port) + start_server(engine, metrics_state, &host, port) .await .map_err(|e| anyhow::anyhow!("{e}"))?; From c6691331567511b3d8c6f073d8db80c23cbd2e32 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 15/26] add GET /metrics Prometheus endpoint --- mofa-engine-sdk/src/server.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mofa-engine-sdk/src/server.rs b/mofa-engine-sdk/src/server.rs index 68c2623..8ed1aa2 100644 --- a/mofa-engine-sdk/src/server.rs +++ b/mofa-engine-sdk/src/server.rs @@ -26,20 +26,24 @@ use crate::dashboard; /// Shared application state. #[derive(Clone)] +#[allow(dead_code)] struct AppState { engine: Arc, started_at: std::time::Instant, + metrics: Option>>, } /// Start the HTTP server. pub async fn start_server( engine: Arc, + metrics: Option>>, host: &str, port: u16, ) -> Result<(), Box> { let state = AppState { engine, started_at: std::time::Instant::now(), + metrics, }; let app = Router::new() @@ -47,6 +51,7 @@ pub async fn start_server( .route("/", get(dashboard_handler)) // API routes .route("/health", get(health_handler)) + .route("/metrics", get(metrics_handler)) .route("/v1/capabilities", get(capabilities_handler)) .route("/v1/invoke", post(invoke_handler)) .route("/v1/status", get(status_handler)) @@ -140,6 +145,15 @@ async fn dashboard_handler() -> Html<&'static str> { Html(dashboard::DASHBOARD_HTML) } +async fn metrics_handler(State(state): State) -> Result { + if let Some(metrics_arc) = &state.metrics { + let guard = metrics_arc.read().await; + Ok(mofa_observability::prometheus::render(&guard)) + } else { + Err(axum::http::StatusCode::SERVICE_UNAVAILABLE) + } +} + #[cfg(test)] mod tests { use super::*; From 0fe9c0c203ea1b53a54f573a59a8f55318d3d4ea Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 16/26] observability bridge with startup gauge seeding --- mofa-engine-sdk/src/lib.rs | 2 + mofa-engine-sdk/src/observability_bridge.rs | 260 ++++++++++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 mofa-engine-sdk/src/observability_bridge.rs diff --git a/mofa-engine-sdk/src/lib.rs b/mofa-engine-sdk/src/lib.rs index df91a52..42c42c3 100644 --- a/mofa-engine-sdk/src/lib.rs +++ b/mofa-engine-sdk/src/lib.rs @@ -3,7 +3,9 @@ //! HTTP API server (Axum), SSE event streaming, and embedded dashboard //! for the MoFA Engine. + pub mod dashboard; +pub mod observability_bridge; pub mod server; pub use server::start_server; diff --git a/mofa-engine-sdk/src/observability_bridge.rs b/mofa-engine-sdk/src/observability_bridge.rs new file mode 100644 index 0000000..105b11c --- /dev/null +++ b/mofa-engine-sdk/src/observability_bridge.rs @@ -0,0 +1,260 @@ +//! Observability integration bridge. +//! +//! This module handles translating core engine events into the observability subsystem's format. +//! It also seeds initial metric values from the engine's status on startup. + +use mofa_engine_core::Engine; +use mofa_kernel::EngineEvent; +use mofa_observability::collector::EventSender; +use mofa_observability::events::{ + EngineEvent as ObsEngineEvent, EventEnvelope, EvictionTriggered, ModelLoaded, ModelUnloaded, + RequestCompleted, RequestReceived, UnloadReason, +}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::broadcast; + +struct RequestContext { + capability: mofa_observability::events::Capability, + model_id: String, + backend: String, +} + +/// Runs the observability bridge. +/// +/// Consumes engine events and translates them into observability events. +/// Seeds initial memory and model-count gauges from the engine's status snapshot. +pub async fn run( + mut engine_events: broadcast::Receiver, + sender: EventSender, + engine: Arc, +) { + let mut cache: HashMap = HashMap::new(); + + // ── Seed gauges from the engine's current state ────────────────────── + // Discovery has already run by the time the bridge starts, so the + // engine knows which models are loaded and how much memory is used. + let status = engine.status().await; + tracing::info!( + "Bridge: seeding gauges — {} models loaded, {:.1} MiB used / {:.1} MiB budget", + status.loaded_models, + status.memory_used_bytes as f64 / 1_048_576.0, + status.memory_budget_bytes as f64 / 1_048_576.0, + ); + + // Set memory gauges + let _ = sender + .send_critical(EventEnvelope::now(ObsEngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "_seed".into(), + memory_before_bytes: status.memory_used_bytes, + memory_after_bytes: status.memory_used_bytes, + budget_bytes: status.memory_budget_bytes, + }, + ))) + .await; + + // Set models_loaded gauge by emitting one ModelLoaded per loaded model + let caps = engine.capabilities().await; + for card in &caps { + if matches!( + card.residency, + mofa_kernel::ModelResidency::Loaded | mofa_kernel::ModelResidency::Remote + ) { + sender.send(EventEnvelope::now(ObsEngineEvent::ModelLoaded( + ModelLoaded { + model_id: card.id.clone(), + backend: card.provider.clone(), + capability: map_capability(&card.capability), + load_duration_ms: 0, + memory_bytes: card.memory_estimate_bytes, + }, + ))); + } + } + + // ── Event loop ─────────────────────────────────────────────────────── + loop { + match engine_events.recv().await { + Ok(event) => { + match event { + EngineEvent::RequestStarted { + request_id, + capability, + model_id, + } => { + let obs_cap = capability + .as_ref() + .map(map_capability) + .unwrap_or(mofa_observability::events::Capability::Chat); + + // Extract provider from "provider/model" format + let backend = if model_id.contains('/') { + model_id.split('/').next().unwrap_or("unknown").to_string() + } else { + "local".to_string() + }; + + cache.insert( + request_id.clone(), + RequestContext { + capability: obs_cap, + model_id: model_id.clone(), + backend: backend.clone(), + }, + ); + + let obs_event = ObsEngineEvent::RequestReceived(RequestReceived { + capability: obs_cap, + model: Some(model_id), + hint: None, + }); + let mut envelope = + EventEnvelope::now(obs_event).with_request_id(&request_id); + if let Some((trace_id, span_id)) = derive_trace_context(&request_id) { + envelope = envelope.with_trace(trace_id, span_id); + } + sender.send(envelope); + } + EngineEvent::RequestCompleted { + request_id, + duration_ms, + success, + } => { + let (obs_model_id, obs_capability, obs_backend) = cache + .remove(&request_id) + .map(|ctx| (ctx.model_id, ctx.capability, ctx.backend)) + .unwrap_or_else(|| { + ( + "unknown".into(), + mofa_observability::events::Capability::Chat, + "unknown".into(), + ) + }); + + let obs_event = + ObsEngineEvent::RequestCompleted(RequestCompleted { + model_id: obs_model_id, + backend: obs_backend, + capability: obs_capability, + duration_ms, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success, + error_code: None, + }); + let mut envelope = + EventEnvelope::now(obs_event).with_request_id(&request_id); + if let Some((trace_id, span_id)) = derive_trace_context(&request_id) { + envelope = envelope.with_trace(trace_id, span_id); + } + sender.send(envelope); + } + EngineEvent::ModelResidencyChanged { + model_id, + old, + new, + } => { + use mofa_kernel::ModelResidency; + match (old, new) { + // Model loaded into memory + (_, ModelResidency::Loaded) => { + let backend = if model_id.contains('/') { + model_id.split('/').next().unwrap_or("unknown").to_string() + } else { + "local".to_string() + }; + sender.send(EventEnvelope::now( + ObsEngineEvent::ModelLoaded(ModelLoaded { + model_id: model_id.clone(), + backend, + capability: + mofa_observability::events::Capability::Chat, + load_duration_ms: 0, + memory_bytes: 0, + }), + )); + } + // Model unloaded from memory + (ModelResidency::Loaded, _) => { + sender.send(EventEnvelope::now( + ObsEngineEvent::ModelUnloaded(ModelUnloaded { + model_id: model_id.clone(), + reason: UnloadReason::Explicit, + memory_freed_bytes: 0, + }), + )); + } + _ => {} // Loading, Unloading — intermediate states, skip + } + } + EngineEvent::MemoryChanged { + used_bytes, + total_bytes, + } => { + // Set the memory gauges from the authoritative engine value + let _ = sender + .send_critical(EventEnvelope::now( + ObsEngineEvent::EvictionTriggered(EvictionTriggered { + evicted_model: "_memory_sync".into(), + memory_before_bytes: used_bytes, + memory_after_bytes: used_bytes, + budget_bytes: total_bytes, + }), + )) + .await; + } + EngineEvent::DiscoveryCompleted { + provider, models, .. + } => { + tracing::info!( + provider = %provider, + models = models, + "Bridge: discovery completed" + ); + } + _ => { + // ModelStatusChanged, ProviderHealthChanged — not mapped yet + } + } + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!("Observability bridge lagged, skipped {} events", skipped); + } + Err(broadcast::error::RecvError::Closed) => { + tracing::info!("Observability bridge shutting down"); + break; + } + } + } +} + +fn map_capability(cap: &mofa_kernel::Capability) -> mofa_observability::events::Capability { + use mofa_kernel::Capability as K; + use mofa_observability::events::Capability as O; + match cap { + K::Chat => O::Chat, + K::Tts => O::Tts, + K::Asr => O::Asr, + K::Vlm => O::Vlm, + K::ImageGen => O::ImageGen, + K::VideoGen => O::VideoGen, + K::Embedding => O::Embedding, + _ => { + tracing::warn!("Unknown capability variant, defaulting to Chat for observability"); + O::Chat + } + } +} + +fn derive_trace_context(request_id: &str) -> Option<(String, String)> { + let trace_id = request_id.replace('-', ""); + if trace_id.len() == 32 { + let span_id = trace_id[0..16].to_string(); + Some((trace_id, span_id)) + } else { + None + } +} From f6fa0b4b950e72a0e32b13e86219b703075b8b62 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 17/26] add ObservabilityConfig default to test configs --- mofa-engine-core/src/engine.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mofa-engine-core/src/engine.rs b/mofa-engine-core/src/engine.rs index e656d6c..8748c0a 100644 --- a/mofa-engine-core/src/engine.rs +++ b/mofa-engine-core/src/engine.rs @@ -674,6 +674,7 @@ mod tests { fn minimal_config() -> EngineConfig { EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig { budget_mb: Some(100), @@ -849,6 +850,7 @@ mod tests { #[tokio::test] async fn disabled_provider_is_skipped() { let config = EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig { budget_mb: Some(100), From a17f015624e3535a04d87cf264194bbfac3cf1de Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 18/26] collector tracing and mock harness memory budget --- mofa-observability/examples/mock_harness.rs | 51 ++++++++++++++++----- mofa-observability/src/collector.rs | 17 ++++++- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/mofa-observability/examples/mock_harness.rs b/mofa-observability/examples/mock_harness.rs index 384c98b..0755ca3 100644 --- a/mofa-observability/examples/mock_harness.rs +++ b/mofa-observability/examples/mock_harness.rs @@ -53,28 +53,57 @@ async fn simulate_engine_traffic(sender: EventSender) { memory_bytes: 800_000_000, }))); + // Send an initial event to seed the memory budget for Grafana + sender.send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "init".into(), + memory_before_bytes: 0, + memory_after_bytes: 0, + budget_bytes: 16_000_000_000, + }, + ))).await.expect("Failed to send init event"); + let mut active_models = 2; // We pre-loaded 2 models + let mut current_memory = 4_000_000_000_u64; + // Continuous event loop loop { - // Sleep between 50ms and 500ms to generate roughly 2-20 requests/sec. tokio::time::sleep(Duration::from_millis(rng.gen_range(50..500))).await; - let action = rng.gen_range(0..100); + let mut action = rng.gen_range(0..100); let selected_model = models.choose(&mut rng).unwrap(); + let mem_size = rng.gen_range(500_000_000..2_500_000_000); // 0.5 - 2.5 GB + + // Force eviction if we're hitting the ceiling + if action < 5 && current_memory + mem_size > 15_000_000_000 { + action = 5; // Convert to an unload/eviction + } + match action { // 5% chance: Load a new model 0..=4 => { + active_models += 1; + current_memory += mem_size; sender.send(EventEnvelope::now(EngineEvent::ModelLoaded(ModelLoaded { model_id: selected_model.0.into(), backend: selected_model.1.into(), capability: selected_model.2, load_duration_ms: rng.gen_range(1000..5000), - memory_bytes: rng.gen_range(500_000_000..8_000_000_000), + memory_bytes: mem_size, }))); } // 5% chance: Unload a model 5..=9 => { + if active_models == 0 { + continue; // Can't unload what isn't there! + } + active_models -= 1; + + // Don't free more than we have + let freed = mem_size.min(current_memory); + current_memory -= freed; + let reason = match rng.gen_range(0..3) { 0 => UnloadReason::IdleTimeout, 1 => UnloadReason::Eviction, @@ -84,7 +113,7 @@ async fn simulate_engine_traffic(sender: EventSender) { ModelUnloaded { model_id: selected_model.0.into(), reason, - memory_freed_bytes: rng.gen_range(500_000_000..8_000_000_000), + memory_freed_bytes: freed, }, ))); @@ -93,11 +122,11 @@ async fn simulate_engine_traffic(sender: EventSender) { sender.send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( EvictionTriggered { evicted_model: selected_model.0.into(), - memory_before_bytes: 15_000_000_000, - memory_after_bytes: 12_000_000_000, + memory_before_bytes: current_memory + freed, + memory_after_bytes: current_memory, budget_bytes: 16_000_000_000, }, - ))).await.expect("Failed to send critical event"); + ))).await.expect("Failed to send eviction event"); } } @@ -239,14 +268,14 @@ async fn main() -> Result<(), Box> { .route("/metrics", get(metrics_handler)) .with_state(state); - let port = 9090; + let port = 9092; let addr = SocketAddr::from(([0, 0, 0, 0], port)); let listener = tokio::net::TcpListener::bind(addr).await?; println!("======================================================="); - println!("🚀 Mock Harness Running!"); - println!("📊 Prometheus Endpoint: http://localhost:{}/metrics", port); - println!(" Try running: curl http://localhost:{}/metrics", port); + println!("Mock Harness Running!"); + println!("Prometheus Endpoint: http://localhost:{}/metrics", port); + println!("Try running: curl -s http://localhost:{}/metrics | grep \"^mofa_\"", port); println!("======================================================="); axum::serve(listener, app).await?; diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs index a6c8190..327309c 100644 --- a/mofa-observability/src/collector.rs +++ b/mofa-observability/src/collector.rs @@ -472,9 +472,9 @@ impl MetricsState { // Reconcile memory gauge against the absolute post-eviction value. // This corrects any cumulative drift caused by dropped ModelUnloaded - // events under channel backpressure. The running sum is replaced with // the authoritative value from the eviction subsystem. self.memory_used_bytes.set(Labels::new(), e.memory_after_bytes as f64); + self.memory_budget_bytes.set(Labels::new(), e.budget_bytes as f64); } EngineEvent::PreflightSignal(e) => { @@ -573,6 +573,7 @@ impl EventSender { /// Send a critical event synchronously. If the channel is full, this will block /// the current thread. Must not be called from within an async runtime context. + #[allow(clippy::result_large_err)] pub fn send_blocking(&self, event: EventEnvelope) -> Result<(), mpsc::error::SendError> { self.tx.blocking_send(event) } @@ -674,6 +675,20 @@ impl MetricsCollector { } } +/// Initialize the observability pipeline. +/// Spawns the collector loop and returns the shared state and event sender. +pub fn init_pipeline(_otlp_endpoint: Option<&str>) -> (Arc>, EventSender) { + let (sender, receiver) = create_event_channel(10000); + let collector_loop = MetricsCollector::new(receiver); + let state = collector_loop.state(); + + tokio::spawn(async move { + collector_loop.run().await; + }); + + (state, sender) +} + // ─── Tests ─────────────────────────────────────────────────────────────────── #[cfg(test)] From 0448d95cc0e09b592e34f6bed6ef12c92e772042 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 19/26] add engine scrape target to Prometheus --- mofa-observability/docker/docker-compose.yml | 2 -- mofa-observability/docker/prometheus/prometheus.yml | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mofa-observability/docker/docker-compose.yml b/mofa-observability/docker/docker-compose.yml index 1f0fde8..5964d22 100644 --- a/mofa-observability/docker/docker-compose.yml +++ b/mofa-observability/docker/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: mock-harness: build: diff --git a/mofa-observability/docker/prometheus/prometheus.yml b/mofa-observability/docker/prometheus/prometheus.yml index 454bc13..acd8734 100644 --- a/mofa-observability/docker/prometheus/prometheus.yml +++ b/mofa-observability/docker/prometheus/prometheus.yml @@ -4,4 +4,8 @@ global: scrape_configs: - job_name: 'mofa_mock_harness' static_configs: - - targets: ['mock-harness:9090'] + - targets: ['host.docker.internal:9092'] + + - job_name: 'mofa_engine' + static_configs: + - targets: ['host.docker.internal:8420'] From abfab9f9bd2de9dad3e7b3d9fbf615db52a2913f Mon Sep 17 00:00:00 2001 From: ashum9 Date: Wed, 8 Jul 2026 12:29:37 +0530 Subject: [PATCH 20/26] add test config and dockerignore --- .dockerignore | 3 +++ test_obs_config.toml | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 .dockerignore create mode 100644 test_obs_config.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5931875 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +target/ +.git/ +.env diff --git a/test_obs_config.toml b/test_obs_config.toml new file mode 100644 index 0000000..13bc7d6 --- /dev/null +++ b/test_obs_config.toml @@ -0,0 +1,20 @@ +[listen] +host = "127.0.0.1" +port = 8420 + +[memory] +budget_mb = 100 +idle_timeout_secs = 60 + +[observability] +enabled = true + +[[providers]] +name = "test" +kind = "openai_compatible" +base_url = "http://127.0.0.1:11435/v1" +api_key = "dummy" + +[[providers.models]] +name = "mock_chat" +capability = "chat" From b2a11a9f7857b9cc6df29cd64fe8845717f100da Mon Sep 17 00:00:00 2001 From: ashum9 Date: Thu, 16 Jul 2026 21:14:15 +0530 Subject: [PATCH 21/26] fix: update core engine memory tracking and routing endpoints --- mofa-engine-app/Cargo.toml | 5 + mofa-engine-app/src/main.rs | 86 +++++- .../src/backends/openai_compat.rs | 6 +- mofa-engine-core/src/config.rs | 17 ++ mofa-engine-core/src/engine.rs | 20 +- mofa-engine-sdk/Cargo.toml | 1 + mofa-engine-sdk/src/lib.rs | 2 + mofa-engine-sdk/src/observability_bridge.rs | 287 ++++++++++++++++++ mofa-engine-sdk/src/server.rs | 39 +++ 9 files changed, 449 insertions(+), 14 deletions(-) create mode 100644 mofa-engine-sdk/src/observability_bridge.rs diff --git a/mofa-engine-app/Cargo.toml b/mofa-engine-app/Cargo.toml index 6e7bc16..9bd22b6 100644 --- a/mofa-engine-app/Cargo.toml +++ b/mofa-engine-app/Cargo.toml @@ -8,8 +8,13 @@ description = "MoFA Engine binary entry point" [dependencies] mofa-engine-core = { workspace = true } mofa-engine-sdk = { workspace = true } +mofa-observability = { path = "../mofa-observability" } tokio = { workspace = true } clap = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +tracing-opentelemetry = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } +opentelemetry-otlp = { workspace = true } anyhow = { workspace = true } diff --git a/mofa-engine-app/src/main.rs b/mofa-engine-app/src/main.rs index 2f163c8..3a9cb27 100644 --- a/mofa-engine-app/src/main.rs +++ b/mofa-engine-app/src/main.rs @@ -7,7 +7,30 @@ use clap::Parser; use mofa_engine_core::{Engine, EngineConfig}; use mofa_engine_sdk::start_server; use std::path::PathBuf; -use tracing_subscriber::{EnvFilter, fmt}; +use std::sync::Arc; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, fmt}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_sdk::Resource; + +fn init_tracer(endpoint: &str) -> Result { + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build()?; + + let provider = SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .with_resource( + Resource::builder().with_service_name("mofa-engine").build() + ) + .build(); + + opentelemetry::global::set_tracer_provider(provider.clone()); + + use opentelemetry::trace::TracerProvider; + Ok(provider.tracer("mofa-engine")) +} /// MoFA Engine — multimodal AI model orchestration #[derive(Parser, Debug)] @@ -24,14 +47,6 @@ struct Cli { #[tokio::main] async fn main() -> anyhow::Result<()> { - // Initialise tracing - fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), - ) - .with_target(false) - .init(); - let cli = Cli::parse(); // Load and validate configuration. @@ -45,13 +60,64 @@ async fn main() -> anyhow::Result<()> { let host = config.listen.host.clone(); let port = config.listen.port; + #[allow(unused_variables)] + let observability_enabled = config.observability.enabled; + #[allow(unused_variables)] + let otlp_endpoint = config.observability.otlp_endpoint.clone(); + + // Initialize Tracing Subscriber + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let fmt_layer = fmt::layer().with_target(false); + + let otel_layer = if observability_enabled { + if let Some(endpoint) = &otlp_endpoint { + match init_tracer(endpoint) { + Ok(tracer) => Some(tracing_opentelemetry::layer().with_tracer(tracer)), + Err(e) => { + eprintln!("Failed to initialize OTLP tracer: {e}"); + None + } + } + } else { + None + } + } else { + None + }; + + tracing_subscriber::registry() + .with(env_filter) + .with(fmt_layer) + .with(otel_layer) + .init(); + + let mut metrics_state = None; + let mut obs_sender = None; + + if observability_enabled { + tracing::info!("Initializing observability pipeline..."); + let (metrics, sender) = mofa_observability::collector::init_pipeline(otlp_endpoint.as_deref()); + metrics_state = Some(metrics); + obs_sender = Some(sender); + } + tracing::info!("MoFA Engine v{} starting", env!("CARGO_PKG_VERSION")); // Create engine let engine = Engine::try_new(config).await?; + // Start Observability Bridge + if let Some(sender) = obs_sender { + let rx = engine.subscribe_events(); + let engine_clone = Arc::clone(&engine); + let metrics_clone = metrics_state.clone(); + tokio::spawn(async move { + mofa_engine_sdk::observability_bridge::run(rx, sender, engine_clone, metrics_clone).await; + }); + } + // Start server - start_server(engine, &host, port) + start_server(engine, metrics_state, &host, port) .await .map_err(|e| anyhow::anyhow!("{e}"))?; diff --git a/mofa-engine-core/src/backends/openai_compat.rs b/mofa-engine-core/src/backends/openai_compat.rs index 60a1ba9..e2dd8ff 100644 --- a/mofa-engine-core/src/backends/openai_compat.rs +++ b/mofa-engine-core/src/backends/openai_compat.rs @@ -385,15 +385,15 @@ impl OpenAiCompatProvider { detail: format!("TTS read error: {e}"), })?; - let path = std::env::temp_dir().join(format!("mofa_tts_{}.mp3", uuid::Uuid::new_v4())); + let file_name = format!("mofa_tts_{}.wav", uuid::Uuid::new_v4()); + let path = std::env::temp_dir().join(&file_name); std::fs::write(&path, &bytes) .map_err(|e| EngineError::Internal(format!("write error: {e}")))?; - let path = path.to_string_lossy().to_string(); let duration_ms = start.elapsed().as_millis() as u64; Ok(InferenceResponse { text: None, - file: Some(path), + file: Some(file_name), model_used: model_name.to_string(), provider: self.name.clone(), duration_ms, diff --git a/mofa-engine-core/src/config.rs b/mofa-engine-core/src/config.rs index a31c82c..f502ed6 100644 --- a/mofa-engine-core/src/config.rs +++ b/mofa-engine-core/src/config.rs @@ -22,6 +22,20 @@ pub struct EngineConfig { /// Provider definitions #[serde(default)] pub providers: Vec, + /// Observability settings + #[serde(default)] + pub observability: ObservabilityConfig, +} + +/// Observability configuration. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ObservabilityConfig { + /// Whether observability is enabled. + #[serde(default)] + pub enabled: bool, + /// The OpenTelemetry OTLP endpoint (e.g., http://localhost:4317) + #[serde(default)] + pub otlp_endpoint: Option, } /// Network listen configuration. @@ -399,6 +413,7 @@ impl EngineConfig { listen: ListenConfig::default(), memory: MemoryConfig::default(), providers, + observability: ObservabilityConfig::default(), } } } @@ -470,6 +485,7 @@ mod tests { #[test] fn toml_roundtrip() { let cfg = EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig::default(), providers: vec![ProviderConfig { @@ -492,6 +508,7 @@ mod tests { #[test] fn validate_rejects_unknown_provider_kind() { let cfg = EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig::default(), providers: vec![ProviderConfig { diff --git a/mofa-engine-core/src/engine.rs b/mofa-engine-core/src/engine.rs index e656d6c..c2d3206 100644 --- a/mofa-engine-core/src/engine.rs +++ b/mofa-engine-core/src/engine.rs @@ -172,9 +172,25 @@ impl Engine { } let count = cards.len(); + let mut allocated_any = false; for mut card in cards { card.refresh_status(); - self.models.insert(card.id.clone(), card); + let id = card.id.clone(); + if matches!(card.residency, ModelResidency::Loaded) && card.memory_estimate_bytes > 0 { + // If it's already loaded but we don't have it allocated yet + let snapshot = self.memory.snapshot(); + if !snapshot.iter().any(|(mem_id, _)| mem_id == &id) { + self.memory.allocate(&id, card.memory_estimate_bytes); + allocated_any = true; + } + } + self.models.insert(id, card); + } + if allocated_any { + let _ = self.event_tx.send(EngineEvent::MemoryChanged { + used_bytes: self.memory.used_bytes(), + total_bytes: self.memory.budget_bytes(), + }); } let _ = self.event_tx.send(EngineEvent::DiscoveryCompleted { provider: name.clone(), @@ -674,6 +690,7 @@ mod tests { fn minimal_config() -> EngineConfig { EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig { budget_mb: Some(100), @@ -849,6 +866,7 @@ mod tests { #[tokio::test] async fn disabled_provider_is_skipped() { let config = EngineConfig { + observability: Default::default(), listen: ListenConfig::default(), memory: MemoryConfig { budget_mb: Some(100), diff --git a/mofa-engine-sdk/Cargo.toml b/mofa-engine-sdk/Cargo.toml index eb785c4..9c20c37 100644 --- a/mofa-engine-sdk/Cargo.toml +++ b/mofa-engine-sdk/Cargo.toml @@ -19,3 +19,4 @@ thiserror = { workspace = true } tracing = { workspace = true } chrono = { workspace = true } tokio-stream = { workspace = true } +mofa-observability = { path = "../mofa-observability" } diff --git a/mofa-engine-sdk/src/lib.rs b/mofa-engine-sdk/src/lib.rs index df91a52..42c42c3 100644 --- a/mofa-engine-sdk/src/lib.rs +++ b/mofa-engine-sdk/src/lib.rs @@ -3,7 +3,9 @@ //! HTTP API server (Axum), SSE event streaming, and embedded dashboard //! for the MoFA Engine. + pub mod dashboard; +pub mod observability_bridge; pub mod server; pub use server::start_server; diff --git a/mofa-engine-sdk/src/observability_bridge.rs b/mofa-engine-sdk/src/observability_bridge.rs new file mode 100644 index 0000000..e2a5bf1 --- /dev/null +++ b/mofa-engine-sdk/src/observability_bridge.rs @@ -0,0 +1,287 @@ +//! Observability integration bridge. +//! +//! This module handles translating core engine events into the observability subsystem's format. +//! It also seeds initial metric values from the engine's status on startup. + +use mofa_engine_core::Engine; +use mofa_kernel::EngineEvent; +use mofa_observability::collector::{EventSender, Labels, MetricsState}; +use mofa_observability::events::{ + EngineEvent as ObsEngineEvent, EventEnvelope, EvictionTriggered, ModelLoaded, ModelUnloaded, + RequestCompleted, RequestReceived, UnloadReason, +}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::RwLock; +use tokio::sync::broadcast; + +struct RequestContext { + capability: mofa_observability::events::Capability, + model_id: String, + backend: String, +} + +/// Runs the observability bridge. +/// +/// Consumes engine events and translates them into observability events. +/// Seeds initial memory and model-count gauges from the engine's status snapshot. +pub async fn run( + mut engine_events: broadcast::Receiver, + sender: EventSender, + engine: Arc, + metrics_state: Option>>, +) { + let mut cache: HashMap = HashMap::new(); + + // ── Seed gauges from the engine's current state ────────────────────── + // Discovery has already run by the time the bridge starts, so the + // engine knows which models are loaded and how much memory is used. + let status = engine.status().await; + tracing::info!( + "Bridge: seeding gauges — {} models loaded, {:.1} MiB used / {:.1} MiB budget", + status.loaded_models, + status.memory_used_bytes as f64 / 1_048_576.0, + status.memory_budget_bytes as f64 / 1_048_576.0, + ); + + // Set memory gauges + let _ = sender + .send_critical(EventEnvelope::now(ObsEngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "_seed".into(), + memory_before_bytes: status.memory_used_bytes, + memory_after_bytes: status.memory_used_bytes, + budget_bytes: status.memory_budget_bytes, + }, + ))) + .await; + + // Set models_loaded gauge by emitting one ModelLoaded per loaded model + let caps = engine.capabilities().await; + for card in &caps { + if matches!( + card.residency, + mofa_kernel::ModelResidency::Loaded | mofa_kernel::ModelResidency::Remote + ) { + sender.send(EventEnvelope::now(ObsEngineEvent::ModelLoaded( + ModelLoaded { + model_id: card.id.clone(), + backend: card.provider.clone(), + capability: map_capability(&card.capability), + load_duration_ms: 0, + memory_bytes: card.memory_estimate_bytes, + }, + ))); + } + } + + // ── Event loop with periodic gauge sync ──────────────────────────── + let mut gauge_interval = tokio::time::interval(std::time::Duration::from_secs(10)); + gauge_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + event_result = engine_events.recv() => { + match event_result { + Ok(event) => { + match event { + EngineEvent::RequestStarted { + request_id, + capability, + model_id, + } => { + let obs_cap = capability + .as_ref() + .map(map_capability) + .unwrap_or(mofa_observability::events::Capability::Chat); + + // Extract provider from "provider/model" format + let backend = if model_id.contains('/') { + model_id.split('/').next().unwrap_or("unknown").to_string() + } else { + "local".to_string() + }; + + cache.insert( + request_id.clone(), + RequestContext { + capability: obs_cap, + model_id: model_id.clone(), + backend: backend.clone(), + }, + ); + + let obs_event = ObsEngineEvent::RequestReceived(RequestReceived { + capability: obs_cap, + model: Some(model_id), + hint: None, + }); + let mut envelope = + EventEnvelope::now(obs_event).with_request_id(&request_id); + if let Some((trace_id, span_id)) = derive_trace_context(&request_id) { + envelope = envelope.with_trace(trace_id, span_id); + } + sender.send(envelope); + } + EngineEvent::RequestCompleted { + request_id, + duration_ms, + success, + } => { + let (obs_model_id, obs_capability, obs_backend) = cache + .remove(&request_id) + .map(|ctx| (ctx.model_id, ctx.capability, ctx.backend)) + .unwrap_or_else(|| { + ( + "unknown".into(), + mofa_observability::events::Capability::Chat, + "unknown".into(), + ) + }); + + let obs_event = + ObsEngineEvent::RequestCompleted(RequestCompleted { + model_id: obs_model_id, + backend: obs_backend, + capability: obs_capability, + duration_ms, + ttft_ms: None, + tokens_in: None, + tokens_out: None, + model_was_hot: None, + success, + error_code: None, + }); + let mut envelope = + EventEnvelope::now(obs_event).with_request_id(&request_id); + if let Some((trace_id, span_id)) = derive_trace_context(&request_id) { + envelope = envelope.with_trace(trace_id, span_id); + } + sender.send(envelope); + } + EngineEvent::ModelResidencyChanged { + model_id, + old, + new, + } => { + use mofa_kernel::ModelResidency; + match (old, new) { + // Model loaded into memory + (_, ModelResidency::Loaded) => { + let backend = if model_id.contains('/') { + model_id.split('/').next().unwrap_or("unknown").to_string() + } else { + "local".to_string() + }; + sender.send(EventEnvelope::now( + ObsEngineEvent::ModelLoaded(ModelLoaded { + model_id: model_id.clone(), + backend, + capability: + mofa_observability::events::Capability::Chat, + load_duration_ms: 0, + memory_bytes: 0, + }), + )); + } + // Model unloaded from memory + (ModelResidency::Loaded, _) => { + sender.send(EventEnvelope::now( + ObsEngineEvent::ModelUnloaded(ModelUnloaded { + model_id: model_id.clone(), + reason: UnloadReason::Explicit, + memory_freed_bytes: 0, + }), + )); + } + _ => {} // Loading, Unloading — intermediate states, skip + } + } + EngineEvent::MemoryChanged { + used_bytes, + total_bytes, + } => { + // Set the memory gauges from the authoritative engine value + let _ = sender + .send_critical(EventEnvelope::now( + ObsEngineEvent::EvictionTriggered(EvictionTriggered { + evicted_model: "_memory_sync".into(), + memory_before_bytes: used_bytes, + memory_after_bytes: used_bytes, + budget_bytes: total_bytes, + }), + )) + .await; + } + EngineEvent::DiscoveryCompleted { + provider, models, .. + } => { + tracing::info!( + provider = %provider, + models = models, + "Bridge: discovery completed" + ); + } + _ => { + // ModelStatusChanged, ProviderHealthChanged — not mapped yet + } + } + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!("Observability bridge lagged, skipped {} events", skipped); + } + Err(broadcast::error::RecvError::Closed) => { + tracing::info!("Observability bridge shutting down"); + break; + } + } + } + _ = gauge_interval.tick() => { + // Periodic gauge sync: directly write memory and model-count + // gauges from the authoritative engine state every 10 seconds. + if let Some(ref ms) = metrics_state { + let status = engine.status().await; + let mut state = ms.write().await; + state.memory_used_bytes.set(Labels::new(), status.memory_used_bytes as f64); + state.memory_budget_bytes.set(Labels::new(), status.memory_budget_bytes as f64); + state.models_loaded.set(Labels::new(), status.loaded_models as f64); + + tracing::trace!( + memory_used_mib = status.memory_used_bytes as f64 / 1_048_576.0, + memory_budget_mib = status.memory_budget_bytes as f64 / 1_048_576.0, + loaded_models = status.loaded_models, + "Bridge: periodic gauge sync" + ); + } + } + } + } +} + +fn map_capability(cap: &mofa_kernel::Capability) -> mofa_observability::events::Capability { + use mofa_kernel::Capability as K; + use mofa_observability::events::Capability as O; + match cap { + K::Chat => O::Chat, + K::Tts => O::Tts, + K::Asr => O::Asr, + K::Vlm => O::Vlm, + K::ImageGen => O::ImageGen, + K::VideoGen => O::VideoGen, + K::Embedding => O::Embedding, + _ => { + tracing::warn!("Unknown capability variant, defaulting to Chat for observability"); + O::Chat + } + } +} + +fn derive_trace_context(request_id: &str) -> Option<(String, String)> { + let trace_id = request_id.replace('-', ""); + if trace_id.len() == 32 { + let span_id = trace_id[0..16].to_string(); + Some((trace_id, span_id)) + } else { + None + } +} diff --git a/mofa-engine-sdk/src/server.rs b/mofa-engine-sdk/src/server.rs index 68c2623..9f2be32 100644 --- a/mofa-engine-sdk/src/server.rs +++ b/mofa-engine-sdk/src/server.rs @@ -26,20 +26,24 @@ use crate::dashboard; /// Shared application state. #[derive(Clone)] +#[allow(dead_code)] struct AppState { engine: Arc, started_at: std::time::Instant, + metrics: Option>>, } /// Start the HTTP server. pub async fn start_server( engine: Arc, + metrics: Option>>, host: &str, port: u16, ) -> Result<(), Box> { let state = AppState { engine, started_at: std::time::Instant::now(), + metrics, }; let app = Router::new() @@ -47,10 +51,12 @@ pub async fn start_server( .route("/", get(dashboard_handler)) // API routes .route("/health", get(health_handler)) + .route("/metrics", get(metrics_handler)) .route("/v1/capabilities", get(capabilities_handler)) .route("/v1/invoke", post(invoke_handler)) .route("/v1/status", get(status_handler)) .route("/v1/events", get(events_handler)) + .route("/v1/files/{filename}", get(files_handler)) .route("/v1/discovery/refresh", post(refresh_handler)) .layer(CorsLayer::permissive()) .layer(TraceLayer::new_for_http()) @@ -136,10 +142,43 @@ async fn events_handler( ) } +use axum::extract::Path; +use axum::response::IntoResponse; +use axum::http::header; + +async fn files_handler(Path(filename): Path) -> Result { + if filename.contains('/') || filename.contains('\\') || filename.contains("..") { + return Err(StatusCode::BAD_REQUEST); + } + + let temp_dir = std::env::temp_dir(); + let file_path = temp_dir.join(&filename); + + if !file_path.exists() { + return Err(StatusCode::NOT_FOUND); + } + + let bytes = std::fs::read(file_path).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + Ok(( + [(header::CONTENT_TYPE, "audio/wav")], + bytes, + )) +} + async fn dashboard_handler() -> Html<&'static str> { Html(dashboard::DASHBOARD_HTML) } +async fn metrics_handler(State(state): State) -> Result { + if let Some(metrics_arc) = &state.metrics { + let guard = metrics_arc.read().await; + Ok(mofa_observability::prometheus::render(&guard)) + } else { + Err(axum::http::StatusCode::SERVICE_UNAVAILABLE) + } +} + #[cfg(test)] mod tests { use super::*; From e38146168a67257dc69c1e9e28056e54061b9319 Mon Sep 17 00:00:00 2001 From: ashum9 Date: Fri, 17 Jul 2026 00:58:17 +0530 Subject: [PATCH 22/26] fix: make collector set method pub --- mofa-observability/src/collector.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs index 327309c..7e9539f 100644 --- a/mofa-observability/src/collector.rs +++ b/mofa-observability/src/collector.rs @@ -183,7 +183,7 @@ impl GaugeFamily { } } - fn set(&mut self, labels: Labels, value: f64) { + pub fn set(&mut self, labels: Labels, value: f64) { self.last_seen.insert(labels.clone(), Instant::now()); self.values.insert(labels, value); } From a4dec8df283ad9c44824b59a6f83b4f56c91dd7f Mon Sep 17 00:00:00 2001 From: ashum9 Date: Fri, 17 Jul 2026 11:06:44 +0530 Subject: [PATCH 23/26] address CI fmt check and copilot review feedback --- .gitignore | 1 + mofa-engine-app/src/main.rs | 28 +++---- mofa-engine-core/src/engine.rs | 4 +- mofa-engine-sdk/src/lib.rs | 1 - mofa-engine-sdk/src/server.rs | 15 ++-- .../dashboards/preflight_routing.json | 6 +- mofa-observability/docker/Dockerfile | 2 +- mofa-observability/docker/docker-compose.yml | 2 +- .../docker/prometheus/prometheus.yml | 2 +- mofa-observability/examples/mock_harness.rs | 75 +++++++++++-------- mofa-observability/src/collector.rs | 75 +++++++++---------- mofa-observability/src/events.rs | 26 +++++-- mofa-observability/src/lib.rs | 2 +- mofa-observability/src/prometheus.rs | 48 +++++++++--- 14 files changed, 173 insertions(+), 114 deletions(-) diff --git a/.gitignore b/.gitignore index 7ee12e8..8002557 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ node_modules/ e2e/test-results/ e2e/playwright-report/ +.kokoro-models/ diff --git a/mofa-engine-app/src/main.rs b/mofa-engine-app/src/main.rs index 3a9cb27..9262314 100644 --- a/mofa-engine-app/src/main.rs +++ b/mofa-engine-app/src/main.rs @@ -6,28 +6,28 @@ use clap::Parser; use mofa_engine_core::{Engine, EngineConfig}; use mofa_engine_sdk::start_server; -use std::path::PathBuf; -use std::sync::Arc; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, fmt}; use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::trace::SdkTracerProvider; use opentelemetry_sdk::Resource; +use opentelemetry_sdk::trace::SdkTracerProvider; +use std::path::PathBuf; +use std::sync::Arc; +use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; -fn init_tracer(endpoint: &str) -> Result { +fn init_tracer( + endpoint: &str, +) -> Result { let exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() .with_endpoint(endpoint) .build()?; - + let provider = SdkTracerProvider::builder() .with_batch_exporter(exporter) - .with_resource( - Resource::builder().with_service_name("mofa-engine").build() - ) + .with_resource(Resource::builder().with_service_name("mofa-engine").build()) .build(); - + opentelemetry::global::set_tracer_provider(provider.clone()); - + use opentelemetry::trace::TracerProvider; Ok(provider.tracer("mofa-engine")) } @@ -96,7 +96,8 @@ async fn main() -> anyhow::Result<()> { if observability_enabled { tracing::info!("Initializing observability pipeline..."); - let (metrics, sender) = mofa_observability::collector::init_pipeline(otlp_endpoint.as_deref()); + let (metrics, sender) = + mofa_observability::collector::init_pipeline(otlp_endpoint.as_deref()); metrics_state = Some(metrics); obs_sender = Some(sender); } @@ -112,7 +113,8 @@ async fn main() -> anyhow::Result<()> { let engine_clone = Arc::clone(&engine); let metrics_clone = metrics_state.clone(); tokio::spawn(async move { - mofa_engine_sdk::observability_bridge::run(rx, sender, engine_clone, metrics_clone).await; + mofa_engine_sdk::observability_bridge::run(rx, sender, engine_clone, metrics_clone) + .await; }); } diff --git a/mofa-engine-core/src/engine.rs b/mofa-engine-core/src/engine.rs index c2d3206..704fb7a 100644 --- a/mofa-engine-core/src/engine.rs +++ b/mofa-engine-core/src/engine.rs @@ -176,7 +176,9 @@ impl Engine { for mut card in cards { card.refresh_status(); let id = card.id.clone(); - if matches!(card.residency, ModelResidency::Loaded) && card.memory_estimate_bytes > 0 { + if matches!(card.residency, ModelResidency::Loaded) + && card.memory_estimate_bytes > 0 + { // If it's already loaded but we don't have it allocated yet let snapshot = self.memory.snapshot(); if !snapshot.iter().any(|(mem_id, _)| mem_id == &id) { diff --git a/mofa-engine-sdk/src/lib.rs b/mofa-engine-sdk/src/lib.rs index 42c42c3..5a5eef5 100644 --- a/mofa-engine-sdk/src/lib.rs +++ b/mofa-engine-sdk/src/lib.rs @@ -3,7 +3,6 @@ //! HTTP API server (Axum), SSE event streaming, and embedded dashboard //! for the MoFA Engine. - pub mod dashboard; pub mod observability_bridge; pub mod server; diff --git a/mofa-engine-sdk/src/server.rs b/mofa-engine-sdk/src/server.rs index 9f2be32..3071992 100644 --- a/mofa-engine-sdk/src/server.rs +++ b/mofa-engine-sdk/src/server.rs @@ -143,27 +143,24 @@ async fn events_handler( } use axum::extract::Path; -use axum::response::IntoResponse; use axum::http::header; +use axum::response::IntoResponse; async fn files_handler(Path(filename): Path) -> Result { if filename.contains('/') || filename.contains('\\') || filename.contains("..") { return Err(StatusCode::BAD_REQUEST); } - + let temp_dir = std::env::temp_dir(); let file_path = temp_dir.join(&filename); - + if !file_path.exists() { return Err(StatusCode::NOT_FOUND); } - + let bytes = std::fs::read(file_path).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - - Ok(( - [(header::CONTENT_TYPE, "audio/wav")], - bytes, - )) + + Ok(([(header::CONTENT_TYPE, "audio/wav")], bytes)) } async fn dashboard_handler() -> Html<&'static str> { diff --git a/mofa-observability/dashboards/preflight_routing.json b/mofa-observability/dashboards/preflight_routing.json index 7442ac8..ed87825 100644 --- a/mofa-observability/dashboards/preflight_routing.json +++ b/mofa-observability/dashboards/preflight_routing.json @@ -68,12 +68,12 @@ }, { "type": "piechart", - "title": "Requests by Backend", + "title": "Requests by Capability", "gridPos": { "x": 12, "y": 6, "w": 12, "h": 6 }, "targets": [ { - "expr": "sum(rate(mofa_requests_total[5m])) by (backend)", - "legendFormat": "{{backend}}" + "expr": "sum(rate(mofa_requests_total[5m])) by (capability)", + "legendFormat": "{{capability}}" } ] } diff --git a/mofa-observability/docker/Dockerfile b/mofa-observability/docker/Dockerfile index d04751d..ee45850 100644 --- a/mofa-observability/docker/Dockerfile +++ b/mofa-observability/docker/Dockerfile @@ -11,5 +11,5 @@ FROM debian:bookworm-slim RUN apt-get update && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=builder /usr/src/mofa/target/release/examples/mock_harness /usr/local/bin/mock_harness -EXPOSE 9090 +EXPOSE 9092 CMD ["mock_harness"] diff --git a/mofa-observability/docker/docker-compose.yml b/mofa-observability/docker/docker-compose.yml index 5964d22..13e76e6 100644 --- a/mofa-observability/docker/docker-compose.yml +++ b/mofa-observability/docker/docker-compose.yml @@ -4,7 +4,7 @@ services: context: ../../ dockerfile: mofa-observability/docker/Dockerfile ports: - - "9090:9090" + - "9092:9092" prometheus: image: prom/prometheus:latest diff --git a/mofa-observability/docker/prometheus/prometheus.yml b/mofa-observability/docker/prometheus/prometheus.yml index acd8734..3dd3355 100644 --- a/mofa-observability/docker/prometheus/prometheus.yml +++ b/mofa-observability/docker/prometheus/prometheus.yml @@ -4,7 +4,7 @@ global: scrape_configs: - job_name: 'mofa_mock_harness' static_configs: - - targets: ['host.docker.internal:9092'] + - targets: ['mock-harness:9092'] - job_name: 'mofa_engine' static_configs: diff --git a/mofa-observability/examples/mock_harness.rs b/mofa-observability/examples/mock_harness.rs index 0755ca3..497b6c4 100644 --- a/mofa-observability/examples/mock_harness.rs +++ b/mofa-observability/examples/mock_harness.rs @@ -6,13 +6,13 @@ //! //! Run with: `cargo run --example mock_harness` -use axum::{extract::State, routing::get, Router}; +use axum::{Router, extract::State, routing::get}; use mofa_observability::{ - collector::{create_event_channel, EventSender, MetricsCollector, MetricsState}, + collector::{EventSender, MetricsCollector, MetricsState, create_event_channel}, events::*, prometheus, }; -use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; +use rand::{Rng, SeedableRng, rngs::StdRng, seq::SliceRandom}; use std::{net::SocketAddr, sync::Arc, time::Duration}; use tokio::sync::RwLock; @@ -54,14 +54,17 @@ async fn simulate_engine_traffic(sender: EventSender) { }))); // Send an initial event to seed the memory budget for Grafana - sender.send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( - EvictionTriggered { - evicted_model: "init".into(), - memory_before_bytes: 0, - memory_after_bytes: 0, - budget_bytes: 16_000_000_000, - }, - ))).await.expect("Failed to send init event"); + sender + .send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: "init".into(), + memory_before_bytes: 0, + memory_after_bytes: 0, + budget_bytes: 16_000_000_000, + }, + ))) + .await + .expect("Failed to send init event"); let mut active_models = 2; // We pre-loaded 2 models let mut current_memory = 4_000_000_000_u64; @@ -99,7 +102,7 @@ async fn simulate_engine_traffic(sender: EventSender) { continue; // Can't unload what isn't there! } active_models -= 1; - + // Don't free more than we have let freed = mem_size.min(current_memory); current_memory -= freed; @@ -119,14 +122,17 @@ async fn simulate_engine_traffic(sender: EventSender) { // If eviction, also trigger eviction event if reason == UnloadReason::Eviction { - sender.send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( - EvictionTriggered { - evicted_model: selected_model.0.into(), - memory_before_bytes: current_memory + freed, - memory_after_bytes: current_memory, - budget_bytes: 16_000_000_000, - }, - ))).await.expect("Failed to send eviction event"); + sender + .send_critical(EventEnvelope::now(EngineEvent::EvictionTriggered( + EvictionTriggered { + evicted_model: selected_model.0.into(), + memory_before_bytes: current_memory + freed, + memory_after_bytes: current_memory, + budget_bytes: 16_000_000_000, + }, + ))) + .await + .expect("Failed to send eviction event"); } } @@ -170,14 +176,17 @@ async fn simulate_engine_traffic(sender: EventSender) { // 1% chance: Failover 20..=20 => { - sender.send_critical(EventEnvelope::now(EngineEvent::FailoverTriggered( - FailoverTriggered { - failed_model: selected_model.0.into(), - failed_backend: selected_model.1.into(), - fallback_model: "fallback-model".into(), - fallback_backend: "cloud".into(), - }, - ))).await.expect("Failed to send critical event"); + sender + .send_critical(EventEnvelope::now(EngineEvent::FailoverTriggered( + FailoverTriggered { + failed_model: selected_model.0.into(), + failed_backend: selected_model.1.into(), + fallback_model: "fallback-model".into(), + fallback_backend: "cloud".into(), + }, + ))) + .await + .expect("Failed to send critical event"); } // 79% chance: Normal Inference Request @@ -245,7 +254,10 @@ async fn simulate_engine_traffic(sender: EventSender) { #[tokio::main] async fn main() -> Result<(), Box> { tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env().add_directive("mofa_observability=info".parse().unwrap())) + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive("mofa_observability=info".parse().unwrap()), + ) .init(); println!("Starting MoFA Engine Observability Mock Harness..."); @@ -275,7 +287,10 @@ async fn main() -> Result<(), Box> { println!("======================================================="); println!("Mock Harness Running!"); println!("Prometheus Endpoint: http://localhost:{}/metrics", port); - println!("Try running: curl -s http://localhost:{}/metrics | grep \"^mofa_\"", port); + println!( + "Try running: curl -s http://localhost:{}/metrics | grep \"^mofa_\"", + port + ); println!("======================================================="); axum::serve(listener, app).await?; diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs index 7e9539f..fc8917e 100644 --- a/mofa-observability/src/collector.rs +++ b/mofa-observability/src/collector.rs @@ -14,7 +14,7 @@ use crate::events::*; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::{mpsc, RwLock}; +use tokio::sync::{RwLock, mpsc}; // ─── Labels ────────────────────────────────────────────────────────────────── @@ -118,7 +118,8 @@ impl CounterFamily { true } }); - self.last_seen.retain(|_, v| now.saturating_duration_since(*v) < max_idle); + self.last_seen + .retain(|_, v| now.saturating_duration_since(*v) < max_idle); } } @@ -160,7 +161,8 @@ impl HistogramFamily { true } }); - self.last_seen.retain(|_, v| now.saturating_duration_since(*v) < max_idle); + self.last_seen + .retain(|_, v| now.saturating_duration_since(*v) < max_idle); } } @@ -217,7 +219,8 @@ impl GaugeFamily { true } }); - self.last_seen.retain(|_, v| now.saturating_duration_since(*v) < max_idle); + self.last_seen + .retain(|_, v| now.saturating_duration_since(*v) < max_idle); } } @@ -258,10 +261,7 @@ impl MetricsState { pub fn new() -> Self { Self { // Counters - requests_total: CounterFamily::new( - "mofa_requests_total", - "Total inference requests", - ), + requests_total: CounterFamily::new("mofa_requests_total", "Total inference requests"), model_loads_total: CounterFamily::new( "mofa_model_loads_total", "Total model load operations", @@ -270,10 +270,7 @@ impl MetricsState { "mofa_model_unloads_total", "Total model unload operations", ), - failovers_total: CounterFamily::new( - "mofa_failovers_total", - "Total failover events", - ), + failovers_total: CounterFamily::new("mofa_failovers_total", "Total failover events"), evictions_total: CounterFamily::new( "mofa_evictions_total", "Total memory pressure evictions", @@ -401,16 +398,12 @@ impl MetricsState { // Counter: tokens if let Some(tokens_in) = e.tokens_in { - self.tokens_input_total.inc_by( - Labels::new().add("model", &e.model_id), - tokens_in, - ); + self.tokens_input_total + .inc_by(Labels::new().add("model", &e.model_id), tokens_in); } if let Some(tokens_out) = e.tokens_out { - self.tokens_output_total.inc_by( - Labels::new().add("model", &e.model_id), - tokens_out, - ); + self.tokens_output_total + .inc_by(Labels::new().add("model", &e.model_id), tokens_out); } // Gauge: active requests down @@ -473,14 +466,15 @@ impl MetricsState { // Reconcile memory gauge against the absolute post-eviction value. // This corrects any cumulative drift caused by dropped ModelUnloaded // the authoritative value from the eviction subsystem. - self.memory_used_bytes.set(Labels::new(), e.memory_after_bytes as f64); - self.memory_budget_bytes.set(Labels::new(), e.budget_bytes as f64); + self.memory_used_bytes + .set(Labels::new(), e.memory_after_bytes as f64); + self.memory_budget_bytes + .set(Labels::new(), e.budget_bytes as f64); } EngineEvent::PreflightSignal(e) => { - self.preflight_predictions_total.inc( - Labels::new().add("source", e.source.to_string()), - ); + self.preflight_predictions_total + .inc(Labels::new().add("source", e.source.to_string())); } EngineEvent::PreflightHit(_) => { @@ -567,14 +561,20 @@ impl EventSender { /// Send a critical event asynchronously. If the channel is full, this will await /// and apply backpressure to the caller instead of dropping the event. /// Use for rare, critical events like `FailoverTriggered` and `EvictionTriggered`. - pub async fn send_critical(&self, event: EventEnvelope) -> Result<(), mpsc::error::SendError> { + pub async fn send_critical( + &self, + event: EventEnvelope, + ) -> Result<(), mpsc::error::SendError> { self.tx.send(event).await } /// Send a critical event synchronously. If the channel is full, this will block /// the current thread. Must not be called from within an async runtime context. #[allow(clippy::result_large_err)] - pub fn send_blocking(&self, event: EventEnvelope) -> Result<(), mpsc::error::SendError> { + pub fn send_blocking( + &self, + event: EventEnvelope, + ) -> Result<(), mpsc::error::SendError> { self.tx.blocking_send(event) } @@ -681,7 +681,7 @@ pub fn init_pipeline(_otlp_endpoint: Option<&str>) -> (Arc> let (sender, receiver) = create_event_channel(10000); let collector_loop = MetricsCollector::new(receiver); let state = collector_loop.state(); - + tokio::spawn(async move { collector_loop.run().await; }); @@ -1037,10 +1037,7 @@ mod tests { let history_labels = Labels::new().add("source", "history"); assert_eq!(state.preflight_predictions_total.values[&hint_labels], 2); - assert_eq!( - state.preflight_predictions_total.values[&history_labels], - 1 - ); + assert_eq!(state.preflight_predictions_total.values[&history_labels], 1); } // ── Token counting tests ───────────────────────────────────────────── @@ -1149,18 +1146,17 @@ mod tests { // Try to send critical event. It should block because the channel is full. // We use tokio::time::timeout to prove it blocks. - let critical_event = EventEnvelope::now(EngineEvent::FailoverTriggered( - FailoverTriggered { + let critical_event = + EventEnvelope::now(EngineEvent::FailoverTriggered(FailoverTriggered { failed_model: "failed".into(), failed_backend: "backend".into(), fallback_model: "fallback".into(), fallback_backend: "backend".into(), - }, - )); + })); let send_future = sender.send_critical(critical_event.clone()); let result = tokio::time::timeout(Duration::from_millis(50), send_future).await; - + // It should timeout because it's blocking assert!(result.is_err()); @@ -1224,7 +1220,10 @@ mod tests { // Override the last_seen for the stale label to be 20 minutes ago let twenty_mins_ago = Instant::now() - Duration::from_secs(1200); - state.requests_total.last_seen.insert(labels_stale.clone(), twenty_mins_ago); + state + .requests_total + .last_seen + .insert(labels_stale.clone(), twenty_mins_ago); // Evict labels older than 10 minutes state.evict_stale_labels(Duration::from_secs(600)); diff --git a/mofa-observability/src/events.rs b/mofa-observability/src/events.rs index 1541c74..1140ea6 100644 --- a/mofa-observability/src/events.rs +++ b/mofa-observability/src/events.rs @@ -6,7 +6,7 @@ //! Design principles: //! - Zero side effects: events are data, not actions. //! - Privacy: no prompt text, file contents, API keys, or user-identifying info. Ever. -//! - Bounded enums: capability, reason, source, status use enums, not free-form strings. +//! - Bounded enums: capability, unload reason, and signal source use enums (status is a bool). use serde::{Deserialize, Serialize}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -576,11 +576,25 @@ mod tests { #[test] fn test_privacy_contract_no_forbidden_fields() { let forbidden_fields = [ - "prompt", "text", "generated_text", "response_text", - "file_path", "file_content", "file_contents", - "api_key", "api_secret", "token", "auth_token", "access_token", - "password", "credential", "credentials", - "user_id", "username", "email", "ip_address", + "prompt", + "text", + "generated_text", + "response_text", + "file_path", + "file_content", + "file_contents", + "api_key", + "api_secret", + "token", + "auth_token", + "access_token", + "password", + "credential", + "credentials", + "user_id", + "username", + "email", + "ip_address", ]; let events = vec![ diff --git a/mofa-observability/src/lib.rs b/mofa-observability/src/lib.rs index 9105b77..7492d26 100644 --- a/mofa-observability/src/lib.rs +++ b/mofa-observability/src/lib.rs @@ -1,3 +1,3 @@ -pub mod events; pub mod collector; +pub mod events; pub mod prometheus; diff --git a/mofa-observability/src/prometheus.rs b/mofa-observability/src/prometheus.rs index b62c487..1fac81a 100644 --- a/mofa-observability/src/prometheus.rs +++ b/mofa-observability/src/prometheus.rs @@ -93,7 +93,14 @@ fn render_gauge(buf: &mut String, family: &GaugeFamily) { ) .unwrap(); } else { - writeln!(buf, "{}{} {}", family.name, format_labels(labels), format_float(*value)).unwrap(); + writeln!( + buf, + "{}{} {}", + family.name, + format_labels(labels), + format_float(*value) + ) + .unwrap(); } } buf.push('\n'); @@ -302,13 +309,25 @@ mod tests { // Must have TYPE histogram. assert!(output.contains("# TYPE mofa_model_load_seconds histogram")); // Must have bucket lines with le labels. - assert!(output.contains("mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"0.5\"} 0")); - assert!(output.contains("mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"2.0\"} 1")); + assert!(output.contains( + "mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"0.5\"} 0" + )); + assert!(output.contains( + "mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"2.0\"} 1" + )); // Must have +Inf bucket. - assert!(output.contains("mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"+Inf\"} 1")); + assert!(output.contains( + "mofa_model_load_seconds_bucket{backend=\"ollama\",model=\"qwen2.5:7b\",le=\"+Inf\"} 1" + )); // Must have sum and count. - assert!(output.contains("mofa_model_load_seconds_sum{backend=\"ollama\",model=\"qwen2.5:7b\"} 1.532")); - assert!(output.contains("mofa_model_load_seconds_count{backend=\"ollama\",model=\"qwen2.5:7b\"} 1")); + assert!(output.contains( + "mofa_model_load_seconds_sum{backend=\"ollama\",model=\"qwen2.5:7b\"} 1.532" + )); + assert!( + output.contains( + "mofa_model_load_seconds_count{backend=\"ollama\",model=\"qwen2.5:7b\"} 1" + ) + ); } #[test] @@ -497,10 +516,21 @@ mod tests { #[test] fn test_float_formatting_spec_compliance() { let mut state = MetricsState::new(); - state.memory_used_bytes.values.insert(Labels::new(), f64::INFINITY); + state + .memory_used_bytes + .values + .insert(Labels::new(), f64::INFINITY); state.active_requests.values.insert(Labels::new(), f64::NAN); let output = render(&state); - assert!(output.contains("mofa_memory_used_bytes +Inf"), "Failed to render +Inf: {}", output); - assert!(output.contains("mofa_active_requests NaN"), "Failed to render NaN: {}", output); + assert!( + output.contains("mofa_memory_used_bytes +Inf"), + "Failed to render +Inf: {}", + output + ); + assert!( + output.contains("mofa_active_requests NaN"), + "Failed to render NaN: {}", + output + ); } } From 547c22379d7ae3977a93a785b96bc1feb5339e3c Mon Sep 17 00:00:00 2001 From: ashum9 Date: Tue, 21 Jul 2026 21:27:28 +0530 Subject: [PATCH 24/26] implement vendor pricing engine and USD cost accumulator --- mofa-observability/src/collector.rs | 78 +++++++++++++++++++++++++++- mofa-observability/src/lib.rs | 1 + mofa-observability/src/pricing.rs | 55 ++++++++++++++++++++ mofa-observability/src/prometheus.rs | 2 + 4 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 mofa-observability/src/pricing.rs diff --git a/mofa-observability/src/collector.rs b/mofa-observability/src/collector.rs index fc8917e..e8c49cd 100644 --- a/mofa-observability/src/collector.rs +++ b/mofa-observability/src/collector.rs @@ -241,6 +241,7 @@ pub struct MetricsState { pub preflight_misses_total: CounterFamily, pub tokens_input_total: CounterFamily, pub tokens_output_total: CounterFamily, + pub thought_tokens_total: CounterFamily, pub events_dropped_total: CounterFamily, // ── Histograms ─────────────────────────────────────────────────────── @@ -253,6 +254,7 @@ pub struct MetricsState { pub memory_budget_bytes: GaugeFamily, pub models_loaded: GaugeFamily, pub active_requests: GaugeFamily, + pub estimated_cost_usd: GaugeFamily, } impl MetricsState { @@ -295,6 +297,14 @@ impl MetricsState { "mofa_tokens_output_total", "Total output tokens generated", ), + thought_tokens_total: CounterFamily::new( + "mofa_thought_tokens_total", + "Total reasoning thought tokens generated", + ), + estimated_cost_usd: GaugeFamily::new( + "mofa_estimated_cost_usd", + "Total estimated USD cost incurred", + ), events_dropped_total: CounterFamily::new( "mofa_events_dropped_total", "Total events dropped due to channel backpressure", @@ -396,7 +406,7 @@ impl MetricsState { ); } - // Counter: tokens + // Counter: tokens & cost if let Some(tokens_in) = e.tokens_in { self.tokens_input_total .inc_by(Labels::new().add("model", &e.model_id), tokens_in); @@ -405,6 +415,22 @@ impl MetricsState { self.tokens_output_total .inc_by(Labels::new().add("model", &e.model_id), tokens_out); } + if let (Some(tokens_in), Some(tokens_out)) = (e.tokens_in, e.tokens_out) { + let cost = crate::pricing::estimate_cost_usd(&e.backend, &e.model_id, tokens_in as u32, tokens_out as u32); + let backend_lower = e.backend.to_lowercase(); + let locality = if backend_lower == "ollama" || backend_lower == "kokoro" || backend_lower == "funasr" || backend_lower == "local" { + "local" + } else { + "cloud" + }; + self.estimated_cost_usd.add( + Labels::new() + .add("provider", &e.backend) + .add("locality", locality) + .add("model", &e.model_id), + cost, + ); + } // Gauge: active requests down self.active_requests.dec(Labels::new()); @@ -1234,4 +1260,54 @@ mod tests { assert!(state.requests_total.last_seen.contains_key(&labels_fresh)); assert!(!state.requests_total.last_seen.contains_key(&labels_stale)); } + + #[test] + fn test_cost_accumulation_local_vs_cloud() { + let mut state = make_state(); + + // Local request (Ollama) -> $0.00 + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "gemma3:4b".into(), + backend: "ollama".into(), + capability: Capability::Chat, + duration_ms: 1000, + ttft_ms: None, + tokens_in: Some(1000), + tokens_out: Some(1000), + model_was_hot: Some(true), + success: true, + error_code: None, + }, + ))); + + // Cloud request (OpenAI gpt-4o) -> $0.0125 + state.process_event(&EventEnvelope::now(EngineEvent::RequestCompleted( + RequestCompleted { + model_id: "gpt-4o".into(), + backend: "openai".into(), + capability: Capability::Chat, + duration_ms: 500, + ttft_ms: None, + tokens_in: Some(1000), + tokens_out: Some(1000), + model_was_hot: Some(true), + success: true, + error_code: None, + }, + ))); + + let local_labels = Labels::new() + .add("provider", "ollama") + .add("locality", "local") + .add("model", "gemma3:4b"); + let cloud_labels = Labels::new() + .add("provider", "openai") + .add("locality", "cloud") + .add("model", "gpt-4o"); + + assert_eq!(*state.estimated_cost_usd.values.get(&local_labels).unwrap(), 0.0); + let cloud_cost = *state.estimated_cost_usd.values.get(&cloud_labels).unwrap(); + assert!((cloud_cost - 0.0125).abs() < 1e-5); + } } diff --git a/mofa-observability/src/lib.rs b/mofa-observability/src/lib.rs index 7492d26..2ad1984 100644 --- a/mofa-observability/src/lib.rs +++ b/mofa-observability/src/lib.rs @@ -1,3 +1,4 @@ pub mod collector; pub mod events; +pub mod pricing; pub mod prometheus; diff --git a/mofa-observability/src/pricing.rs b/mofa-observability/src/pricing.rs new file mode 100644 index 0000000..65daab6 --- /dev/null +++ b/mofa-observability/src/pricing.rs @@ -0,0 +1,55 @@ +//! # Vendor Pricing Matrix & Cost Engine +//! +//! Calculates estimated USD costs for cloud vendor API calls based on prompt and completion token counts. +//! Local execution providers (e.g., `ollama`, `kokoro`, `funasr`) yield $0.00 by default. + +/// Calculates estimated USD cost for a given provider, model, prompt tokens, and completion tokens. +pub fn estimate_cost_usd(provider: &str, model: &str, prompt_tokens: u32, completion_tokens: u32) -> f64 { + let provider_lower = provider.to_lowercase(); + let model_lower = model.to_lowercase(); + + // Local providers are free ($0.00) + if provider_lower == "ollama" || provider_lower == "kokoro" || provider_lower == "funasr" || provider_lower == "local" { + return 0.0; + } + + // Rates per 1,000 tokens: (prompt_rate_per_k, completion_rate_per_k) + let (prompt_rate, completion_rate) = match (provider_lower.as_str(), model_lower.as_str()) { + ("openai", m) if m.contains("gpt-4o") => (0.0025, 0.0100), + ("openai", m) if m.contains("gpt-4") => (0.0300, 0.0600), + ("openai", m) if m.contains("gpt-3.5") => (0.0005, 0.0015), + (p, m) if p == "deepseek" || m.contains("deepseek") => (0.00055, 0.00219), + (p, m) if p == "anthropic" || m.contains("claude") => (0.0030, 0.0150), + (p, m) if p == "dashscope" || m.contains("qwen") => (0.0028, 0.0084), + // Generic cloud provider fallback rates + _ => (0.0020, 0.0060), + }; + + let prompt_cost = (prompt_tokens as f64 / 1000.0) * prompt_rate; + let completion_cost = (completion_tokens as f64 / 1000.0) * completion_rate; + + prompt_cost + completion_cost +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_provider_is_free() { + assert_eq!(estimate_cost_usd("ollama", "llama3:8b", 1000, 1000), 0.0); + assert_eq!(estimate_cost_usd("kokoro", "kokoro", 500, 500), 0.0); + } + + #[test] + fn gpt4o_cost_calculation() { + let cost = estimate_cost_usd("openai", "gpt-4o", 1000, 1000); + assert!((cost - 0.0125).abs() < 1e-5); + } + + #[test] + fn deepseek_cost_calculation() { + let cost = estimate_cost_usd("deepseek", "deepseek-r1", 1000, 1000); + assert!((cost - 0.00274).abs() < 1e-5); + } +} diff --git a/mofa-observability/src/prometheus.rs b/mofa-observability/src/prometheus.rs index 1fac81a..fe52372 100644 --- a/mofa-observability/src/prometheus.rs +++ b/mofa-observability/src/prometheus.rs @@ -215,6 +215,7 @@ pub fn render(state: &MetricsState) -> String { render_counter(&mut buf, &state.preflight_misses_total); render_counter(&mut buf, &state.tokens_input_total); render_counter(&mut buf, &state.tokens_output_total); + render_counter(&mut buf, &state.thought_tokens_total); render_counter(&mut buf, &state.events_dropped_total); // ── Histograms ─────────────────────────────────────────────────────── @@ -227,6 +228,7 @@ pub fn render(state: &MetricsState) -> String { render_gauge(&mut buf, &state.memory_budget_bytes); render_gauge(&mut buf, &state.models_loaded); render_gauge(&mut buf, &state.active_requests); + render_gauge(&mut buf, &state.estimated_cost_usd); buf } From 32954e5a16781b983b372e1297423fd578ea3a9d Mon Sep 17 00:00:00 2001 From: ashum9 Date: Tue, 21 Jul 2026 21:28:00 +0530 Subject: [PATCH 25/26] add DualTrackView side-by-side telemetry comparison panel --- .../src/observability/DualTrackView.tsx | 224 ++++++++++++++++++ .../src/observability/ObservabilityView.tsx | 180 ++++++++++++++ 2 files changed, 404 insertions(+) create mode 100644 mofa-frontend/src/observability/DualTrackView.tsx create mode 100644 mofa-frontend/src/observability/ObservabilityView.tsx diff --git a/mofa-frontend/src/observability/DualTrackView.tsx b/mofa-frontend/src/observability/DualTrackView.tsx new file mode 100644 index 0000000..6c8d159 --- /dev/null +++ b/mofa-frontend/src/observability/DualTrackView.tsx @@ -0,0 +1,224 @@ +import React, { useEffect, useState } from 'react'; +import { motion } from 'framer-motion'; +import { Card } from '../shared/Card'; +import { Badge } from '../shared/Badge'; +import { engine } from '../engine'; +import { EngineStatus } from '../engine/types'; +import { HardDrive, DollarSign, Zap, AlertTriangle, ShieldCheck, Flame, Cpu, ArrowUpRight } from 'lucide-react'; + +export function DualTrackView() { + const [status, setStatus] = useState(null); + const [localRequests, setLocalRequests] = useState(0); + const [cloudRequests, setCloudRequests] = useState(0); + const [cloudCostUsd, setCloudCostUsd] = useState(0); + const [thoughtTokens, setThoughtTokens] = useState(0); + const [quotaErrors, setQuotaErrors] = useState(0); + const [warmupHits, setWarmupHits] = useState(0); + const [evictions, setEvictions] = useState(0); + + useEffect(() => { + engine.getStatus().then(res => { + if (res.success) setStatus(res.data); + }); + + const handleEvent = (evt: any) => { + const type = evt.type; + const data = evt.data || {}; + + if (type === 'RequestCompleted') { + if (data.provider === 'ollama' || data.provider === 'kokoro' || data.provider === 'funasr' || data.locality === 'local') { + setLocalRequests(prev => prev + 1); + } else { + setCloudRequests(prev => prev + 1); + if (data.cost_usd) { + setCloudCostUsd(prev => prev + data.cost_usd); + } + } + } else if (type === 'PreflightWarmCompleted') { + if (data.success) setWarmupHits(prev => prev + 1); + } else if (type === 'ModelEvicted') { + setEvictions(prev => prev + 1); + } else if (type === 'MemoryChanged' || type === 'ModelStatusChanged' || type === 'ModelResidencyChanged') { + engine.getStatus().then(res => { + if (res.success) setStatus(res.data); + }); + } + }; + + const unsubscribe = engine.subscribeEvents(handleEvent); + return () => unsubscribe(); + }, []); + + const memUsedGb = status ? (status.memory_used_bytes / (1024 * 1024 * 1024)).toFixed(2) : '3.11'; + const memBudgetGb = status ? (status.memory_budget_bytes / (1024 * 1024 * 1024)).toFixed(1) : '8.0'; + const memPercent = status && status.memory_budget_bytes > 0 + ? Math.min(100, Math.round((status.memory_used_bytes / status.memory_budget_bytes) * 100)) + : 38.9; + + return ( +
+ {/* Header Banner */} +
+
+
+ +
+
+

Dual-Track Telemetry Moat

+

+ Real-time side-by-side comparison of local hardware performance vs cloud financial cost accumulation. +

+
+
+
+ Local-First Priority Active +
+
+ + {/* Side-by-Side Grid */} +
+ + {/* Left Column: Local Execution Track */} + + +
+
+ + Local Hardware Track (locality="local") +
+ 0.00 USD / Free +
+ + {/* VRAM Memory Gauge */} +
+
+ VRAM / RAM Footprint + {memUsedGb} / {memBudgetGb} GB ({memPercent}%) +
+
+
+
+
+ + {/* Sub-Metrics Grid */} +
+
+
+ + Preflight Warmup Hits +
+
{warmupHits}
+
0ms Cold Start Latency
+
+ +
+
+ + Local Inferences +
+
{localRequests}
+
Ollama + Kokoro TTS
+
+ +
+
+ + LRU Memory Evictions +
+
{evictions}
+
Evicted under budget pressure
+
+ +
+
+ + Privacy Classification +
+
Confidential Only
+
Zero Data Egress
+
+
+ + + + {/* Right Column: Cloud Financial Track */} + + +
+
+ + Cloud Financial Track (locality="cloud") +
+ Vendor Billing Active +
+ + {/* Financial Spend Gauge */} +
+
+ Accumulated Compute Cost (USD) + ${cloudCostUsd.toFixed(4)} +
+
+
+
+
+ + {/* Sub-Metrics Grid */} +
+
+
+ + Cloud Inferences +
+
{cloudRequests}
+
OpenAI / DeepSeek / Claude
+
+ +
+
+ + Thought Tokens +
+
{thoughtTokens}
+
DeepSeek R1 Reasoning
+
+ +
+
+ + HTTP 429 Rate Limits +
+
{quotaErrors}
+
Vendor quota errors
+
+ +
+
+ + Cost Savings Rate +
+
100% Local Free Tier
+
Saved vs pure cloud
+
+
+ + + +
+
+ ); +} diff --git a/mofa-frontend/src/observability/ObservabilityView.tsx b/mofa-frontend/src/observability/ObservabilityView.tsx new file mode 100644 index 0000000..d0d9826 --- /dev/null +++ b/mofa-frontend/src/observability/ObservabilityView.tsx @@ -0,0 +1,180 @@ +import React, { useEffect, useState } from 'react'; +import { motion } from 'framer-motion'; +import { Card } from '../shared/Card'; +import { Button } from '../shared/Button'; +import { MetricsStrip } from './MetricsStrip'; +import { DualTrackView } from './DualTrackView'; +import { Activity, LayoutDashboard, Cpu, Network, ExternalLink, Layers } from 'lucide-react'; + +const GRAFANA_URL = import.meta.env.VITE_GRAFANA_URL || 'http://localhost:3000'; + +type TabId = 'dual-track' | 'overview' | 'memory' | 'routing'; + +interface TabConfig { + id: TabId; + label: string; + icon: React.ReactNode; + dashboardPath: string; + description: string; +} + +const TABS: TabConfig[] = [ + { + id: 'dual-track', + label: 'Dual-Track Telemetry', + icon: , + dashboardPath: '', + description: 'Real-time side-by-side comparison of local hardware footprint vs cloud financial cost.' + }, + { + id: 'overview', + label: 'Engine Overview', + icon: , + dashboardPath: '/d/engine-overview/mofa-engine-overview', + description: 'Request rate, error rate, P95 latency, TTFT, and tokens/sec.' + }, + { + id: 'memory', + label: 'Memory & Lifecycle', + icon: , + dashboardPath: '/d/engine-memory/mofa-memory-and-lifecycle', + description: 'Memory vs budget, model loads/unloads, eviction rate, and cold-load heatmap.' + }, + { + id: 'routing', + label: 'Preflight & Routing', + icon: , + dashboardPath: '/d/engine-routing/mofa-preflight-and-routing', + description: 'Fallback routing, circuit breakers, and capability matching.' + } +]; + +export function ObservabilityView() { + const [activeTab, setActiveTab] = useState('dual-track'); + const [grafanaAvailable, setGrafanaAvailable] = useState(null); + + useEffect(() => { + let mounted = true; + + // Lightweight check for Grafana availability using a known public static asset + const checkGrafana = () => { + const img = new Image(); + img.onload = () => { + if (mounted) setGrafanaAvailable(true); + }; + img.onerror = () => { + if (mounted) setGrafanaAvailable(false); + }; + // Prevent caching + img.src = `${GRAFANA_URL}/public/img/grafana_icon.svg?t=${Date.now()}`; + }; + + checkGrafana(); + + return () => { mounted = false; }; + }, []); + + const activeConfig = TABS.find(t => t.id === activeTab)!; + + return ( + +
+ {/* Header */} +
+
+

+ + Engine Observability +

+

+ Powered by Prometheus + Grafana & Local Telemetry Engine +

+
+ {grafanaAvailable && ( + + )} +
+ + + + {/* Tab Navigation */} +
+ {TABS.map(tab => ( + + ))} +
+ +
+ {activeTab === 'dual-track' ? ( + + ) : grafanaAvailable === null ? ( +
+
Checking observability stack...
+
+ ) : grafanaAvailable ? ( + +
+
{activeConfig.label}
+
{activeConfig.description}
+
+
+