From 0e75e13185c7630bf5fcd33957379fdf6ee2a97d Mon Sep 17 00:00:00 2001 From: ashum9 Date: Tue, 21 Jul 2026 20:57:45 +0530 Subject: [PATCH] add empirically verified event, request, and metrics specifications --- docs/contracts/engine_event_schema.json | 135 ++++++++++++++++++ docs/contracts/inference_request_schema.json | 99 +++++++++++++ docs/contracts/metrics_spec.md | 138 +++++++++++++++++++ 3 files changed, 372 insertions(+) create mode 100644 docs/contracts/engine_event_schema.json create mode 100644 docs/contracts/inference_request_schema.json create mode 100644 docs/contracts/metrics_spec.md diff --git a/docs/contracts/engine_event_schema.json b/docs/contracts/engine_event_schema.json new file mode 100644 index 0000000..e1cd172 --- /dev/null +++ b/docs/contracts/engine_event_schema.json @@ -0,0 +1,135 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "EngineEvent", + "description": "Exhaustive JSON Schema for all 10 EngineEvent variants in MoFA Engine, verified against origin/engine mofa-kernel/src/types.rs.", + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string", + "enum": [ + "model_status_changed", + "model_residency_changed", + "request_started", + "request_completed", + "memory_changed", + "model_evicted", + "preflight_warm_started", + "preflight_warm_completed", + "provider_health_changed", + "discovery_completed" + ] + }, + "timestamp": { + "type": "integer", + "description": "Unix timestamp in milliseconds when event was emitted." + } + }, + "oneOf": [ + { + "title": "ModelStatusChanged", + "description": "Emitted when model status transitions (e.g. Cold -> Hot)", + "properties": { + "type": { "const": "model_status_changed" }, + "model_id": { "type": "string", "example": "ollama/gemma3:4b" }, + "old": { "type": "string", "enum": ["cold", "warming", "hot", "busy", "failed"] }, + "new": { "type": "string", "enum": ["cold", "warming", "hot", "busy", "failed"] } + }, + "required": ["type", "model_id", "old", "new"] + }, + { + "title": "ModelResidencyChanged", + "description": "Emitted when model residency changes (e.g. Unloaded -> Loaded)", + "properties": { + "type": { "const": "model_residency_changed" }, + "model_id": { "type": "string", "example": "ollama/gemma3:4b" }, + "old": { "type": "string", "enum": ["unknown", "unloaded", "loading", "loaded", "unloading", "remote"] }, + "new": { "type": "string", "enum": ["unknown", "unloaded", "loading", "loaded", "unloading", "remote"] } + }, + "required": ["type", "model_id", "old", "new"] + }, + { + "title": "RequestStarted", + "description": "Emitted when inference request processing starts", + "properties": { + "type": { "const": "request_started" }, + "request_id": { "type": "string", "example": "req-9842a1" }, + "capability": { "type": ["string", "null"], "enum": ["chat", "tts", "asr", "image_gen", "video_gen", "vlm", "embedding", null] }, + "model_id": { "type": "string", "example": "ollama/gemma3:4b" } + }, + "required": ["type", "request_id", "model_id"] + }, + { + "title": "RequestCompleted", + "description": "Emitted when inference request finishes successfully or with error", + "properties": { + "type": { "const": "request_completed" }, + "request_id": { "type": "string", "example": "req-9842a1" }, + "duration_ms": { "type": "integer", "example": 1850 }, + "success": { "type": "boolean", "example": true } + }, + "required": ["type", "request_id", "duration_ms", "success"] + }, + { + "title": "MemoryChanged", + "description": "Emitted when overall system RAM/VRAM footprint allocation changes", + "properties": { + "type": { "const": "memory_changed" }, + "used_bytes": { "type": "integer", "example": 3339000000 }, + "total_bytes": { "type": "integer", "example": 8589934592 } + }, + "required": ["type", "used_bytes", "total_bytes"] + }, + { + "title": "ModelEvicted", + "description": "Emitted when LRU memory pressure forces a model eviction from VRAM", + "properties": { + "type": { "const": "model_evicted" }, + "model_id": { "type": "string", "example": "ollama/llama3.2:1b" }, + "reason": { "type": "string", "example": "memory_pressure" } + }, + "required": ["type", "model_id", "reason"] + }, + { + "title": "PreflightWarmStarted", + "description": "Emitted when preflight predictor initiates predictive model warming", + "properties": { + "type": { "const": "preflight_warm_started" }, + "model_id": { "type": "string", "example": "kokoro/kokoro" }, + "source": { "type": "string", "enum": ["hint", "subscription", "history"] } + }, + "required": ["type", "model_id", "source"] + }, + { + "title": "PreflightWarmCompleted", + "description": "Emitted when predictive warming finishes loading the model", + "properties": { + "type": { "const": "preflight_warm_completed" }, + "model_id": { "type": "string", "example": "kokoro/kokoro" }, + "success": { "type": "boolean", "example": true } + }, + "required": ["type", "model_id", "success"] + }, + { + "title": "ProviderHealthChanged", + "description": "Emitted when provider health check or circuit breaker state changes", + "properties": { + "type": { "const": "provider_health_changed" }, + "provider": { "type": "string", "example": "ollama" }, + "health": { "type": "string", "enum": ["healthy", "degraded", "unavailable", "unknown"] } + }, + "required": ["type", "provider", "health"] + }, + { + "title": "DiscoveryCompleted", + "description": "Emitted when provider model discovery task finishes", + "properties": { + "type": { "const": "discovery_completed" }, + "provider": { "type": "string", "example": "ollama" }, + "models": { "type": "integer", "example": 2 }, + "success": { "type": "boolean", "example": true } + }, + "required": ["type", "provider", "models", "success"] + } + ] +} diff --git a/docs/contracts/inference_request_schema.json b/docs/contracts/inference_request_schema.json new file mode 100644 index 0000000..ca31100 --- /dev/null +++ b/docs/contracts/inference_request_schema.json @@ -0,0 +1,99 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "InferenceRequest", + "description": "Exhaustive JSON Schema for InferenceRequest payload contract verified against origin/engine mofa-kernel/src/types.rs.", + "type": "object", + "required": ["messages"], + "properties": { + "capability": { + "type": ["string", "null"], + "enum": ["chat", "tts", "asr", "image_gen", "video_gen", "vlm", "embedding", null], + "description": "Target capability to invoke. Null defaults to 'chat'." + }, + "model": { + "type": ["string", "null"], + "description": "Canonical 'provider/model' ID or unique short model name. Null triggers deterministic auto-routing." + }, + "app_id": { + "type": ["string", "null"], + "description": "Application scope identifier (e.g. 'mofa-fm') used by Markov preflight predictor." + }, + "session_id": { + "type": ["string", "null"], + "description": "Session correlation identifier for multi-turn inference tracking." + }, + "fallback_policy": { + "type": "string", + "enum": ["capability_only", "disabled", "allow_named"], + "default": "capability_only", + "description": "Fallback routing behavior strategy. 'capability_only' stays within target capability; 'disabled' prevents fallback." + }, + "locality": { + "type": "string", + "enum": ["auto", "prefer_local", "local_only"], + "default": "auto", + "description": "Routing locality guardrail. 'local_only' restricts to local hardware for privacy; 'prefer_local' ranks local ahead of cloud." + }, + "prefer": { + "type": ["string", "null"], + "enum": ["local", "cloud", null], + "description": "Legacy hard routing preference flag." + }, + "data_class": { + "type": ["string", "null"], + "enum": ["confidential", "public", null], + "description": "Data privacy classification tier. 'confidential' strictly blocks cloud fallback." + }, + "reasoning": { + "type": ["object", "null"], + "description": "Deep reasoning configuration for models like DeepSeek R1.", + "properties": { + "effort": { + "type": "string", + "enum": ["low", "medium", "high"], + "default": "medium" + }, + "include": { + "type": "boolean", + "default": true, + "description": "Whether to return reasoning thought tokens in response." + } + } + }, + "messages": { + "type": "array", + "description": "Input messages sequence.", + "items": { + "type": "object", + "required": ["role", "content"], + "properties": { + "role": { + "type": "string", + "enum": ["system", "user", "assistant"], + "example": "user" + }, + "content": { + "type": "string", + "example": "Write a 30-second podcast script about quantum computing." + } + } + } + }, + "input_file": { + "type": ["string", "null"], + "description": "Absolute path to input audio/video/image file on engine host (for ASR, VLM, document processing)." + }, + "params": { + "type": ["object", "null"], + "description": "Extra provider pass-through parameters (e.g. voice, speed, temperature, max_tokens)." + }, + "hint_next": { + "type": ["string", "null"], + "description": "Next anticipated capability (e.g. 'tts') for cross-capability preflight warming." + }, + "request_id": { + "type": "string", + "description": "Unique correlation request ID. Auto-generated by engine if omitted." + } + } +} diff --git a/docs/contracts/metrics_spec.md b/docs/contracts/metrics_spec.md new file mode 100644 index 0000000..abbfd95 --- /dev/null +++ b/docs/contracts/metrics_spec.md @@ -0,0 +1,138 @@ +# Metric Naming & Tagging Taxonomy Specification + +**Document Version:** v1.1.0-EMPIRICALLY-VERIFIED +**Path:** `docs/contracts/metrics_spec.md` +**Mandatory Requirement:** Every metric emitted by `mofa-observability` MUST carry `provider=""` and `locality="local" | "cloud"` labels. + +--- + +## 1. Metric Naming Rules & Label Standards + +1. **Namespace Prefix**: All metrics MUST begin with `mofa_`. +2. **Mandatory Labels**: + * `provider`: String identifier of the serving backend (e.g., `ollama`, `kokoro`, `openai`, `deepseek`). + * `locality`: Enum string `local` or `cloud`. +3. **Dual-Track Moat Principle**: Local execution metrics (RAM/VRAM memory footprint, cold start durations, warmup hits) and Cloud metrics (USD cost, token count, 429 errors) MUST be recorded in distinct metric series allowing direct side-by-side comparison in Grafana. + +--- + +## 2. Complete 18-Metric Family Dictionary + +### 2.1 Financial & Token Consumption Metrics + +#### 1. `mofa_estimated_cost_usd` +* **Type**: Counter +* **Description**: Accumulated USD financial compute cost. +* **Labels**: `provider`, `locality`, `model` +* **Prometheus Exposition Example**: + ```text + # HELP mofa_estimated_cost_usd Total estimated USD cost incurred. + # TYPE mofa_estimated_cost_usd counter + mofa_estimated_cost_usd{provider="openai",locality="cloud",model="gpt-4o"} 0.042 + mofa_estimated_cost_usd{provider="ollama",locality="local",model="gemma3:4b"} 0.000 + ``` + +#### 2. `mofa_tokens_total` +* **Type**: Counter +* **Description**: Total tokens processed by engine. +* **Labels**: `provider`, `locality`, `model`, `type="prompt" | "completion"` +* **PromQL Query Example**: + ```promql + sum(rate(mofa_tokens_total[5m])) by (provider, locality, type) + ``` + +#### 3. `mofa_thought_tokens_total` +* **Type**: Counter +* **Description**: Deep reasoning thought tokens generated by reasoning models (e.g. DeepSeek R1). +* **Labels**: `provider`, `locality`, `model` + +--- + +### 2.2 Local Execution & Memory Lifecycle Metrics + +#### 4. `mofa_cold_start_seconds` +* **Type**: Histogram +* **Description**: Measures time elapsed from model load trigger to first ready inference. +* **Labels**: `provider`, `locality`, `model` +* **Buckets**: `[0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0]` + +#### 5. `mofa_warmup_hits_total` +* **Type**: Counter +* **Description**: Preflight model warming hit counters. +* **Labels**: `provider`, `locality`, `source="hint" | "subscription" | "markov"` + +#### 6. `mofa_memory_usage_bytes` +* **Type**: Gauge +* **Description**: Host RAM and GPU VRAM footprint allocated. +* **Labels**: `provider`, `locality`, `type="ram" | "vram"`, `status="reserved" | "observed"` +* **Prometheus Exposition Example**: + ```text + # HELP mofa_memory_usage_bytes Current memory usage in bytes. + # TYPE mofa_memory_usage_bytes gauge + mofa_memory_usage_bytes{provider="ollama",locality="local",type="vram",status="observed"} 3339000000 + ``` + +#### 7. `mofa_model_residency_status` +* **Type**: Gauge +* **Description**: Runtime state of model in memory (`0=unloaded`, `1=loading`, `2=loaded`, `3=remote`). +* **Labels**: `provider`, `locality`, `model` + +#### 8. `mofa_idle_evictions_total` +* **Type**: Counter +* **Description**: Count of LRU model evictions from VRAM due to memory budget limits. +* **Labels**: `provider`, `locality`, `model` + +--- + +### 2.3 System Health, Request Throughput & Observability + +#### 9. `mofa_requests_total` +* **Type**: Counter +* **Description**: Cumulative requests processed by engine. +* **Labels**: `provider`, `locality`, `capability`, `status="200" | "400" | "404" | "422" | "500"` + +#### 10. `mofa_request_duration_seconds` +* **Type**: Histogram +* **Description**: End-to-end request latency distribution. +* **Labels**: `provider`, `locality`, `capability` +* **Buckets**: `[0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]` + +#### 11. `mofa_provider_health` +* **Type**: Gauge +* **Description**: Binary health status indicator (`1` for healthy, `0` for unhealthy). +* **Labels**: `provider`, `locality` + +#### 12. `mofa_circuit_breaker_state` +* **Type**: Gauge +* **Description**: Circuit breaker status (`0=closed`, `1=open`, `2=half_open`). +* **Labels**: `provider` + +#### 13. `mofa_active_connections` +* **Type**: Gauge +* **Description**: Active HTTP REST and SSE client streaming connection count. +* **Labels**: `protocol="rest" | "sse"` + +#### 14. `mofa_preflight_predictions_total` +* **Type**: Counter +* **Description**: Markov preflight prediction evaluation outcomes. +* **Labels**: `scope_id`, `result="approved" | "rejected"` + +#### 15. `mofa_cloud_quota_errors_total` +* **Type**: Counter +* **Description**: Count of HTTP 429 rate limit / quota exhaustion errors from cloud providers. +* **Labels**: `provider` + +#### 16. `mofa_sse_events_emitted_total` +* **Type**: Counter +* **Description**: Total SSE telemetry events emitted to client dashboards. +* **Labels**: `event_type` + +#### 17. `mofa_label_evictions_total` +* **Type**: Counter +* **Description**: Total high-cardinality metric tags garbage-collected by telemetry engine. +* **Labels**: `collector` + +#### 18. `mofa_quality_gate_evaluations_total` +* **Type**: Counter +* **Description**: VLM and ffprobe video quality evaluation outcomes. +* **Labels**: `pipeline`, `result="pass" | "fail"`