mofa-org · ashum9 · Jul 22, 2026 · Jul 21, 2026
diff --git a/docs/contracts/engine_event_schema.json b/docs/contracts/engine_event_schema.json
@@ -0,0 +1,135 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "EngineEvent",
+  "description": "Exhaustive JSON Schema for all 10 EngineEvent variants in MoFA Engine, verified against origin/engine mofa-kernel/src/types.rs.",
+  "type": "object",
+  "required": ["type"],
+  "properties": {
+    "type": {
+      "type": "string",
+      "enum": [
+        "model_status_changed",
+        "model_residency_changed",
+        "request_started",
+        "request_completed",
+        "memory_changed",
+        "model_evicted",
+        "preflight_warm_started",
+        "preflight_warm_completed",
+        "provider_health_changed",
+        "discovery_completed"
+      ]
+    },
+    "timestamp": {
+      "type": "integer",
+      "description": "Unix timestamp in milliseconds when event was emitted."
+    }
+  },
+  "oneOf": [
+    {
+      "title": "ModelStatusChanged",
+      "description": "Emitted when model status transitions (e.g. Cold -> Hot)",
+      "properties": {
+        "type": { "const": "model_status_changed" },
+        "model_id": { "type": "string", "example": "ollama/gemma3:4b" },
+        "old": { "type": "string", "enum": ["cold", "warming", "hot", "busy", "failed"] },
+        "new": { "type": "string", "enum": ["cold", "warming", "hot", "busy", "failed"] }
+      },
+      "required": ["type", "model_id", "old", "new"]
+    },
+    {
+      "title": "ModelResidencyChanged",
+      "description": "Emitted when model residency changes (e.g. Unloaded -> Loaded)",
+      "properties": {
+        "type": { "const": "model_residency_changed" },
+        "model_id": { "type": "string", "example": "ollama/gemma3:4b" },
+        "old": { "type": "string", "enum": ["unknown", "unloaded", "loading", "loaded", "unloading", "remote"] },
+        "new": { "type": "string", "enum": ["unknown", "unloaded", "loading", "loaded", "unloading", "remote"] }
+      },
+      "required": ["type", "model_id", "old", "new"]
+    },
+    {
+      "title": "RequestStarted",
+      "description": "Emitted when inference request processing starts",
+      "properties": {
+        "type": { "const": "request_started" },
+        "request_id": { "type": "string", "example": "req-9842a1" },
+        "capability": { "type": ["string", "null"], "enum": ["chat", "tts", "asr", "image_gen", "video_gen", "vlm", "embedding", null] },
+        "model_id": { "type": "string", "example": "ollama/gemma3:4b" }
+      },
+      "required": ["type", "request_id", "model_id"]
+    },
+    {
+      "title": "RequestCompleted",
+      "description": "Emitted when inference request finishes successfully or with error",
+      "properties": {
+        "type": { "const": "request_completed" },
+        "request_id": { "type": "string", "example": "req-9842a1" },
+        "duration_ms": { "type": "integer", "example": 1850 },
+        "success": { "type": "boolean", "example": true }
+      },
+      "required": ["type", "request_id", "duration_ms", "success"]
+    },
+    {
+      "title": "MemoryChanged",
+      "description": "Emitted when overall system RAM/VRAM footprint allocation changes",
+      "properties": {
+        "type": { "const": "memory_changed" },
+        "used_bytes": { "type": "integer", "example": 3339000000 },
+        "total_bytes": { "type": "integer", "example": 8589934592 }
+      },
+      "required": ["type", "used_bytes", "total_bytes"]
+    },
+    {
+      "title": "ModelEvicted",
+      "description": "Emitted when LRU memory pressure forces a model eviction from VRAM",
+      "properties": {
+        "type": { "const": "model_evicted" },
+        "model_id": { "type": "string", "example": "ollama/llama3.2:1b" },
+        "reason": { "type": "string", "example": "memory_pressure" }
+      },
+      "required": ["type", "model_id", "reason"]
+    },
+    {
+      "title": "PreflightWarmStarted",
+      "description": "Emitted when preflight predictor initiates predictive model warming",
+      "properties": {
+        "type": { "const": "preflight_warm_started" },
+        "model_id": { "type": "string", "example": "kokoro/kokoro" },
+        "source": { "type": "string", "enum": ["hint", "subscription", "history"] }
+      },
+      "required": ["type", "model_id", "source"]
+    },
+    {
+      "title": "PreflightWarmCompleted",
+      "description": "Emitted when predictive warming finishes loading the model",
+      "properties": {
+        "type": { "const": "preflight_warm_completed" },
+        "model_id": { "type": "string", "example": "kokoro/kokoro" },
+        "success": { "type": "boolean", "example": true }
+      },
+      "required": ["type", "model_id", "success"]
+    },
+    {
+      "title": "ProviderHealthChanged",
+      "description": "Emitted when provider health check or circuit breaker state changes",
+      "properties": {
+        "type": { "const": "provider_health_changed" },
+        "provider": { "type": "string", "example": "ollama" },
+        "health": { "type": "string", "enum": ["healthy", "degraded", "unavailable", "unknown"] }
+      },
+      "required": ["type", "provider", "health"]
+    },
+    {
+      "title": "DiscoveryCompleted",
+      "description": "Emitted when provider model discovery task finishes",
+      "properties": {
+        "type": { "const": "discovery_completed" },
+        "provider": { "type": "string", "example": "ollama" },
+        "models": { "type": "integer", "example": 2 },
+        "success": { "type": "boolean", "example": true }
+      },
+      "required": ["type", "provider", "models", "success"]
+    }
+  ]
+}
diff --git a/docs/contracts/inference_request_schema.json b/docs/contracts/inference_request_schema.json
@@ -0,0 +1,99 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "InferenceRequest",
+  "description": "Exhaustive JSON Schema for InferenceRequest payload contract verified against origin/engine mofa-kernel/src/types.rs.",
+  "type": "object",
+  "required": ["messages"],
+  "properties": {
+    "capability": {
+      "type": ["string", "null"],
+      "enum": ["chat", "tts", "asr", "image_gen", "video_gen", "vlm", "embedding", null],
+      "description": "Target capability to invoke. Null defaults to 'chat'."
+    },
+    "model": {
+      "type": ["string", "null"],
+      "description": "Canonical 'provider/model' ID or unique short model name. Null triggers deterministic auto-routing."
+    },
+    "app_id": {
+      "type": ["string", "null"],
+      "description": "Application scope identifier (e.g. 'mofa-fm') used by Markov preflight predictor."
+    },
+    "session_id": {
+      "type": ["string", "null"],
+      "description": "Session correlation identifier for multi-turn inference tracking."
+    },
+    "fallback_policy": {
+      "type": "string",
+      "enum": ["capability_only", "disabled", "allow_named"],
+      "default": "capability_only",
+      "description": "Fallback routing behavior strategy. 'capability_only' stays within target capability; 'disabled' prevents fallback."
+    },
+    "locality": {
+      "type": "string",
+      "enum": ["auto", "prefer_local", "local_only"],
+      "default": "auto",
+      "description": "Routing locality guardrail. 'local_only' restricts to local hardware for privacy; 'prefer_local' ranks local ahead of cloud."
+    },
+    "prefer": {
+      "type": ["string", "null"],
+      "enum": ["local", "cloud", null],
+      "description": "Legacy hard routing preference flag."
+    },
+    "data_class": {
+      "type": ["string", "null"],
+      "enum": ["confidential", "public", null],
+      "description": "Data privacy classification tier. 'confidential' strictly blocks cloud fallback."
+    },
+    "reasoning": {
+      "type": ["object", "null"],
+      "description": "Deep reasoning configuration for models like DeepSeek R1.",
+      "properties": {
+        "effort": {
+          "type": "string",
+          "enum": ["low", "medium", "high"],
+          "default": "medium"
+        },
+        "include": {
+          "type": "boolean",
+          "default": true,
+          "description": "Whether to return reasoning thought tokens in response."
+        }
+      }
+    },
+    "messages": {
+      "type": "array",
+      "description": "Input messages sequence.",
+      "items": {
+        "type": "object",
+        "required": ["role", "content"],
+        "properties": {
+          "role": {
+            "type": "string",
+            "enum": ["system", "user", "assistant"],
+            "example": "user"
+          },
+          "content": {
+            "type": "string",
+            "example": "Write a 30-second podcast script about quantum computing."
+          }
+        }
+      }
+    },
+    "input_file": {
+      "type": ["string", "null"],
+      "description": "Absolute path to input audio/video/image file on engine host (for ASR, VLM, document processing)."
+    },
+    "params": {
+      "type": ["object", "null"],
+      "description": "Extra provider pass-through parameters (e.g. voice, speed, temperature, max_tokens)."
+    },
+    "hint_next": {
+      "type": ["string", "null"],
+      "description": "Next anticipated capability (e.g. 'tts') for cross-capability preflight warming."
+    },
+    "request_id": {
+      "type": "string",
+      "description": "Unique correlation request ID. Auto-generated by engine if omitted."
+    }
+  }
+}
diff --git a/docs/contracts/metrics_spec.md b/docs/contracts/metrics_spec.md
@@ -0,0 +1,138 @@
+# Metric Naming & Tagging Taxonomy Specification
+
+**Document Version:** v1.1.0-EMPIRICALLY-VERIFIED  
+**Path:** `docs/contracts/metrics_spec.md`  
+**Mandatory Requirement:** Every metric emitted by `mofa-observability` MUST carry `provider="<id>"` and `locality="local" | "cloud"` labels.
+
+---
+
+## 1. Metric Naming Rules & Label Standards
+
+1. **Namespace Prefix**: All metrics MUST begin with `mofa_`.
+2. **Mandatory Labels**:
+   * `provider`: String identifier of the serving backend (e.g., `ollama`, `kokoro`, `openai`, `deepseek`).
+   * `locality`: Enum string `local` or `cloud`.
+3. **Dual-Track Moat Principle**: Local execution metrics (RAM/VRAM memory footprint, cold start durations, warmup hits) and Cloud metrics (USD cost, token count, 429 errors) MUST be recorded in distinct metric series allowing direct side-by-side comparison in Grafana.
+
+---
+
+## 2. Complete 18-Metric Family Dictionary
+
+### 2.1 Financial & Token Consumption Metrics
+
+#### 1. `mofa_estimated_cost_usd`
+* **Type**: Counter
+* **Description**: Accumulated USD financial compute cost.
+* **Labels**: `provider`, `locality`, `model`
+* **Prometheus Exposition Example**:
+  ```text
+  # HELP mofa_estimated_cost_usd Total estimated USD cost incurred.
+  # TYPE mofa_estimated_cost_usd counter
+  mofa_estimated_cost_usd{provider="openai",locality="cloud",model="gpt-4o"} 0.042
+  mofa_estimated_cost_usd{provider="ollama",locality="local",model="gemma3:4b"} 0.000
+  ```
+
+#### 2. `mofa_tokens_total`
+* **Type**: Counter
+* **Description**: Total tokens processed by engine.
+* **Labels**: `provider`, `locality`, `model`, `type="prompt" | "completion"`
+* **PromQL Query Example**:
+  ```promql
+  sum(rate(mofa_tokens_total[5m])) by (provider, locality, type)
+  ```
+
+#### 3. `mofa_thought_tokens_total`
+* **Type**: Counter
+* **Description**: Deep reasoning thought tokens generated by reasoning models (e.g. DeepSeek R1).
+* **Labels**: `provider`, `locality`, `model`
+
+---
+
+### 2.2 Local Execution & Memory Lifecycle Metrics
+
+#### 4. `mofa_cold_start_seconds`
+* **Type**: Histogram
+* **Description**: Measures time elapsed from model load trigger to first ready inference.
+* **Labels**: `provider`, `locality`, `model`
+* **Buckets**: `[0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0]`
+
+#### 5. `mofa_warmup_hits_total`
+* **Type**: Counter
+* **Description**: Preflight model warming hit counters.
+* **Labels**: `provider`, `locality`, `source="hint" | "subscription" | "markov"`
+
+#### 6. `mofa_memory_usage_bytes`
+* **Type**: Gauge
+* **Description**: Host RAM and GPU VRAM footprint allocated.
+* **Labels**: `provider`, `locality`, `type="ram" | "vram"`, `status="reserved" | "observed"`
+* **Prometheus Exposition Example**:
+  ```text
+  # HELP mofa_memory_usage_bytes Current memory usage in bytes.
+  # TYPE mofa_memory_usage_bytes gauge
+  mofa_memory_usage_bytes{provider="ollama",locality="local",type="vram",status="observed"} 3339000000
+  ```
+
+#### 7. `mofa_model_residency_status`
+* **Type**: Gauge
+* **Description**: Runtime state of model in memory (`0=unloaded`, `1=loading`, `2=loaded`, `3=remote`).
+* **Labels**: `provider`, `locality`, `model`
+
+#### 8. `mofa_idle_evictions_total`
+* **Type**: Counter
+* **Description**: Count of LRU model evictions from VRAM due to memory budget limits.
+* **Labels**: `provider`, `locality`, `model`
+
+---
+
+### 2.3 System Health, Request Throughput & Observability
+
+#### 9. `mofa_requests_total`
+* **Type**: Counter
+* **Description**: Cumulative requests processed by engine.
+* **Labels**: `provider`, `locality`, `capability`, `status="200" | "400" | "404" | "422" | "500"`
+
+#### 10. `mofa_request_duration_seconds`
+* **Type**: Histogram
+* **Description**: End-to-end request latency distribution.
+* **Labels**: `provider`, `locality`, `capability`
+* **Buckets**: `[0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]`
+
+#### 11. `mofa_provider_health`
+* **Type**: Gauge
+* **Description**: Binary health status indicator (`1` for healthy, `0` for unhealthy).
+* **Labels**: `provider`, `locality`
+
+#### 12. `mofa_circuit_breaker_state`
+* **Type**: Gauge
+* **Description**: Circuit breaker status (`0=closed`, `1=open`, `2=half_open`).
+* **Labels**: `provider`
+
+#### 13. `mofa_active_connections`
+* **Type**: Gauge
+* **Description**: Active HTTP REST and SSE client streaming connection count.
+* **Labels**: `protocol="rest" | "sse"`
+
+#### 14. `mofa_preflight_predictions_total`
+* **Type**: Counter
+* **Description**: Markov preflight prediction evaluation outcomes.
+* **Labels**: `scope_id`, `result="approved" | "rejected"`
+
+#### 15. `mofa_cloud_quota_errors_total`
+* **Type**: Counter
+* **Description**: Count of HTTP 429 rate limit / quota exhaustion errors from cloud providers.
+* **Labels**: `provider`
+
+#### 16. `mofa_sse_events_emitted_total`
+* **Type**: Counter
+* **Description**: Total SSE telemetry events emitted to client dashboards.
+* **Labels**: `event_type`
+
+#### 17. `mofa_label_evictions_total`
+* **Type**: Counter
+* **Description**: Total high-cardinality metric tags garbage-collected by telemetry engine.
+* **Labels**: `collector`
+
+#### 18. `mofa_quality_gate_evaluations_total`
+* **Type**: Counter
+* **Description**: VLM and ffprobe video quality evaluation outcomes.
+* **Labels**: `pipeline`, `result="pass" | "fail"`