feat: unified LLM config, OpenRouter, safety guards, multi-model eval

config.example.json

@@ -7,10 +7,7 @@
   "ENABLE_REDIS_CACHE": true,
   "LLM_PROVIDER": "anthropic",
   "LLM_MODEL": null,
-  "OPENAI_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY",
-  "ANTHROPIC_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY",
-  "MISTRAL_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY",
-  "GEMINI_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY",
+  "LLM_API_KEY": "REPLACE_ME",
   "OLLAMA_BASE_URL": "http://localhost:11434",
   "TIME_WINDOW_INITIAL_MINUTES": 10,
   "TIME_WINDOW_EXPANSIONS": "60,360,1440",

db/schema.sql

@@ -98,6 +98,10 @@ CREATE INDEX IF NOT EXISTS investigations_conv_idx       ON investigations (conv
 ALTER TABLE investigations
     ADD COLUMN IF NOT EXISTS conversation_id UUID REFERENCES conversations(id) ON DELETE SET NULL;
 
+-- FSM state snapshot for resume-from-serialization (replaces message-replay).
+ALTER TABLE investigations
+    ADD COLUMN IF NOT EXISTS state_json JSONB;
+
 -- Investigation steps: individual ReAct thought → action → observation records
 CREATE TABLE IF NOT EXISTS investigation_steps (
     id               SERIAL PRIMARY KEY,

docker/config.docker.json

@@ -2,10 +2,8 @@
   "REPI_ENV": "production",
   "LOG_LEVEL": "INFO",
   "DATABASE_URL": "postgresql+asyncpg://repi_user:password_here@db:5432/repi",
-"LLM_PROVIDER": "mistral",
-  "LLM_API_KEY": "",
-  "OPENAI_API_KEY": "",
-  "ANTHROPIC_API_KEY": "",
-  "MISTRAL_API_KEY": "",
-  "GEMINI_API_KEY": ""
+  "REDIS_URL": "redis://redis:6379",
+  "ENABLE_REDIS_CACHE": true,
+  "LLM_PROVIDER": "mistral",
+  "LLM_API_KEY": ""
 }

eval/ragas_eval.py

@@ -180,31 +180,31 @@ def _make_ragas_llm(provider: str | None = None):
     else:
         providers = ["mistral", "gemini"]
 
+    api_key = settings.LLM_API_KEY
+    if not api_key:
+        raise RuntimeError(
+            "No evaluator LLM available. Configure LLM_API_KEY in .repi/config.json"
+        )
+
+    base_urls = {
+        "gemini": "https://generativelanguage.googleapis.com/v1beta/openai/",
+        "mistral": "https://api.mistral.ai/v1",
+    }
+    default_models = {
+        "gemini": "gemini-2.0-flash",
+        "mistral": "mistral-small-2506",
+    }
+
     for p in providers:
-        if p == "gemini":
-            api_key = settings.GEMINI_API_KEY or settings.GOOGLE_API_KEY
-            if not api_key:
-                continue
-            client = OpenAI(
-                api_key=api_key,
-                base_url="https://generativelanguage.googleapis.com/v1beta/openai/",
-            )
-            print(f"  [config] RAGAS evaluator: Gemini (gemini-2.0-flash)")
-            return llm_factory(model="gemini-2.0-flash", provider="openai", client=client)
-
-        elif p == "mistral":
-            api_key = settings.MISTRAL_API_KEY
-            if not api_key:
-                continue
-            client = OpenAI(
-                api_key=api_key,
-                base_url="https://api.mistral.ai/v1",
-            )
-            print(f"  [config] RAGAS evaluator: Mistral (mistral-small-2506)")
-            return llm_factory(model="mistral-small-2506", provider="openai", client=client)
+        if p not in base_urls:
+            continue
+        client = OpenAI(api_key=api_key, base_url=base_urls[p])
+        model = default_models[p]
+        print(f"  [config] RAGAS evaluator: {p} ({model})")
+        return llm_factory(model=model, provider="openai", client=client)
 
     raise RuntimeError(
-        "No evaluator LLM available. Configure GEMINI_API_KEY or MISTRAL_API_KEY in .repi/config.json"
+        f"No supported RAGAS evaluator provider in {providers}. Use 'gemini' or 'mistral'."
     )
 
 
@@ -215,7 +215,7 @@ def _make_ragas_embeddings(provider: str | None = None):
     import warnings
     warnings.filterwarnings("ignore", message=".*LangchainEmbeddingsWrapper.*deprecated.*")
 
-    api_key = settings.MISTRAL_API_KEY
+    api_key = settings.LLM_API_KEY
     if not api_key:
         return None
 

eval/results/mistral_self.json

@@ -0,0 +1,229 @@
+{
+  "pass_threshold": 0.8,
+  "summary": {
+    "passed": 2,
+    "failed": 2,
+    "errored": 0,
+    "average_score": 0.768
+  },
+  "results": [
+    {
+      "dataset": "dataset_1_cascading_inventory_migration",
+      "query": "why did checkout break friday night",
+      "status": "fail",
+      "aggregate_score": 0.68,
+      "model_under_test": "mistral-large-latest",
+      "judge_model": "mistral-large-latest",
+      "criteria": [
+        {
+          "name": "trigger_identification",
+          "score": 0.9,
+          "explanation": "The trigger service is correctly identified as 'inventory-svc'. The log line references 'Applying migration 0042_add_warehouse_id.sql', which is semantically equivalent to the required 'Migration 0042 added column warehouse_id NOT NULL to skus table' but does not explicitly mention the NOT NULL constraint or the table name. This is a minor omission but still captures the core concept of the migration."
+        },
+        {
+          "name": "root_cause_accuracy",
+          "score": 1.0,
+          "explanation": "The root_cause explanation mentions all required concepts: 'migration' (Postgres migration), 'warehouse_id' (warehouse_id column), 'NOT NULL' (NOT NULL column), 'retry' (retry storm), and 'pool exhaustion' (HTTP client pool EXHAUSTED). The explanation is comprehensive and semantically correct."
+        },
+        {
+          "name": "propagation_chain",
+          "score": 1.0,
+          "explanation": "The propagation_chain includes all required hops in the correct order. It mentions 'migration 0042 / warehouse_id NOT NULL' (implicitly via 'Migration 0042 added column warehouse_id NOT NULL'), '500 / null value / warehouse_id' (constraint violation and 500 error), 'retry' (retry 1/5 and retry storm), 'pool exhausted' (HTTP client pool EXHAUSTED), and 'checkout / cart timeout' (Checkout failed for cart_id). The chain is complete and accurate."
+        },
+        {
+          "name": "red_herring_handling",
+          "score": 0.5,
+          "explanation": "The ruled_out_hypotheses address some red herrings but miss key ones. It correctly rules out 'gateway', 'read-states-svc', 'message-svc', 'presence-svc', 'cdn-edge', and 'network outage'. However, it does not explicitly address 'pricing-svc', 'payment-svc', or 'notification-svc', which are required to be ruled out. This is a significant omission."
+        },
+        {
+          "name": "confidence_calibration",
+          "score": 0.0,
+          "explanation": "Expected confidence 'high', got 'medium'."
+        }
+      ],
+      "raw_answer_truncated": "{\n  \"incident_window\": {\n    \"start\": \"2026-06-26T22:00:00+00:00\",\n    \"end\": \"2026-06-27T06:00:00+00:00\"\n  },\n  \"affected_services\": [\n    \"inventory-svc\",\n    \"cart-svc\",\n    \"gateway\"\n  ],\n  \"trigger_event\": {\n    \"chunk_id\": \"84a2a432-447c-48b7-95d7-c537f427f68c\",\n    \"service\": \"inventory-svc\",\n    \"timestamp\": \"2026-06-26T22:00:14+00:00\",\n    \"log_line\": \"Applying migration 0042_add_warehouse_id.sql\"\n  },\n  \"propagation_chain\": [\n    {\n      \"service\": \"inventory-svc\",\n      \"chunk_id\": \"e",
+      "stats": {
+        "iterations_used": 4,
+        "reflections_used": 1,
+        "chunks_gathered": 27,
+        "tools_called": [
+          "find_logs_by_id",
+          "scan_window",
+          "search_logs"
+        ],
+        "compile_source": "llm",
+        "compile_attempts": 1,
+        "floor_adjustments": [
+          "no gaps listed despite non-low confidence (\u22653 chunk citations \u2014 leaving as-is)",
+          "affected_services ['gateway'] never appeared in tool results or any chunk text (downgraded confidence one notch as a precaution)"
+        ],
+        "gathering_exit_reason": "Root cause and impact confirmed: unbackfilled Postgres migration in `inventory-svc` caused constraint violations, cascading to `cart-svc` and breaking checkout. Gateway propagated errors without logging them."
+      },
+      "judge_parse_attempts": 1
+    },
+    {
+      "dataset": "dataset_2_insufficient_logging",
+      "query": "why is the nightly report failing",
+      "status": "fail",
+      "aggregate_score": 0.633,
+      "model_under_test": "mistral-large-latest",
+      "judge_model": "mistral-large-latest",
+      "criteria": [
+        {
+          "name": "trigger_identification",
+          "score": 0.5,
+          "explanation": "The answer identifies the trigger event as the cron-runner job start, but it does not mention either of the acceptable trigger events: (1) report-svc log with 'SIGKILL' or (2) cron-runner log mentioning 'exceeded timeout'. While the propagation chain correctly includes these events, the trigger_event field itself does not meet the criteria. The explanation is partially correct but misses the explicit requirement for the trigger event to be one of the two specified log lines."
+        },
+        {
+          "name": "root_cause_accuracy",
+          "score": 0.8,
+          "explanation": "The root_cause field does not explicitly state 'unable_to_determine' or 'hypothesis_only', but it avoids asserting a specific root cause as fact. It provides a plausible hypothesis based on the evidence (timeout and SIGKILL) but does not overreach. This is mostly correct, though it could be clearer about the uncertainty."
+        },
+        {
+          "name": "red_herring_handling",
+          "score": 0.5,
+          "explanation": "The ruled_out_hypotheses field addresses some key possibilities but misses others. It correctly rules out Postgres issues and OOM killer, but it does not explicitly consider 'code crash' (weakly ruled out due to no stack trace), 'deadlock' (cannot confirm or rule out), or 'external kill' (consistent with SIGKILL but not confirmed). The handling is partially correct but omits critical hypotheses."
+        },
+        {
+          "name": "confidence_calibration",
+          "score": 0.0,
+          "explanation": "Expected confidence 'low', got 'medium'."
+        },
+        {
+          "name": "gap_awareness",
+          "score": 1.0,
+          "explanation": "Gaps list includes content matching expected gaps."
+        },
+        {
+          "name": "hallucination_avoidance",
+          "score": 1.0,
+          "explanation": "The root_cause field avoids asserting any of the prohibited terms as confirmed facts. It does not mention 'memory leak', 'OOM (as a fact)', 'GC pressure', or specific code paths/dependencies. The answer is fully correct in this regard."
+        }
+      ],
+      "raw_answer_truncated": "{\n  \"incident_window\": {\n    \"start\": \"2026-06-25T02:00:00+00:00\",\n    \"end\": \"2026-06-25T03:15:00+00:00\"\n  },\n  \"affected_services\": [\n    \"cron-runner\",\n    \"report-svc\"\n  ],\n  \"trigger_event\": {\n    \"chunk_id\": \"fb724a62-08cc-4390-a505-d017adc1ae06\",\n    \"service\": \"cron-runner\",\n    \"timestamp\": \"2026-06-25T02:00:00+00:00\",\n    \"log_line\": \"Triggering job nightly_reports \\u2192 report-svc\"\n  },\n  \"propagation_chain\": [\n    {\n      \"service\": \"cron-runner\",\n      \"chunk_id\": \"27ac94ca-8331-42",
+      "stats": {
+        "iterations_used": 7,
+        "reflections_used": 2,
+        "chunks_gathered": 25,
+        "tools_called": [
+          "find_logs_by_id",
+          "scan_window",
+          "search_logs"
+        ],
+        "compile_source": "llm",
+        "compile_attempts": 1,
+        "floor_adjustments": [],
+        "gathering_exit_reason": "All plausible root causes have been investigated. The evidence shows the nightly report job timed out after 1 hour (as configured), and the downstream `report-svc` process was later killed by SIGKILL. No OOM, Postgres, or other resource exhaustion logs were found. Further tool calls are unlikely to yield new evidence."
+      },
+      "judge_parse_attempts": 1
+    },
+    {
+      "dataset": "dataset_3_jwt_key_rotation_noise",
+      "query": "why are users getting 401 errors last monday morning",
+      "status": "pass",
+      "aggregate_score": 0.9,
+      "model_under_test": "mistral-large-latest",
+      "judge_model": "mistral-large-latest",
+      "criteria": [
+        {
+          "name": "trigger_identification",
+          "score": 0.9,
+          "explanation": "The trigger service is correctly identified as auth-svc, and the log line references 'JWT key rotation triggered', which is one of the required semantic matches. However, the log line does not explicitly mention any of the other exact phrases like 'key_id=k-2026-05' or 'Public key push to verification-svc timed out', though the context is clear from the propagation chain and root cause. This is a minor omission."
+        },
+        {
+          "name": "root_cause_accuracy",
+          "score": 1.0,
+          "explanation": "The root cause explanation mentions all required concepts: JWT, key rotation, k-2026-05, verification-svc, and public key (implied by 'signing key' and 'synchronize the new key'). The explanation is semantically correct and complete."
+        },
+        {
+          "name": "propagation_chain",
+          "score": 0.6,
+          "explanation": "The propagation chain includes all required hops in the correct order. auth-svc mentions 'Active signing key changed: k-2025 \u2192 k-2026-05', which semantically matches 'key rotation / k-2026-05 / push timed out'. verification-svc mentions 'unknown key_id=k-2026-05'. billing-svc mentions 'Checkout failed: upstream 401 Unauthorized', which semantically matches '401 / checkout failed'. However, api-gateway is missing from the chain, which is a significant omission."
+        },
+        {
+          "name": "red_herring_handling",
+          "score": 0.9,
+          "explanation": "The ruled_out_hypotheses correctly address the red herrings user-svc and cache-svc, explicitly stating no causal link to the 401 failures. The explanations are clear and accurate. However, the hypotheses are not explicitly named as 'user-svc' or 'cache-svc' in the hypothesis field, though the content makes it clear. This is a minor omission."
+        },
+        {
+          "name": "confidence_calibration",
+          "score": 1.0,
+          "explanation": "Confidence correctly set to 'high'."
+        },
+        {
+          "name": "hallucination_avoidance",
+          "score": 1.0,
+          "explanation": "The root cause does not name any of the prohibited terms as THE root cause. It correctly identifies auth-svc's key rotation as the root cause and treats other services as downstream effects. No hallucinations are present."
+        }
+      ],
+      "raw_answer_truncated": "{\n  \"incident_window\": {\n    \"start\": \"2026-06-22T09:15:00+00:00\",\n    \"end\": \"2026-06-22T09:33:02+00:00\"\n  },\n  \"affected_services\": [\n    \"verification-svc\",\n    \"billing-svc\",\n    \"api-gateway\",\n    \"auth-svc\"\n  ],\n  \"trigger_event\": {\n    \"chunk_id\": \"0743d05c-1fce-4af8-8a7f-15e091be78e5\",\n    \"service\": \"auth-svc\",\n    \"timestamp\": \"2026-06-22T09:15:02+00:00\",\n    \"log_line\": \"JWT key rotation triggered: scheduled policy rotate-every-<NUM>d\"\n  },\n  \"propagation_chain\": [\n    {\n      \"servic",
+      "stats": {
+        "iterations_used": 4,
+        "reflections_used": 1,
+        "chunks_gathered": 38,
+        "tools_called": [
+          "get_timeline",
+          "scan_window",
+          "search_logs"
+        ],
+        "compile_source": "llm",
+        "compile_attempts": 1,
+        "floor_adjustments": [],
+        "gathering_exit_reason": "Root cause identified: `verification-svc` failed to synchronize with the new JWT signing key (`k-2026-05`) after the `auth-svc` key rotation event at 09:15:02 UTC, causing it to reject valid JWTs and trigger 401 errors for users. The evidence is conclusive and no further investigation is required."
+      },
+      "judge_parse_attempts": 1
+    },
+    {
+      "dataset": "dataset_4_discord_gateway_cascade",
+      "query": "why are users getting disconnected from the gateway",
+      "status": "pass",
+      "aggregate_score": 0.86,
+      "model_under_test": "mistral-large-latest",
+      "judge_model": "mistral-large-latest",
+      "criteria": [
+        {
+          "name": "trigger_identification",
+          "score": 1.0,
+          "explanation": "The trigger service is correctly identified as 'read-states-svc', and the log line explicitly references 'Cassandra timeout' (OperationTimedOutException). This fully matches the criterion."
+        },
+        {
+          "name": "root_cause_accuracy",
+          "score": 0.8,
+          "explanation": "The root_cause explanation mentions Cassandra, read-states, timeout, and reconnect, but it does not explicitly reference the 'pool' concept (e.g., connection pool exhaustion or degradation). While the propagation chain hints at pool issues, the root_cause itself omits this detail."
+        },
+        {
+          "name": "propagation_chain",
+          "score": 0.8,
+          "explanation": "The propagation chain is mostly correct and includes all required hops in the right order. However, it misses explicit mentions of 'BATCH APPLY / OperationTimedOutException' for read-states-svc and 'mark_read / timeout / disconnect' for the first gateway hop. The reconnect storm and thread pool saturation are well-covered, and the 502 mention for cdn-edge is accurate."
+        },
+        {
+          "name": "red_herring_handling",
+          "score": 0.7,
+          "explanation": "The ruled_out_hypotheses correctly address the presence-svc red herring, noting its lack of involvement in the incident. However, it does not explicitly state that presence-svc showed elevated activity *caused by* the gateway reconnect storm (a key nuance in the criterion). The other red herrings are well-handled."
+        },
+        {
+          "name": "confidence_calibration",
+          "score": 1.0,
+          "explanation": "Confidence correctly set to 'high'."
+        }
+      ],
+      "raw_answer_truncated": "{\n  \"incident_window\": {\n    \"start\": \"2026-06-10T02:15:00+00:00\",\n    \"end\": \"2026-06-10T02:30:00+00:00\"\n  },\n  \"affected_services\": [\n    \"gateway\",\n    \"read-states-svc\",\n    \"message-svc\",\n    \"cdn-edge\"\n  ],\n  \"trigger_event\": {\n    \"chunk_id\": \"9938c84a-5eeb-40d9-8650-3fd49e217be1\",\n    \"service\": \"read-states-svc\",\n    \"timestamp\": \"2026-06-10T02:15:00+00:00\",\n    \"log_line\": \"Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=<IP>, consisten",
+      "stats": {
+        "iterations_used": 2,
+        "reflections_used": 0,
+        "chunks_gathered": 64,
+        "tools_called": [
+          "scan_window",
+          "search_logs"
+        ],
+        "compile_source": "llm",
+        "compile_attempts": 1,
+        "floor_adjustments": [
+          "no gaps listed despite non-low confidence (\u22653 chunk citations \u2014 leaving as-is)"
+        ],
+        "gathering_exit_reason": "Root cause identified: `read-states-svc` Cassandra write timeouts caused gateway disconnections. Timeline and cascading effects are fully mapped."
+      },
+      "judge_parse_attempts": 1
+    }
+  ]
+}