From d96bc9d389fd2faf70c594f02cf18cf9059cb6b7 Mon Sep 17 00:00:00 2001 From: Varun Singh Date: Mon, 29 Jun 2026 00:38:31 +0530 Subject: [PATCH 1/3] feat: added base structure + types for fsm --- db/schema.sql | 4 + repi/investigation/react_loop.py | 1638 +++++++++++++++-------------- repi/investigation/state.py | 169 +++ repi/models/schema.py | 2 + tests/investigation/test_state.py | 137 +++ 5 files changed, 1174 insertions(+), 776 deletions(-) create mode 100644 repi/investigation/state.py create mode 100644 tests/investigation/test_state.py diff --git a/db/schema.sql b/db/schema.sql index 3379f5d..6c174b8 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -98,6 +98,10 @@ CREATE INDEX IF NOT EXISTS investigations_conv_idx ON investigations (conv ALTER TABLE investigations ADD COLUMN IF NOT EXISTS conversation_id UUID REFERENCES conversations(id) ON DELETE SET NULL; +-- FSM state snapshot for resume-from-serialization (replaces message-replay). +ALTER TABLE investigations + ADD COLUMN IF NOT EXISTS state_json JSONB; + -- Investigation steps: individual ReAct thought → action → observation records CREATE TABLE IF NOT EXISTS investigation_steps ( id SERIAL PRIMARY KEY, diff --git a/repi/investigation/react_loop.py b/repi/investigation/react_loop.py index 54578cb..0325906 100644 --- a/repi/investigation/react_loop.py +++ b/repi/investigation/react_loop.py @@ -3,46 +3,36 @@ import logging import asyncio import time -import re -from datetime import datetime, timedelta from uuid import UUID -from typing import List, Dict, Any, Optional, Callable, Awaitable -from dataclasses import dataclass, field +from typing import Any, Optional, Callable, Awaitable import asyncpg -from repi.core.dates import DateHandler, default_date_handler as _dh +from repi.core.dates import default_date_handler as _dh from repi.llm.adapters import LLMRateLimitError, LLMBadRequestError from repi.llm.provider import LLMProvider, Message from repi.investigation.tools import ToolCall, ToolResult, TOOL_SCHEMAS -from repi.retrieval.heuristics import cluster_logs from repi.investigation.store import InvestigationStore from repi.intent.resolver import resolve as resolve_intent, ResolvedIntent, ClarificationNeeded from repi.investigation.sweep import auto_sweep -from repi.investigation.schema import InvestigationAnswer, validate_answer -from repi.investigation.compiler import compile_answer, synthesize_answer, CompileResult +from repi.investigation.compiler import compile_answer, synthesize_answer +from repi.investigation.state import ( + Phase, + InvestigationState, + LoopDeps, + Thought, + Action, + Observation, + InvestigationStep, + InvestigationResult, +) logger = logging.getLogger(__name__) -# The ReAct loop only gathers evidence. The final InvestigationAnswer is -# produced by `repi.investigation.compiler` via a separate LLM call. -# -# `DONE_GATHERING_TOOL` is the LLM's voluntary exit signal for the gathering -# phase. Dispatcher-only — not in `TOOL_SCHEMAS`. Args: optional {"reason": "..."}. -# -# `LEGACY_SUBMIT_TOOL` is also accepted: if a model emits `submit_answer`, it -# is treated as a done-gathering signal (its args are discarded; the compiler -# produces the real answer). DONE_GATHERING_TOOL = "done_gathering" LEGACY_SUBMIT_TOOL = "submit_answer" - -# Reflection turn injected every N action steps: the LLM steps back, -# summarises hypotheses + evidence, and picks the highest-value next action. -# Pure thought — no tool call on this turn. Termination is gated: only call -# `done_gathering` after multiple distinct lines of inquiry have all returned -# no useful evidence. REFLECTION_PROMPT = ( "Stop. Before your next action, reflect:\n" "1. What hypotheses have you considered so far?\n" @@ -59,42 +49,691 @@ from repi.llm.json_utils import parse_llm_response # noqa: F401 -@dataclass -class Thought: - content: str - -@dataclass -class Action: - tool_call: ToolCall - -@dataclass -class Observation: - tool_result: ToolResult - -@dataclass -class InvestigationStep: - step_number: int - thought: Thought - action: Optional[Action] = None - observation: Optional[Observation] = None - timestamp: datetime = field(default_factory=_dh.now) - # `kind` classifies special turns. "reflection" = forced re-plan (no tool call); - # None = normal thought → action → observation step. - kind: Optional[str] = None - -@dataclass -class InvestigationResult: - id: str - query: str - steps: list[InvestigationStep] - answer: str - evidence_chunk_ids: list[str] - confidence: str - duration_seconds: float - evidence: list[dict] = field(default_factory=list) - stats: dict = field(default_factory=dict) +# ─── Observation compaction ────────────────────────────────────────────────── + +MAX_OBS_ITEMS = 10 +MAX_OBS_TEXT_CHARS = 300 +MAX_OBS_TOTAL_CHARS = 6000 + + +def _compact_observation(result: Any) -> str: + def _walk(node: Any, max_items: int, max_chars: int) -> Any: + if isinstance(node, dict): + return {k: _walk(v, max_items, max_chars) for k, v in node.items()} + if isinstance(node, list): + clipped = [_walk(x, max_items, max_chars) for x in node[:max_items]] + if len(node) > max_items: + clipped.append(f"... {len(node) - max_items} more items truncated") + return clipped + if isinstance(node, str) and len(node) > max_chars: + return node[:max_chars] + "...[truncated]" + return node + + s = json.dumps(_walk(result, MAX_OBS_ITEMS, MAX_OBS_TEXT_CHARS), default=str) + if len(s) > MAX_OBS_TOTAL_CHARS: + s = json.dumps(_walk(result, 3, 120), default=str) + if len(s) > MAX_OBS_TOTAL_CHARS: + s = json.dumps({ + "note": "observation too large; showing a truncated excerpt", + "excerpt": s[:MAX_OBS_TOTAL_CHARS], + }) + return s + + +def _extract_chunks(tool_result: Any) -> list[dict]: + chunks: list[dict] = [] + seen: set[str] = set() + + def _maybe_append(item: Any) -> None: + if isinstance(item, dict): + cid = item.get("chunk_id") + if cid and cid not in seen: + seen.add(cid) + chunks.append(item) + + if isinstance(tool_result, list): + for item in tool_result: + _maybe_append(item) + elif isinstance(tool_result, dict): + _maybe_append(tool_result) + for value in tool_result.values(): + if isinstance(value, list): + for item in value: + _maybe_append(item) + return chunks + + +# ─── Ledger helpers ────────────────────────────────────────────────────────── + +def _ledger_key(tool_name: str, args: dict) -> str: + try: + normalized = json.dumps(args or {}, sort_keys=True, default=str) + except (TypeError, ValueError): + normalized = repr(args) + return f"{tool_name}::{normalized}" + + +def _ledger_summary(ledger: dict[str, dict]) -> str: + if not ledger: + return "" + lines = [] + for entry in ledger.values(): + lines.append(f"- {entry['tool_name']}({json.dumps(entry['args'], default=str, sort_keys=True)})") + return "TOOLS ALREADY CALLED (do not repeat with identical args):\n" + "\n".join(lines) + + +# ─── Rate limiting ─────────────────────────────────────────────────────────── + +async def _wait_for_rate_limit(deps: LoopDeps): + now = time.time() + deps.llm_call_timestamps = [t for t in deps.llm_call_timestamps if now - t < 60] + while len(deps.llm_call_timestamps) >= deps.llm_max_calls_per_min: + wait_time = 60 - (now - deps.llm_call_timestamps[0]) + 1 + logger.warning(f"Rate limit: Waiting {wait_time:.1f}s...") + await asyncio.sleep(wait_time) + now = time.time() + deps.llm_call_timestamps = [t for t in deps.llm_call_timestamps if now - t < 60] + deps.llm_call_timestamps.append(now) + + +# ─── LLM call with retry ──────────────────────────────────────────────────── + +async def _llm_call_with_retry( + deps: LoopDeps, + messages: list[Message], + step_label: str, +) -> str | None: + for attempt in range(3): + try: + await _wait_for_rate_limit(deps) + return await deps.llm.complete(messages) + except LLMBadRequestError as e: + logger.error(f"{step_label}: non-retryable LLM error: {e}") + return None + except Exception as e: + logger.warning(f"{step_label} attempt {attempt+1}/3 failed: {e}") + if attempt < 2: + delay = 15 * (2 ** attempt) + if isinstance(e, LLMRateLimitError) and e.retry_after: + delay = e.retry_after + 1 + await asyncio.sleep(delay) + return None + + +# ─── System prompt builder ─────────────────────────────────────────────────── + +def _build_system_prompt(known_services: list[str]) -> str: + gathering_tools = {k: v for k, v in TOOL_SCHEMAS.items() if k != LEGACY_SUBMIT_TOOL} + return f"""You are a senior SRE gathering evidence about an incident. +Postgres is the source of truth for this investigation. + +KNOWN SERVICES: {known_services} +TOOLS: {json.dumps(gathering_tools, indent=2)} + +YOUR ROLE: You are an EVIDENCE GATHERER. A separate "compile" step will turn +the evidence you collect into the final structured answer. You do NOT produce +that answer yourself. + +RESPONSE FORMAT (use exactly this shape on every turn — no exceptions): +{{ "thought": "...", "action": {{ "tool": "", "args": {{...}} }} }} + +When you believe you have gathered enough evidence (or that further +investigation will not change the answer), signal exit by calling +`done_gathering`: +{{ "thought": "...", "action": {{ "tool": "done_gathering", "args": {{ "reason": "..." }} }} }} + +GATHERING PRINCIPLES: +1. ENTITY FIRST. If the query mentions a literal identifier (UUID, W3C trace + or span id, ULID, a prefixed ID like ch_xxx / pi_xxx / cus_xxx, an AWS + resource id like i-0abc…, a git SHA, or any other unique token), call + `find_logs_by_id` on the FIRST action turn. Semantic + FTS retrieval + will often miss specific tokens because the English tokenizer fragments + them and embeddings don't preserve literal token identity. +2. ALWAYS correlate logs cross-service. `scan_window` is usually the right + first call when investigating a TIME WINDOW (no entity in play) — it + returns ERRORS plus pre-context for each service that emitted them. +3. Don't repeat tool calls with identical arguments — the dispatcher + dedupes them, but you still waste a turn. +4. If a tool call returns nothing useful, vary the arguments (different + service, wider window, different level filter) before giving up on + that line of inquiry. +5. If two consecutive tool calls return no new evidence, call + `done_gathering` — there is no value in spamming the dispatcher. +6. When NO time window was provided, do NOT pass time_from/time_to to + `search_logs`/`scan_window`; rely on `find_logs_by_id` and unbounded + searches keyed off the entity/service signal you do have. +7. Do NOT emit a "Final Answer:" prefix or fill in any + InvestigationAnswer schema. The compile step will produce that. + +Current UTC: {_dh.to_iso(_dh.now())} +""" + + +# ─── Phase handlers ────────────────────────────────────────────────────────── + +async def handle_resolving(state: InvestigationState, deps: LoopDeps) -> InvestigationState: + now = _dh.now() + clarified_query = state.query + + if state.post_clarification: + last_step_obs = None + if deps.store: + existing_steps = await deps.store.get_steps(state.investigation_id) + last_step = existing_steps[-1] if existing_steps else None + if ( + last_step + and last_step.action + and (last_step.action.get("name") == "ask_user" or last_step.action.get("tool") == "ask_user") + and last_step.observation + and last_step.observation.get("result") + ): + reply = last_step.observation["result"].get("reply", "") + clarified_query = f"{state.query} (User Clarification: {reply})" + + resolution = resolve_intent(clarified_query, deps.known_services, now) + logger.info(f"Intent Resolution for '{clarified_query}': {resolution}") + + if isinstance(resolution, ClarificationNeeded): + if state.post_clarification: + logger.warning( + f"Still ambiguous after clarification ({resolution.missing_dims}). " + "Proceeding with widest-default window (last 24h)." + ) + state.resolved_intent = ResolvedIntent( + time_from=_dh.ago(days=1), + time_to=_dh.now(), + services=[], + symptoms=[], + assumed=[ + "time could not be resolved after clarification — defaulting to last 24 hours", + f"clarification was: {clarified_query}", + ], + ) + state.phase = Phase.SWEEPING + return state + + thought_text = "I need to clarify the request before I can proceed." + action_data = {"name": "ask_user", "args": {"question": resolution.question}} + + if deps.store: + existing_steps = await deps.store.get_steps(state.investigation_id) + last_step = existing_steps[-1] if existing_steps else None + if not last_step or last_step.action.get("args", {}).get("question") != resolution.question: + await deps.store.add_step( + investigation_id=state.investigation_id, + step_number=state.next_step_number, + thought=thought_text, + action=action_data, + ) + await deps.store.set_awaiting_clarification(state.investigation_id, resolution.question) + + step = InvestigationStep( + state.next_step_number, + Thought(thought_text), + Action(ToolCall(name="ask_user", args=action_data["args"])), + ) + state.processed_steps.append(step) + if deps.on_step: + await deps.on_step(step) + + state.pending_question = resolution.question + state.phase = Phase.WAITING_CLARIFICATION + return state + + state.resolved_intent = resolution + state.phase = Phase.SWEEPING + return state + + +async def handle_waiting_clarification(state: InvestigationState, deps: LoopDeps) -> InvestigationState: + state.post_clarification = True + state.pending_question = None + state.phase = Phase.RESOLVING + return state + + +async def handle_sweeping(state: InvestigationState, deps: LoopDeps) -> InvestigationState: + if deps.on_phase_change: + try: + await deps.on_phase_change("gathering") + except Exception: + logger.debug("on_phase_change(gathering) hook raised", exc_info=True) + + state.messages = [ + Message(role="system", content=_build_system_prompt(deps.known_services)), + Message(role="user", content=state.query), + ] + + if state.resolved_intent and deps.pool: + if state.resolved_intent.time_from is None: + priming_lines = ["RAG CONTEXT:"] + priming_lines.append( + f"- entities mentioned: {state.resolved_intent.entities or 'none'}" + ) + priming_lines.append( + f"- services mentioned: {state.resolved_intent.services or 'none'}" + ) + priming_lines.append( + "- no time window — use find_logs_by_id (for entities) " + "or search_logs with null time_from/time_to (for services)." + ) + if state.resolved_intent.assumed: + priming_lines.append("") + priming_lines.append("ASSUMPTIONS:") + priming_lines.extend(f"- {a}" for a in state.resolved_intent.assumed) + state.messages.append(Message(role="user", content="\n".join(priming_lines))) + else: + sweep_results = await auto_sweep( + pool=deps.pool, + time_from=state.resolved_intent.time_from, + time_to=state.resolved_intent.time_to, + exclude_services=[], + ) + sweep_msg = f"SWEEP CONTEXT:\n{_compact_observation(sweep_results)}\n\n" + if state.resolved_intent.assumed: + sweep_msg += "ASSUMPTIONS:\n" + "\n".join(f"- {a}" for a in state.resolved_intent.assumed) + "\n" + state.messages.append(Message(role="user", content=sweep_msg)) + + state.phase = Phase.GATHERING + return state + + +async def handle_reflecting(state: InvestigationState, deps: LoopDeps) -> InvestigationState: + state.messages.append(Message(role="user", content=REFLECTION_PROMPT)) + + raw_reflection = await _llm_call_with_retry( + deps, state.messages, f"Reflection {state.next_step_number}" + ) + + if raw_reflection is None: + logger.error(f"Reflection {state.next_step_number}: LLM call failed after 3 retries, skipping") + if state.messages and state.messages[-1].content == REFLECTION_PROMPT: + state.messages.pop() + state.action_steps_since_reflection = 0 + state.reflections_used += 1 + state.phase = Phase.GATHERING + return state + + if deps.store: + await deps.store.increment_llm_calls(state.investigation_id) + + try: + parsed_reflection = parse_llm_response(raw_reflection) + reflection_thought = parsed_reflection.get("thought", "") or raw_reflection + except Exception: + reflection_thought = raw_reflection + + if not isinstance(reflection_thought, str): + try: + reflection_thought = json.dumps(reflection_thought, default=str) + except (TypeError, ValueError): + reflection_thought = str(reflection_thought) + + reflection_step = InvestigationStep( + state.next_step_number, + Thought(reflection_thought), + None, + None, + kind="reflection", + ) + state.processed_steps.append(reflection_step) + + if deps.store: + await deps.store.add_step( + investigation_id=state.investigation_id, + step_number=state.next_step_number, + thought=reflection_thought, + action=None, + observation=None, + kind="reflection", + ) + + if deps.on_step: + await deps.on_step(reflection_step) + + state.messages.append(Message(role="assistant", content=raw_reflection)) + state.next_step_number += 1 + state.action_steps_since_reflection = 0 + state.reflections_used += 1 + state.phase = Phase.GATHERING + return state + + +async def handle_gathering(state: InvestigationState, deps: LoopDeps) -> InvestigationState: + if state.actions_taken >= deps.max_iterations: + state.gathering_exit_reason = "max_actions_reached" + state.phase = Phase.COMPILING + return state + + # Check if reflection is due + if ( + deps.enable_reflection + and deps.reflection_interval > 0 + and state.reflections_used < deps.max_reflections + and state.action_steps_since_reflection >= deps.reflection_interval + ): + state.phase = Phase.REFLECTING + return state + + # Iteration delay (skip on first step) + if state.next_step_number > 1 and state.actions_taken > 0: + await asyncio.sleep(deps.min_iteration_delay) + + # Graduated finalize prompts + actions_remaining = deps.max_iterations - state.actions_taken + if actions_remaining == 2: + state.messages.append(Message( + role="user", + content=( + "You have 2 actions left before gathering ends. Only " + "issue a tool call if it would materially change the " + "final answer; otherwise call `done_gathering`." + ), + )) + elif actions_remaining == 1: + state.messages.append(Message( + role="user", + content=( + "Last action. The next turn should either call " + "`done_gathering` or one final tool call. After this " + "turn the gathering phase ends." + ), + )) + + # Ensure last message is user/system before calling LLM + if state.messages and state.messages[-1].role == "assistant": + state.messages.append(Message(role="user", content="Continue gathering evidence.")) + + # LLM call + raw_response = await _llm_call_with_retry( + deps, state.messages, f"Step {state.next_step_number}" + ) + + if raw_response is None: + logger.error(f"Step {state.next_step_number}: LLM call failed after 3 retries, ending gathering") + state.gathering_exit_reason = "llm_call_failed_repeatedly" + state.phase = Phase.COMPILING + return state + + if deps.store: + await deps.store.increment_llm_calls(state.investigation_id) + + try: + parsed = parse_llm_response(raw_response) + + _thought_raw = parsed.get("thought", "") + if not isinstance(_thought_raw, str): + try: + _thought_raw = json.dumps(_thought_raw, default=str) + except (TypeError, ValueError): + _thought_raw = str(_thought_raw) + thought = Thought(content=_thought_raw) + action = None + observation = None + signal_done = False + + is_repeat_call = False + if "action" in parsed and isinstance(parsed["action"], dict): + tool_name = parsed["action"].get("tool") + tool_args = parsed["action"].get("args", {}) or {} + + if tool_name in (DONE_GATHERING_TOOL, LEGACY_SUBMIT_TOOL): + signal_done = True + state.gathering_exit_reason = ( + tool_args.get("reason", "model_signaled_done") + if isinstance(tool_args, dict) + else "model_signaled_done" + ) + action = Action(tool_call=ToolCall( + name=DONE_GATHERING_TOOL, + args={"reason": str(state.gathering_exit_reason)}, + )) + elif tool_name: + action = Action(tool_call=ToolCall(name=tool_name, args=tool_args)) + + if tool_name in deps.tools: + lkey = _ledger_key(tool_name, tool_args) + cached = state.tool_call_ledger.get(lkey) + if cached is not None: + is_repeat_call = True + observation = Observation(tool_result=ToolResult( + tool_name=tool_name, + args=tool_args, + result=cached["result"], + )) + logger.info(f"Repeat tool call dedup'd: {tool_name}({tool_args})") + else: + try: + result = await deps.tools[tool_name](**tool_args) + observation = Observation(tool_result=ToolResult( + tool_name=tool_name, + args=tool_args, + result=result, + )) + state.tool_call_ledger[lkey] = { + "tool_name": tool_name, + "args": tool_args, + "result": result, + } + new_chunks = _extract_chunks(result) + if not new_chunks: + state.consecutive_empty_tool_calls += 1 + else: + state.consecutive_empty_tool_calls = 0 + if deps.store: + await deps.store.add_chunks(state.investigation_id, new_chunks) + except Exception as e: + logger.error(f"Tool failed: {e}") + observation = Observation(tool_result=ToolResult( + tool_name=tool_name, args=tool_args, result=None, error=str(e), + )) + state.consecutive_empty_tool_calls += 1 + else: + observation = Observation(tool_result=ToolResult( + tool_name=tool_name, args=tool_args, result=None, + error=f"Unknown tool '{tool_name}'", + )) + + # Null-action guard + if action is None and not signal_done: + if not state.null_action_reprompted: + state.null_action_reprompted = True + state.messages.append(Message(role="assistant", content=raw_response)) + state.messages.append(Message( + role="user", + content=( + "Your previous reply had no tool call. Reply again " + "with a JSON object containing an `action` — either a " + "real tool call or `done_gathering` if you are done." + ), + )) + # Stay in GATHERING, will re-enter this handler + return state + state.gathering_exit_reason = "model_emitted_thought_only_twice" + logger.warning( + "Step %d: model gave no action twice in a row; exiting gathering", + state.next_step_number, + ) + state.phase = Phase.COMPILING + return state + + state.null_action_reprompted = False + + # Persist the step + kind = "signal" if signal_done else None + step = InvestigationStep(state.next_step_number, thought, action, observation, kind=kind) + state.processed_steps.append(step) + + if deps.store: + await deps.store.add_step( + investigation_id=state.investigation_id, + step_number=state.next_step_number, + thought=thought.content, + action=_asdict(action.tool_call) if action else None, + observation=_asdict(observation.tool_result) if observation else None, + kind=kind, + ) + + if deps.on_step: + await deps.on_step(step) + + state.next_step_number += 1 + + if signal_done: + state.phase = Phase.COMPILING + return state + + state.actions_taken += 1 + state.action_steps_since_reflection += 1 + + # Stall detection + if state.consecutive_empty_tool_calls >= 2: + state.gathering_exit_reason = "stalled_no_new_evidence" + logger.info( + "Exiting gathering early: %d consecutive tool calls returned no new chunks", + state.consecutive_empty_tool_calls, + ) + state.phase = Phase.COMPILING + return state + + state.messages.append(Message(role="assistant", content=raw_response)) + if observation and observation.tool_result: + res = observation.tool_result.result if observation.tool_result.result is not None else {"error": observation.tool_result.error} + prefix = "(repeat call — returning cached result)\n" if is_repeat_call else "" + state.messages.append(Message(role="user", content=f"{prefix}Observation:\n{_compact_observation(res)}")) + + if state.tool_call_ledger: + state.messages[0] = Message( + role="system", + content=_build_system_prompt(deps.known_services) + "\n\n" + _ledger_summary(state.tool_call_ledger), + ) + + except Exception as e: + logger.error(f"Step {state.next_step_number} failed during processing: {e}") + if state.messages and state.messages[-1].role == "assistant": + state.messages.pop() + + # Stay in GATHERING for next iteration + return state + + +async def handle_compiling(state: InvestigationState, deps: LoopDeps) -> InvestigationState: + chunks_obj = await deps.store.get_chunks(state.investigation_id) if deps.store else [] + evidence_chunks = [ + { + "chunk_id": c.chunk_id, + "service": c.service, + "timestamp": str(c.timestamp), + "message": c.message, + } + for c in chunks_obj + ] + + if deps.on_phase_change: + try: + await deps.on_phase_change("compiling") + except Exception: + logger.debug("on_phase_change(compiling) hook raised", exc_info=True) + + recent_thoughts = [ + s.thought.content for s in state.processed_steps[-4:] if s.thought and s.thought.content + ] + + try: + compile_result = await compile_answer( + llm=deps.llm, + query=state.query, + resolved_intent=state.resolved_intent, + evidence=evidence_chunks, + tool_ledger=state.tool_call_ledger, + recent_thoughts=recent_thoughts, + known_services=deps.known_services, + ) + final_answer_dict = compile_result.answer + compile_source = compile_result.source + compile_attempts = compile_result.attempts + floor_adjustments = compile_result.floor_adjustments + except Exception as e: + logger.error("Compiler raised; falling back to deterministic synth: %s", e) + final_answer_dict = synthesize_answer( + query=state.query, + resolved_intent=state.resolved_intent, + evidence=evidence_chunks, + tool_ledger=state.tool_call_ledger, + extra_gaps=[f"compile_answer raised: {e}"], + ) + compile_source = "deterministic_exception" + compile_attempts = 0 + floor_adjustments = [] + + compile_thought = ( + f"Compiled answer from {len(evidence_chunks)} evidence chunks across " + f"{len({c.get('service') for c in evidence_chunks if c.get('service')})} services " + f"(source={compile_source}, attempts={compile_attempts}, " + f"exit_reason={state.gathering_exit_reason})" + ) + compile_step = InvestigationStep( + state.next_step_number, + Thought(compile_thought), + None, + None, + kind="compile", + ) + state.processed_steps.append(compile_step) + if deps.store: + await deps.store.add_step( + investigation_id=state.investigation_id, + step_number=state.next_step_number, + thought=compile_thought, + action=None, + observation=None, + kind="compile", + ) + if deps.on_step: + await deps.on_step(compile_step) + + if deps.store: + await deps.store.finalize(state.investigation_id, json.dumps(final_answer_dict)) + + state._compile_result = { + "final_answer_dict": final_answer_dict, + "compile_source": compile_source, + "compile_attempts": compile_attempts, + "floor_adjustments": floor_adjustments, + "evidence_chunks": evidence_chunks, + "chunks_obj": chunks_obj, + } + + if deps.on_phase_change: + try: + await deps.on_phase_change("done") + except Exception: + logger.debug("on_phase_change(done) hook raised", exc_info=True) + + state.phase = Phase.DONE + return state + + +# ─── Phase dispatch table ──────────────────────────────────────────────────── + +HANDLERS: dict[Phase, Callable] = { + Phase.RESOLVING: handle_resolving, + Phase.SWEEPING: handle_sweeping, + Phase.GATHERING: handle_gathering, + Phase.REFLECTING: handle_reflecting, + Phase.COMPILING: handle_compiling, + Phase.WAITING_CLARIFICATION: handle_waiting_clarification, +} + + +# ─── Main class (preserves public interface) ───────────────────────────────── class ReactInvestigationLoop: + # Expose compaction constants on the class for tests that reference them. + MAX_OBS_ITEMS = MAX_OBS_ITEMS + MAX_OBS_TEXT_CHARS = MAX_OBS_TEXT_CHARS + MAX_OBS_TOTAL_CHARS = MAX_OBS_TOTAL_CHARS + def __init__( self, llm: LLMProvider, @@ -113,8 +752,6 @@ def __init__( self.tools = tools self.known_services = known_services self.pool = pool - # Action-step budget; reflection turns and null-action re-prompts - # do not consume it. self.max_iterations = max_iterations self.min_iteration_delay = min_iteration_delay self.store = store @@ -124,103 +761,34 @@ def __init__( self.llm_max_calls_per_min = llm_max_calls_per_min self._llm_call_timestamps: list[float] = [] + # Keep these as class/instance methods for backwards compat with tests. @staticmethod def _ledger_key(tool_name: str, args: dict) -> str: - """Stable hash key for a tool call: name + JSON-with-sorted-keys. - Sorted keys means {"a":1,"b":2} and {"b":2,"a":1} dedupe identically.""" - try: - normalized = json.dumps(args or {}, sort_keys=True, default=str) - except (TypeError, ValueError): - normalized = repr(args) - return f"{tool_name}::{normalized}" + return _ledger_key(tool_name, args) @staticmethod def _ledger_summary(ledger: dict[str, dict]) -> str: - """One-line-per-entry summary of every tool call already issued.""" - if not ledger: - return "" - lines = [] - for entry in ledger.values(): - lines.append(f"- {entry['tool_name']}({json.dumps(entry['args'], default=str, sort_keys=True)})") - return "TOOLS ALREADY CALLED (do not repeat with identical args):\n" + "\n".join(lines) + return _ledger_summary(ledger) async def _wait_for_rate_limit(self): - now = time.time() - self._llm_call_timestamps = [t for t in self._llm_call_timestamps if now - t < 60] - while len(self._llm_call_timestamps) >= self.llm_max_calls_per_min: - wait_time = 60 - (now - self._llm_call_timestamps[0]) + 1 - logger.warning(f"Rate limit: Waiting {wait_time:.1f}s...") - await asyncio.sleep(wait_time) - now = time.time() - self._llm_call_timestamps = [t for t in self._llm_call_timestamps if now - t < 60] - self._llm_call_timestamps.append(now) - - # Caps for what a single observation may contribute to the LLM - # conversation. Tool results are re-sent on EVERY subsequent turn, so an - # uncapped scan_window result multiplies across the whole loop — this is - # the main driver of token-per-minute 429s. The full, untruncated result - # is still persisted to the DB and the tool ledger. - MAX_OBS_ITEMS = 10 - MAX_OBS_TEXT_CHARS = 300 - MAX_OBS_TOTAL_CHARS = 6000 + deps = LoopDeps( + llm=self.llm, tools=self.tools, known_services=self.known_services, + pool=self.pool, store=self.store, + llm_call_timestamps=self._llm_call_timestamps, + llm_max_calls_per_min=self.llm_max_calls_per_min, + ) + await _wait_for_rate_limit(deps) + self._llm_call_timestamps = deps.llm_call_timestamps @classmethod def _compact_observation(cls, result: Any) -> str: - """Serialize a tool result for the LLM conversation, clipping long - lists and long text fields with explicit markers so the model knows - evidence was elided (and can narrow its next query).""" - - def _walk(node: Any, max_items: int, max_chars: int) -> Any: - if isinstance(node, dict): - return {k: _walk(v, max_items, max_chars) for k, v in node.items()} - if isinstance(node, list): - clipped = [_walk(x, max_items, max_chars) for x in node[:max_items]] - if len(node) > max_items: - clipped.append(f"... {len(node) - max_items} more items truncated") - return clipped - if isinstance(node, str) and len(node) > max_chars: - return node[:max_chars] + "...[truncated]" - return node - - s = json.dumps(_walk(result, cls.MAX_OBS_ITEMS, cls.MAX_OBS_TEXT_CHARS), default=str) - if len(s) > cls.MAX_OBS_TOTAL_CHARS: - # Re-walk with much tighter caps rather than slicing the JSON - # string (which would hand the model malformed JSON). - s = json.dumps(_walk(result, 3, 120), default=str) - if len(s) > cls.MAX_OBS_TOTAL_CHARS: - # Pathological shape (e.g. hundreds of keys) — wrap a hard slice - # inside a fresh JSON object so the payload stays parseable. - s = json.dumps({ - "note": "observation too large; showing a truncated excerpt", - "excerpt": s[:cls.MAX_OBS_TOTAL_CHARS], - }) - return s + return _compact_observation(result) def _extract_chunks(self, tool_result: Any) -> list[dict]: - """Collect every dict-with-chunk_id from a tool result. Walks nested - list-valued fields (e.g. scan_window's `logs` and `pre_context_logs`) - so chunk-bearing observations get persisted as evidence regardless of - how the tool wraps its output.""" - chunks: list[dict] = [] - seen: set[str] = set() - - def _maybe_append(item: Any) -> None: - if isinstance(item, dict): - cid = item.get("chunk_id") - if cid and cid not in seen: - seen.add(cid) - chunks.append(item) - - if isinstance(tool_result, list): - for item in tool_result: - _maybe_append(item) - elif isinstance(tool_result, dict): - _maybe_append(tool_result) - for value in tool_result.values(): - if isinstance(value, list): - for item in value: - _maybe_append(item) - return chunks + return _extract_chunks(tool_result) + + def _build_system_prompt(self) -> str: + return _build_system_prompt(self.known_services) async def investigate( self, @@ -235,12 +803,9 @@ async def investigate( self.known_services = known_services start_time = time.time() - - # --- PERSISTENCE: Resume or Create --- + + # --- Persistence: Resume or Create --- investigation_obj = None - existing_steps = [] - evidence_chunks = [] - if self.store: if investigation_id: investigation_obj = await self.store.get_by_id(investigation_id) @@ -248,660 +813,181 @@ async def investigate( investigation_obj = await self.store.get_or_create(query) else: investigation_obj = await self.store.create(query) - existing_steps = await self.store.get_steps(investigation_obj.id) - chunks_obj = await self.store.get_chunks(investigation_obj.id) - evidence_chunks = [ - { - "chunk_id": c.chunk_id, - "service": c.service, - "timestamp": c.timestamp, - "text": c.message - } for c in chunks_obj - ] - - # --- 1. INTENT RESOLUTION --- - resolved_intent = None - - last_step = existing_steps[-1] if existing_steps else None - clarified_query = query - if last_step and last_step.action and (last_step.action.get("name") == "ask_user" or last_step.action.get("tool") == "ask_user"): - if last_step.observation and last_step.observation.get("result"): - reply = last_step.observation["result"].get("reply", "") - clarified_query = f"{query} (User Clarification: {reply})" - logger.info(f"Resuming with clarified query: {clarified_query}") - - post_clarification = clarified_query != query - - if not existing_steps or post_clarification: - now = _dh.now() - resolution = resolve_intent(clarified_query, self.known_services, now) - logger.info(f"Intent Resolution for '{clarified_query}': {resolution}") - - if isinstance(resolution, ClarificationNeeded): - if post_clarification: - # Single-round guarantee: already clarified once — commit with widest defaults - logger.warning( - f"Still ambiguous after clarification ({resolution.missing_dims}). " - "Proceeding with widest-default window (last 24h)." - ) - resolution = ResolvedIntent( - time_from=_dh.ago(days=1), - time_to=_dh.now(), - services=[], - symptoms=[], - assumed=[ - f"time could not be resolved after clarification — defaulting to last 24 hours", - f"clarification was: {clarified_query}", - ], - ) - else: - thought_text = "I need to clarify the request before I can proceed." - action_data = {"name": "ask_user", "args": {"question": resolution.question}} - - if self.store: - # Only add if it's not already the same question (to avoid loops) - if not last_step or last_step.action.get("args", {}).get("question") != resolution.question: - await self.store.add_step( - investigation_id=investigation_obj.id, - step_number=len(existing_steps) + 1, - thought=thought_text, - action=action_data - ) - await self.store.set_awaiting_clarification(investigation_obj.id, resolution.question) - - step = InvestigationStep(len(existing_steps) + 1, Thought(thought_text), Action(ToolCall(name="ask_user", args=action_data["args"]))) - if on_step: await on_step(step) - - return InvestigationResult( - id=str(investigation_obj.id), - query=query, - steps=[step], - answer="Awaiting clarification...", - evidence_chunk_ids=[], - confidence="low", - duration_seconds=time.time() - start_time - ) - resolved_intent = resolution - - # --- 2. AUTO SWEEP --- - # `tool_call_ledger` dedupes identical tool invocations across - # iterations: hash → {tool_name, args, result}. The summary is appended to - # the system message each turn so the LLM knows what's already been tried. - tool_call_ledger: dict[str, dict] = {} - - # Signal the gathering phase has begun (consumed by SSE stream). - if on_phase_change: + # --- Try to resume from serialized state --- + state = None + if investigation_obj and hasattr(investigation_obj, 'state_json') and investigation_obj.state_json: try: - await on_phase_change("gathering") - except Exception: - logger.debug("on_phase_change(gathering) hook raised", exc_info=True) - - messages = [ - Message(role="system", content=self._build_system_prompt()), - Message(role="user", content=query) - ] - - if resolved_intent and self.pool and (not existing_steps or post_clarification): - if resolved_intent.time_from is None: - # No time window — auto_sweep would query the entire corpus and - # drown the LLM in unrelated noise. Inject an entity/service-keyed - # priming message instead and let the LLM pick its first tool. - priming_lines = ["RAG CONTEXT:"] - priming_lines.append( - f"- entities mentioned: {resolved_intent.entities or 'none'}" - ) - priming_lines.append( - f"- services mentioned: {resolved_intent.services or 'none'}" - ) - priming_lines.append( - "- no time window — use find_logs_by_id (for entities) " - "or search_logs with null time_from/time_to (for services)." - ) - if resolved_intent.assumed: - priming_lines.append("") - priming_lines.append("ASSUMPTIONS:") - priming_lines.extend(f"- {a}" for a in resolved_intent.assumed) - messages.append(Message(role="user", content="\n".join(priming_lines))) - else: - sweep_results = await auto_sweep( - pool=self.pool, - time_from=resolved_intent.time_from, - time_to=resolved_intent.time_to, - exclude_services=[] - ) - - sweep_msg = f"SWEEP CONTEXT:\n{self._compact_observation(sweep_results)}\n\n" - if resolved_intent.assumed: - sweep_msg += "ASSUMPTIONS:\n" + "\n".join(f"- {a}" for a in resolved_intent.assumed) + "\n" - - messages.append(Message(role="user", content=sweep_msg)) - - processed_steps = [] - start_at_iteration = 0 - - for s in existing_steps: - thought = Thought(content=s.thought) - action = None - observation = None - - if s.action: - action = Action(tool_call=ToolCall(name=s.action.get("name") or s.action.get("tool"), args=s.action["args"])) - if s.observation: - observation = Observation(tool_result=ToolResult( - tool_name=s.observation.get("tool_name", "unknown"), - args=s.observation.get("args", {}), - result=s.observation.get("result"), - error=s.observation.get("error") - )) - - step = InvestigationStep( - s.step_number, - thought, - action, - observation, - s.created_at, - kind=getattr(s, "kind", None), - ) - processed_steps.append(step) - - llm_payload = {"thought": s.thought} - if action: - llm_payload["action"] = {"tool": action.tool_call.name, "args": action.tool_call.args} - - messages.append(Message(role="assistant", content=json.dumps(llm_payload))) - if observation: - res = observation.tool_result.result if observation.tool_result.result is not None else {"error": observation.tool_result.error} - messages.append(Message(role="user", content=f"Observation:\n{self._compact_observation(res)}")) - - # Seed the ledger from replayed steps so dedupe survives resume. - if action and observation and observation.tool_result.result is not None: - ledger_key = self._ledger_key(action.tool_call.name, action.tool_call.args) - tool_call_ledger.setdefault(ledger_key, { - "tool_name": action.tool_call.name, - "args": action.tool_call.args, - "result": observation.tool_result.result, - }) - - start_at_iteration = max(start_at_iteration, s.step_number) - - # --- Action / reflection counters ----------------------------------- - # `actions_taken` is the only thing that consumes `max_iterations`. - # Reflection turns and null-action re-prompts do NOT decrement the - # action budget — they live on their own counters. - actions_taken = 0 - reflections_used = 0 - action_steps_since_reflection = 0 - consecutive_empty_tool_calls = 0 - for s in existing_steps: - if getattr(s, "kind", None) == "reflection": - action_steps_since_reflection = 0 - reflections_used += 1 - elif getattr(s, "kind", None) is None and s.action: - action_steps_since_reflection += 1 - - next_step_number = start_at_iteration + 1 - gathering_exit_reason = "max_actions_reached" - null_action_reprompted_this_turn = False - - while actions_taken < self.max_iterations: - if next_step_number > start_at_iteration + 1: - await asyncio.sleep(self.min_iteration_delay) - - # --- Reflection turn ------------------------------------------------ - if ( - self.enable_reflection - and self.reflection_interval > 0 - and reflections_used < self.max_reflections - and action_steps_since_reflection >= self.reflection_interval - ): - messages.append(Message(role="user", content=REFLECTION_PROMPT)) - raw_reflection = None - for _refl_retry in range(3): - try: - await self._wait_for_rate_limit() - raw_reflection = await self.llm.complete(messages) - break - except LLMBadRequestError as e: - # Bad payload/auth — retrying the identical request can't succeed. - logger.error(f"Reflection {next_step_number}: non-retryable LLM error: {e}") - break - except Exception as e: - logger.warning(f"Reflection {next_step_number} attempt {_refl_retry+1}/3 failed: {e}") - if _refl_retry < 2: - delay = 15 * (2 ** _refl_retry) - if isinstance(e, LLMRateLimitError) and e.retry_after: - delay = e.retry_after + 1 - await asyncio.sleep(delay) - - if raw_reflection is None: - logger.error(f"Reflection {next_step_number}: LLM call failed after 3 retries, skipping") - if messages and messages[-1].content == REFLECTION_PROMPT: - messages.pop() - action_steps_since_reflection = 0 - reflections_used += 1 - continue - - if self.store and investigation_obj: - await self.store.increment_llm_calls(investigation_obj.id) - - try: - parsed_reflection = parse_llm_response(raw_reflection) - reflection_thought = parsed_reflection.get("thought", "") or raw_reflection - except Exception: - reflection_thought = raw_reflection - - # The reflection prompt invites rich structured reasoning, so the - # LLM sometimes emits `thought` as a dict/list. The DB column is - # TEXT — coerce to a JSON string in that case. - if not isinstance(reflection_thought, str): - try: - reflection_thought = json.dumps(reflection_thought, default=str) - except (TypeError, ValueError): - reflection_thought = str(reflection_thought) - - reflection_step = InvestigationStep( - next_step_number, - Thought(reflection_thought), - None, - None, - kind="reflection", + state = InvestigationState.from_json(json.dumps(investigation_obj.state_json)) + logger.info(f"Resumed investigation {investigation_obj.id} from phase {state.phase.value}") + except Exception as e: + logger.warning(f"Failed to restore state from state_json, falling back: {e}") + state = None + + # --- Fall back to legacy resume (replay from steps) --- + if state is None: + existing_steps = await self.store.get_steps(investigation_obj.id) if self.store else [] + + post_clarification = False + if existing_steps: + last_step = existing_steps[-1] + if last_step.action and ( + last_step.action.get("name") == "ask_user" + or last_step.action.get("tool") == "ask_user" + ): + if last_step.observation and last_step.observation.get("result"): + post_clarification = True + + inv_id = investigation_obj.id if investigation_obj else UUID("00000000-0000-0000-0000-000000000000") + + if existing_steps and not post_clarification: + # Reconstruct state from existing steps (legacy resume path) + state = InvestigationState( + phase=Phase.GATHERING, + investigation_id=inv_id, + query=query, + messages=[ + Message(role="system", content=_build_system_prompt(self.known_services)), + Message(role="user", content=query), + ], + tool_call_ledger={}, ) - processed_steps.append(reflection_step) - - if self.store and investigation_obj: - await self.store.add_step( - investigation_id=investigation_obj.id, - step_number=next_step_number, - thought=reflection_thought, - action=None, - observation=None, - kind="reflection", - ) - - if on_step: - await on_step(reflection_step) - - # Keep the reflection in the rolling conversation so the next - # turn's action is anchored to the re-plan. - messages.append(Message(role="assistant", content=raw_reflection)) - - next_step_number += 1 - action_steps_since_reflection = 0 - reflections_used += 1 - continue - - # --- Graduated finalize prompts --------------------------------- - # As the action budget runs low, escalate toward exit. The loop - # never produces the final answer itself — `done_gathering` (or - # exhausting the budget) hands off to the compiler. - actions_remaining = self.max_iterations - actions_taken - if actions_remaining == 2: - messages.append(Message( - role="user", - content=( - "You have 2 actions left before gathering ends. Only " - "issue a tool call if it would materially change the " - "final answer; otherwise call `done_gathering`." - ), - )) - elif actions_remaining == 1: - messages.append(Message( - role="user", - content=( - "Last action. The next turn should either call " - "`done_gathering` or one final tool call. After this " - "turn the gathering phase ends." - ), - )) - # --- Ensure last message is user/system before calling LLM ----- - if messages and messages[-1].role == "assistant": - messages.append(Message(role="user", content="Continue gathering evidence.")) + for s in existing_steps: + thought = Thought(content=s.thought) + action = None + observation = None - # --- LLM call with retry ----------------------------------------------- - raw_response = None - for _llm_retry in range(3): - try: - await self._wait_for_rate_limit() - raw_response = await self.llm.complete(messages) - break - except LLMBadRequestError as e: - # Bad payload/auth — retrying the identical request can't succeed. - logger.error(f"Step {next_step_number}: non-retryable LLM error: {e}") - break - except Exception as e: - logger.warning(f"Step {next_step_number} LLM call attempt {_llm_retry+1}/3 failed: {e}") - if _llm_retry < 2: - delay = 15 * (2 ** _llm_retry) - if isinstance(e, LLMRateLimitError) and e.retry_after: - delay = e.retry_after + 1 - await asyncio.sleep(delay) - - if raw_response is None: - logger.error(f"Step {next_step_number}: LLM call failed after 3 retries, ending gathering") - gathering_exit_reason = "llm_call_failed_repeatedly" - break - - if self.store and investigation_obj: - await self.store.increment_llm_calls(investigation_obj.id) - - try: - parsed = parse_llm_response(raw_response) - - _thought_raw = parsed.get("thought", "") - if not isinstance(_thought_raw, str): - try: - _thought_raw = json.dumps(_thought_raw, default=str) - except (TypeError, ValueError): - _thought_raw = str(_thought_raw) - thought = Thought(content=_thought_raw) - action = None - observation = None - signal_done = False - - is_repeat_call = False - if "action" in parsed and isinstance(parsed["action"], dict): - tool_name = parsed["action"].get("tool") - tool_args = parsed["action"].get("args", {}) or {} - - if tool_name in (DONE_GATHERING_TOOL, LEGACY_SUBMIT_TOOL): - signal_done = True - gathering_exit_reason = ( - tool_args.get("reason", "model_signaled_done") - if isinstance(tool_args, dict) - else "model_signaled_done" - ) + if s.action: action = Action(tool_call=ToolCall( - name=DONE_GATHERING_TOOL, - args={"reason": str(gathering_exit_reason)}, + name=s.action.get("name") or s.action.get("tool"), + args=s.action["args"], )) - elif tool_name: - action = Action(tool_call=ToolCall(name=tool_name, args=tool_args)) - - if tool_name in self.tools: - ledger_key = self._ledger_key(tool_name, tool_args) - cached = tool_call_ledger.get(ledger_key) - if cached is not None: - is_repeat_call = True - observation = Observation(tool_result=ToolResult( - tool_name=tool_name, - args=tool_args, - result=cached["result"], - )) - logger.info(f"Repeat tool call dedup'd: {tool_name}({tool_args})") - else: - try: - result = await self.tools[tool_name](**tool_args) - observation = Observation(tool_result=ToolResult( - tool_name=tool_name, - args=tool_args, - result=result - )) - tool_call_ledger[ledger_key] = { - "tool_name": tool_name, - "args": tool_args, - "result": result, - } - new_chunks = self._extract_chunks(result) - if not new_chunks: - consecutive_empty_tool_calls += 1 - else: - consecutive_empty_tool_calls = 0 - if self.store: - await self.store.add_chunks(investigation_obj.id, new_chunks) - except Exception as e: - logger.error(f"Tool failed: {e}") - observation = Observation(tool_result=ToolResult( - tool_name=tool_name, args=tool_args, result=None, error=str(e) - )) - consecutive_empty_tool_calls += 1 - else: - observation = Observation(tool_result=ToolResult( - tool_name=tool_name, args=tool_args, result=None, error=f"Unknown tool '{tool_name}'" - )) - - # --- Null-action guard --------------------------------------- - # If the model produced neither a tool call nor a done signal, - # give it ONE chance to recover without spending an action. - if action is None and not signal_done: - if not null_action_reprompted_this_turn: - null_action_reprompted_this_turn = True - messages.append(Message(role="assistant", content=raw_response)) - messages.append(Message( - role="user", - content=( - "Your previous reply had no tool call. Reply again " - "with a JSON object containing an `action` — either a " - "real tool call or `done_gathering` if you are done." - ), + if s.observation: + observation = Observation(tool_result=ToolResult( + tool_name=s.observation.get("tool_name", "unknown"), + args=s.observation.get("args", {}), + result=s.observation.get("result"), + error=s.observation.get("error"), )) - continue - # Already re-prompted once this turn; force exit so the - # compiler can still produce an answer. - gathering_exit_reason = "model_emitted_thought_only_twice" - logger.warning( - "Step %d: model gave no action twice in a row; exiting gathering", - next_step_number, - ) - break - - null_action_reprompted_this_turn = False - - # --- Persist the step (signal or real action) ---------------- - kind = "signal" if signal_done else None - step = InvestigationStep(next_step_number, thought, action, observation, kind=kind) - processed_steps.append(step) - - if self.store: - await self.store.add_step( - investigation_id=investigation_obj.id, - step_number=next_step_number, - thought=thought.content, - action=asdict(action.tool_call) if action else None, - observation=asdict(observation.tool_result) if observation else None, - kind=kind, - ) - - if on_step: - await on_step(step) - - next_step_number += 1 - if signal_done: - break + step = InvestigationStep( + s.step_number, thought, action, observation, + s.created_at, + kind=getattr(s, "kind", None), + ) + state.processed_steps.append(step) + + llm_payload = {"thought": s.thought} + if action: + llm_payload["action"] = {"tool": action.tool_call.name, "args": action.tool_call.args} + state.messages.append(Message(role="assistant", content=json.dumps(llm_payload))) + if observation: + res = observation.tool_result.result if observation.tool_result.result is not None else {"error": observation.tool_result.error} + state.messages.append(Message(role="user", content=f"Observation:\n{_compact_observation(res)}")) + + if action and observation and observation.tool_result.result is not None: + lkey = _ledger_key(action.tool_call.name, action.tool_call.args) + state.tool_call_ledger.setdefault(lkey, { + "tool_name": action.tool_call.name, + "args": action.tool_call.args, + "result": observation.tool_result.result, + }) + + state.next_step_number = max(state.next_step_number, s.step_number + 1) + + # Reconstruct counters + for s in existing_steps: + if getattr(s, "kind", None) == "reflection": + state.action_steps_since_reflection = 0 + state.reflections_used += 1 + elif getattr(s, "kind", None) is None and s.action: + state.action_steps_since_reflection += 1 + else: + # Fresh investigation or post-clarification + state = InvestigationState( + phase=Phase.RESOLVING, + investigation_id=inv_id, + query=query, + messages=[], + tool_call_ledger={}, + post_clarification=post_clarification, + ) - # Real action steps advance the budget and the reflection counter. - actions_taken += 1 - action_steps_since_reflection += 1 + # --- Build LoopDeps --- + deps = LoopDeps( + llm=self.llm, + tools=self.tools, + known_services=self.known_services, + pool=self.pool, + store=self.store, + max_iterations=self.max_iterations, + min_iteration_delay=self.min_iteration_delay, + enable_reflection=self.enable_reflection, + reflection_interval=self.reflection_interval, + max_reflections=self.max_reflections, + llm_max_calls_per_min=self.llm_max_calls_per_min, + on_step=on_step, + on_phase_change=on_phase_change, + llm_call_timestamps=self._llm_call_timestamps, + ) - # Stall detection: two empty tool calls in a row exits gathering. - if consecutive_empty_tool_calls >= 2: - gathering_exit_reason = "stalled_no_new_evidence" - logger.info( - "Exiting gathering early: %d consecutive tool calls returned no new chunks", - consecutive_empty_tool_calls, - ) - break - - messages.append(Message(role="assistant", content=raw_response)) - if observation and observation.tool_result: - res = observation.tool_result.result if observation.tool_result.result is not None else {"error": observation.tool_result.error} - prefix = "(repeat call — returning cached result)\n" if is_repeat_call else "" - messages.append(Message(role="user", content=f"{prefix}Observation:\n{self._compact_observation(res)}")) - - if tool_call_ledger: - messages[0] = Message( - role="system", - content=self._build_system_prompt() + "\n\n" + self._ledger_summary(tool_call_ledger), - ) + # --- Dispatch loop --- + while state.phase not in (Phase.DONE, Phase.WAITING_CLARIFICATION): + handler = HANDLERS[state.phase] + state = await handler(state, deps) - except Exception as e: - logger.error(f"Step {next_step_number} failed during processing: {e}") - if messages and messages[-1].role == "assistant": - messages.pop() - continue - - # --- Compile phase -------------------------------------------------- - # Gathering is done. Hand off to the compiler, which runs a single, - # focused LLM call against the evidence we collected and produces a - # validated InvestigationAnswer. The compiler internally falls back - # to a deterministic synthesis if its own LLM call fails twice. - chunks_obj = await self.store.get_chunks(investigation_obj.id) if self.store else [] - evidence_chunks = [ - { - "chunk_id": c.chunk_id, - "service": c.service, - "timestamp": str(c.timestamp), - "message": c.message, - } - for c in chunks_obj - ] - - if on_phase_change: - try: - await on_phase_change("compiling") - except Exception: - logger.debug("on_phase_change(compiling) hook raised", exc_info=True) + # Persist state snapshot after each transition + if self.store and investigation_obj: + try: + investigation_obj.state_json = json.loads(state.to_json()) + self.store.session.add(investigation_obj) + await self.store.session.commit() + except Exception as e: + logger.debug(f"Failed to persist state_json: {e}") - recent_thoughts = [ - s.thought.content for s in processed_steps[-4:] if s.thought and s.thought.content - ] + self._llm_call_timestamps = deps.llm_call_timestamps - try: - compile_result = await compile_answer( - llm=self.llm, - query=query, - resolved_intent=resolved_intent, - evidence=evidence_chunks, - tool_ledger=tool_call_ledger, - recent_thoughts=recent_thoughts, - known_services=self.known_services, - ) - final_answer_dict = compile_result.answer - compile_source = compile_result.source - compile_attempts = compile_result.attempts - floor_adjustments = compile_result.floor_adjustments - except Exception as e: - logger.error("Compiler raised; falling back to deterministic synth: %s", e) - final_answer_dict = synthesize_answer( + # --- Return result --- + if state.phase == Phase.WAITING_CLARIFICATION: + return InvestigationResult( + id=str(state.investigation_id), query=query, - resolved_intent=resolved_intent, - evidence=evidence_chunks, - tool_ledger=tool_call_ledger, - extra_gaps=[f"compile_answer raised: {e}"], + steps=state.processed_steps, + answer="Awaiting clarification...", + evidence_chunk_ids=[], + confidence="low", + duration_seconds=time.time() - start_time, ) - compile_source = "deterministic_exception" - compile_attempts = 0 - floor_adjustments = [] - - # Persist the compile step so the trace shows the phase boundary. - compile_thought = ( - f"Compiled answer from {len(evidence_chunks)} evidence chunks across " - f"{len({c.get('service') for c in evidence_chunks if c.get('service')})} services " - f"(source={compile_source}, attempts={compile_attempts}, " - f"exit_reason={gathering_exit_reason})" - ) - compile_step = InvestigationStep( - next_step_number, - Thought(compile_thought), - None, - None, - kind="compile", - ) - processed_steps.append(compile_step) - if self.store: - await self.store.add_step( - investigation_id=investigation_obj.id, - step_number=next_step_number, - thought=compile_thought, - action=None, - observation=None, - kind="compile", - ) - if on_step: - await on_step(compile_step) - if self.store and investigation_obj: - await self.store.finalize(investigation_obj.id, json.dumps(final_answer_dict)) + cr = getattr(state, "_compile_result", {}) + final_answer_dict = cr.get("final_answer_dict", {}) + chunks_obj = cr.get("chunks_obj", []) stats = { - "iterations_used": actions_taken, - "reflections_used": reflections_used, - "chunks_gathered": len(evidence_chunks), - "tools_called": sorted({e["tool_name"] for e in tool_call_ledger.values()}), - "compile_source": compile_source, - "compile_attempts": compile_attempts, - "floor_adjustments": floor_adjustments, - "gathering_exit_reason": gathering_exit_reason, + "iterations_used": state.actions_taken, + "reflections_used": state.reflections_used, + "chunks_gathered": len(cr.get("evidence_chunks", [])), + "tools_called": sorted({e["tool_name"] for e in state.tool_call_ledger.values()}), + "compile_source": cr.get("compile_source", "unknown"), + "compile_attempts": cr.get("compile_attempts", 0), + "floor_adjustments": cr.get("floor_adjustments", []), + "gathering_exit_reason": state.gathering_exit_reason, } - if on_phase_change: - try: - await on_phase_change("done") - except Exception: - logger.debug("on_phase_change(done) hook raised", exc_info=True) - return InvestigationResult( - id=str(investigation_obj.id) if investigation_obj else "unknown", + id=str(state.investigation_id), query=query, - steps=processed_steps, + steps=state.processed_steps, answer=json.dumps(final_answer_dict, indent=2), - evidence_chunk_ids=[c.chunk_id for c in chunks_obj] if self.store else [], + evidence_chunk_ids=[c.chunk_id for c in chunks_obj], confidence=final_answer_dict.get("confidence", "low"), duration_seconds=time.time() - start_time, - evidence=evidence_chunks, + evidence=cr.get("evidence_chunks", []), stats=stats, ) - def _build_system_prompt(self) -> str: - # Don't advertise `submit_answer` in the schema — this loop only - # gathers; the compiler produces the final answer. - gathering_tools = {k: v for k, v in TOOL_SCHEMAS.items() if k != LEGACY_SUBMIT_TOOL} - return f"""You are a senior SRE gathering evidence about an incident. -Postgres is the source of truth for this investigation. - -KNOWN SERVICES: {self.known_services} -TOOLS: {json.dumps(gathering_tools, indent=2)} - -YOUR ROLE: You are an EVIDENCE GATHERER. A separate "compile" step will turn -the evidence you collect into the final structured answer. You do NOT produce -that answer yourself. - -RESPONSE FORMAT (use exactly this shape on every turn — no exceptions): -{{ "thought": "...", "action": {{ "tool": "", "args": {{...}} }} }} - -When you believe you have gathered enough evidence (or that further -investigation will not change the answer), signal exit by calling -`done_gathering`: -{{ "thought": "...", "action": {{ "tool": "done_gathering", "args": {{ "reason": "..." }} }} }} - -GATHERING PRINCIPLES: -1. ENTITY FIRST. If the query mentions a literal identifier (UUID, W3C trace - or span id, ULID, a prefixed ID like ch_xxx / pi_xxx / cus_xxx, an AWS - resource id like i-0abc…, a git SHA, or any other unique token), call - `find_logs_by_id` on the FIRST action turn. Semantic + FTS retrieval - will often miss specific tokens because the English tokenizer fragments - them and embeddings don't preserve literal token identity. -2. ALWAYS correlate logs cross-service. `scan_window` is usually the right - first call when investigating a TIME WINDOW (no entity in play) — it - returns ERRORS plus pre-context for each service that emitted them. -3. Don't repeat tool calls with identical arguments — the dispatcher - dedupes them, but you still waste a turn. -4. If a tool call returns nothing useful, vary the arguments (different - service, wider window, different level filter) before giving up on - that line of inquiry. -5. If two consecutive tool calls return no new evidence, call - `done_gathering` — there is no value in spamming the dispatcher. -6. When NO time window was provided, do NOT pass time_from/time_to to - `search_logs`/`scan_window`; rely on `find_logs_by_id` and unbounded - searches keyed off the entity/service signal you do have. -7. Do NOT emit a "Final Answer:" prefix or fill in any - InvestigationAnswer schema. The compile step will produce that. - -Current UTC: {_dh.to_iso(_dh.now())} -""" -def asdict(obj): - from dataclasses import asdict as _asdict - return _asdict(obj) +def _asdict(obj): + from dataclasses import asdict as _asdict_fn + return _asdict_fn(obj) diff --git a/repi/investigation/state.py b/repi/investigation/state.py new file mode 100644 index 0000000..5e15b68 --- /dev/null +++ b/repi/investigation/state.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any, Callable, Awaitable, Optional +from uuid import UUID + +import asyncpg + +from repi.llm.provider import LLMProvider, Message +from repi.investigation.store import InvestigationStore +from repi.investigation.tools import ToolCall, ToolResult +from repi.intent.resolver import ResolvedIntent + + +class Phase(str, Enum): + RESOLVING = "resolving" + SWEEPING = "sweeping" + GATHERING = "gathering" + REFLECTING = "reflecting" + COMPILING = "compiling" + WAITING_CLARIFICATION = "waiting_clarification" + DONE = "done" + + +@dataclass +class Thought: + content: str + + +@dataclass +class Action: + tool_call: ToolCall + + +@dataclass +class Observation: + tool_result: ToolResult + + +@dataclass +class InvestigationStep: + step_number: int + thought: Thought + action: Optional[Action] = None + observation: Optional[Observation] = None + timestamp: datetime = field(default_factory=datetime.utcnow) + kind: Optional[str] = None + + +@dataclass +class InvestigationResult: + id: str + query: str + steps: list[InvestigationStep] + answer: str + evidence_chunk_ids: list[str] + confidence: str + duration_seconds: float + evidence: list[dict] = field(default_factory=list) + stats: dict = field(default_factory=dict) + + +@dataclass +class InvestigationState: + phase: Phase + investigation_id: UUID + query: str + messages: list[Message] + tool_call_ledger: dict[str, dict] + actions_taken: int = 0 + reflections_used: int = 0 + action_steps_since_reflection: int = 0 + consecutive_empty_tool_calls: int = 0 + processed_steps: list[InvestigationStep] = field(default_factory=list) + evidence_chunk_ids: set[str] = field(default_factory=set) + resolved_intent: ResolvedIntent | None = None + gathering_exit_reason: str = "max_actions_reached" + pending_question: str | None = None + next_step_number: int = 1 + null_action_reprompted: bool = False + post_clarification: bool = False + + def to_json(self) -> str: + return json.dumps(self._to_dict(), default=str) + + def _to_dict(self) -> dict: + return { + "phase": self.phase.value, + "investigation_id": str(self.investigation_id), + "query": self.query, + "messages": [{"role": m.role, "content": m.content} for m in self.messages], + "tool_call_ledger": self.tool_call_ledger, + "actions_taken": self.actions_taken, + "reflections_used": self.reflections_used, + "action_steps_since_reflection": self.action_steps_since_reflection, + "consecutive_empty_tool_calls": self.consecutive_empty_tool_calls, + "evidence_chunk_ids": sorted(self.evidence_chunk_ids), + "resolved_intent": _intent_to_dict(self.resolved_intent) if self.resolved_intent else None, + "gathering_exit_reason": self.gathering_exit_reason, + "pending_question": self.pending_question, + "next_step_number": self.next_step_number, + "null_action_reprompted": self.null_action_reprompted, + "post_clarification": self.post_clarification, + } + + @classmethod + def from_json(cls, raw: str) -> InvestigationState: + d = json.loads(raw) + return cls( + phase=Phase(d["phase"]), + investigation_id=UUID(d["investigation_id"]), + query=d["query"], + messages=[Message(role=m["role"], content=m["content"]) for m in d["messages"]], + tool_call_ledger=d.get("tool_call_ledger", {}), + actions_taken=d.get("actions_taken", 0), + reflections_used=d.get("reflections_used", 0), + action_steps_since_reflection=d.get("action_steps_since_reflection", 0), + consecutive_empty_tool_calls=d.get("consecutive_empty_tool_calls", 0), + evidence_chunk_ids=set(d.get("evidence_chunk_ids", [])), + resolved_intent=_intent_from_dict(d["resolved_intent"]) if d.get("resolved_intent") else None, + gathering_exit_reason=d.get("gathering_exit_reason", "max_actions_reached"), + pending_question=d.get("pending_question"), + next_step_number=d.get("next_step_number", 1), + null_action_reprompted=d.get("null_action_reprompted", False), + post_clarification=d.get("post_clarification", False), + ) + + +@dataclass +class LoopDeps: + llm: LLMProvider + tools: dict[str, Callable] + known_services: list[str] + pool: asyncpg.Pool | None + store: InvestigationStore | None + max_iterations: int = 10 + min_iteration_delay: float = 2.0 + enable_reflection: bool = True + reflection_interval: int = 3 + max_reflections: int = 2 + llm_max_calls_per_min: int = 60 + on_step: Callable[[InvestigationStep], Awaitable[None]] | None = None + on_phase_change: Callable[[str], Awaitable[None]] | None = None + llm_call_timestamps: list[float] = field(default_factory=list) + + +def _intent_to_dict(intent: ResolvedIntent) -> dict: + return { + "time_from": intent.time_from.isoformat() if intent.time_from else None, + "time_to": intent.time_to.isoformat() if intent.time_to else None, + "services": intent.services, + "symptoms": intent.symptoms, + "entities": intent.entities, + "assumed": intent.assumed, + } + + +def _intent_from_dict(d: dict) -> ResolvedIntent: + return ResolvedIntent( + time_from=datetime.fromisoformat(d["time_from"]) if d.get("time_from") else None, + time_to=datetime.fromisoformat(d["time_to"]) if d.get("time_to") else None, + services=d.get("services", []), + symptoms=d.get("symptoms", []), + entities=d.get("entities", []), + assumed=d.get("assumed", []), + ) diff --git a/repi/models/schema.py b/repi/models/schema.py index f33481a..fbeb83c 100644 --- a/repi/models/schema.py +++ b/repi/models/schema.py @@ -139,6 +139,8 @@ class Investigation(SQLModel, table=True): conversation_id: Optional[UUID] = Field(default=None, index=True) # Scopes the investigation's retrieval + tools to one project. project_id: Optional[UUID] = Field(default=None, index=True) + # FSM state snapshot for resume-from-serialization. + state_json: Optional[dict[str, Any]] = Field(default=None, sa_column=Column(JSONB)) class InvestigationStep(SQLModel, table=True): __tablename__ = "investigation_steps" diff --git a/tests/investigation/test_state.py b/tests/investigation/test_state.py new file mode 100644 index 0000000..85f4a6d --- /dev/null +++ b/tests/investigation/test_state.py @@ -0,0 +1,137 @@ +"""Tests for InvestigationState serialization and Phase transitions.""" +from __future__ import annotations + +import json +from uuid import uuid4 + +import pytest + +from repi.investigation.state import ( + Phase, + InvestigationState, + InvestigationStep, + Thought, +) +from repi.intent.resolver import ResolvedIntent +from repi.llm.provider import Message +from repi.investigation.react_loop import HANDLERS + + +class TestStateSerializationRoundTrip: + def test_minimal_state_round_trips(self): + state = InvestigationState( + phase=Phase.GATHERING, + investigation_id=uuid4(), + query="show errors", + messages=[Message(role="user", content="show errors")], + tool_call_ledger={}, + ) + restored = InvestigationState.from_json(state.to_json()) + + assert restored.phase == state.phase + assert restored.investigation_id == state.investigation_id + assert restored.query == state.query + assert len(restored.messages) == 1 + assert restored.messages[0].role == "user" + assert restored.messages[0].content == "show errors" + + def test_full_state_round_trips(self): + inv_id = uuid4() + state = InvestigationState( + phase=Phase.REFLECTING, + investigation_id=inv_id, + query="why did auth fail", + messages=[ + Message(role="system", content="sys"), + Message(role="user", content="q"), + Message(role="assistant", content="a"), + ], + tool_call_ledger={ + "search_logs::{\"query\": \"auth\"}": { + "tool_name": "search_logs", + "args": {"query": "auth"}, + "result": [{"chunk_id": "c1"}], + } + }, + actions_taken=3, + reflections_used=1, + action_steps_since_reflection=2, + consecutive_empty_tool_calls=0, + evidence_chunk_ids={"c1", "c2"}, + resolved_intent=ResolvedIntent( + time_from=None, + time_to=None, + services=["auth-svc"], + symptoms=["timeout"], + entities=["abc-123"], + assumed=["no time given"], + ), + gathering_exit_reason="max_actions_reached", + pending_question=None, + next_step_number=5, + null_action_reprompted=True, + post_clarification=False, + ) + + restored = InvestigationState.from_json(state.to_json()) + + assert restored.phase == Phase.REFLECTING + assert restored.investigation_id == inv_id + assert restored.actions_taken == 3 + assert restored.reflections_used == 1 + assert restored.action_steps_since_reflection == 2 + assert restored.evidence_chunk_ids == {"c1", "c2"} + assert restored.resolved_intent.services == ["auth-svc"] + assert restored.resolved_intent.entities == ["abc-123"] + assert restored.next_step_number == 5 + assert restored.null_action_reprompted is True + assert len(restored.messages) == 3 + assert len(restored.tool_call_ledger) == 1 + + def test_waiting_clarification_round_trips(self): + state = InvestigationState( + phase=Phase.WAITING_CLARIFICATION, + investigation_id=uuid4(), + query="show errors", + messages=[], + tool_call_ledger={}, + pending_question="Which service?", + ) + restored = InvestigationState.from_json(state.to_json()) + + assert restored.phase == Phase.WAITING_CLARIFICATION + assert restored.pending_question == "Which service?" + + def test_resolved_intent_with_timestamps_round_trips(self): + from datetime import datetime + state = InvestigationState( + phase=Phase.SWEEPING, + investigation_id=uuid4(), + query="errors last hour", + messages=[], + tool_call_ledger={}, + resolved_intent=ResolvedIntent( + time_from=datetime(2026, 6, 29, 10, 0, 0), + time_to=datetime(2026, 6, 29, 11, 0, 0), + services=["api"], + ), + ) + restored = InvestigationState.from_json(state.to_json()) + + assert restored.resolved_intent.time_from.year == 2026 + assert restored.resolved_intent.time_to.hour == 11 + + +class TestTransitionTable: + """Every phase in the dispatch table has a handler registered.""" + + def test_all_non_terminal_phases_have_handlers(self): + for phase in Phase: + if phase == Phase.DONE: + assert phase not in HANDLERS + else: + assert phase in HANDLERS, f"Missing handler for {phase}" + + def test_handler_count_matches_non_terminal_phases(self): + non_terminal = [p for p in Phase if p != Phase.DONE] + assert len(HANDLERS) == len(non_terminal) From 288e03686cfc64134184d26c8f3ffa38db0b9cad Mon Sep 17 00:00:00 2001 From: Varun Singh Date: Mon, 29 Jun 2026 22:38:19 +0530 Subject: [PATCH 2/3] feat: unified LLM config, OpenRouter integration, safety guards, multi-model eval - Unify 6 per-provider API key fields into single LLM_API_KEY with backward-compatible migration via model_validator - Replace if/elif factory chain with registry-based provider pattern - Add OpenRouter as OpenAI-compatible provider (reuses OpenAICompatProvider with custom base_url) - Add API safety guards: mask secrets in GET /config, sanitize exceptions, input sanitization for prompt injection, tighten CORS from wildcard to localhost - Add slowapi rate limiting on POST /investigate (10/min) and POST /chat (20/min) - Add --provider, --model, --api-key flags to eval runner for multi-model testing - Add eval/run_multi_model.py orchestrator with --mock dry-run support - Change REFLECTION_INTERVAL default from 3 to 2 for fairer model comparison --- config.example.json | 5 +- docker/config.docker.json | 6 +- eval/ragas_eval.py | 46 ++++---- eval/run_evals.py | 145 ++++++++++++++------------ eval/run_multi_model.py | 173 +++++++++++++++++++++++++++++++ pyproject.toml | 1 + repi/api/__init__.py | 15 ++- repi/api/chat.py | 8 +- repi/api/config.py | 20 +++- repi/api/investigate.py | 20 ++-- repi/api/limiter.py | 4 + repi/cli.py | 31 ++---- repi/core/config.py | 31 ++++-- repi/investigation/react_loop.py | 16 ++- repi/investigation/sanitize.py | 23 ++++ repi/llm/adapters.py | 27 +++-- repi/llm/factory.py | 89 ++++++++-------- tests/test_cli_config.py | 10 +- tests/test_config_isolation.py | 29 +++--- uv.lock | 116 +++++++++++++++++++++ 20 files changed, 599 insertions(+), 216 deletions(-) create mode 100644 eval/run_multi_model.py create mode 100644 repi/api/limiter.py create mode 100644 repi/investigation/sanitize.py diff --git a/config.example.json b/config.example.json index 327ed03..62d7689 100644 --- a/config.example.json +++ b/config.example.json @@ -7,10 +7,7 @@ "ENABLE_REDIS_CACHE": true, "LLM_PROVIDER": "anthropic", "LLM_MODEL": null, - "OPENAI_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY", - "ANTHROPIC_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY", - "MISTRAL_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY", - "GEMINI_API_KEY": "REPLACE_ME_OR_LEAVE_EMPTY", + "LLM_API_KEY": "REPLACE_ME", "OLLAMA_BASE_URL": "http://localhost:11434", "TIME_WINDOW_INITIAL_MINUTES": 10, "TIME_WINDOW_EXPANSIONS": "60,360,1440", diff --git a/docker/config.docker.json b/docker/config.docker.json index edeb54d..3ebeb0d 100644 --- a/docker/config.docker.json +++ b/docker/config.docker.json @@ -5,9 +5,5 @@ "REDIS_URL": "redis://redis:6379", "ENABLE_REDIS_CACHE": true, "LLM_PROVIDER": "mistral", - "LLM_API_KEY": "", - "OPENAI_API_KEY": "", - "ANTHROPIC_API_KEY": "", - "MISTRAL_API_KEY": "", - "GEMINI_API_KEY": "" + "LLM_API_KEY": "" } diff --git a/eval/ragas_eval.py b/eval/ragas_eval.py index b1214ae..bf95dda 100644 --- a/eval/ragas_eval.py +++ b/eval/ragas_eval.py @@ -180,31 +180,31 @@ def _make_ragas_llm(provider: str | None = None): else: providers = ["mistral", "gemini"] + api_key = settings.LLM_API_KEY + if not api_key: + raise RuntimeError( + "No evaluator LLM available. Configure LLM_API_KEY in .repi/config.json" + ) + + base_urls = { + "gemini": "https://generativelanguage.googleapis.com/v1beta/openai/", + "mistral": "https://api.mistral.ai/v1", + } + default_models = { + "gemini": "gemini-2.0-flash", + "mistral": "mistral-small-2506", + } + for p in providers: - if p == "gemini": - api_key = settings.GEMINI_API_KEY or settings.GOOGLE_API_KEY - if not api_key: - continue - client = OpenAI( - api_key=api_key, - base_url="https://generativelanguage.googleapis.com/v1beta/openai/", - ) - print(f" [config] RAGAS evaluator: Gemini (gemini-2.0-flash)") - return llm_factory(model="gemini-2.0-flash", provider="openai", client=client) - - elif p == "mistral": - api_key = settings.MISTRAL_API_KEY - if not api_key: - continue - client = OpenAI( - api_key=api_key, - base_url="https://api.mistral.ai/v1", - ) - print(f" [config] RAGAS evaluator: Mistral (mistral-small-2506)") - return llm_factory(model="mistral-small-2506", provider="openai", client=client) + if p not in base_urls: + continue + client = OpenAI(api_key=api_key, base_url=base_urls[p]) + model = default_models[p] + print(f" [config] RAGAS evaluator: {p} ({model})") + return llm_factory(model=model, provider="openai", client=client) raise RuntimeError( - "No evaluator LLM available. Configure GEMINI_API_KEY or MISTRAL_API_KEY in .repi/config.json" + f"No supported RAGAS evaluator provider in {providers}. Use 'gemini' or 'mistral'." ) @@ -215,7 +215,7 @@ def _make_ragas_embeddings(provider: str | None = None): import warnings warnings.filterwarnings("ignore", message=".*LangchainEmbeddingsWrapper.*deprecated.*") - api_key = settings.MISTRAL_API_KEY + api_key = settings.LLM_API_KEY if not api_key: return None diff --git a/eval/run_evals.py b/eval/run_evals.py index 6715097..befba1d 100644 --- a/eval/run_evals.py +++ b/eval/run_evals.py @@ -6,6 +6,7 @@ uv run python eval/run_evals.py uv run python eval/run_evals.py --dataset dataset_1 uv run python eval/run_evals.py --judge-provider openai --judge-model gpt-4o + uv run python eval/run_evals.py --provider openrouter --model anthropic/claude-sonnet-4-20250514 --api-key sk-... uv run python eval/run_evals.py --no-reflection """ from __future__ import annotations @@ -68,6 +69,15 @@ def _parse_args(argv: list[str]) -> dict: elif argv[i] == "--judge-api-key" and i + 1 < len(argv): args["judge_api_key"] = argv[i + 1] i += 1 + elif argv[i] == "--provider" and i + 1 < len(argv): + args["provider"] = argv[i + 1] + i += 1 + elif argv[i] == "--model" and i + 1 < len(argv): + args["model"] = argv[i + 1] + i += 1 + elif argv[i] == "--api-key" and i + 1 < len(argv): + args["api_key"] = argv[i + 1] + i += 1 elif argv[i] == "--out" and i + 1 < len(argv): args["out_path"] = argv[i + 1] i += 1 @@ -75,79 +85,73 @@ def _parse_args(argv: list[str]) -> dict: return args +_PROVIDER_KEY_ALIASES = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "mistral": "MISTRAL_API_KEY", + "gemini": "GEMINI_API_KEY", +} + + +def _resolve_key_for_provider(provider: str) -> str: + """Look up an API key for a specific provider from raw config.json. + + Checks the legacy per-provider field first (e.g. GEMINI_API_KEY), + then falls back to LLM_API_KEY. This lets the eval harness use a + different provider's key without --judge-api-key when the raw config + still has per-provider keys from before the unification.""" + import json + from repi.core.config import CONFIG_PATH, settings + raw: dict = {} + try: + raw = json.loads(CONFIG_PATH.read_text()) + except Exception: + pass + alias = _PROVIDER_KEY_ALIASES.get(provider) + if alias: + val = raw.get(alias) + if val: + return val + return settings.LLM_API_KEY or "" + + def create_judge(args: dict, mut_provider_name: str) -> LLMJudge: """Create the judge LLM provider. Self-grading (judge provider == model-under-test provider) is disallowed - unless `--judge-provider` is passed explicitly to opt in. The auto path - prefers OpenAI → Anthropic → Gemini, picking the first provider whose - API key is configured AND that differs from the MUT. + Requires `--judge-provider` (and optionally `--judge-api-key`). + Falls back to LLM_API_KEY from config when no explicit key is given. """ - from repi.llm.adapters import ( - OpenAIProvider, AnthropicProvider, MistralProvider, - GeminiProvider, OllamaProvider, - ) + from repi.llm.factory import create_provider judge_provider_name = args.get("judge_provider") + if not judge_provider_name: + raise RuntimeError( + "No --judge-provider specified. Pass --judge-provider " + "--judge-api-key to configure the eval judge." + ) - if judge_provider_name: - # Explicit override — caller takes responsibility for any self-grading. - api_key = args.get("judge_api_key", "") - # If no --judge-api-key, look up the key for this provider from settings. - if not api_key: - from repi.core.config import settings as _s - api_key = { - "openai": _s.OPENAI_API_KEY, - "anthropic": _s.ANTHROPIC_API_KEY, - "mistral": _s.MISTRAL_API_KEY, - "gemini": _s.GEMINI_API_KEY, - }.get(judge_provider_name.lower(), "") or "" - model = args.get("judge_model") - providers = { - "openai": lambda: OpenAIProvider(api_key=api_key, model=model or "gpt-4o"), - "anthropic": lambda: AnthropicProvider(api_key=api_key, model=model or "claude-3-5-sonnet-20240620"), - "mistral": lambda: MistralProvider(api_key=api_key, model=model or "mistral-large-latest"), - "gemini": lambda: GeminiProvider(api_key=api_key, model=model or "gemini-1.5-pro"), - "ollama": lambda: OllamaProvider(model=model or "mistral"), - } - factory = providers.get(judge_provider_name.lower()) - if not factory: - raise ValueError(f"Unknown judge provider: {judge_provider_name}") - if judge_provider_name.lower() == mut_provider_name.lower(): - print( - f" [config] WARNING: --judge-provider matches MUT provider " - f"({mut_provider_name}). Self-grading risk." - ) - judge = LLMJudge(factory()) - judge.provider_name = judge_provider_name.lower() - return judge - - # Auto-pick: find the first provider != MUT that has a key configured. - from repi.core.config import settings - mut = mut_provider_name.lower() - preferences = [ - ("openai", settings.OPENAI_API_KEY, lambda k: OpenAIProvider(api_key=k, model="gpt-4o")), - ("anthropic", settings.ANTHROPIC_API_KEY, lambda k: AnthropicProvider(api_key=k, model="claude-3-5-sonnet-20240620")), - ("gemini", settings.GEMINI_API_KEY, lambda k: GeminiProvider(api_key=k, model="gemini-2.0-flash")), - ("mistral", settings.MISTRAL_API_KEY, lambda k: MistralProvider(api_key=k, model="mistral-large-latest")), - ] - for name, key, factory in preferences: - if name == mut: - continue - if key: - print(f" [config] judge provider auto-selected: {name} (MUT: {mut})") - judge = LLMJudge(factory(key)) - judge.provider_name = name - return judge - - # No alternative provider available — hard fail rather than self-grade. - raise RuntimeError( - f"Judge provider auto-selection failed: model-under-test is '{mut}' and no " - "alternative provider key is configured. Configure a second provider key " - "(OPENAI_API_KEY / ANTHROPIC_API_KEY / GEMINI_API_KEY in .repi/config.json) " - "OR pass --judge-provider --judge-api-key explicitly to opt " - "into self-grading." + api_key = args.get("judge_api_key", "") + if not api_key: + api_key = _resolve_key_for_provider(judge_provider_name.lower()) + + model = args.get("judge_model") + + if judge_provider_name.lower() == mut_provider_name.lower(): + print( + f" [config] WARNING: --judge-provider matches MUT provider " + f"({mut_provider_name}). Self-grading risk." + ) + + judge = LLMJudge( + create_provider( + provider=judge_provider_name, + api_key=api_key or None, + model=model, + ) ) + judge.provider_name = judge_provider_name.lower() + return judge # ─── Runner ────────────────────────────────────────────────────────────────── @@ -328,13 +332,20 @@ async def main(): _s.ENABLE_REFLECTION = False print(" [config] reflection disabled (--no-reflection)") + from repi.core.config import settings as _settings + + if args.get("provider"): + _settings.LLM_PROVIDER = args["provider"] + if args.get("model"): + _settings.LLM_MODEL = args["model"] + if args.get("api_key"): + _settings.LLM_API_KEY = args["api_key"] + dataset_filter = args.get("dataset_filter") container = get_container() + container.refresh_llm() await container.init_db() - # Look up the MUT provider before picking a judge so the auto-selector - # can guarantee judge != MUT. - from repi.core.config import settings as _settings mut_provider = _settings.LLM_PROVIDER judge = create_judge(args, mut_provider_name=mut_provider) diff --git a/eval/run_multi_model.py b/eval/run_multi_model.py new file mode 100644 index 0000000..ce8b7d1 --- /dev/null +++ b/eval/run_multi_model.py @@ -0,0 +1,173 @@ +""" +Multi-model eval orchestrator — runs the eval suite across a list of models +and prints a comparison table. + +Usage: + uv run python eval/run_multi_model.py --judge-provider gemini + uv run python eval/run_multi_model.py --mock + uv run python eval/run_multi_model.py --models models.json --judge-provider gemini +""" +from __future__ import annotations +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).parent.parent + +DEFAULT_MODELS = [ + {"provider": "openrouter", "model": "mistralai/mistral-large-latest"}, + {"provider": "openrouter", "model": "anthropic/claude-sonnet-4-20250514"}, + {"provider": "openrouter", "model": "openai/gpt-4o"}, +] + +MOCK_RESULTS = { + "summary": {"passed": 3, "failed": 1, "errored": 0, "average_score": 0.78}, + "results": [ + {"dataset": "dataset_1_cascading_inventory_migration", "status": "pass", + "aggregate_score": 0.85, "stats": {"iterations_used": 6}}, + {"dataset": "dataset_2_insufficient_logging", "status": "fail", + "aggregate_score": 0.55, "stats": {"iterations_used": 8}}, + {"dataset": "dataset_3_jwt_key_rotation_noise", "status": "pass", + "aggregate_score": 0.90, "stats": {"iterations_used": 4}}, + {"dataset": "dataset_4_discord_gateway_cascade", "status": "pass", + "aggregate_score": 0.82, "stats": {"iterations_used": 7}}, + ], +} + + +def _parse_args(argv: list[str]) -> dict: + args: dict = {} + i = 0 + while i < len(argv): + if argv[i] == "--mock": + args["mock"] = True + elif argv[i] == "--models" and i + 1 < len(argv): + args["models_path"] = argv[i + 1] + i += 1 + elif argv[i] == "--judge-provider" and i + 1 < len(argv): + args["judge_provider"] = argv[i + 1] + i += 1 + elif argv[i] == "--judge-model" and i + 1 < len(argv): + args["judge_model"] = argv[i + 1] + i += 1 + elif argv[i] == "--judge-api-key" and i + 1 < len(argv): + args["judge_api_key"] = argv[i + 1] + i += 1 + elif argv[i] == "--api-key" and i + 1 < len(argv): + args["api_key"] = argv[i + 1] + i += 1 + i += 1 + return args + + +def _load_models(args: dict) -> list[dict]: + path = args.get("models_path") + if path: + return json.loads(Path(path).read_text()) + return DEFAULT_MODELS + + +def _short_name(model_id: str) -> str: + return model_id.rsplit("/", 1)[-1][:28] + + +def _run_eval(model_cfg: dict, args: dict, out_path: Path) -> dict | None: + cmd = [ + sys.executable, str(ROOT / "eval" / "run_evals.py"), + "--provider", model_cfg["provider"], + "--model", model_cfg["model"], + "--out", str(out_path), + ] + if args.get("api_key"): + cmd += ["--api-key", args["api_key"]] + if args.get("judge_provider"): + cmd += ["--judge-provider", args["judge_provider"]] + if args.get("judge_model"): + cmd += ["--judge-model", args["judge_model"]] + if args.get("judge_api_key"): + cmd += ["--judge-api-key", args["judge_api_key"]] + + print(f"\n Running: {' '.join(cmd[:6])}...") + result = subprocess.run(cmd, capture_output=False) + if result.returncode != 0: + print(f" eval returned exit code {result.returncode}") + + if out_path.exists(): + return json.loads(out_path.read_text()) + return None + + +def _print_table(all_results: dict[str, dict]): + datasets = set() + for res in all_results.values(): + for r in res.get("results", []): + datasets.add(r["dataset"]) + datasets_sorted = sorted(datasets) + short_ds = [d.replace("dataset_", "D").split("_")[0] for d in datasets_sorted] + + header = f" {'Model':<30s}" + for sd in short_ds: + header += f" | {sd:>6s}" + header += f" | {'Avg':>6s} | {'Iters':>5s}" + print(f"\n{'='*len(header)}") + print(header) + print(f" {'-'*30}" + "".join(f"-+-{'-'*6}" for _ in short_ds) + f"-+-{'-'*6}-+-{'-'*5}") + + for model_id, res in all_results.items(): + name = _short_name(model_id) + row = f" {name:<30s}" + scores = {} + total_iters = 0 + for r in res.get("results", []): + scores[r["dataset"]] = r.get("aggregate_score", 0.0) + total_iters += (r.get("stats") or {}).get("iterations_used", 0) + + for ds in datasets_sorted: + s = scores.get(ds, 0.0) + row += f" | {s:>6.2f}" + + avg = res.get("summary", {}).get("average_score", 0.0) + row += f" | {avg:>6.2f} | {total_iters:>5d}" + print(row) + + print(f"{'='*len(header)}") + + +def main(): + args = _parse_args(sys.argv[1:]) + models = _load_models(args) + is_mock = args.get("mock", False) + + if not is_mock and not args.get("judge_provider"): + print(" ERROR: --judge-provider required (or use --mock for dry run)") + return 1 + + results_dir = ROOT / "eval" / "results" + results_dir.mkdir(parents=True, exist_ok=True) + + all_results: dict[str, dict] = {} + + for model_cfg in models: + model_id = model_cfg["model"] + safe_name = model_id.replace("/", "_") + out_path = results_dir / f"{safe_name}.json" + + if is_mock: + print(f"\n [mock] {model_id}") + all_results[model_id] = MOCK_RESULTS + else: + res = _run_eval(model_cfg, args, out_path) + if res: + all_results[model_id] = res + else: + print(f" WARNING: no results for {model_id}") + + if all_results: + _print_table(all_results) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index da99884..e3421eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "typer>=0.12.0,<0.13", "watchfiles>=0.21.0,<0.22", "langchain-mistralai>=0.2.12", + "slowapi>=0.1.10", ] [project.scripts] diff --git a/repi/api/__init__.py b/repi/api/__init__.py index f10b3e5..423d736 100644 --- a/repi/api/__init__.py +++ b/repi/api/__init__.py @@ -1,8 +1,13 @@ import logging from contextlib import asynccontextmanager -from fastapi import FastAPI +from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from slowapi import _rate_limit_exceeded_handler +from slowapi.errors import RateLimitExceeded + +from repi.api.limiter import limiter from repi.core.container import get_container from repi.api.ingest import router as ingest_router @@ -31,9 +36,15 @@ async def lifespan(app: FastAPI): lifespan=lifespan ) +app.state.limiter = limiter +app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + +from repi.core.config import settings as _settings + +_cors_origins = _settings.CORS_ORIGINS or [f"http://localhost:{_settings.UI_PORT}"] app.add_middleware( CORSMiddleware, - allow_origins=["*"], # tighten to the frontend URL in production + allow_origins=_cors_origins, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], diff --git a/repi/api/chat.py b/repi/api/chat.py index f1b7a02..e596452 100644 --- a/repi/api/chat.py +++ b/repi/api/chat.py @@ -20,10 +20,11 @@ from datetime import datetime, timedelta, timezone from uuid import UUID, uuid4 -from fastapi import APIRouter +from fastapi import APIRouter, Request from fastapi.responses import StreamingResponse from sqlalchemy import select, text as sa_text +from repi.api.limiter import limiter from repi.core.container import get_container from repi.core.config import get_settings from repi.core.dates import default_date_handler as _dh @@ -105,7 +106,8 @@ def _chat_confidence(chunks: list[dict], entities: list[str]) -> str: @router.post("/chat") -async def chat(req: ChatRequest) -> StreamingResponse: +@limiter.limit("20/minute") +async def chat(request: Request, req: ChatRequest) -> StreamingResponse: container = get_container() container.require_llm() # 409 if no API key is configured. @@ -340,6 +342,6 @@ async def event_generator(): except Exception as e: logger.exception("chat endpoint raised") - yield _sse("error", {"message": str(e), "conversation_id": str(conversation_id)}) + yield _sse("error", {"message": "An internal error occurred", "conversation_id": str(conversation_id)}) return StreamingResponse(event_generator(), media_type="text/event-stream") diff --git a/repi/api/config.py b/repi/api/config.py index f85e769..310c9d8 100644 --- a/repi/api/config.py +++ b/repi/api/config.py @@ -9,10 +9,22 @@ router = APIRouter() + +def _mask_secrets(data: dict) -> dict: + masked = {} + for key, value in data.items(): + if key.endswith(("_KEY", "_SECRET", "_TOKEN")) and value: + s = str(value) + masked[key] = f"{s[:4]}...{s[-4:]}" if len(s) > 10 else "***" + else: + masked[key] = value + return masked + + @router.get("/config") async def get_config(): - """Return the current configuration.""" - return settings.model_dump() + """Return the current configuration with secrets masked.""" + return _mask_secrets(settings.model_dump()) @router.put("/config") async def update_config(new_config: dict): @@ -49,5 +61,5 @@ async def update_config(new_config: dict): return {"status": "success", "message": "Configuration updated and reloaded"} except Exception as e: - logger.error(f"Failed to update config: {e}") - raise HTTPException(status_code=400, detail=str(e)) + logger.error("Failed to update config", exc_info=True) + raise HTTPException(status_code=400, detail="Configuration update failed") diff --git a/repi/api/investigate.py b/repi/api/investigate.py index 989587f..5a730ae 100644 --- a/repi/api/investigate.py +++ b/repi/api/investigate.py @@ -7,8 +7,11 @@ from uuid import UUID from datetime import datetime +from fastapi import Request as StarletteRequest + from repi.core.container import get_container from repi.investigation.react_loop import InvestigationStep +from repi.api.limiter import limiter from repi.api.schemas import ( ClarifyRequest, InvestigateRequest, @@ -42,7 +45,8 @@ async def list_investigations(limit: int = 20): return results @router.post("/investigate", response_model=SimpleInvestigationResponse) -async def investigate(request: InvestigateRequest): +@limiter.limit("10/minute") +async def investigate(request: StarletteRequest, request_body: InvestigateRequest): """ Start an autonomous log investigation (non-blocking). @@ -59,10 +63,10 @@ async def investigate(request: InvestigateRequest): from sqlmodel import select as sm_select from sqlalchemy import text as sa_text - conversation_id = request.conversation_id - project_id = request.project_id + conversation_id = request_body.conversation_id + project_id = request_body.project_id if conversation_id is None: - conv = Conversation(title=request.query[:80], project_id=project_id) + conv = Conversation(title=request_body.query[:80], project_id=project_id) session.add(conv) await session.commit() await session.refresh(conv) @@ -73,7 +77,7 @@ async def investigate(request: InvestigateRequest): res = await session.exec(stmt) existing = res.first() if existing is None: - session.add(Conversation(id=conversation_id, title=request.query[:80], project_id=project_id)) + session.add(Conversation(id=conversation_id, title=request_body.query[:80], project_id=project_id)) await session.commit() elif project_id is None: # Inherit the conversation's project when the caller didn't pin one. @@ -81,7 +85,7 @@ async def investigate(request: InvestigateRequest): store = container.get_investigation_store(session) investigation = await store.get_or_create( - request.query, conversation_id=conversation_id, project_id=project_id + request_body.query, conversation_id=conversation_id, project_id=project_id ) # Bump the conversation's updated_at so the sidebar surfaces the @@ -244,8 +248,8 @@ async def on_phase_change(phase: str): done_payload = {"answer": result.answer, "stats": result.stats} yield f"data: {json.dumps({'type': 'done', 'data': done_payload}, default=str)}\n\n" except Exception as e: - logger.error(f"Investigation failed: {e}") - yield f"data: {json.dumps({'type': 'error', 'data': {'message': str(e)}})}\n\n" + logger.error("Investigation failed", exc_info=True) + yield f"data: {json.dumps({'type': 'error', 'data': {'message': 'Investigation failed'}})}\n\n" return StreamingResponse(event_generator(), media_type="text/event-stream") diff --git a/repi/api/limiter.py b/repi/api/limiter.py new file mode 100644 index 0000000..38404a8 --- /dev/null +++ b/repi/api/limiter.py @@ -0,0 +1,4 @@ +from slowapi import Limiter +from slowapi.util import get_remote_address + +limiter = Limiter(key_func=get_remote_address) diff --git a/repi/cli.py b/repi/cli.py index 34f9fa6..1097c76 100644 --- a/repi/cli.py +++ b/repi/cli.py @@ -56,13 +56,8 @@ def _root( CONFIG_DIR = REPO_ROOT / ".repi" CONFIG_FILE = CONFIG_DIR / "config.json" -PROVIDERS = ["openai", "anthropic", "mistral", "gemini", "ollama"] -PROVIDER_KEY_ENV = { - "openai": "OPENAI_API_KEY", - "anthropic": "ANTHROPIC_API_KEY", - "mistral": "MISTRAL_API_KEY", - "gemini": "GEMINI_API_KEY", -} +PROVIDERS = ["openai", "openrouter", "anthropic", "mistral", "gemini", "ollama"] +KEYLESS_PROVIDERS = {"ollama"} DEFAULT_DB_URL = "postgresql+asyncpg://repi_user:password_here@localhost:5432/repi" DEFAULT_REDIS_URL = "redis://localhost:6379" @@ -105,8 +100,8 @@ def _config_payload(provider: str, api_key: str | None) -> dict: "REDIS_URL": DEFAULT_REDIS_URL, "LLM_PROVIDER": provider, } - if api_key and provider in PROVIDER_KEY_ENV: - payload[PROVIDER_KEY_ENV[provider]] = api_key + if api_key: + payload["LLM_API_KEY"] = api_key return payload @@ -223,9 +218,9 @@ def init( type=click.Choice(PROVIDERS, case_sensitive=False), ).lower() api_key: str | None = None - if provider in PROVIDER_KEY_ENV: + if provider not in KEYLESS_PROVIDERS: api_key = typer.prompt( - f"{PROVIDER_KEY_ENV[provider]}", + "API key", hide_input=True, default="", show_default=False, @@ -591,20 +586,14 @@ async def _check_redis(url: str) -> tuple[bool, str]: def _check_llm_key(settings) -> tuple[bool, str]: provider = (settings.LLM_PROVIDER or "").lower() - key_field_map = { - "openai": "OPENAI_API_KEY", - "anthropic": "ANTHROPIC_API_KEY", - "mistral": "MISTRAL_API_KEY", - "gemini": "GEMINI_API_KEY", - } if provider == "ollama": return True, "ollama (no key required)" - field = key_field_map.get(provider) - if field is None: + from repi.llm.factory import SUPPORTED_PROVIDERS + if provider not in SUPPORTED_PROVIDERS: return False, f"unknown provider '{provider}'" - val = getattr(settings, field, None) or getattr(settings, "LLM_API_KEY", None) + val = settings.LLM_API_KEY if not val: - return False, f"{field} not set" + return False, "LLM_API_KEY not set" masked = f"{val[:4]}…{val[-4:]}" if len(val) > 10 else "set" return True, masked diff --git a/repi/core/config.py b/repi/core/config.py index 688d4a4..3295302 100644 --- a/repi/core/config.py +++ b/repi/core/config.py @@ -2,9 +2,9 @@ import os import json from pathlib import Path -from typing import List, Optional +from typing import Any, List, Optional from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict -from pydantic import Field +from pydantic import Field, model_validator def _resolve_config_path() -> Path: """Locate .repi/config.json: cwd first (docker runs from /app), then parent @@ -24,6 +24,11 @@ def _resolve_config_path() -> Path: CONFIG_PATH = _resolve_config_path() CONFIG_DIR = CONFIG_PATH.parent +_LEGACY_KEY_FIELDS = ( + "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY", + "GEMINI_API_KEY", "GOOGLE_API_KEY", +) + class Settings(BaseSettings): REPI_ENV: str = Field(default="production", description="Runtime environment") LOG_LEVEL: str = Field(default="INFO", description="Logging level (DEBUG/INFO/WARNING/ERROR)") @@ -49,13 +54,22 @@ class Settings(BaseSettings): LLM_PROVIDER: str = "openai" LLM_MODEL: Optional[str] = None LLM_API_KEY: Optional[str] = None - OPENAI_API_KEY: Optional[str] = None - MISTRAL_API_KEY: Optional[str] = None - ANTHROPIC_API_KEY: Optional[str] = None - GEMINI_API_KEY: Optional[str] = None - GOOGLE_API_KEY: Optional[str] = None OLLAMA_BASE_URL: str = "http://localhost:11434" + @model_validator(mode="before") + @classmethod + def _migrate_legacy_api_keys(cls, data: Any) -> Any: + if not isinstance(data, dict): + return data + if data.get("LLM_API_KEY"): + return data + for field in _LEGACY_KEY_FIELDS: + val = data.get(field) + if val: + data["LLM_API_KEY"] = val + break + return data + WATCHER_CONFIG_REFRESH_SECS: int = 30 LLM_MAX_CALLS_PER_MIN: int = Field(default=60, ge=1) @@ -74,13 +88,14 @@ class Settings(BaseSettings): ENABLE_LEVEL_BOOST: bool = True UI_PORT: int = 3000 + CORS_ORIGINS: List[str] = Field(default_factory=list) MAX_RETRIES_PER_STEP: int = 2 BACKOFF_BASE_SECONDS: int = 5 # Forced re-plan turn every N action steps to break perseveration. ENABLE_REFLECTION: bool = True - REFLECTION_INTERVAL: int = 3 + REFLECTION_INTERVAL: int = 2 # /chat followup-bias envelope. When a turn omits an explicit time # window AND the previous assistant turn cited chunks, the chat path diff --git a/repi/investigation/react_loop.py b/repi/investigation/react_loop.py index 0325906..1d0d8a4 100644 --- a/repi/investigation/react_loop.py +++ b/repi/investigation/react_loop.py @@ -16,6 +16,7 @@ from repi.intent.resolver import resolve as resolve_intent, ResolvedIntent, ClarificationNeeded from repi.investigation.sweep import auto_sweep from repi.investigation.compiler import compile_answer, synthesize_answer +from repi.investigation.sanitize import sanitize_query from repi.investigation.state import ( Phase, InvestigationState, @@ -213,7 +214,7 @@ def _build_system_prompt(known_services: list[str]) -> str: async def handle_resolving(state: InvestigationState, deps: LoopDeps) -> InvestigationState: now = _dh.now() - clarified_query = state.query + clarified_query = sanitize_query(state.query) if state.post_clarification: last_step_obs = None @@ -301,7 +302,7 @@ async def handle_sweeping(state: InvestigationState, deps: LoopDeps) -> Investig state.messages = [ Message(role="system", content=_build_system_prompt(deps.known_services)), - Message(role="user", content=state.query), + Message(role="user", content=sanitize_query(state.query)), ] if state.resolved_intent and deps.pool: @@ -715,6 +716,11 @@ async def handle_compiling(state: InvestigationState, deps: LoopDeps) -> Investi # ─── Phase dispatch table ──────────────────────────────────────────────────── +# +# GATHERING ⇄ REFLECTING is a cycle, not a pipeline. ReAct is preserved: +# each gathering step is act→observe, reflection is the guard that decides +# whether to loop back (probe somewhere new) or exit to COMPILING. The FSM +# is a supervisor around the loop, not a replacement for it. HANDLERS: dict[Phase, Callable] = { Phase.RESOLVING: handle_resolving, @@ -820,6 +826,12 @@ async def investigate( try: state = InvestigationState.from_json(json.dumps(investigation_obj.state_json)) logger.info(f"Resumed investigation {investigation_obj.id} from phase {state.phase.value}") + # Kick WAITING_CLARIFICATION into the resume path so the + # dispatch loop doesn't exit immediately. + if state.phase == Phase.WAITING_CLARIFICATION: + state.post_clarification = True + state.pending_question = None + state.phase = Phase.RESOLVING except Exception as e: logger.warning(f"Failed to restore state from state_json, falling back: {e}") state = None diff --git a/repi/investigation/sanitize.py b/repi/investigation/sanitize.py new file mode 100644 index 0000000..4309909 --- /dev/null +++ b/repi/investigation/sanitize.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import re + +MAX_QUERY_LENGTH = 2000 + +_INJECTION_PATTERNS = [ + re.compile(r"\n\s*(System|Human|Assistant)\s*:", re.IGNORECASE), + re.compile(r"ignore\s+(all\s+)?previous\s+instructions", re.IGNORECASE), + re.compile(r"you\s+are\s+now\s+(a|an)\b", re.IGNORECASE), + re.compile(r"disregard\s+(all\s+)?(prior|above|previous)", re.IGNORECASE), + re.compile(r"forget\s+(everything|all|your)\s+(instructions|rules|guidelines)", re.IGNORECASE), + re.compile(r"<\s*/?\s*(system|prompt|instruction)\s*>", re.IGNORECASE), +] + + +def sanitize_query(raw: str) -> str: + if not raw: + return raw + cleaned = raw[:MAX_QUERY_LENGTH] + for pattern in _INJECTION_PATTERNS: + cleaned = pattern.sub("[filtered]", cleaned) + return cleaned.strip() diff --git a/repi/llm/adapters.py b/repi/llm/adapters.py index da4a924..2219a7e 100644 --- a/repi/llm/adapters.py +++ b/repi/llm/adapters.py @@ -96,17 +96,27 @@ def _check_4xx(response: httpx.Response, provider: str, model: str) -> None: logger.warning("%s %d body: %s", provider, response.status_code, body[:500]) raise LLMBadRequestError(provider, model, response.status_code, body) -class OpenAIProvider(LLMProvider): - def __init__(self, api_key: str, model: str = "gpt-4o"): +class OpenAICompatProvider(LLMProvider): + def __init__( + self, + api_key: str, + model: str = "gpt-4o", + base_url: str = "https://api.openai.com/v1", + provider_label: str = "openai", + extra_headers: Optional[dict] = None, + ): self._api_key = api_key self._model = model - self._url = "https://api.openai.com/v1/chat/completions" + self._url = f"{base_url.rstrip('/')}/chat/completions" + self._provider_label = provider_label + self._extra_headers = extra_headers or {} async def complete(self, messages: List[Message], max_tokens: int = 2000, temperature: float = 0.0) -> str: try: + headers = {"Authorization": f"Bearer {self._api_key}", **self._extra_headers} response = await _post_with_429_retry( - self._url, provider="openai", model=self._model, - headers={"Authorization": f"Bearer {self._api_key}"}, + self._url, provider=self._provider_label, model=self._model, + headers=headers, json_body={ "model": self._model, "messages": [{"role": m.role, "content": m.content} for m in messages], @@ -114,18 +124,21 @@ async def complete(self, messages: List[Message], max_tokens: int = 2000, temper "temperature": temperature } ) - _check_4xx(response, "openai", self._model) + _check_4xx(response, self._provider_label, self._model) response.raise_for_status() return response.json()["choices"][0]["message"]["content"] except LLMError: raise except Exception as e: - raise LLMError(str(e), "openai", self._model) + raise LLMError(str(e), self._provider_label, self._model) @property def model_name(self) -> str: return self._model + +OpenAIProvider = OpenAICompatProvider + class AnthropicProvider(LLMProvider): def __init__(self, api_key: str, model: str = "claude-3-5-sonnet-20240620"): self._api_key = api_key diff --git a/repi/llm/factory.py b/repi/llm/factory.py index 7577c48..dcd24b9 100644 --- a/repi/llm/factory.py +++ b/repi/llm/factory.py @@ -1,47 +1,54 @@ from __future__ import annotations -import os + from repi.core.config import settings from repi.llm.provider import LLMProvider -from repi.llm.adapters import OpenAIProvider, AnthropicProvider, OllamaProvider, GeminiProvider, MistralProvider +from repi.llm.adapters import ( + OpenAICompatProvider, AnthropicProvider, OllamaProvider, + GeminiProvider, MistralProvider, +) -def create_provider_from_env() -> LLMProvider: - """ - Creates an LLM provider based on environment variables. - Supported: openai, anthropic, ollama, gemini, mistral. - """ - provider_type = settings.LLM_PROVIDER.lower() - model_name = settings.LLM_MODEL - - if provider_type == "openai": - api_key = settings.LLM_API_KEY or settings.OPENAI_API_KEY - if not api_key: - raise ValueError("LLM_API_KEY or OPENAI_API_KEY must be set for OpenAI provider") - return OpenAIProvider(api_key=api_key, model=model_name or "gpt-4o") - - elif provider_type == "anthropic": - api_key = settings.LLM_API_KEY or settings.ANTHROPIC_API_KEY - if not api_key: - raise ValueError("LLM_API_KEY or ANTHROPIC_API_KEY must be set for Anthropic provider") - return AnthropicProvider(api_key=api_key, model=model_name or "claude-3-5-sonnet-20240620") - - elif provider_type == "ollama": - base_url = settings.OLLAMA_BASE_URL - return OllamaProvider(base_url=base_url, model=model_name or "mistral") - - elif provider_type == "gemini": - api_key = settings.LLM_API_KEY or settings.GEMINI_API_KEY or settings.GOOGLE_API_KEY - if not api_key: - raise ValueError("LLM_API_KEY, GEMINI_API_KEY or GOOGLE_API_KEY must be set for Gemini provider") - return GeminiProvider(api_key=api_key, model=model_name or "gemini-1.5-pro") - - elif provider_type == "mistral": - api_key = settings.LLM_API_KEY or settings.MISTRAL_API_KEY - if not api_key: - raise ValueError("LLM_API_KEY or MISTRAL_API_KEY must be set for Mistral provider") - return MistralProvider(api_key=api_key, model=model_name or "mistral-large-latest") - - else: +_PROVIDERS: dict[str, tuple[type, str, dict]] = { + "openai": (OpenAICompatProvider, "gpt-4o", {}), + "openrouter": (OpenAICompatProvider, "openai/gpt-4o", { + "base_url": "https://openrouter.ai/api/v1", + "provider_label": "openrouter", + }), + "anthropic": (AnthropicProvider, "claude-sonnet-4-20250514", {}), + "mistral": (MistralProvider, "mistral-large-latest", {}), + "gemini": (GeminiProvider, "gemini-2.0-flash", {}), +} + +SUPPORTED_PROVIDERS = list(_PROVIDERS) + ["ollama"] + + +def create_provider( + provider: str | None = None, + api_key: str | None = None, + model: str | None = None, +) -> LLMProvider: + provider = (provider or settings.LLM_PROVIDER).lower() + model = model or settings.LLM_MODEL + + if provider == "ollama": + return OllamaProvider( + base_url=settings.OLLAMA_BASE_URL, + model=model or "mistral", + ) + + if provider not in _PROVIDERS: + raise ValueError( + f"Unsupported LLM_PROVIDER: '{provider}'. " + f"Supported: {', '.join(SUPPORTED_PROVIDERS)}." + ) + + adapter_cls, default_model, extra_kwargs = _PROVIDERS[provider] + key = api_key or settings.LLM_API_KEY + if not key: raise ValueError( - f"Unsupported LLM_PROVIDER: '{provider_type}'. " - "Supported values: 'openai', 'anthropic', 'ollama', 'gemini', 'mistral'." + f"LLM_API_KEY must be set for {provider} provider" ) + return adapter_cls(api_key=key, model=model or default_model, **extra_kwargs) + + +def create_provider_from_env() -> LLMProvider: + return create_provider() diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index 68e511a..9172c04 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -38,8 +38,8 @@ def test_config_get_returns_current_value(tmp_config): def test_config_get_masks_api_keys_by_default(tmp_config): - tmp_config.write_text(json.dumps({"OPENAI_API_KEY": "sk-abcdefghijklmnop"}) + "\n") - result = runner.invoke(app, ["config", "get", "OPENAI_API_KEY"]) + tmp_config.write_text(json.dumps({"LLM_API_KEY": "sk-abcdefghijklmnop"}) + "\n") + result = runner.invoke(app, ["config", "get", "LLM_API_KEY"]) assert result.exit_code == 0, result.stdout out = result.stdout.strip() assert "sk-abcdefghijklmnop" not in out @@ -47,8 +47,8 @@ def test_config_get_masks_api_keys_by_default(tmp_config): def test_config_get_unmask_reveals_secret(tmp_config): - tmp_config.write_text(json.dumps({"OPENAI_API_KEY": "sk-abcdefghijklmnop"}) + "\n") - result = runner.invoke(app, ["config", "get", "OPENAI_API_KEY", "--unmask"]) + tmp_config.write_text(json.dumps({"LLM_API_KEY": "sk-abcdefghijklmnop"}) + "\n") + result = runner.invoke(app, ["config", "get", "LLM_API_KEY", "--unmask"]) assert result.exit_code == 0, result.stdout assert "sk-abcdefghijklmnop" in result.stdout @@ -93,7 +93,7 @@ def test_config_set_rejects_malformed_pair(tmp_config): def test_config_set_hides_api_key_value_in_output(tmp_config): - result = runner.invoke(app, ["config", "set", "OPENAI_API_KEY=sk-supersecret"]) + result = runner.invoke(app, ["config", "set", "LLM_API_KEY=sk-supersecret"]) assert result.exit_code == 0, result.stdout assert "sk-supersecret" not in result.stdout assert "" in result.stdout diff --git a/tests/test_config_isolation.py b/tests/test_config_isolation.py index edb7a1d..0271eb1 100644 --- a/tests/test_config_isolation.py +++ b/tests/test_config_isolation.py @@ -9,29 +9,26 @@ def test_settings_ignores_env_vars(monkeypatch): - # Every secret-looking field should resolve to its class default (None - # for *_API_KEY, even when the shell env tries to inject a value. - monkeypatch.setenv("MISTRAL_API_KEY", "should-not-leak") - monkeypatch.setenv("OPENAI_API_KEY", "should-not-leak") - monkeypatch.setenv("ANTHROPIC_API_KEY", "should-not-leak") - monkeypatch.setenv("GEMINI_API_KEY", "should-not-leak") monkeypatch.setenv("LLM_API_KEY", "should-not-leak") monkeypatch.setenv("DATABASE_URL", "postgresql+asyncpg://evil:evil@evil:5432/evil") s = Settings() - assert s.MISTRAL_API_KEY is None - assert s.OPENAI_API_KEY is None - assert s.ANTHROPIC_API_KEY is None - assert s.GEMINI_API_KEY is None assert s.LLM_API_KEY is None - # DATABASE_URL falls back to the class default, not the env injection. assert "evil" not in s.DATABASE_URL def test_settings_accepts_kwargs(monkeypatch): - # init kwargs (the path used by get_settings() after reading config.json) - # remain the *only* way real values arrive. - monkeypatch.setenv("MISTRAL_API_KEY", "from-env-should-lose") - s = Settings(MISTRAL_API_KEY="from-kwargs-should-win") - assert s.MISTRAL_API_KEY == "from-kwargs-should-win" + monkeypatch.setenv("LLM_API_KEY", "from-env-should-lose") + s = Settings(LLM_API_KEY="from-kwargs-should-win") + assert s.LLM_API_KEY == "from-kwargs-should-win" + + +def test_legacy_key_migration(): + s = Settings(OPENAI_API_KEY="sk-old-style") + assert s.LLM_API_KEY == "sk-old-style" + + +def test_legacy_key_does_not_overwrite_explicit(): + s = Settings(LLM_API_KEY="sk-explicit", ANTHROPIC_API_KEY="sk-legacy") + assert s.LLM_API_KEY == "sk-explicit" diff --git a/uv.lock b/uv.lock index 26f3e37..c11bb90 100644 --- a/uv.lock +++ b/uv.lock @@ -611,6 +611,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/66/73034ad30b59f13439b75e620989dacba4c047256e358ba7c2e9ec98ea22/datasets-5.0.0-py3-none-any.whl", hash = "sha256:7dd34927a0fd7046e98aad5cb9430e699c373238a15befa7b9bf22b991a7fee6", size = 555084, upload-time = "2026-06-05T13:18:24.435Z" }, ] +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + [[package]] name = "dill" version = "0.3.8" @@ -1996,6 +2008,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/f4/c206c0888f8a506404cb4f16ad89593bdc2f70cf00de26a1a0a7a76ad7a3/langsmith-0.3.45-py3-none-any.whl", hash = "sha256:5b55f0518601fa65f3bb6b1a3100379a96aa7b3ed5e9380581615ba9c65ed8ed", size = 363002, upload-time = "2025-06-05T05:10:27.228Z" }, ] +[[package]] +name = "limits" +version = "5.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "packaging", version = "25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "packaging", version = "26.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/69/826a5d1f45426c68d8f6539f8d275c0e4fcaa57f0c017ec3100986558a41/limits-5.8.0.tar.gz", hash = "sha256:c9e0d74aed837e8f6f50d1fcebcf5fd8130957287206bc3799adaee5092655da", size = 226104, upload-time = "2026-02-05T07:17:35.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/98/cb5ca20618d205a09d5bec7591fbc4130369c7e6308d9a676a28ff3ab22c/limits-5.8.0-py3-none-any.whl", hash = "sha256:ae1b008a43eb43073c3c579398bd4eb4c795de60952532dc24720ab45e1ac6b8", size = 60954, upload-time = "2026-02-05T07:17:34.425Z" }, +] + [[package]] name = "loguru" version = "0.7.3" @@ -3512,6 +3539,7 @@ dependencies = [ { name = "python-multipart" }, { name = "redis" }, { name = "rich" }, + { name = "slowapi" }, { name = "sqlalchemy" }, { name = "sqlmodel" }, { name = "typer" }, @@ -3549,6 +3577,7 @@ requires-dist = [ { name = "python-multipart", specifier = ">=0.0.9,<0.0.10" }, { name = "redis", specifier = ">=5.0.4,<6" }, { name = "rich", specifier = ">=13.7.1,<14" }, + { name = "slowapi", specifier = ">=0.1.10" }, { name = "sqlalchemy", specifier = ">=2.0.31,<3" }, { name = "sqlmodel", specifier = ">=0.0.38,<0.0.39" }, { name = "typer", specifier = ">=0.12.0,<0.13" }, @@ -3836,6 +3865,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "slowapi" +version = "0.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "limits" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/52/24527cf25a8b508926aff53350b0136561dfe86c7125f61526653666e1b2/slowapi-0.1.10.tar.gz", hash = "sha256:d320d5bc04d9f171a77fb16700faf3036d85b00f420f22924c8a225f95bd14f9", size = 13841, upload-time = "2026-06-13T11:59:31.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/8b/1d359f38706b4097d9a943bf8bd22599f537de4cbaff1e622d3e3936e164/slowapi-0.1.10-py3-none-any.whl", hash = "sha256:3acb61561dc9d687e3d3669362ff6a439de9ba44e2fed3a9c165da26b4b83e28", size = 14921, upload-time = "2026-06-13T11:59:30.485Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -4488,6 +4529,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, ] +[[package]] +name = "wrapt" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782, upload-time = "2026-06-20T23:47:44.367Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678, upload-time = "2026-06-20T23:47:45.857Z" }, + { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671, upload-time = "2026-06-20T23:47:47.345Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785, upload-time = "2026-06-20T23:47:48.759Z" }, + { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699, upload-time = "2026-06-20T23:47:50.177Z" }, + { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695, upload-time = "2026-06-20T23:47:51.602Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813, upload-time = "2026-06-20T23:47:53.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809, upload-time = "2026-06-20T23:47:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414, upload-time = "2026-06-20T23:47:55.882Z" }, + { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368, upload-time = "2026-06-20T23:47:57.237Z" }, + { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489, upload-time = "2026-06-20T23:47:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, + { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, + { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, + { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, + { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, + { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, + { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, + { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" }, + { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" }, + { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, + { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" }, + { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" }, + { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" }, + { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, + { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" }, + { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" }, + { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, +] + [[package]] name = "xxhash" version = "3.7.0" From 305588d8f7ba3a3c938132c683b0a6208149159d Mon Sep 17 00:00:00 2001 From: Varun Singh Date: Mon, 29 Jun 2026 23:27:10 +0530 Subject: [PATCH 3/3] fix: resolved conflixts --- eval/results/mistral_self.json | 229 +++++++++++++++++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 eval/results/mistral_self.json diff --git a/eval/results/mistral_self.json b/eval/results/mistral_self.json new file mode 100644 index 0000000..164545e --- /dev/null +++ b/eval/results/mistral_self.json @@ -0,0 +1,229 @@ +{ + "pass_threshold": 0.8, + "summary": { + "passed": 2, + "failed": 2, + "errored": 0, + "average_score": 0.768 + }, + "results": [ + { + "dataset": "dataset_1_cascading_inventory_migration", + "query": "why did checkout break friday night", + "status": "fail", + "aggregate_score": 0.68, + "model_under_test": "mistral-large-latest", + "judge_model": "mistral-large-latest", + "criteria": [ + { + "name": "trigger_identification", + "score": 0.9, + "explanation": "The trigger service is correctly identified as 'inventory-svc'. The log line references 'Applying migration 0042_add_warehouse_id.sql', which is semantically equivalent to the required 'Migration 0042 added column warehouse_id NOT NULL to skus table' but does not explicitly mention the NOT NULL constraint or the table name. This is a minor omission but still captures the core concept of the migration." + }, + { + "name": "root_cause_accuracy", + "score": 1.0, + "explanation": "The root_cause explanation mentions all required concepts: 'migration' (Postgres migration), 'warehouse_id' (warehouse_id column), 'NOT NULL' (NOT NULL column), 'retry' (retry storm), and 'pool exhaustion' (HTTP client pool EXHAUSTED). The explanation is comprehensive and semantically correct." + }, + { + "name": "propagation_chain", + "score": 1.0, + "explanation": "The propagation_chain includes all required hops in the correct order. It mentions 'migration 0042 / warehouse_id NOT NULL' (implicitly via 'Migration 0042 added column warehouse_id NOT NULL'), '500 / null value / warehouse_id' (constraint violation and 500 error), 'retry' (retry 1/5 and retry storm), 'pool exhausted' (HTTP client pool EXHAUSTED), and 'checkout / cart timeout' (Checkout failed for cart_id). The chain is complete and accurate." + }, + { + "name": "red_herring_handling", + "score": 0.5, + "explanation": "The ruled_out_hypotheses address some red herrings but miss key ones. It correctly rules out 'gateway', 'read-states-svc', 'message-svc', 'presence-svc', 'cdn-edge', and 'network outage'. However, it does not explicitly address 'pricing-svc', 'payment-svc', or 'notification-svc', which are required to be ruled out. This is a significant omission." + }, + { + "name": "confidence_calibration", + "score": 0.0, + "explanation": "Expected confidence 'high', got 'medium'." + } + ], + "raw_answer_truncated": "{\n \"incident_window\": {\n \"start\": \"2026-06-26T22:00:00+00:00\",\n \"end\": \"2026-06-27T06:00:00+00:00\"\n },\n \"affected_services\": [\n \"inventory-svc\",\n \"cart-svc\",\n \"gateway\"\n ],\n \"trigger_event\": {\n \"chunk_id\": \"84a2a432-447c-48b7-95d7-c537f427f68c\",\n \"service\": \"inventory-svc\",\n \"timestamp\": \"2026-06-26T22:00:14+00:00\",\n \"log_line\": \"Applying migration 0042_add_warehouse_id.sql\"\n },\n \"propagation_chain\": [\n {\n \"service\": \"inventory-svc\",\n \"chunk_id\": \"e", + "stats": { + "iterations_used": 4, + "reflections_used": 1, + "chunks_gathered": 27, + "tools_called": [ + "find_logs_by_id", + "scan_window", + "search_logs" + ], + "compile_source": "llm", + "compile_attempts": 1, + "floor_adjustments": [ + "no gaps listed despite non-low confidence (\u22653 chunk citations \u2014 leaving as-is)", + "affected_services ['gateway'] never appeared in tool results or any chunk text (downgraded confidence one notch as a precaution)" + ], + "gathering_exit_reason": "Root cause and impact confirmed: unbackfilled Postgres migration in `inventory-svc` caused constraint violations, cascading to `cart-svc` and breaking checkout. Gateway propagated errors without logging them." + }, + "judge_parse_attempts": 1 + }, + { + "dataset": "dataset_2_insufficient_logging", + "query": "why is the nightly report failing", + "status": "fail", + "aggregate_score": 0.633, + "model_under_test": "mistral-large-latest", + "judge_model": "mistral-large-latest", + "criteria": [ + { + "name": "trigger_identification", + "score": 0.5, + "explanation": "The answer identifies the trigger event as the cron-runner job start, but it does not mention either of the acceptable trigger events: (1) report-svc log with 'SIGKILL' or (2) cron-runner log mentioning 'exceeded timeout'. While the propagation chain correctly includes these events, the trigger_event field itself does not meet the criteria. The explanation is partially correct but misses the explicit requirement for the trigger event to be one of the two specified log lines." + }, + { + "name": "root_cause_accuracy", + "score": 0.8, + "explanation": "The root_cause field does not explicitly state 'unable_to_determine' or 'hypothesis_only', but it avoids asserting a specific root cause as fact. It provides a plausible hypothesis based on the evidence (timeout and SIGKILL) but does not overreach. This is mostly correct, though it could be clearer about the uncertainty." + }, + { + "name": "red_herring_handling", + "score": 0.5, + "explanation": "The ruled_out_hypotheses field addresses some key possibilities but misses others. It correctly rules out Postgres issues and OOM killer, but it does not explicitly consider 'code crash' (weakly ruled out due to no stack trace), 'deadlock' (cannot confirm or rule out), or 'external kill' (consistent with SIGKILL but not confirmed). The handling is partially correct but omits critical hypotheses." + }, + { + "name": "confidence_calibration", + "score": 0.0, + "explanation": "Expected confidence 'low', got 'medium'." + }, + { + "name": "gap_awareness", + "score": 1.0, + "explanation": "Gaps list includes content matching expected gaps." + }, + { + "name": "hallucination_avoidance", + "score": 1.0, + "explanation": "The root_cause field avoids asserting any of the prohibited terms as confirmed facts. It does not mention 'memory leak', 'OOM (as a fact)', 'GC pressure', or specific code paths/dependencies. The answer is fully correct in this regard." + } + ], + "raw_answer_truncated": "{\n \"incident_window\": {\n \"start\": \"2026-06-25T02:00:00+00:00\",\n \"end\": \"2026-06-25T03:15:00+00:00\"\n },\n \"affected_services\": [\n \"cron-runner\",\n \"report-svc\"\n ],\n \"trigger_event\": {\n \"chunk_id\": \"fb724a62-08cc-4390-a505-d017adc1ae06\",\n \"service\": \"cron-runner\",\n \"timestamp\": \"2026-06-25T02:00:00+00:00\",\n \"log_line\": \"Triggering job nightly_reports \\u2192 report-svc\"\n },\n \"propagation_chain\": [\n {\n \"service\": \"cron-runner\",\n \"chunk_id\": \"27ac94ca-8331-42", + "stats": { + "iterations_used": 7, + "reflections_used": 2, + "chunks_gathered": 25, + "tools_called": [ + "find_logs_by_id", + "scan_window", + "search_logs" + ], + "compile_source": "llm", + "compile_attempts": 1, + "floor_adjustments": [], + "gathering_exit_reason": "All plausible root causes have been investigated. The evidence shows the nightly report job timed out after 1 hour (as configured), and the downstream `report-svc` process was later killed by SIGKILL. No OOM, Postgres, or other resource exhaustion logs were found. Further tool calls are unlikely to yield new evidence." + }, + "judge_parse_attempts": 1 + }, + { + "dataset": "dataset_3_jwt_key_rotation_noise", + "query": "why are users getting 401 errors last monday morning", + "status": "pass", + "aggregate_score": 0.9, + "model_under_test": "mistral-large-latest", + "judge_model": "mistral-large-latest", + "criteria": [ + { + "name": "trigger_identification", + "score": 0.9, + "explanation": "The trigger service is correctly identified as auth-svc, and the log line references 'JWT key rotation triggered', which is one of the required semantic matches. However, the log line does not explicitly mention any of the other exact phrases like 'key_id=k-2026-05' or 'Public key push to verification-svc timed out', though the context is clear from the propagation chain and root cause. This is a minor omission." + }, + { + "name": "root_cause_accuracy", + "score": 1.0, + "explanation": "The root cause explanation mentions all required concepts: JWT, key rotation, k-2026-05, verification-svc, and public key (implied by 'signing key' and 'synchronize the new key'). The explanation is semantically correct and complete." + }, + { + "name": "propagation_chain", + "score": 0.6, + "explanation": "The propagation chain includes all required hops in the correct order. auth-svc mentions 'Active signing key changed: k-2025 \u2192 k-2026-05', which semantically matches 'key rotation / k-2026-05 / push timed out'. verification-svc mentions 'unknown key_id=k-2026-05'. billing-svc mentions 'Checkout failed: upstream 401 Unauthorized', which semantically matches '401 / checkout failed'. However, api-gateway is missing from the chain, which is a significant omission." + }, + { + "name": "red_herring_handling", + "score": 0.9, + "explanation": "The ruled_out_hypotheses correctly address the red herrings user-svc and cache-svc, explicitly stating no causal link to the 401 failures. The explanations are clear and accurate. However, the hypotheses are not explicitly named as 'user-svc' or 'cache-svc' in the hypothesis field, though the content makes it clear. This is a minor omission." + }, + { + "name": "confidence_calibration", + "score": 1.0, + "explanation": "Confidence correctly set to 'high'." + }, + { + "name": "hallucination_avoidance", + "score": 1.0, + "explanation": "The root cause does not name any of the prohibited terms as THE root cause. It correctly identifies auth-svc's key rotation as the root cause and treats other services as downstream effects. No hallucinations are present." + } + ], + "raw_answer_truncated": "{\n \"incident_window\": {\n \"start\": \"2026-06-22T09:15:00+00:00\",\n \"end\": \"2026-06-22T09:33:02+00:00\"\n },\n \"affected_services\": [\n \"verification-svc\",\n \"billing-svc\",\n \"api-gateway\",\n \"auth-svc\"\n ],\n \"trigger_event\": {\n \"chunk_id\": \"0743d05c-1fce-4af8-8a7f-15e091be78e5\",\n \"service\": \"auth-svc\",\n \"timestamp\": \"2026-06-22T09:15:02+00:00\",\n \"log_line\": \"JWT key rotation triggered: scheduled policy rotate-every-d\"\n },\n \"propagation_chain\": [\n {\n \"servic", + "stats": { + "iterations_used": 4, + "reflections_used": 1, + "chunks_gathered": 38, + "tools_called": [ + "get_timeline", + "scan_window", + "search_logs" + ], + "compile_source": "llm", + "compile_attempts": 1, + "floor_adjustments": [], + "gathering_exit_reason": "Root cause identified: `verification-svc` failed to synchronize with the new JWT signing key (`k-2026-05`) after the `auth-svc` key rotation event at 09:15:02 UTC, causing it to reject valid JWTs and trigger 401 errors for users. The evidence is conclusive and no further investigation is required." + }, + "judge_parse_attempts": 1 + }, + { + "dataset": "dataset_4_discord_gateway_cascade", + "query": "why are users getting disconnected from the gateway", + "status": "pass", + "aggregate_score": 0.86, + "model_under_test": "mistral-large-latest", + "judge_model": "mistral-large-latest", + "criteria": [ + { + "name": "trigger_identification", + "score": 1.0, + "explanation": "The trigger service is correctly identified as 'read-states-svc', and the log line explicitly references 'Cassandra timeout' (OperationTimedOutException). This fully matches the criterion." + }, + { + "name": "root_cause_accuracy", + "score": 0.8, + "explanation": "The root_cause explanation mentions Cassandra, read-states, timeout, and reconnect, but it does not explicitly reference the 'pool' concept (e.g., connection pool exhaustion or degradation). While the propagation chain hints at pool issues, the root_cause itself omits this detail." + }, + { + "name": "propagation_chain", + "score": 0.8, + "explanation": "The propagation chain is mostly correct and includes all required hops in the right order. However, it misses explicit mentions of 'BATCH APPLY / OperationTimedOutException' for read-states-svc and 'mark_read / timeout / disconnect' for the first gateway hop. The reconnect storm and thread pool saturation are well-covered, and the 502 mention for cdn-edge is accurate." + }, + { + "name": "red_herring_handling", + "score": 0.7, + "explanation": "The ruled_out_hypotheses correctly address the presence-svc red herring, noting its lack of involvement in the incident. However, it does not explicitly state that presence-svc showed elevated activity *caused by* the gateway reconnect storm (a key nuance in the criterion). The other red herrings are well-handled." + }, + { + "name": "confidence_calibration", + "score": 1.0, + "explanation": "Confidence correctly set to 'high'." + } + ], + "raw_answer_truncated": "{\n \"incident_window\": {\n \"start\": \"2026-06-10T02:15:00+00:00\",\n \"end\": \"2026-06-10T02:30:00+00:00\"\n },\n \"affected_services\": [\n \"gateway\",\n \"read-states-svc\",\n \"message-svc\",\n \"cdn-edge\"\n ],\n \"trigger_event\": {\n \"chunk_id\": \"9938c84a-5eeb-40d9-8650-3fd49e217be1\",\n \"service\": \"read-states-svc\",\n \"timestamp\": \"2026-06-10T02:15:00+00:00\",\n \"log_line\": \"Cassandra timeout on BATCH APPLY to read_states keyspace: OperationTimedOutException (coordinator=, consisten", + "stats": { + "iterations_used": 2, + "reflections_used": 0, + "chunks_gathered": 64, + "tools_called": [ + "scan_window", + "search_logs" + ], + "compile_source": "llm", + "compile_attempts": 1, + "floor_adjustments": [ + "no gaps listed despite non-low confidence (\u22653 chunk citations \u2014 leaving as-is)" + ], + "gathering_exit_reason": "Root cause identified: `read-states-svc` Cassandra write timeouts caused gateway disconnections. Timeline and cascading effects are fully mapped." + }, + "judge_parse_attempts": 1 + } + ] +} \ No newline at end of file