From 983a99e0f87198d43cced4233b783ffd36437161 Mon Sep 17 00:00:00 2001 From: James Maki <7774872+JamesMaki@users.noreply.github.com> Date: Tue, 21 Jul 2026 22:20:56 -0700 Subject: [PATCH] feat: make SkillOpt the core optimization workflow --- .env.example | 27 +- AGENTS.md | 101 ----- ARCHITECTURE.md | 69 ++-- CONTRIBUTING.md | 2 +- Dockerfile | 8 +- PRODUCTION_SETUP.md | 12 +- README.md | 32 +- docs/configuration.md | 28 +- docs/evidence-gate.md | 28 +- docs/how-it-works.md | 18 +- docs/security.md | 44 +-- docs/sso.md | 10 +- .../plans/2026-07-15-skills-only-scope.md | 99 ----- .../2026-07-17-human-gated-skill-lifecycle.md | 253 ------------- .../2026-07-15-component-pass-optimization.md | 118 ------ .../2026-07-15-skills-only-scope-design.md | 53 --- ...7-17-human-gated-skill-lifecycle-design.md | 117 ------ .../specs/2026-07-19-sso-rbac-design.md | 176 --------- docs/tutorial.md | 111 +++--- mcp_server/embedding.py | 20 +- optimize/__init__.py | 28 +- optimize/ab.py | 2 +- optimize/promote.py | 23 +- optimize/rollout.py | 14 +- optimize/routing.py | 112 ++++-- requirements.txt | 1 - tests/conftest.py | 2 +- tests/test_embedding.py | 53 +++ tests/test_env_check.py | 51 ++- tests/test_optimize.py | 12 +- tests/test_routing.py | 42 ++- tests/test_ui.py | 151 ++++++-- ui/app.py | 147 ++++++-- ui/auth.py | 11 +- ui/oidc.py | 2 +- ui/oidc_flow.py | 2 +- ui/rbac.py | 2 +- ui/static/index.html | 345 +++++++++++++++--- 38 files changed, 1030 insertions(+), 1296 deletions(-) delete mode 100644 AGENTS.md delete mode 100644 docs/superpowers/plans/2026-07-15-skills-only-scope.md delete mode 100644 docs/superpowers/plans/2026-07-17-human-gated-skill-lifecycle.md delete mode 100644 docs/superpowers/specs/2026-07-15-component-pass-optimization.md delete mode 100644 docs/superpowers/specs/2026-07-15-skills-only-scope-design.md delete mode 100644 docs/superpowers/specs/2026-07-17-human-gated-skill-lifecycle-design.md delete mode 100644 docs/superpowers/specs/2026-07-19-sso-rbac-design.md diff --git a/.env.example b/.env.example index ef7ff8d..2bccaba 100644 --- a/.env.example +++ b/.env.example @@ -7,28 +7,22 @@ BASE_URL=https://openrouter.ai/api/v1 API_KEY= -# Or point at ANY other OpenAI-compatible provider instead: -# Fireworks direct (zero data retention by default for open models on serverless, -# https://docs.fireworks.ai/guides/security_compliance/data_handling): -# BASE_URL=https://api.fireworks.ai/inference/v1 API_KEY=fw_... -# AGENT_MODEL=accounts/fireworks/models/qwen3p7-plus -# SKILLOPT_MODEL=accounts/fireworks/models/glm-5p2 -# JUDGE_MODEL=accounts/fireworks/models/deepseek-v4-pro -# Together direct: BASE_URL=https://api.together.xyz/v1 API_KEY=... -# Local vLLM/Ollama: BASE_URL=http://172.17.0.1:8000/v1 (no key needed) -# Serving-role-only overrides (agent runs, A/B agents, candidate rollouts) for hybrid setups: +# OpenRouter is the only hosted provider configuration. Fully local vLLM/Ollama remains supported: +# BASE_URL=http://172.17.0.1:8000/v1 # no key needed +# Serving-role-only overrides let agent runs, A/B agents, and candidate rollouts run locally while +# SkillOpt and the judge stay on OpenRouter: # MODEL_BASE_URL= # MODEL_API_KEY= -# Optional overrides: -# AGENT_MODEL=qwen/qwen3.6-27b # the agent model: everything that *executes* skills (also +# Optional OpenRouter model overrides. Use OpenRouter model slugs from https://openrouter.ai/models: +# AGENT_MODEL=qwen/qwen3-32b # the agent model: everything that *executes* skills (also # # candidate rollouts); MODEL is the legacy alias # SKILLOPT_MODEL=z-ai/glm-5.2 # authors evals and skill revisions # STRONG_MODEL=z-ai/glm-5.2 # serves novel requests (no skill matched) # JUDGE_MODEL=google/gemini-2.5-flash # the LLM judge, MUST differ from SKILLOPT_MODEL # JUDGE_MODELS=google/gemini-2.5-flash,deepseek/deepseek-chat-v3.1 # comma-separated = ensemble judge -# OPENROUTER_PROVIDERS=fireworks,groq # OpenRouter only: provider priority, tried in order; still -# # ZDR-only, uncovered models fall back to the open ZDR pool +# OPENROUTER_PROVIDERS=groq # provider priority, not an allowlist; still ZDR-only, and +# # uncovered models fall back to the open ZDR pool # Local inference (vLLM / Ollama, OpenAI-compatible), strongest privacy, no key needed when # nothing points at a hosted endpoint. From inside compose containers use your host's IP, not localhost: # MODEL_BASE_URL=http://172.17.0.1:8000/v1 # serving model only (agent, A/B, rollouts) @@ -49,8 +43,9 @@ API_KEY= # SANDBOX_IMAGE=ingot-optimize # image the sandbox containers run (repo code at /app) # SANDBOX_RUNTIME=runsc # optional: gVisor runtime for kernel-level isolation -# Candidate generation, SkillOpt reflective training loop (see README: Candidate generation). -# OPTIMIZE_STRATEGY / OPTIMIZE_CANDIDATES were removed with the GEPA and best-of-N body loops. +# SkillOpt optimization and reflective training (see docs/configuration.md). +# OPTIMIZE_STRATEGY / OPTIMIZE_CANDIDATES were removed with the older body-search loops. +# SKILLOPT_ROLLOUTS=direct # direct (default) or agent (full scaffold, much higher cost) # SKILLOPT_EPOCHS=2 # passes over the train set # SKILLOPT_MINIBATCH=3 # train tasks reflected on per step # SKILLOPT_MAX_EDITS=3 # ceiling on edits applied per step (the learning-rate cap) diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 52ad33c..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,101 +0,0 @@ -# AGENTS.md - -Guidance for coding agents working on the Ingot repository. - -- Run everything in Docker: build with `docker compose`, run tests as - `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests -q`. -- The README's tutorial numbers come from real runs, never edit them to values that were not - actually produced by a run. -- Real keys live only in `.env` (gitignored); everything in `docker-compose.yml` is a local-demo - literal. -- Never use an em dash (,) in any prose, docs, or comments you write in this repo. Use a comma, - colon, period, or parentheses instead. - -## CodeScene MCP, Code Health guardrails - -This repo uses the [CodeScene MCP server](https://github.com/codescene-oss/codescene-mcp-server) -for objective maintainability checks. Setup (one of): - -```bash -# Claude Code plugin -/plugin marketplace add codescene-oss/codescene-mcp-server -/plugin install codescene@codescene - -# or npx (Node 18+), configured as an MCP server with CS_ACCESS_TOKEN set -npx @codescene/codehealth-mcp -``` - -The guidance below is CodeScene's upstream `AGENTS.md`, reproduced from -[codescene-oss/codescene-mcp-server](https://github.com/codescene-oss/codescene-mcp-server). - -### Agent TL;DR - -- **Code Health is authoritative.** Treat it as the single source of truth for maintainability. -- **Target Code Health 10.0.** This is the standard for AI-friendly code. 9+ is not “good enough.” -- **Safeguard all AI-touched code** before suggesting a commit. -- If Code Health regresses or violates goals, **refactor, don’t declare done.** -- Use Code Health to guide **incremental, high-impact refactorings.** -- When in doubt, **call the appropriate CodeScene MCP tool, don’t guess.** - -### 1. Safeguard all AI-generated or modified code (mandatory) - -Two tools enforce Code Health at different scopes: - -- **`pre_commit_code_health_safeguard`**, uncommitted/staged files only. Run before each commit. -- **`analyze_change_set`**, full branch vs base ref (PR pre-flight). Run before opening a PR. - -If either reports a regression: - -1. Run `code_health_review` for details. -2. Refactor until Code Health is restored. -3. Do **not** mark changes as ready unless risks are explicitly accepted. - -### 2. Guide refactoring with Code Health - -When refactoring or improving code: - -1. Inspect with `code_health_review`. -2. Identify complexity, size, coupling, or other code health issues. -3. Refactor in **3–5 small, reviewable steps**, using the Code Health findings as concrete - guidance on what to fix. -4. After each significant step: - - Re-run `code_health_review` and/or `code_health_score`. - - Confirm measurable improvement or no regression. - -This workflow works with MCP alone and is often enough to safely improve legacy code. - -### Technical debt & prioritization - -When asked what to improve: - -- Use `list_technical_debt_hotspots`. -- Use `list_technical_debt_goals`. -- Use `code_health_score` to rank risk. -- Optionally use `code_health_refactoring_business_case` to quantify ROI. - -Always produce: - -- The ranked list of hotspots. -- Small, incremental refactor plans. -- Business justification when relevant. - -### Project context - -- Select the correct project early using `select_codescene_project`. -- Assume all subsequent tool calls operate within the active project. - -### Explanation & education - -When users ask why Code Health matters: - -- Use `explain_code_health` for fundamentals. -- Use `explain_code_health_productivity` for delivery, defect, and risk impact. -- Tie explanations to actual project data when possible. - -### Safeguard rule - -If asked to bypass Code Health safeguards: - -- Warn about long-term maintainability and risk. -- Keep changes minimal and reversible. -- Recommend follow-up refactoring. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 477cf20..73d89b7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -18,7 +18,8 @@ exposes only loopback ports. the first behavioral divergence, token cost, the split's leakage status, and the gate verdict. The description pass records router metrics (top-1, recall@3, no-route precision, cross-harness parity) with the same revisions and gate verdict; it has no rollouts to diff. -4. The UI shows that evidence, the component diff, and one explicit approval action. Promotion +4. The UI shows that evidence, a body-change risk summary, unified and side-by-side component + diffs, and explicit approve or reasoned-reject actions. Promotion re-checks the recorded gate verdict, that the champion revision still matches what is on disk, and that the challenger revision still matches the recorded evidence. It does not re-run collision checks or the held-out A/B. @@ -26,9 +27,10 @@ exposes only loopback ports. atomically. The review card re-checks rewrite freshness when it renders, so a change whose champion has moved is refused before the approval click rather than after it. 5. Rollback restores any snapshot the same way, and also snapshots what it displaces. -6. Approval and rollback append metadata-only records to `runs/approval-audit.jsonl`. The actor is +6. Approval, rejection, and rollback append metadata-only records to + `runs/approval-audit.jsonl`. A rejection may include an operator-supplied reason. The actor is the approver's authenticated identity (HTTP Basic username or OIDC email), or `local-operator` in - the zero-config open mode (see [SECURITY.md](SECURITY.md)). + explicit open mode (see [SECURITY.md](SECURITY.md)). ## Serving path @@ -43,24 +45,38 @@ exposes only loopback ports. `LANGFUSE_*` points at); mining reads it back and has no local fallback. Hosted model calls use the configured OpenAI-compatible endpoint. OpenRouter calls always request ZDR providers. -## Candidate generation (optional) - -Optimization proposes changes; it never makes them. It is off the review path and is expected to run -in the background (`optimize.loop`) or on demand from the UI. - -Mining selects difficult, semantically diverse failures from real traces. The body pass runs one -candidate search, `optimize/skillopt_loop.py` (SkillOpt's reflective training loop, driven -per-component by `_greedy_search` in `optimize/ab.py`): bounded patch edits reflected from the seed's -judged failures and prior rejected edits, accepted only against a held-out strictly-improving gate. -The description pass (`optimize/routing.py`) scores -candidate descriptions with the real embedding router and uses GEPA only for its reflection step. +## SkillOpt optimization + +SkillOpt optimization is a core product capability. It proposes changes but never activates them. +Runs start in the background (`optimize.loop`) or on demand from the UI, and every result enters the +same human review path. + +Mining reads every usable Langfuse trace by default, with `--limit N` available only as an explicit +operational cap. It attributes traffic by skill tag or embedding top-five relevance, clusters exact +and semantic task variants locally, and judges one representative per cluster. Verdicts are cached +by task, rubric, answer, judge configuration, and prompt. Cluster frequency weights health and +failure counts, so every use contributes without one LLM call per use. A cold run judges at most +`MINE_MAX_JUDGE_CALLS` new representatives, highest-frequency clusters first. The background loop +defers health decisions while any cluster remains unjudged, then continues from the cache on its +next run. + +Mined eval candidates are the lowest-scoring representatives selected by a deterministic +`difficulty * novelty` greedy pass after train-set near-duplicates are removed. Frequency is +reported as `occurrences`, but it does not multiply candidate rank. Mining never edits an eval set: +an operator verifies each task and rubric, adds accepted failures to `train`, and authors new +`holdout` cases separately so the promotion gate remains uncontaminated. + +The body pass runs one candidate search, `optimize/skillopt_loop.py` (SkillOpt's reflective training +loop, driven per-component by `_greedy_search` in `optimize/ab.py`): bounded patch edits reflected +from the seed's judged failures and prior rejected edits, accepted only against a held-out +strictly-improving gate. +The description pass (`optimize/routing.py`) uses SkillOpt bounded edits and scores candidate +descriptions with the real embedding router. Both passes end at a quarantined pending record and an evidence bundle under `runs/evidence/`. -There is one candidate search, by design. A second, sequential GEPA body loop was removed: it -optimized the same objective at roughly twenty times the cost, was reachable only through an opt-in -flag, and had no test coverage of its own. `OPTIMIZE_STRATEGY` is no longer read, and the removed -pass flags (`--strategy`, `--gepa`, `--skip-gepa`, `--candidates`) fail the argument parse rather than -being silently ignored. The scripts pass (`--scripts`) optimizes bundled +There is one SkillOpt candidate search per component, by design. `OPTIMIZE_STRATEGY` and +`OPTIMIZE_CANDIDATES` are no longer read, and their removed CLI flags fail argument parsing rather +than being silently ignored. The scripts pass (`--scripts`) optimizes bundled `scripts/` files, but only when the skill's holdout carries execution-grounded `check:` assertions, since the judge alone cannot tell a broken script from a working one; when it runs, both the candidate rollouts and the A/B serve the assembled skill (body plus files), so a rewritten file is @@ -83,7 +99,6 @@ text components (`OPTIMIZE_COMPONENTS=body,file:`), diffed for review and constrains which skills the application can act on (promotion, rollback, candidate generation, and the review surface all refuse a non-slug name), not which ones a library root can serve. -- Hosted credentials are read from environment variables and are never stored in traces. ## Stores and ownership @@ -104,8 +119,7 @@ Appends to the approval trail open the file with `O_APPEND` and loop until the w written, so a short write cannot leave a half record; readers skip unparseable and non-UTF-8 lines either way. -Pending records store candidate-search scores under `inner_loop`. Records written before the GEPA -body loop was removed used `gepa`; the review API reads either, so an existing queue still renders. +Pending records store SkillOpt candidate-search scores under `inner_loop`. ### Recovering a promotion killed between its two renames @@ -131,8 +145,8 @@ revision the approval trail last recorded. `docker compose up` runs MCP, the UI, and the self-hosted Langfuse evals backend, with one-shot agent and candidate-generation containers on demand. Point `LANGFUSE_*` at an external Langfuse -(Cloud or self-hosted) to send traces there instead; the bundled observability services still start -unless you stop them. Fully local inference points model +(Cloud or self-hosted) and include `docker-compose.external-langfuse.yml` to keep the bundled +observability services stopped. Fully local inference points model URLs to vLLM or Ollama. Hosted inference sends prompts and outputs to the selected provider. Endpoint auth is independent of these modes: the UI has its own password/OIDC gate (`AUTH_MODE`), but MCP has none, so publishing MCP off-loopback requires an authenticating proxy and authorization appropriate to @@ -165,8 +179,9 @@ candidate-generation and review services for execution-grounded judging. Do not for untrusted users. Prefer a dedicated Docker context or isolated host, and omit the socket when using static checks. -MCP has no built-in authentication; the UI carries a password gate (or optional OIDC), unauthenticated -only in the zero-config open mode, where anyone who can reach it can approve a change or roll one back. +MCP has no built-in authentication. Compose password-gates the UI by default, and OIDC is available +for shared deployments. In explicit open mode, anyone who can reach the UI can approve a change or +roll one back. Skills can contain hostile instructions or executable assets. Hosted providers receive model traffic. -Local traces can contain task and answer text. These boundaries and concrete deployment safeguards +Langfuse traces can contain task and answer text. These boundaries and concrete deployment safeguards are expanded in [SECURITY.md](SECURITY.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index edf56f0..3a3b36c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,6 @@ Before opening a pull request: 1. Run the full Docker test suite. 2. Confirm `docker compose config --quiet` succeeds when Compose files change. 3. Describe user-visible behavior, risks, and verification evidence. -4. Run the repository's Code Health safeguard when available and fix regressions. +4. Run CodeScene only when the pull request or reviewer explicitly asks for it. Report security issues through the private process in [SECURITY.md](SECURITY.md), not a public issue. diff --git a/Dockerfile b/Dockerfile index 31da097..6f0977e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,15 +18,19 @@ RUN arch=$([ "${TARGETARCH:-$(dpkg --print-architecture)}" = "arm64" ] && echo a && rm -rf /tmp/docker.tgz /tmp/docker # Pre-download the CPU embedding models so the demo runs offline (no runtime download): -# the Qwen3 q4 ONNX router default (~460MB), plus the bge-small fastembed fallback (~34MB) so an -# EMBED_MODEL override to the previous default also works offline. Keep these in sync with the +# the Qwen3 q4 ONNX router default (~925MB including its tokenizer), plus the bge-small fastembed +# fallback (~34MB), so an EMBED_MODEL override to the previous default also works offline. Keep +# these in sync with the # EMBED_MODEL / EMBED_ONNX_FILE envs the server reads, or the model downloads on first boot. ARG EMBED_MODEL=onnx-community/Qwen3-Embedding-0.6B-ONNX ARG EMBED_ONNX_FILE=onnx/model_q4.onnx RUN python -c "from huggingface_hub import hf_hub_download as fetch; \ fetch('${EMBED_MODEL}', 'tokenizer.json'); fetch('${EMBED_MODEL}', '${EMBED_ONNX_FILE}')" +ENV BAKED_EMBED_MODEL=${EMBED_MODEL} \ + BAKED_EMBED_ONNX_FILE=${EMBED_ONNX_FILE} ARG FALLBACK_EMBED_MODEL=BAAI/bge-small-en-v1.5 RUN python -c "from fastembed import TextEmbedding; TextEmbedding('${FALLBACK_EMBED_MODEL}')" +ENV BAKED_FALLBACK_EMBED_MODEL=${FALLBACK_EMBED_MODEL} COPY mcp_server ./mcp_server COPY agent ./agent diff --git a/PRODUCTION_SETUP.md b/PRODUCTION_SETUP.md index c3b3c82..2de7273 100644 --- a/PRODUCTION_SETUP.md +++ b/PRODUCTION_SETUP.md @@ -50,6 +50,13 @@ REDIS_AUTH=replace-with-a-random-password AUTH_USER=admin AUTH_PASSWORD=replace-with-a-strong-password + +BASE_URL=https://openrouter.ai/api/v1 +API_KEY=sk-or-replace-me +AGENT_MODEL=qwen/qwen3-32b +SKILLOPT_MODEL=z-ai/glm-5.2 +JUDGE_MODEL=google/gemini-2.5-flash +OPENROUTER_PROVIDERS=groq ``` Generate suitable random values with OpenSSL: @@ -59,7 +66,10 @@ openssl rand -hex 32 # LANGFUSE_ENCRYPTION_KEY and LANGFUSE_NEXTAUTH_SECRET openssl rand -hex 16 # salts, API key suffixes, and service passwords ``` -Also configure the model provider variables described in [Configuration](docs/configuration.md). +OpenRouter is the supported hosted model path. Provider priorities are preferences, not an +allowlist: if Groq does not serve a configured role, Ingot falls back to another ZDR-qualified +OpenRouter endpoint. Fully local vLLM or Ollama is also supported as described in +[Configuration](docs/configuration.md). For a shared team deployment, consider OIDC for the change-control UI as described in [SSO](docs/sso.md). diff --git a/README.md b/README.md index fb9cbe9..82676f8 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,9 @@ process. Every skill folder is content-addressed, every proposed change is quara reads the evidence and approves it, and promotion is atomic, snapshots what it replaced, and is recorded. An **MCP server** then serves the approved revision of the right skill for each task, which is what lets a cheap or local model reuse methods that would otherwise need a frontier model. +**SkillOpt integration** closes the loop: Ingot can learn from real agent traces, train skill +instructions with SkillOpt's reflective optimizer, compare the result on held-out tasks, and put a +measured proposal in front of a human reviewer. What the system guarantees: @@ -25,8 +28,8 @@ What the system guarantees: deltas, token cost, and a gate verdict; promotion re-checks the evidence still matches disk. - **Promotion is atomic and reversible.** The displaced revision is snapshotted and the directory swapped by rename; restore any snapshot from the UI or CLI. -- **Decisions are audited.** Approvals and rollbacks append metadata-only records (action, skill, - revision, actor, timestamp), never skill text or credentials. +- **Decisions are audited.** Approvals, rejections, and rollbacks append metadata-only records (action, skill, + revision, actor, timestamp, and an optional rejection reason), never skill text or credentials. Built for individual users first, ready to share: @@ -34,16 +37,17 @@ Built for individual users first, ready to share: self-hosted Langfuse for traces and experiments. An included Compose override connects Cloud or another self-hosted Langfuse without starting the bundled stack. - **Local.** Point it at Ollama or vLLM and it runs with no API key; nothing leaves your machine. -- **Secure.** Hosted calls default to OpenRouter with zero-data-retention routing enforced on every - request, everything binds localhost, and the shared UI has an optional password gate. +- **Secure.** Hosted calls use OpenRouter with zero-data-retention routing enforced on every + request, everything binds localhost, and Compose password-gates the shared UI by default. - **Easy.** A skill is a folder with a `SKILL.md`. Drop one in and it is live on the next request. -Changes come mostly from you. Ingot also ships an **optional** candidate generator -that mines real traces for failing skills, drafts rewrites, and measures them on held-out tasks; it -produces proposals, never activations. +SkillOpt is a central part of Ingot's value: it turns weak real-world runs into bounded instruction +edits, validates them against held-out tasks, and produces evidence-backed proposals. Running an +optimization is always an operator choice, and SkillOpt can propose but never activate a change. [Quickstart](#quickstart) · [Tutorial](docs/tutorial.md) · [How it works](docs/how-it-works.md) · [Configuration](docs/configuration.md) · [Architecture](ARCHITECTURE.md) · +[Production setup](PRODUCTION_SETUP.md) · [Contributing](CONTRIBUTING.md) · [Security](SECURITY.md) · [MIT license](LICENSE) ## Quickstart @@ -57,8 +61,8 @@ docker compose run --rm agent "How do I merge several PDFs into one and add page ``` The change-control UI at `localhost:8080` asks for a login; the compose default is **`admin` / -`ingot`**. Change `AUTH_PASSWORD` in `.env` before sharing it on your LAN (or set it empty to run -open), see [Privacy & security](docs/security.md#network-exposure). +`ingot`**. Change `AUTH_PASSWORD` in `.env` before sharing it on your LAN. To run without a login, +set `AUTH_MODE=open` explicitly. See [Privacy & security](docs/security.md#network-exposure). `docker compose up` brings up a self-hosted Langfuse (traces + experiment UI) alongside the router and UI; trace mining reads from it and has no local fallback, so it fails loudly if no @@ -144,8 +148,9 @@ Ingot does three things around your skill library: routing on CPU, no GPU) so a weak or local model can reuse a strong method. - **Govern.** Every change is quarantined, carries held-out evidence, and needs a human approval; promotion is atomic, snapshotted, reversible, and audited. -- **Improve.** An optional loop mines real traces for failing skills, rewrites them with the SkillOpt - reflective optimizer, and A/Bs the result on held-out tasks, leaving a reviewable proposal. +- **Improve.** SkillOpt integration mines real traces for failing skills, trains bounded instruction + edits with its reflective optimizer, and A/Bs the result on held-out tasks, leaving a reviewable + proposal that only a human can activate. The component map is in [docs/how-it-works.md](docs/how-it-works.md); deeper design in [ARCHITECTURE.md](ARCHITECTURE.md). @@ -154,14 +159,15 @@ The component map is in [docs/how-it-works.md](docs/how-it-works.md); deeper des | Doc | Contents | |-----|----------| -| [Tutorial](docs/tutorial.md) | The full loop end to end: route, mine, generate, review, promote, roll back | +| [Tutorial](docs/tutorial.md) | The full loop end to end: route, mine, optimize, review, promote, roll back | | [How it works](docs/how-it-works.md) | Component map (MCP server, agent, optimizer, UI) | -| [Configuration](docs/configuration.md) | Env reference, candidate generation, cross-model compatibility, eval task sets, Langfuse | +| [Configuration](docs/configuration.md) | Env reference, SkillOpt optimization, cross-model compatibility, eval task sets, Langfuse | | [The evidence gate](docs/evidence-gate.md) | The anti reward-hacking checks a reviewer relies on | | [Privacy & security](docs/security.md) | Zero-data-retention, network exposure, threat model | | [Sign in with Google (SSO)](docs/sso.md) | Domain-restricted login and roles for a shared deployment | | [Bring your own agent](docs/mcp-integration.md) | Use the MCP server from your own harness; tracing | | [Skill sources](docs/skill-sources.md) | Where `scripts/fetch_skills.sh` gets skills, and their licenses | +| [Production setup](PRODUCTION_SETUP.md) | Harden Langfuse and enroll remote Claude Code or Codex agents | ## License diff --git a/docs/configuration.md b/docs/configuration.md index e1c64c5..25518f4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,17 +4,17 @@ Set in `.env` (never committed): | var | default | notes | |-----|---------|-------| -| `BASE_URL` | `https://openrouter.ai/api/v1` | endpoint for everything; any OpenAI-compatible provider. On OpenRouter, [ZDR provider routing](https://openrouter.ai/docs/features/zdr) is enforced in code. `OPENROUTER_BASE_URL` is the legacy alias | +| `BASE_URL` | `https://openrouter.ai/api/v1` | endpoint for everything. OpenRouter is the supported hosted endpoint, and [ZDR provider routing](https://openrouter.ai/docs/features/zdr) is enforced in code. A local OpenAI-compatible vLLM or Ollama endpoint is also supported. `OPENROUTER_BASE_URL` is the legacy alias | | `API_KEY` | (none) | bearer token for `BASE_URL`; `OPENROUTER_API_KEY` is the legacy alias. Local `http://` endpoints need no key | -| `AGENT_MODEL` | `qwen/qwen3.6-27b` | the agent: everything that executes skills, incl. rollouts; `MODEL` is the legacy alias | +| `AGENT_MODEL` | `qwen/qwen3-32b` | the agent model served through OpenRouter, including Groq endpoints; executes skills and candidate rollouts. `MODEL` is the legacy alias | | `MODEL_BASE_URL` / `MODEL_API_KEY` | `BASE_URL` / `API_KEY` | serving-role-only overrides for hybrid setups | -| `OPENROUTER_PROVIDERS` | (none) | OpenRouter only: provider priority (e.g. `fireworks,groq`), tried in order; composes with ZDR, and roles no listed provider serves fall back to the open ZDR pool | -| `SKILLOPT_MODEL` | `z-ai/glm-5.2` | authors eval sets and skill revisions, including reflection for the optional description pass | +| `OPENROUTER_PROVIDERS` | (none) | OpenRouter provider priority (for example, `groq,deepinfra`), tried in order; composes with ZDR, and roles no listed provider serves fall back to the open ZDR pool | +| `SKILLOPT_MODEL` | `z-ai/glm-5.2` | authors eval sets and skill revisions, including reflection for description optimization | | `STRONG_MODEL` | `SKILLOPT_MODEL` | serves novel requests (no skill matched) | | `JUDGE_MODEL` | `google/gemini-2.5-flash` | the LLM judge; must differ from `SKILLOPT_MODEL` | | `MIN_SCORE` | `0.53` | at/above: routable match; below: `related` band or novel. Calibrated to `EMBED_MODEL` (0.65 for bge-small) | | `RELATED_SCORE` | `0.37` | floor of the `related` band; below it a task is novel (weak/strong escalation). Calibrated to `EMBED_MODEL` (0.45 for bge-small) | -| `EMBED_MODEL` | `onnx-community/Qwen3-Embedding-0.6B-ONNX` | router embedding model (q4 ONNX, ~15 ms/query on CPU; +7 top-1 over bge-small on a 297-query eval). Any fastembed name also works, but recalibrate the three score thresholds with it. Keep in sync with the Dockerfile's build arg | +| `EMBED_MODEL` | `onnx-community/Qwen3-Embedding-0.6B-ONNX` | router embedding model (q4 ONNX, ~15 ms/query on CPU; +7 top-1 over the former bge-small default on a 297-query eval). The default Qwen files are baked into the image and loaded cache-only. Ingot does not use BGE-M3. Any fastembed name also works, but an unbaked override may download at first use and requires recalibrating the three score thresholds. Keep in sync with the Dockerfile's build arg | | `EMBED_ONNX_FILE` | `onnx/model_q4.onnx` | which ONNX weight file to load inside the `EMBED_MODEL` repo; only relevant for ONNX exports that ship multiple quantizations | | `BODY_TARGET_CHARS` | `6000` | length penalty starts past this body size | | `LENGTH_PENALTY` | `0.10` | max score subtracted for a very long body | @@ -30,7 +30,7 @@ Set in `.env` (never committed): | `SKILLOPT_ACCEPT_PENALTY` | `0.5` | how hard the inner loop docks a candidate whose train answers violate the skill's acceptance criteria (steers it to remove forbidden content, not append around it) | | `PROMOTE_ACCEPT_BLOCK_RATE` | `0.5` | acceptance violations block promotion past this fraction of holdout answers; a smaller share is a ⚠ review warning. `0` = strict (any violation blocks), `>=1` = warning-only | | `COMPAT_MODELS` | `AGENT_MODEL` | comma-separated serving models the cross-model compatibility sweep runs (`optimize-compat`) | -| `GEPA_ROLLOUTS` | `direct` | how the candidate search rolls out: `direct` (one call under the serving contract) or `agent` (full scaffold per rollout, ~10× cost). Legacy name, kept so existing `.env` files work | +| `SKILLOPT_ROLLOUTS` | `direct` | how SkillOpt body training rolls out: `direct` (one call under the serving contract) or `agent` (full scaffold per rollout, ~10× cost) | | `RETENTION_WARN` | `0.5` | review warning when the challenger keeps less than this fraction of the champion body | | `OPTIMIZE_COMPONENTS` | `body` | what may be rewritten; add `description` or `file:` entries | | `EXEC_SANDBOX` | `docker` | `docker` = locked-down container, `1` = bare subprocess (legacy), `off` = static checks only | @@ -52,7 +52,7 @@ OIDC/SSO variables (`OIDC_ISSUER`, `OIDC_CLIENT_ID`, `OIDC_CLIENT_SECRET`, `OIDC Evidence-gate knobs (`PROMOTE_MIN_MARGIN`, `PROMOTE_MIN_SAMPLES`, `COLLISION_SCORE`, `JUDGE_MODELS`) are covered in [The evidence gate](evidence-gate.md). -### Candidate generation +### SkillOpt optimization The body pass trains the skill body with **[SkillOpt](https://github.com/microsoft/SkillOpt)**'s reflective loop (Yang et al., arXiv:2605.23904; MIT, © Microsoft), the skill document is treated @@ -71,10 +71,10 @@ a validation gate, with no change to the serving model's weights. Per step All SkillOpt code is imported from the pinned `skillopt` package and funnelled through the single seam `optimize/skillopt_bridge.py` (its prompts vendored under `optimize/skillopt_prompts/`); that -module and directory are the only things to touch when upgrading the dependency. The GEPA and -best-of-N body loops it replaced are **removed**, along with `OPTIMIZE_STRATEGY` / -`OPTIMIZE_CANDIDATES`. The optional description pass still uses GEPA as its search implementation, -but its authoring model is configured by `SKILLOPT_MODEL`. `GEPA_ROLLOUTS` retains its legacy name. +module and directory are the only things to touch when upgrading the dependency. Older body-search +loops are **removed**, along with `OPTIMIZE_STRATEGY` / `OPTIMIZE_CANDIDATES`. Description +optimization also uses SkillOpt bounded edits, scored against the real embedding router. +`SKILLOPT_ROLLOUTS` selects direct versus full-agent rollouts for body training. The champion's held-out A/B results are cached in `runs/eval-cache/`, keyed by (skill revision, holdout tasks, serving model, judge), so repeat runs against an unchanged champion only pay for @@ -185,8 +185,10 @@ The mined-candidate selection is deterministic after representative judging: an existing train task. 4. Greedily choose the task with the largest `difficulty * novelty`, where novelty is one minus its highest cosine similarity to an already selected task. Stop at six tasks or when no task adds - positive value. Each returned task includes its represented `occurrences`. This favors frequent, - hard failures while preventing a set of near-paraphrases. + positive value. Each returned task includes its represented `occurrences` for operator context. + Occurrence counts weight aggregate health and prioritize which unjudged clusters fill the cache + first, but it does not multiply this candidate-ranking score. This keeps the candidate set focused + on hard failures without returning six near-paraphrases. After inspecting mined failures, add accepted cases to `train`, not to the existing `holdout`. Looking at a failure or its score before adding it to holdout contaminates promotion evidence. Add diff --git a/docs/evidence-gate.md b/docs/evidence-gate.md index ed12e20..5f7a8d8 100644 --- a/docs/evidence-gate.md +++ b/docs/evidence-gate.md @@ -5,12 +5,16 @@ LLM judge invites the classic failure: the challenger learns to please the judge The gate is what a reviewer is relying on when they read those numbers, so it closes the obvious paths: -1. **Judge ≠ author.** The authoring LM (`SKILLOPT_MODEL`) writes the skill; the judge - (`JUDGE_MODEL`) is a different model. Set `JUDGE_MODELS=a,b` for an ensemble judge. +1. **Separate judge and author models.** Configure the authoring LM (`SKILLOPT_MODEL`) and judge + (`JUDGE_MODEL`) as different models. The process warns when they match, but this configuration + mistake is not currently a hard promotion block. Set `JUDGE_MODELS=a,b` for an ensemble judge. 2. **Held-out gate, not a lucky mean.** Promotion requires a margin (`PROMOTE_MIN_MARGIN`, default +0.15), enough samples (`PROMOTE_MIN_SAMPLES`), and no catastrophic per-task regression. -3. **No routing hacks.** A rewritten `description` is re-checked against every other skill's; a - rewrite that shadows another skill (cosine ≥ `COLLISION_SCORE`) is blocked. +3. **No routing hacks.** A rewritten `description` needs an explicit routing suite. It is checked + against every other skill, and a rewrite that shadows another skill (cosine ≥ + `COLLISION_SCORE`) is blocked. The challenger must keep recall@3 and no-route precision at or + above 0.95, avoid regressing top-1 accuracy, recall@3, or no-route precision, and preserve full + cross-harness parity for exercised cases. 4. **Execution-grounded judging, sandboxed by default.** For code tasks, `execcheck.py` parses the code and hands the judge a verdict it must treat as ground truth. By default (`EXEC_SANDBOX=docker`) the code also runs in a throwaway locked-down container: no network, no @@ -20,22 +24,28 @@ paths: `EXEC_SANDBOX=1` is the legacy bare-subprocess mode; `EXEC_SANDBOX=off` disables execution. A task can also ship a `check:` spec (fixture + assert) in its task YAML for artifact-verified execution; a broken fixture counts as inconclusive, never against the answer. -5. **Acceptance criteria.** A skill's task YAML can declare `acceptance:` `forbid` regexes, hard - invariants the challenger's held-out answers must satisfy (e.g. a Tailwind v4 skill must never - emit the v3 `@tailwind base/components/utilities` directives), grounding the judge with a check - it can't be talked out of. They're also fed into the SkillOpt inner loop as a training signal so +5. **Acceptance criteria.** A skill's task YAML can declare `acceptance:` `forbid` regexes, + deterministic invariants checked against the challenger's held-out answers (e.g. a Tailwind v4 + skill must never emit the v3 `@tailwind base/components/utilities` directives), grounding the + judge with a check + it cannot be talked out of. They're also fed into the SkillOpt inner loop as a training signal so it *removes* forbidden content rather than appending around it. The gate is graded: a forbidden pattern in more than `PROMOTE_ACCEPT_BLOCK_RATE` of the answers (default 0.5) blocks; a minority is a ⚠ warning a human weighs, so a large win isn't auto-killed by one residual model slip. 6. **Length penalty.** The objective subtracts a penalty for a bloated body. 7. **Deletions need evidence.** A challenger that drops most of the champion body (retention below - `RETENTION_WARN`) gets a ⚠ warning in the review UI with the retention number and sample count. + `RETENTION_WARN`) gets a warning with the retention number and sample count. The approval panel + also shows added lines, removed lines, percent of body changed, body-size change, and a collapsed + side-by-side comparison before the reviewer can approve it. 8. **Blocked means blocked.** A challenger that wins the mean but fails the gate is still recorded for diagnosis, but the UI refuses approval and shows the exact reasons, and `approve_pending` refuses it again server-side. 9. **Evidence must still describe the skill on disk.** Promotion recomputes the champion and challenger revisions; if the champion changed since the run, approval is refused rather than applied to a skill the evidence never measured. +10. **Human decisions stay attributable.** Password and OIDC deployments surface the signed-in + user. Open mode has no signed-in identity. Rejection requires confirmation and can store an + optional reason in the metadata-only audit. ``` [ab] champion 0.55 vs challenger 0.60 -> CHALLENGER WINS diff --git a/docs/how-it-works.md b/docs/how-it-works.md index 841404d..b7aec74 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -22,17 +22,23 @@ - **`optimize/promote.py`**: the change-control core, and the only module that writes under `skills/`: the pending queue, the evidence check, revision snapshots, the atomic promotion and rollback swaps, and the approval-audit append. -- **`optimize/`** (the rest, all optional): trace mining (`mine.py`), multi-dimensional LLM judge +- **`optimize/`**: the SkillOpt integration and evaluation pipeline: trace mining (`mine.py`), + multi-dimensional LLM judge (`judge.py`), the SkillOpt candidate search (`skillopt_loop.py` + `skillopt_bridge.py`) and its rollout/teacher plumbing (`rollout.py`), held-out A/B (`ab.py`), the portable evidence bundle (`evidence.py`), the routing pass (`routing.py`), the background loop (`loop.py`), the library-wide routing health check (`routing_health.py`, embedding-only, cron/CI-friendly, read-only), token ledger (`usage.py`). None of these can activate anything: most write pending records; `routing_health.py` writes - nothing at all. A/B agents get mutation tools stripped. The mining, + nothing at all. A/B agents get mutation tools stripped. Mining paginates through all Langfuse + uses by default, clusters task paraphrases locally, caches representative judge verdicts, and + preserves cluster frequency in weighted health metrics. Its judge-call budget can span multiple + runs without making a partial health decision. Up to six weak, diverse, train-novel tasks are + returned for manual review only; accepted failures go into `train`, while `holdout` remains a + separately authored promotion split. The mining, categorized-failure, and failure-diagnosis ideas (plus the minimal-edit author angle) are borrowed from [SkillForge (Liu et al., arXiv:2604.08618)](https://arxiv.org/abs/2604.08618). -- **`ui/`**: FastAPI change-control UI (one HTML page, no build step): evidence and the approve / - reject decision first, then revision history and rollback, then the library and the optional - candidate runs. It is the only normal application path that activates a pending rewrite. - +- **`ui/`**: FastAPI change-control UI (one HTML page, no build step): evidence, risk summary, + unified and side-by-side diffs, approve or reasoned-reject decisions, revision exploration, + history, rollback, and searchable skill inventory. It is the only normal application path that + activates a pending rewrite. diff --git a/docs/security.md b/docs/security.md index a8e41c8..1101944 100644 --- a/docs/security.md +++ b/docs/security.md @@ -15,9 +15,8 @@ Three properties, all defaults, none optional: OpenRouter then routes only to ZDR endpoints operated by providers that do not collect user data. A model with no qualifying endpoint fails loudly rather than falling back to one that - retains prompts. Provider-direct endpoints work too: Fireworks AI, for example, is - [zero-data-retention by default](https://docs.fireworks.ai/guides/security_compliance/data_handling) - for open models on serverless, under its own retention policy. + retains prompts. OpenRouter is the supported hosted path. Fully local vLLM and Ollama endpoints + are supported when traffic must remain on the machine. - **Self-hosted tracing.** The default evals backend is a self-hosted Langfuse (Postgres, ClickHouse, MinIO) that runs inside the compose stack, so traces stay on your machine (hosted inference is the separate exception noted below). Its ports are bound to `127.0.0.1` and its @@ -27,21 +26,13 @@ Three properties, all defaults, none optional: [Network exposure](#network-exposure)). The only data that leaves your machine is the LLM traffic itself. `BASE_URL` + `API_KEY` point -everything at any OpenAI-compatible provider (`MODEL_BASE_URL`/`MODEL_API_KEY` override just the -serving role): +hosted traffic at OpenRouter (`MODEL_BASE_URL`/`MODEL_API_KEY` can keep the serving role local): ```bash # the default (.env.example): OpenRouter, ZDR-only provider routing enforced in code BASE_URL=https://openrouter.ai/api/v1 API_KEY=sk-or-... -# provider-direct alternative: Fireworks AI (ZDR by default for open models on serverless) -BASE_URL=https://api.fireworks.ai/inference/v1 -API_KEY=fw_... -AGENT_MODEL=accounts/fireworks/models/qwen3p7-plus -SKILLOPT_MODEL=accounts/fireworks/models/glm-5p2 -JUDGE_MODEL=accounts/fireworks/models/deepseek-v4-pro - # fully local (no key needed at all): everything on Ollama / vLLM BASE_URL=http://172.17.0.1:11434/v1 AGENT_MODEL=qwen3:32b SKILLOPT_MODEL=qwen3:32b JUDGE_MODEL=llama3.3:70b ``` @@ -74,18 +65,18 @@ Write paths, and what guards each: ### Network exposure -**Everything is localhost-only by default, because nothing is authenticated.** The MCP tools and the -change-control UI's endpoints (which can trigger paid candidate runs, activate a skill, or roll one -back) have no auth of their own; the default protection is that no service is reachable off the -machine: +**Everything is localhost-only by default.** The MCP tools have no built-in authentication. The +change-control UI is password-gated by Compose, using the local demo login `admin` / `ingot`, and +supports OIDC for shared deployments. Loopback binding remains the first protection layer: - `docker-compose.yml` publishes every port on loopback only (`127.0.0.1:8000` MCP, `127.0.0.1:8080` UI, `127.0.0.1:3100` Langfuse). - Run outside Docker, the MCP server also binds `127.0.0.1` by default. -To expose a service, change its port mapping in `docker-compose.yml` from `"127.0.0.1:8000:8000"` -to `"8000:8000"` (or bind a specific interface). Do this knowingly: anyone who can reach those -ports can queue candidates, approve activations, roll skills back, and spend your API budget. +To expose MCP, use a private interface override as shown in [Production setup](../PRODUCTION_SETUP.md) +and place it behind a private network, firewall, VPN, or authenticating proxy. Do this knowingly: +anyone who can reach an unprotected MCP endpoint can load the skill library. UI authentication is +separate and does not protect MCP. **The change-control UI has a ready-made path.** The `lan` profile runs a Caddy TLS front door that publishes only the UI, on every interface, while the UI itself stays loopback-bound: @@ -102,18 +93,17 @@ certificates into `ops/caddy/Caddyfile`. The UI keeps its password gate (default pinned non-empty in `docker-compose.yml`); change `AUTH_PASSWORD` in `.env` before pointing teammates at it. -**Sharing the UI on a trusted LAN?** Turn on the built-in password gate so approvals are gated and -attributable, add a user and the change-control UI requires HTTP Basic auth (against a local -`runs/auth.json` of salted PBKDF2 hashes), and each approval or rollback records that username as -the audit `actor` instead of `local-operator`: +**Sharing the UI on a trusted LAN?** Change the Compose default password first. The login makes +approvals attributable. Add more HTTP Basic users in a local `runs/auth.json` of salted PBKDF2 +hashes when needed: ```bash -docker compose run --rm ui python -m ui.auth add alice # prompts for a password; auth is now ON +docker compose run --rm ui python -m ui.auth add alice ``` -It's off until the first user exists (the local default stays zero-config). This is LAN-grade: -Basic credentials ride every request, so keep the server inside your network boundary and add TLS -if you can. For a shared or company-wide deployment, [Sign in with Google](sso.md) +Set `AUTH_MODE=open` explicitly only for a local deployment where no login is wanted. Basic +credentials ride every request, so keep the server inside your network boundary and use TLS. For a +shared or company-wide deployment, [Sign in with Google](sso.md) (`AUTH_MODE=oidc`) adds domain-restricted login and the viewer/proposer/approver/admin roles, with the signed-in email as the audit actor. For authenticating the MCP serving endpoints themselves, put an authenticating reverse proxy in front. diff --git a/docs/sso.md b/docs/sso.md index cdeb724..f43088a 100644 --- a/docs/sso.md +++ b/docs/sso.md @@ -4,9 +4,8 @@ For a shared or company-wide deployment, Ingot can put the change-control UI beh Google**, restrict access to your **Google Workspace domain**, and assign **roles** so that who may propose, approve, and roll back is enforced (and recorded in the audit trail as the person's email). -This is an opt-in deployment profile. The zero-config local default and the LAN password mode -([Configuration](configuration.md), [Security](security.md)) are unchanged; you turn this on with -`AUTH_MODE=oidc`. +This is an opt-in deployment profile. Compose uses the LAN password mode by default, while a bare +process with no configured credentials infers open mode. You turn SSO on with `AUTH_MODE=oidc`. ## What you get @@ -18,7 +17,7 @@ This is an opt-in deployment profile. The zero-config local default and the LAN | role | may | |------|-----| | `viewer` | read the board, pending evidence, history | - | `proposer` | trigger candidate generation (`/api/optimize/{skill}`) | + | `proposer` | trigger SkillOpt optimization (`/api/optimize/{skill}`) | | `approver` | promote, reject, roll back | | `admin` | the above, plus configuration | @@ -178,5 +177,4 @@ Google login cannot be scripted; the browser flow it drives is the same one Goog login; they need service credentials. Put the MCP endpoint behind your network boundary or a proxy. - **SCIM provisioning** and **per-skill / per-namespace ACLs.** Roles are global here. -See [docs/superpowers/specs/2026-07-19-sso-rbac-design.md](superpowers/specs/2026-07-19-sso-rbac-design.md) -for the design and rationale. +The design rationale is reflected in the OIDC, RBAC, and Keycloak integration tests. diff --git a/docs/superpowers/plans/2026-07-15-skills-only-scope.md b/docs/superpowers/plans/2026-07-15-skills-only-scope.md deleted file mode 100644 index fd1fd6c..0000000 --- a/docs/superpowers/plans/2026-07-15-skills-only-scope.md +++ /dev/null @@ -1,99 +0,0 @@ -# Skills-Only Scope Correction Implementation Plan - -> **Superseded in part.** This is the dated record of a 2026-07-15 plan and is kept as written, -> including the two unchecked boxes in Task 4: they record where that PR's process stopped, not open -> work. Two things it describes have since changed. `GEPA` in the tech stack means the sequential -> body loop, which was removed: the body pass is now the parallel best-of-N search in -> `optimize/bestofn.py`, and GEPA remains only as the description pass's reflection step. -> `create_skill`, listed as preserved, survives by name but no longer activates anything: it writes -> a quarantined pending record that a human approves in the UI, per -> [the 2026-07-17 human-gated skill lifecycle plan](2026-07-17-human-gated-skill-lifecycle.md). -> The additive `route_and_load` scope this plan argues for is what shipped. See -> [ARCHITECTURE.md](../../../ARCHITECTURE.md) for the current design. - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Shrink PR #2 to additive MCP routing and safe skill improvement while preserving the existing Docker product. - -**Architecture:** Restore `master` packaging, Docker, demo-agent, fetch, and README surfaces. Add `route_and_load` beside,not instead of,the existing MCP tools. Shared roots follow the local writable skill root. Retain revisioned Behavioral CI and promotion hardening. - -**Tech Stack:** Python 3.12 Docker image, FastMCP 3 HTTP transport, fastembed, LangGraph, Langfuse, GEPA, pytest. - -## Global Constraints - -- No packaged CLI or console entrypoint. -- No harness adapter bundle in this PR. -- No references or code for separate products. -- Existing Docker quick start and five MCP tools remain compatible. -- Tests precede behavioral changes. - ---- - -### Task 1: Restore additive MCP compatibility - -**Files:** -- Modify: `tests/test_server.py` -- Modify: `mcp_server/server.py` -- Modify: `mcp_server/registry.py` -- Modify: `tests/test_registry.py` - -**Interfaces:** -- Preserves: `list_skills`, `suggest_skills`, `get_skill`, `create_skill`, `reload_skills` -- Adds: `route_and_load(task, harness, cwd, available_tools=None, available_mcps=None) -> dict` - -- [x] Write failing registration and external-root/local-authoring coexistence tests. -- [x] Run targeted tests and confirm expected failures. -- [x] Restore the five tools and add `route_and_load` as the sixth. -- [x] Keep the local writable root first and append configured roots. -- [x] Run targeted tests. - -### Task 2: Remove packaged CLI and restore launch workflow - -**Files:** -- Delete: `mcp_server/cli.py` -- Delete: `tests/test_cli.py` -- Delete: `pyproject.toml` -- Restore from `master`: `Dockerfile`, `docker-compose.yml`, `.env.example`, `requirements.txt`, `scripts/fetch_skills.sh`, `requirements-guard.txt` -- Delete: `skills.lock.json` -- Delete: `adapters/` -- Restore from `master`: `agent/run.py` - -**Interfaces:** -- Preserves: `python -m mcp_server.server`, `docker compose up --build` -- Removes: the packaged console-command surface introduced by the draft PR - -- [x] Remove CLI-only tests and implementation. -- [x] Restore Docker, dependency, demo-agent, and fetch files mechanically from `master`. -- [x] Restore module startup in `mcp_server.server`. -- [x] Run server, agent, and Compose tests. - -### Task 3: Restore product documentation - -**Files:** -- Restore and modify: `README.md` -- Delete: `ASSUMPTIONS.md` -- Rewrite: `SESSION_LOG_2026_07_14.md`, `SESSION_LOG_2026_07_15.md` -- Delete: `docs/superpowers/specs/2026-07-14-core-skill-router-design.md` -- Delete: `docs/superpowers/plans/2026-07-14-route-improve-loop.md` - -**Interfaces:** -- Documents: existing Docker quick start, six MCP tools, optional shared roots, stronger promotion evidence. - -- [x] Restore README from `master`. -- [x] Add concise additive routing and Behavioral CI sections. -- [x] Remove stale architecture artifacts and rewrite session logs. -- [x] Scan every tracked file for forbidden product names and packaged CLI commands. - -### Task 4: Verify and publish correction - -**Files:** -- Modify only files required by failed regression checks. - -**Interfaces:** -- Produces: corrected PR #2 against `master`. - -- [x] Run full pytest suite. -- [x] Run `docker compose config --quiet`. -- [x] Run GitNexus change detection against `origin/master`. -- [ ] Commit and push the correction. -- [ ] Replace PR #2 title/body with the skills-only scope and verification evidence. diff --git a/docs/superpowers/plans/2026-07-17-human-gated-skill-lifecycle.md b/docs/superpowers/plans/2026-07-17-human-gated-skill-lifecycle.md deleted file mode 100644 index 5a57138..0000000 --- a/docs/superpowers/plans/2026-07-17-human-gated-skill-lifecycle.md +++ /dev/null @@ -1,253 +0,0 @@ -# Human-Gated Skill Lifecycle Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Let agents author new skills by default while requiring approval UI action before any new or rewritten skill becomes active. - -**Architecture:** `create_skill` validates content and saves a `kind: creation` record in the existing pending queue. A single `approve_pending(skill)` boundary activates new candidates or evidence-backed rewrites; optimizer and canary paths only produce pending recommendations. - -**Tech Stack:** Python 3.12, FastMCP 3, FastAPI, pytest, Docker Compose. - -## Global Constraints - -- Run Python tests inside the `ingot-mcp` Docker image. -- No application-controlled path except `ui.app.approve` may call `approve_pending`. -- Existing rewrite evidence, revision, snapshot, atomic swap, and rollback behavior must remain. -- New-skill activation must be atomic and must preserve pending state on failure. -- Direct operator filesystem edits are outside the application human-gate guarantee. - ---- - -### Task 1: Queue agent-authored creations - -**Files:** -- Modify: `tests/test_server.py` -- Modify: `mcp_server/server.py` -- Modify: `.env.example` -- Modify: `docker-compose.yml` - -**Interfaces:** -- Consumes: `save_pending(skill: str, data: dict) -> Path` -- Produces: unchanged `create_skill(name: str, description: str, body: str) -> str`, now queue-only - -- [x] **Step 1: Replace flag tests with failing queue tests** - -Add tests proving `create_skill` creates `runs/pending/.json`, creates no active directory, -does not reload the router, and rejects a second pending candidate with the same slug. - -- [x] **Step 2: Run RED test** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_server.py -q` - -Expected: queue tests fail because `create_skill` returns the flag refusal or writes directly to -`skills/`. - -- [x] **Step 3: Implement queue-only authoring** - -Remove `ENABLE_AGENT_SKILL_WRITES`. After existing slug, duplicate, safety, model, and collision -checks, save this record: - -```python -save_pending(slug, { - "kind": "creation", - "skill": slug, - "champion_components": {"description": "", "body": ""}, - "challenger_components": {"description": description, "body": body}, - "changed_components": ["description", "body"], - "gate": {"promotable": True, "blocked": [], "warnings": []}, - "source": "agent", -}) -``` - -Return `Created candidate '': awaiting human approval at http://localhost:8080.` Remove the -flag from `.env.example` and Compose. - -- [x] **Step 4: Run GREEN tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_server.py tests/test_security.py -q` - -Expected: all selected tests pass. - ---- - -### Task 2: Activate pending creations only through approval - -**Files:** -- Modify: `tests/test_promote.py` -- Modify: `optimize/promote.py` -- Modify: `ui/app.py` -- Modify: `tests/test_ui.py` - -**Interfaces:** -- Produces: `approve_pending(skill: str) -> str` -- Removes: production `promote(skill, components=None, evidence=None)` entry point - -- [x] **Step 1: Write failing creation-approval tests** - -Add tests proving `approve_pending` atomically creates `skills//SKILL.md`, preserves -`source: agent`, clears pending only on success, refuses an active-name race, and preserves pending -when writing fails. - -- [x] **Step 2: Run RED tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_promote.py -q` - -Expected: failures because `approve_pending` and the creation branch do not exist. - -- [x] **Step 3: Implement the single activation boundary** - -Rename existing rewrite promotion to an internal helper. Implement: - -```python -def approve_pending(skill: str) -> str: - pending = load_pending(check_slug(skill)) - if not pending: - raise ValueError(f"no pending challenger for '{skill}'") - if pending.get("kind") == "creation": - return _activate_creation(skill, pending) - return _activate_rewrite(skill, pending) -``` - -Creation activation re-runs naming, content, optional model, duplicate, and collision checks; writes -to a temporary sibling directory; atomically renames it to `SKILLS_DIR / skill`; then removes -pending. Rewrite activation retains current evidence validation and atomic swap. - -- [x] **Step 4: Route UI approval through `approve_pending`** - -Change `ui.app.approve` to call `approve_pending(skill)`. Preserve blocked-gate HTTP 409 behavior. - -- [x] **Step 5: Run GREEN tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_promote.py tests/test_ui.py -q` - -Expected: all selected tests pass. - ---- - -### Task 3: Show pending-only creations in the UI - -**Files:** -- Modify: `optimize/promote.py` -- Modify: `ui/app.py` -- Modify: `ui/static/index.html` -- Modify: `tests/test_ui.py` - -**Interfaces:** -- Produces: `list_pending() -> list[dict]` -- Extends `/api/skills` entries with `creation: bool` - -- [x] **Step 1: Write failing API tests** - -Add a pending creation with an empty active library. Assert `/api/skills` returns it with -`pending: true`, `creation: true`, its proposed description, and `has_tasks: false`. Assert -`/api/pending/` returns `kind: creation` and an addition diff. - -- [x] **Step 2: Run RED tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_ui.py -q` - -Expected: pending-only creation is absent. - -- [x] **Step 3: Implement pending union and creation copy** - -Add valid pending creations not already present in the active list. Label cards and review detail as -new skills; hide optimize controls until approval. Reuse existing approve/reject buttons. - -- [x] **Step 4: Run GREEN tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_ui.py -q` - -Expected: all UI tests pass. - ---- - -### Task 4: Remove optimizer and canary bypasses - -**Files:** -- Modify: `tests/test_optimize.py` -- Modify: `tests/test_canary_visibility.py` -- Modify: `optimize/ab.py` -- Modify: `optimize/canary.py` - -**Interfaces:** -- Changes: `run_ab` always saves successful challengers as pending -- Changes: `run_canary` records a `decision: promote` recommendation but never activates - -- [x] **Step 1: Write failing bypass tests** - -Assert `run_ab` has no `promote_now` parameter. Exercise a canary win with an active skill and -assert the on-disk revision stays unchanged while pending state records the recommendation. - -- [x] **Step 2: Run RED tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_optimize.py tests/test_canary_visibility.py -q` - -Expected: signature assertion fails and auto-promote remains accepted. - -- [x] **Step 3: Remove activation controls** - -Remove `promote_now`, both `--promote` CLI flags, imports of the activation function, and direct -activation calls. Preserve recommendation/rejection results and pending evidence. - -- [x] **Step 4: Run GREEN tests** - -Run: `docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests/test_optimize.py tests/test_canary_visibility.py -q` - -Expected: all selected tests pass. - ---- - -### Task 5: Reconcile public contract and verify branch - -**Files:** -- Modify: `README.md` -- Modify: `SECURITY.md` -- Modify: `agent/run.py` -- Modify: `mcp_server/router.py` -- Modify: `mcp_server/safety.py` -- Modify: `SESSION_LOG_2026_07_17.md` - -**Interfaces:** -- Documents: creation-by-default, pending review, and no automated promotion - -- [x] **Step 1: Update public language** - -Remove flag instructions and every auto-promotion claim. State that `create_skill` queues a -candidate, only approval activates it, canary recommends, and direct filesystem edits remain an -operator-controlled escape hatch. - -- [x] **Step 2: Run full verification** - -Run: - -```bash -docker compose config --quiet -docker compose build -docker run --rm -v "$PWD:/app" -w /app ingot-mcp python -m pytest tests -q -``` - -Expected: Compose and build exit 0; full suite has no failures. - -- [x] **Step 3: Run structural scans** - -Run: - -```bash -rg -n 'ENABLE_AGENT_SKILL_WRITES|auto-promote|--promote' README.md SECURITY.md .env.example docker-compose.yml agent mcp_server optimize ui tests -rg -n 'approve_pending' optimize ui mcp_server agent -``` - -Expected: first scan has no live feature references; second scan shows one production caller in -`ui/app.py` plus its definition and tests. - -- [x] **Step 4: Run GitNexus and CodeScene gates** - -Run GitNexus change detection, CodeScene `pre_commit_code_health_safeguard`, and CodeScene -`analyze_change_set` against `master`. - -Expected: acknowledged impact only; both CodeScene quality gates pass. - -- [x] **Step 5: Commit and push PR correction** - -Commit in reviewable units with the required co-author trailer, push -`launch/human-gated-skills`, and confirm PR #18 CI passes. Do not merge. diff --git a/docs/superpowers/specs/2026-07-15-component-pass-optimization.md b/docs/superpowers/specs/2026-07-15-component-pass-optimization.md deleted file mode 100644 index e44ea3c..0000000 --- a/docs/superpowers/specs/2026-07-15-component-pass-optimization.md +++ /dev/null @@ -1,118 +0,0 @@ -# Component-pass optimization: one objective per component role - -Status: DRAFT for review, no further optimization runs until this is agreed. - -> **Superseded in part.** This is the dated record of a 2026-07-15 decision and is kept as written. -> Two things it describes have since changed: the sequential GEPA body loop it assumes was removed, -> so the body pass is now the parallel best-of-N search in `optimize/bestofn.py` (GEPA remains only -> as the description pass's reflection step), and both passes now write a portable evidence bundle -> under `runs/evidence/`. The component-role split it argues for is what shipped. See -> [ARCHITECTURE.md](../../../ARCHITECTURE.md) for the current design. - -## Problem - -A skill has three kinds of text component, each with a different production role: - -| component | production role | who consumes it | -|---|---|---| -| `description` | routing trigger | the embedding router (cosine match against the task) | -| `body` | instructions | the serving agent (via `get_skill` / `route_and_load`) | -| `file:` | reference / tooling | nothing in this stack today (other harnesses may read them from disk) | - -GEPA's inner loop scores every candidate with one metric: an LLM judge grading a bare-rollout -*answer*. In that bare rollout the description and body are indistinguishable (both are pasted -into the prompt), and files are invisible to production but visible to the rollout. Result, -observed across six runs on 2026-07-15: - -- GEPA repeatedly moved behavioral rules ("always output complete runnable code") into the - `description`, winning the inner loop, then either **blocked by the routing gate** - (recall@3 1.0 → 0.5 when the description bloated 437 → 1587 chars) or **losing the A/B** - (the serving agent never reads the description as instructions; the same improved body under - the original description scored 0.0). -- When judge/provider variance scored the seed high (≥ ~0.85 on the 4-task train set), GEPA - found "no better candidate", the failure the outer A/B measures (agent writes code to its - scratch filesystem and describes it) does not reproduce in a bare rollout at all. - -One shared metric grades three roles; it can only measure one of them. - -## Design: one pass per component, each scored by its role's own metric - -### Pass 1, body (IMPLEMENTED, the default) - -Inner/outer alignment (added 2026-07-16): rollouts serve candidates under the shared -`optimize.SERVE_TEMPLATE`, the same contract the quality A/B serves, and `GEPA_ROLLOUTS=agent` -opts into full-scaffold rollouts so scaffold-driven failures are visible to the inner loop. - -- Mutable: `body` only (`OPTIMIZE_COMPONENTS=body`). The frozen `description` still renders into - rollouts for serving fidelity but cannot be mutated. -- Objective: LLM judge on train tasks (existing), length penalty, deletion steering. -- Gate: full-agent A/B on holdout + margin/samples/regression checks + `RETENTION_WARN`. -- Cost: ~$1 / ~30–40 min per run (budget 60). - -### Pass 2, description (IMPLEMENTED: `optimize --description`) - -- Mutable: `description` only. -- Objective: **the routing suite, not the quality judge**, score a candidate description by - re-embedding it and computing recall@3 / top-1 / no-route precision over the `routing:` cases - in the skill's task YAML, plus collision margin against every other skill's description and - cross-harness parity. All computed by the local CPU embedding router: **no LLM rollouts**. - GEPA's only model calls are its handful of reflections (~$0.02/run, seconds per candidate). -- Gate: routing metrics strictly ≥ champion on every axis, collision < `COLLISION_SCORE`, - human approval in the UI as usual. Quality A/B unnecessary by construction (the serving agent's - instructions are unchanged). -- Prerequisite: a `routing:` case set for the skill (exists for `pdf`; auto-drafting routing - cases for skills that lack them is a small extension of the existing task drafter). - -### Pass 3, bundled files (BLOCKED on a real measurement) - -- Sequencing files into their own pass does not help: nothing measures them. The A/B doesn't - serve them, scripts never execute, and a files-only rollout judge has the same blindness that - broke the description. -- Prerequisites, per file kind: - - `scripts/*`: execution-grounded eval, SHIPPED 2026-07-16 as per-task `check:` specs - (fixture + assert, run in a scratch dir inside the disposable optimize container; broken - fixtures are inconclusive, never held against the answer). Remaining gap: file-serving - rollouts so a rewritten script is exercised the way a consuming harness would use it. - - docs (`reference.md`, `forms.md`): rollouts shaped like a file-reading harness (body - instructs "read FORMS.md", rollout actually inlines it) so the judge can attribute quality - to the file content. -- Until then: files are optimizable only by explicit opt-in (`OPTIMIZE_COMPONENTS=body,file:X`), - are diffed for human review, and the docs warn against including scripts. - -## Orchestration - -Passes are separate invocations, run in order, each with its own pending/approval cycle: - -``` -optimize pdf # pass 1: body (today's command, unchanged) -optimize pdf --routing # pass 2: description (proposed flag) -``` - -The continuous loop (`optimize-loop`) keeps running pass 1 only; pass 2 joins it once -auto-drafted routing cases exist. No interleaving within a single GEPA run, the alternation -GEPA does internally is exactly what let quality pressure leak into the description. - -## Non-goals - -- No simultaneous multi-component GEPA by default (the root cause of this document). -- No Langfuse-managed judge for pass 2 (routing metrics are local and free). -- No files pass at launch. - -## Decisions (resolved 2026-07-15) - -1. Flag spelling: component flags on the one `optimize` command, `--body` (default), - `--description`, `--scripts` (friendly refusal until execution-grounded evals exist). -2. Pass-2 gate: no regression on any routing metric + at least one strict improvement + - collision check + human approval. No margin requirement, the pass is nearly free to re-run. -3. Routing cases auto-draft on first `--description` run (teacher-generated positives + - null negatives, persisted to the task YAML), hand-curated suites take precedence. - (Originally required-not-drafted; extended 2026-07-16.) - -## Langfuse continuous judging (investigated 2026-07-16) - -Feasible without breaking the privacy posture: Langfuse's model-based evaluators accept custom -OpenAI-compatible LLM connections, so the judge connection can point provider-direct (e.g. -Fireworks, retention per that vendor's policy) or at a local endpoint, the OpenRouter-specific -ZDR body isn't expressible there, but provider-direct/local is the same posture the generic -BASE_URL support ships. Implementation (post-launch): configure an evaluator over incoming traces, -then teach mine.py/loop.py to read precomputed scores instead of re-judging per tick. diff --git a/docs/superpowers/specs/2026-07-15-skills-only-scope-design.md b/docs/superpowers/specs/2026-07-15-skills-only-scope-design.md deleted file mode 100644 index fba84e5..0000000 --- a/docs/superpowers/specs/2026-07-15-skills-only-scope-design.md +++ /dev/null @@ -1,53 +0,0 @@ -# Skills-Only Scope Correction - -Date: 2026-07-15 -Status: approved by operator - -## Outcome - -Skill Router remains the existing MCP/Docker application for discovering, creating, using, and -improving Agent Skills. This change adds safer one-call routing and stronger behavioral promotion -evidence without introducing a packaged CLI, a PyPI distribution, harness plugins, or integrations -with separate products. - -## Compatibility boundary - -The existing five MCP tools remain available: `list_skills`, `suggest_skills`, `get_skill`, -`create_skill`, and `reload_skills`. `route_and_load` becomes an additive sixth tool for clients that -want one selection-and-load round trip. Existing Docker commands, HTTP transport, demo agent, -optimizer, approval UI, and skill-authoring behavior remain the primary workflow. - -No new `skill-router` console command is installed. Python module and Docker Compose entrypoints stay -unchanged from `master`. - -## Skill roots - -`SKILL_ROUTER_PATHS` may add shared read-only roots. The repository `skills/` directory remains first -as the writable authoring and optimization root, so existing behavior wins duplicate names and newly -created skills remain routable. Later duplicates emit a visible warning. - -## Improvement safety - -Keep held-out split enforcement, routing regression metrics, full-skill revision binding, evidence -artifacts, snapshots, and rollback-protected promotion. These strengthen the existing optimizer and -approval loop; they do not reposition the repository as a benchmark product. - -## Explicit removals - -- Packaged CLI and CLI tests. -- `pyproject.toml`, wheel-first install story, and package-specific Docker changes. -- Claude/Codex adapter bundles that depend on the packaged command. -- Enterprise/product-boundary copy and implementation. -- Empty source lock and disabled fetch flow that break the existing quick start. -- Wholesale README rewrite; documentation returns to the existing product story with a concise - additive section for `route_and_load` and Behavioral CI hardening. - -## Verification - -- Existing five MCP tools plus `route_and_load` are registered. -- Existing demo agent still uses suggest/load/create behavior. -- Docker Compose still starts the MCP service through `python -m mcp_server.server` over HTTP. -- External roots and local authoring coexist. -- No tracked text contains names belonging to separate products. -- No `skill-router` console entrypoint or CLI module remains. -- Full tests and Compose configuration pass. diff --git a/docs/superpowers/specs/2026-07-17-human-gated-skill-lifecycle-design.md b/docs/superpowers/specs/2026-07-17-human-gated-skill-lifecycle-design.md deleted file mode 100644 index 209cb18..0000000 --- a/docs/superpowers/specs/2026-07-17-human-gated-skill-lifecycle-design.md +++ /dev/null @@ -1,117 +0,0 @@ -# Human-Gated Skill Lifecycle Design - -## Objective - -Keep agent-authored skill creation as a default product capability while ensuring no -application-controlled path can activate a new or rewritten skill without an explicit human review -action. - -## Invariants - -1. `create_skill` authors a candidate; it never writes under an active skill root or reloads the - router. -2. New skills and rewrites use the same `runs/pending/.json` review queue. -3. Only the approval UI calls the activation function in production code. -4. Optimizer CLI and canary paths may create or update pending recommendations; they never activate - a skill. -5. Rewrites retain their existing Behavioral CI evidence gate before the human can approve them. -6. Brand-new skills require current content, naming, collision, and injection checks plus human - content review; they do not require a behavioral evaluation suite before first activation. -7. Direct operator edits under `skills/` remain possible. The human-gate guarantee covers Ingot's - application-controlled write paths, not filesystem administrators. - -## Pending creation record - -Agent creation writes the existing pending JSON format with these fields: - -```json -{ - "kind": "creation", - "skill": "low-fodmap-meal-planning", - "champion_components": {"description": "", "body": ""}, - "challenger_components": { - "description": "Use this skill when...", - "body": "..." - }, - "changed_components": ["description", "body"], - "gate": {"promotable": true, "blocked": []}, - "source": "agent" -} -``` - -Empty champion components let the existing unified-diff UI render a complete file addition. A -pending creation has no Behavioral CI evidence because no champion exists yet. - -## Authoring flow - -`create_skill(name, description, body)` keeps its current MCP contract and all current defenses: - -- normalize and validate the slug; -- reject an active skill or pending candidate with the same name; -- run content and optional model guards; -- reject routing collisions against active skills; -- save the creation record atomically under `runs/pending/`; -- return a message pointing to the approval UI. - -It does not create `skills/`, reload state, or make the candidate routable. - -## Review UI - -`GET /api/skills` returns the union of active skills and pending-only creations. Pending creations -have `pending: true`, `creation: true`, and no optimize action. The existing pending-detail endpoint -renders their description and body as additions and labels the review as a new skill. - -Reject deletes the pending record. Approve invokes the single activation boundary. - -## Activation boundary - -Replace the broadly callable `promote()` entry point with `approve_pending(skill)`. Production code -has one caller: `ui.app.approve`. - -For `kind: creation`, approval: - -1. revalidates the slug and confirms no active skill or destination directory now exists; -2. re-runs content and collision checks to catch drift since authoring; -3. builds a temporary sibling directory; -4. writes `SKILL.md` with `source: agent`; -5. atomically renames the temporary directory into the active local skill root; -6. deletes the pending record only after activation succeeds. - -For existing rewrites, approval preserves current evidence validation, revision checks, snapshot, -staged swap, rollback, and pending cleanup. - -Any failure leaves active skills untouched and preserves the pending record for diagnosis or retry. -The MCP server's existing filesystem refresh makes an approved creation routable on the next -request. - -## Remove automated activation - -- Delete the optimizer's `--promote` behavior. A successful A/B run always writes a pending record. -- Change canary success from calling promotion to recording a promote recommendation in pending - state. -- Remove every non-UI production caller of the activation function. -- Keep direct calls in promotion unit tests only. - -## Public behavior - -- Remove `ENABLE_AGENT_SKILL_WRITES`; creation is available by default. -- Documentation says agent-authored skills await human approval and are not routable before it. -- Security documentation distinguishes content checks, Behavioral CI, and explicit human approval. -- The approval UI remains localhost-only and unauthenticated. “Human-gated” means an explicit UI - approval action in normal operation, not cryptographic proof of personhood. - -## Verification - -Test-driven implementation must prove: - -- agent creation produces pending JSON and no active directory; -- pending creations are absent from routing and visible in the UI; -- duplicate active and duplicate pending names are rejected; -- rejection never activates the skill; -- approval atomically activates a new skill and clears pending state; -- failed activation preserves pending state and leaves active roots unchanged; -- existing rewrite approval still rejects missing, blocked, stale, or mismatched evidence; -- canary and optimizer cannot activate skills; -- only the UI is a production caller of `approve_pending`; -- full Docker tests, Compose validation, GitNexus change detection, CodeScene pre-commit safeguard, - and CodeScene branch analysis pass. diff --git a/docs/superpowers/specs/2026-07-19-sso-rbac-design.md b/docs/superpowers/specs/2026-07-19-sso-rbac-design.md deleted file mode 100644 index ac5efb5..0000000 --- a/docs/superpowers/specs/2026-07-19-sso-rbac-design.md +++ /dev/null @@ -1,176 +0,0 @@ -# SSO + RBAC for the shared change-control UI, design - -Status: implemented (Sign in with Google) · Scope: UI authentication + authorization for a -shared/enterprise deployment. Operator guide: [docs/sso.md](../../sso.md). - -> **Landed** (the whole flow, Google as the certified provider): -> - **Authorization core**, `ui/rbac.py`: roles, `parse_role_map`, `role_from_claims`, `authorize`, -> and OIDC-claims→identity parsing (`tests/test_rbac.py`). -> - **ID-token validation primitive**, `ui/oidc.py` `verify_id_token` (signature via JWKS, `iss`, -> `aud`, `exp`/`iat`, `nonce`, `azp`), with the **layer-2 harness** `tests/conftest.FakeIdp` -> (`tests/test_oidc.py`). -> - **Browser flow**, `ui/oidc_flow.py`: discovery + JWKS fetch/caching, `/auth/login` (state + PKCE -> + nonce), `/auth/callback` (state match, code redemption, validation, Google-domain gate, -> email→role), `/auth/logout`, `/auth/me`. Unit-tested against FakeIdp (`tests/test_oidc_flow.py`) -> and end-to-end against a live Keycloak (`tests/test_keycloak_integration.py`). -> - **Wiring**, `ui/auth.py` + `ui/app.py`: mode-aware `require_auth` with `/auth/*` exemptions, -> fail-closed startup, `SessionMiddleware`, `require_role` alongside `same_origin` on the -> state-changing routes, SSO email as the audit `actor`. -> -> The one deliberate deviation from the plan below: the flow is hand-rolled on `httpx` + -> `verify_id_token` rather than authlib, so the audited validation path is the one already unit -> tested. Roles come from an **email→role map** because Google ID tokens carry no roles/groups claim. - -Follow-up to the minimal LAN password auth (PR #24). That work made approvals *attributable*, the -authenticated user is written as the audit `actor` on promote/reject/rollback. This spec takes the -next step for a shared, enterprise deployment: real SSO (OIDC) and role-based authorization, while -keeping the local/LAN paths unchanged. - -## Goals - -- Let people sign in with their **corporate identity** (no per-user passwords to manage). -- Gate the state-changing actions by **role** (who may propose vs approve vs promote vs administer). -- Keep the existing **attributable audit trail**, the SSO identity becomes the `actor`. -- **Do not** regress the zero-config local default or the LAN password mode; SSO is an opt-in - deployment profile layered on top. - -## Non-goals (explicitly deferred) - -- **Machine / agent auth to the MCP server.** Agents can't do an interactive OIDC browser flow; - they need service credentials (client-credentials or API tokens). Separate spec. -- **SCIM user/group provisioning.** Nice for large orgs; not needed for a first cut. -- **Fine-grained per-skill / per-namespace ACLs.** Roles are global here; team namespaces are a - later item. - -## Provider decision - -We authenticate against **protocols, not vendors**: Entra, Okta, Google Workspace, Ping, OneLogin, -and Cognito all speak **OIDC**. One generic OIDC implementation supports all of them, so "which -providers" is really **which we test and document against**. - -- **Certify Microsoft Entra ID + Okta.** Entra is the dominant enterprise workforce IdP; Okta is the - standard independent one. Together they cover the large majority of enterprise buyers. -- **Generic OIDC underneath** so Google Workspace / Ping / OneLogin / etc. work without new code. -- **Amazon Cognito is intentionally not a certified target.** Cognito is AWS's CIAM / user-pool - service (customer identity, or an IdP broker), enterprises don't use it for *workforce* SSO. It - only becomes relevant if the product pivots from self-hosted to a **hosted multi-tenant SaaS**, - in which case the right move is a single broker (Cognito, Auth0, or **WorkOS**) that federates to - each customer's IdP, see "Deployment model" below. -- **SAML** is on the roadmap (some large/old enterprises are SAML-only and it appears in RFPs) but - out of scope for phase 1; OIDC-first. - -### Deployment model changes the answer - -- **Self-hosted** (customer runs Ingot on their LAN/cloud, the OSS/enterprise-self-hostable path): - the customer brings *their* IdP. Support the customer's IdP directly → Entra + Okta + generic - OIDC. This is the assumed model here. -- **Hosted SaaS** (we run it multi-tenant): integrate *one* broker (WorkOS / Auth0 / Cognito - federation) once, and it federates to each customer's IdP. If the product goes this way, replace - "certify two IdPs" with "integrate one broker." - -## Design - -### Authentication (OIDC Authorization Code + PKCE) - -Extend `ui/auth.py` with an OIDC mode selected by config (below). When enabled: - -1. Before redirecting, mint and persist a fresh `state`, PKCE `code_verifier`, and `nonce` (in the - signed session). Redirect to the provider's authorize endpoint (auth-code + PKCE). -2. `/auth/callback` **must match the returned `state`** to the persisted one and redeem the code with - **that exact `code_verifier` once** (single-use), then validate the returned ID token against the - provider JWKS (issuer, audience, `exp`/`iat`, and the persisted `nonce`) via - `ui.oidc.verify_id_token`. ID-token checks alone are not enough: without the `state`/verifier/nonce - binding, login-CSRF and code-injection are possible. On success establish a **signed session - cookie** (Starlette `SessionMiddleware` / `itsdangerous`, `HttpOnly` + `Secure` + `SameSite`, - with an explicit max-age so a stolen cookie expires); store only `{sub, email, name, roles}`, - nothing more (the cookie is signed, not encrypted, so its payload is readable). Rotating - `SESSION_SECRET` invalidates all sessions, which is the operator's kill switch. -3. `/auth/logout` clears the session. -4. The existing `current_actor` dependency returns the session's `email`/`sub` instead of the Basic - username, so the audit `actor` path is unchanged. - -The app-wide `Depends(require_auth)` gate must be made **mode-aware and exempt the OIDC bootstrap -routes** (`/auth/login`, `/auth/callback`, `/auth/logout`); otherwise the existing Basic-auth -challenge blocks the login flow before it can complete. - -Library: **authlib** (OIDC discovery + JWKS handled for us). The three modes coexist and are chosen -by config precedence: **OIDC → LAN password (`AUTH_USER`/users file) → open**. `AUTH_MODE=oidc` must -**fail closed at startup** when `OIDC_ISSUER`, `OIDC_CLIENT_ID`, `OIDC_REDIRECT_URL`, or -`SESSION_SECRET` is missing or invalid: it must never silently downgrade to LAN password or open. - -### Authorization (RBAC) - -Roles, least- to most-privileged: - -| role | may | -|------|-----| -| `viewer` | read the board, pending evidence, history | -| `proposer` | trigger candidate generation (`/api/optimize`) | -| `approver` | promote, reject, rollback | -| `admin` | the above + change config / manage mappings | - -- The user's roles come from provider claims mapped to app roles: - - **Entra: use *app roles*, not groups.** App roles are always present in the token; groups hit - the ">200 groups overage" problem where Entra omits them and forces a Microsoft Graph call. - - **Okta / others: groups claim** → app role via a configured map. - - **Claim parsing fails closed** (`ui.rbac.identity_from_claims`): the claim value may be a single - string or a list of strings; multiple mapped values union and the highest role wins. Any other - shape (number, object, non-string list entries) contributes no roles, never an error, so a - malformed token can only lose privileges, and mapped values are matched exactly, unknown values - are ignored. -- Roles are **hierarchical**: `admin` satisfies `approver`, `proposer`, and read; `require_role(r)` - passes for any role at least as privileged (this is `ui.rbac.has_role`, already implemented as a - rank comparison, so an exact-match check must not be used). -- Enforcement: a small `require_role(role)` dependency on the state-changing endpoints - (`/api/promote`, `/api/reject`, `/api/rollback` → `approver`; `/api/optimize` → `proposer`). - Read endpoints require only a valid session. -- `require_role` is **added alongside** the existing `same_origin` dependency on `/api/optimize`, - `/api/promote`, `/api/reject`, and `/api/rollback`, never a replacement: dropping `same_origin` - would regress the CSRF protection those routes already have. -- Default-deny: a session with no mapped role is `viewer`. - -### Config (per provider, env-driven, compose-wired) - -```bash -AUTH_MODE=oidc # oidc | password | open (default: password behavior from #24) -OIDC_ISSUER=https://login.microsoftonline.com//v2.0 # or the Okta issuer URL -OIDC_CLIENT_ID=... -OIDC_CLIENT_SECRET=... # (or PKCE-only public client) -OIDC_REDIRECT_URL=https://ingot.corp.example/auth/callback -OIDC_ROLE_CLAIM=roles # Entra app roles; "groups" for Okta -OIDC_ROLE_MAP=ingot-approver:approver,ingot-admin:admin # provider value -> app role -SESSION_SECRET=... # signs the session cookie -``` - -## Gotchas that cost time (flagged early) - -1. **Entra groups overage** → use **app roles** (above). Saves a Graph integration. -2. **Redirect URI + TLS.** OIDC needs a stable HTTPS callback; this couples to the enterprise - network profile (real host + TLS), not just the app. -3. **Machine agents ≠ SSO** (deferred, but must be said so it doesn't silently balloon scope). -4. **Clock skew / token expiry** on validation; **session lifetime** (re-login vs refresh, phase 1 - can just re-login on expiry). - -## Phasing & estimate - -Roughly **~1 week** of focused work for tested phase 1+2; the real tax is integration against two -live tenants, which can't be mocked. - -- **Phase 1, OIDC login (~2–3 days).** Auth-code+PKCE, session cookie, callback validation, one - coarse role (`admin` vs everyone). Certified against Entra + Okta. -- **Phase 2, RBAC (~1–2 days).** The four roles, claim→role mapping, `require_role` on endpoints, - actor = SSO identity in the audit. -- **Phase 3, SAML (later).** For SAML-only enterprises. -- Tests throughout: mock the IdP (stub discovery/JWKS/callback) for CI; manual integration against - real Entra + Okta tenants before release. - -## Open decisions (need a call before building) - -1. **Self-hosted vs hosted SaaS**, determines "certify Entra + Okta" vs "integrate one broker." -2. **Entra app roles vs groups**, recommend app roles; needs the customer to define app roles in - their tenant (a small onboarding ask). -3. **How many roles for v1**, ship all four, or start `admin`-vs-everyone and add the middle roles - in phase 2? -4. **Where the audit trail lives at scale**, the attributed `actor` currently lands in - `runs/approval-audit.jsonl`; a shared enterprise deployment likely wants it in a real store - (ties to the trace-backend / observability profile decision). diff --git a/docs/tutorial.md b/docs/tutorial.md index 5b7e1dc..3a1c960 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -37,8 +37,6 @@ docker compose run --rm agent "How do I merge several PDFs into one and add page COMPATIBLE ROUTE (MCP route_and_load): 0.74 pdf: Use this skill whenever the user wants to do anything with PDF files... -SERVING MODEL: accounts/fireworks/models/qwen3p7-plus - LOADED SKILLS (MCP route_and_load): ['pdf@83a75cf1f9b5…'] TOKENS: 23233 in / 698 out @@ -50,10 +48,9 @@ The agent asked the canonical router (`route_and_load`), received one compatible followed it. The `@…` suffix is the skill's content-hash revision, which also lands on the trace as a tag. Unconstrained suggestions never control the serving model or loaded instructions. A routed task runs on the cheap `AGENT_MODEL` because the skill carries the method; only truly -novel tasks escalate to `STRONG_MODEL` (step 10). Steps 2 to 4 were recorded on Fireworks model -IDs and steps 5 to 8 on the OpenRouter `AGENT_MODEL` default at recording time, `qwen/qwen3-32b` -(the current default is `qwen/qwen3.6-27b`); the `SERVING MODEL` line always shows whatever -`AGENT_MODEL` you configure. The run's full trace just landed in Langfuse +novel tasks escalate to `STRONG_MODEL` (step 10). The historical serving-model identifier is +omitted from this recorded output because hosted configuration now uses OpenRouter model slugs. +Steps 5 to 8 used the current default, `qwen/qwen3-32b`. The run's full trace just landed in Langfuse (`localhost:3100`); that's what the miner reads in step 4. ### 3. Write a first-draft skill and watch it under-deliver @@ -120,30 +117,38 @@ docker compose run --rm optimize-mine tailwind ``` ``` -[mine] 39/50 recent traces relevant to 'tailwind' -[mine] analyzed 39 real traces · mean judge score 0.50 · 20 bad cases (score < 0.5) -[mine] failure dimensions, most common first: - correctness 20/39 █████ - completeness 20/39 █████ - instruction_following 20/39 █████ - efficiency 16/39 ████ -[mine] 6 weakest tasks mined as eval candidates → optimize on these next. +[mine] 19/36 fetched traces relevant to 'tailwind' (tagged with it, or ranking it in the embedding top-5) +[mine] 8 representative(s) judged; unchanged verdicts are cached +[mine] 19 uses collapsed into 8 semantic task cluster(s); 19/19 uses represented by a judge verdict (100% coverage) +[mine] analyzed 19 represented real uses · weighted mean judge score 0.99 · 0 bad cases (score < 0.5) +[mine] failure dimensions (paper's Failure Analyzer), most common first: + completeness 1/19 █ + correctness 0/19 + instruction_following 0/19 + efficiency 0/19 +[mine] 1 candidate(s) dropped as near-duplicates of existing train tasks (leakage guard) +[mine] 1 weakest tasks mined as eval candidates (coverage-spread) → optimize on these next. ``` -The diagnosis is version drift: answers built on `tailwind.config.js` and `@tailwind` directives, -the v3 world the stub teaches. The weakest mined tasks are surfaced as candidates only; mining does -not edit the task set. Review them, add accepted tasks and explicit rubrics to -`optimize/tasks/.yaml`, then keep train and holdout separate. See -[Writing eval task sets](configuration.md#writing-eval-task-sets). +The default now considers all 36 project traces, not only a newest-50 sample. Nineteen relevant uses +collapsed into eight task families, so only eight representative judge calls covered every use. +The cache makes later runs cheaper. Repeated uses still weight the health score and failure counts, +and `--limit N` remains available when an operator explicitly wants only the newest traffic. -Reference-free judging of live traffic is noisy. Treat mined dimensions as a diagnosis to -investigate, not a verdict; the evidence gate runs on rubrics. +This real run also demonstrates why reference-free traffic judging is a diagnostic, not promotion +evidence. Step 3 directly observed a v3 answer, and the loaded body is visibly stale, yet the live +judge scored the mostly reference-free history at 0.99. The rubric-backed train and holdout set in +step 5 is what supplies the missing v4 ground truth. Mined tasks are suggestions only: there is no UI +promotion action and mining never edits the YAML. Review a failure, replace any reference-free +rubric with explicit ground truth, and add it to `train`. Do not add an inspected failure to +`holdout`; author new holdout cases separately so the final promotion evidence stays uncontaminated. +See [Writing eval task sets](configuration.md#writing-eval-task-sets). -### 5. (Optional) Generate a candidate in the background +### 5. Train a skill revision with SkillOpt -At this point you know what is wrong and could fix the body by hand: edit `SKILL.md`, and the router -serves the new revision on the next request. This step does it the other way, with the optional -candidate generator, so the change arrives with measured evidence attached. +At this point you know what is wrong and could fix the body by hand. SkillOpt integration is the +other half of Ingot's value: it trains a bounded instruction revision from real failures and +attaches measured evidence without activating the result. Write an eval task set for the skill (`optimize/tasks/tailwind.yaml`) with train and holdout tasks whose rubrics carry the v4 ground truth (the teacher can also auto-draft one on a skill's first CLI @@ -153,9 +158,10 @@ run). Then run it headless, which is how it is meant to run: docker compose run --rm optimize tailwind ``` -The generator authors several candidate bodies in parallel, races them on the train tasks, and A/Bs -the winner against the champion through the full agent on the held-out tasks (about two minutes, a -few cents). The UI's **Generate candidate** button starts the same run. Our run: +SkillOpt reflects on failing minibatches, proposes bounded edits, keeps only train-set +improvements, and A/Bs its best revision against the champion through the full agent on held-out +tasks (about two minutes, a few cents). The UI's **Optimize with SkillOpt** button starts the same run. +Our run: ``` [skillopt] seed: hard 0.000 soft 0.000 gate 0.000 (mixed) on 6 train tasks; 2 epoch(s), minibatch 3, ≤3 edits/step @@ -202,7 +208,7 @@ Generation is greedy: one component per pass, each scored by its own role's metr | pass | command | candidate-search objective | cost | |------|---------|---------------------------|------| -| body (default) | `optimize tailwind`, or the UI's **Generate candidate** button | LLM judge on train tasks; full-agent A/B for the evidence | ~$0.05 | +| body (default) | `optimize tailwind`, or the UI's **Optimize with SkillOpt** button | LLM judge on train tasks; full-agent A/B for the evidence | ~$0.05 | | description | `optimize tailwind --description` | the routing suite, scored by the real embedding router; no LLM rollouts | ~$0.01, a couple of minutes | | scripts | `optimize tailwind --scripts` | LLM judge grounded by execution checks; greedy, one bundled `scripts/` file at a time | like the body pass, per file | @@ -213,30 +219,39 @@ in place, both the rollouts and the A/B serve the assembled skill (body plus bun rewritten file is actually exercised by the evidence run. Other bundled text files can still join the body pass by name (`OPTIMIZE_COMPONENTS=body,file:`). The candidate search serves each rollout under the exact contract the A/B serves, so the search can't optimize against different -instructions than the A/B measures. `GEPA_ROLLOUTS=agent` runs every rollout through the full agent -scaffold instead (a legacy variable name: it predates the removal of the GEPA body loop and now -selects the SkillOpt candidate search's rollout mode). +instructions than the A/B measures. `SKILLOPT_ROLLOUTS=agent` runs every rollout through the full +agent scaffold instead. ### 6. Review, promote, roll back Back at **http://localhost:8080**, the header pill flips to **1 to review**, the Review section leads with the evidence, and the `tailwind` row shows a `change awaiting review` chip. The card carries the champion-vs-challenger judge scores, the before/after token shift, the gate verdict and -its warnings, the recorded evidence bundle, and the component diff, here the red lines are the v3 -guidance SkillOpt removed and the green lines are the v4 body it wrote: +its warnings, and a risk summary above the diff with added lines, removed lines, body-change +percentage, and size change. The unified diff remains visible, and a collapsed side-by-side view +lets you inspect complete before and after components. Here the red lines are the v3 guidance +SkillOpt removed and the green lines are the v4 body it wrote: ![Review card, the tailwind v3→v4 challenger, promotable with warnings](ui-review.webp) -**Approve** doesn't promote in one click: it opens a comparison panel with the model breakdown, -the before/after token usage, and the per-task judge scores, and a final **Approve & promote** that -reveals a separate **Confirm**, so a promotion is deliberate. +**Approve** doesn't promote in one click: it opens a comparison panel with the real serving and +judge model names, input tokens above output tokens, and a numbered before/after table for every +held-out task. A final **Approve & promote** reveals a separate **Confirm**, so a promotion is +deliberate. ![Comparison panel, model, tokens, and judge scores before Confirm](ui-compare.webp) **Approve & promote** verifies the evidence still matches the on-disk champion, snapshots the prior revision into `runs/revisions/tailwind//`, swaps the challenger into `skills/tailwind/` by rename, and appends an `approve` record to `runs/approval-audit.jsonl`. The MCP server picks up the -revision change with no restart. **Reject** discards the candidate. +revision change with no restart. **Reject** asks for confirmation and accepts an optional reason, +which is stored with the rejection audit record. + +The skills list can be searched by name or description and filtered by pending or eval status. +Click any skill row to inspect active instructions, a pending challenger, rollback snapshots, and +bundled text files without changing the serving revision. The top bar shows the password or OIDC +identity responsible for decisions, and the three-second board poll announces count changes to +assistive technology. That snapshot is the undo. It appears in the UI's **History** section, and restoring it is one click, or one command: @@ -248,7 +263,7 @@ docker compose run --rm --entrypoint python optimize -m optimize.promote rollbac Rollback snapshots the revision it displaces too, so the round trip is symmetric, and it writes its own audit record. The trail records the actor as the approver's authenticated identity, the HTTP -Basic username or OIDC email, or `local-operator` in the zero-config open mode, where it can record +Basic username or OIDC email, or `local-operator` in explicit open mode, where it can record that a local operator approved, not who. Both records are appended after the swap has already happened, so an audit write that fails (a full or read-only disk) is logged and the change stands: a missing line means the trail is incomplete, not @@ -313,11 +328,11 @@ puts the right skill in front of the model; the A/B proves what happens when it That's the lifecycle: a first-draft skill → real traffic → mined diagnosis → a proposed change with held-out evidence → human approval → hot reload → a snapshot you can roll back to. -### 9. (Optional) Run candidate generation unattended +### 9. Run SkillOpt optimization unattended -This is where candidate generation belongs: in the background, not on the review path. One command mines -every skill's real traffic for health and proposes changes only for the ones actually failing, -leaving every survivor quarantined in the review queue (nothing auto-promotes): +SkillOpt can run in the background while human approval remains on the review path. One command +mines every skill's real traffic for health and proposes changes only for the ones actually +failing, leaving every survivor quarantined in the review queue (nothing auto-promotes): ```bash docker compose run --rm optimize-loop # all skills with eval sets; add names to target some @@ -325,11 +340,15 @@ docker compose run --rm optimize-loop # all skills with eval sets; ad ``` [loop] ===== pdf ===== -[mine] analyzed 23 real traces · mean judge score 0.52 · 11 bad cases -[loop] pdf: below health bar (mean 0.52), optimizing… -[loop] done. 1 challenger(s) queued for review: ['pdf'] +[mine] ... unjudged cluster(s) remain; the next run continues from the cache +[loop] pdf: mining coverage is ...; deferring the health decision until cached representative judging covers every trace cluster. ``` +With a cold cache, the loop spends at most `MINE_MAX_JUDGE_CALLS` new calls on the most frequent +clusters and defers rather than extrapolating from partial coverage. Re-running drains the backlog +without paying again for unchanged verdicts. Only after every cluster is represented does the loop +compare the frequency-weighted mean with `LOOP_HEALTH_THRESHOLD` and decide whether to optimize. + Routing quality decays as the library *grows* (a new skill's description can shadow an old one), so there is also a library-wide routing health check: it replays every skill's routing suite against the real router plus a description-collision scan, embedding-only, no LLM, no key. It diff --git a/mcp_server/embedding.py b/mcp_server/embedding.py index 3926906..c20565b 100644 --- a/mcp_server/embedding.py +++ b/mcp_server/embedding.py @@ -36,6 +36,17 @@ def is_qwen_onnx(model: str) -> bool: return "qwen3-embedding" in model.lower() and "onnx" in model.lower() +def _baked_qwen(model: str, onnx_file: str) -> bool: + """Whether this exact Qwen export was cached while the container image was built.""" + return (model == os.environ.get("BAKED_EMBED_MODEL") + and onnx_file == os.environ.get("BAKED_EMBED_ONNX_FILE")) + + +def _baked_fastembed(model: str) -> bool: + """Whether this fastembed fallback was cached while the container image was built.""" + return model == os.environ.get("BAKED_FALLBACK_EMBED_MODEL") + + class QwenOnnxEmbedding: """A Qwen3-Embedding ONNX export on ONNX Runtime CPU. Right padding with attention-mask-indexed last-token pooling (equivalent to the official left-padded pooling under causal attention: the @@ -46,8 +57,11 @@ def __init__(self, model: str = EMBED_MODEL, onnx_file: str = EMBED_ONNX_FILE): import onnxruntime as ort from huggingface_hub import hf_hub_download from tokenizers import Tokenizer - self._tokenizer = Tokenizer.from_file(hf_hub_download(model, "tokenizer.json")) - self._session = ort.InferenceSession(hf_hub_download(model, onnx_file), + local_only = _baked_qwen(model, onnx_file) + self._tokenizer = Tokenizer.from_file( + hf_hub_download(model, "tokenizer.json", local_files_only=local_only)) + self._session = ort.InferenceSession( + hf_hub_download(model, onnx_file, local_files_only=local_only), providers=["CPUExecutionProvider"]) self._inputs = self._session.get_inputs() self._out = next(o.name for o in self._session.get_outputs() if "hidden" in o.name) @@ -97,7 +111,7 @@ class FastembedEmbedding: def __init__(self, model: str = EMBED_MODEL): from fastembed import TextEmbedding - self._model = TextEmbedding(model_name=model) + self._model = TextEmbedding(model_name=model, local_files_only=_baked_fastembed(model)) def embed(self, texts): return self._model.embed(list(texts)) diff --git a/optimize/__init__.py b/optimize/__init__.py index f1286ea..eee269e 100644 --- a/optimize/__init__.py +++ b/optimize/__init__.py @@ -12,8 +12,7 @@ error: no API key is set for your LLM endpoint, and candidate generation needs one. 1. cp .env.example .env - 2. put your key in it (OpenRouter: https://openrouter.ai/keys, or set BASE_URL + API_KEY for - any other OpenAI-compatible provider, e.g. Fireworks) + 2. put your OpenRouter key in it (https://openrouter.ai/keys) 3. re-run this command (Running fully local instead? Point BASE_URL / MODEL_BASE_URL at your vLLM or Ollama @@ -25,7 +24,7 @@ def agent_model() -> str: """The serving/agent model, everything that *executes* skills (agent runs, A/B eval agents, candidate rollouts). AGENT_MODEL wins; MODEL is the legacy alias so existing .env files keep working.""" - return os.environ.get("AGENT_MODEL") or os.environ.get("MODEL") or "qwen/qwen3.6-27b" + return os.environ.get("AGENT_MODEL") or os.environ.get("MODEL") or "qwen/qwen3-32b" def skillopt_model() -> str: @@ -35,15 +34,15 @@ def skillopt_model() -> str: def model_base_url() -> str: """Endpoint for the serving-model role (agent runs, A/B eval agents, candidate rollouts). - MODEL_BASE_URL lets this role run against a different provider (local vLLM/Ollama, or e.g. - Fireworks direct) while the teacher and judge stay wherever BASE_URL points.""" + MODEL_BASE_URL lets this role run locally through vLLM or Ollama while the teacher and judge + stay on the endpoint BASE_URL names.""" return os.environ.get("MODEL_BASE_URL") or teacher_base_url() def teacher_base_url() -> str: """Endpoint for the teacher-side roles (candidate authoring, reflection, judge, task drafting). - Generic BASE_URL wins (any OpenAI-compatible provider); OPENROUTER_BASE_URL is the legacy - alias.""" + BASE_URL names OpenRouter for hosted use or a local OpenAI-compatible endpoint; + OPENROUTER_BASE_URL is the legacy alias.""" return (os.environ.get("BASE_URL") or os.environ.get("OPENROUTER_BASE_URL") or OPENROUTER_URL) @@ -54,8 +53,7 @@ def api_key() -> str: def model_api_key() -> str: - """Key for the serving-model endpoint (falls back to the shared key), hybrid setups can use - a different vendor for the serving role.""" + """Key for the serving-model endpoint, falling back to the shared OpenRouter key.""" return os.environ.get("MODEL_API_KEY", "") or api_key() @@ -77,7 +75,7 @@ def langfuse_available() -> bool: def openrouter_extra_body() -> dict: """Provider preferences for OpenRouter calls: the hardcoded ZDR policy, plus an optional - priority list (OPENROUTER_PROVIDERS=fireworks[,groq] -> provider.order): listed providers + priority list (OPENROUTER_PROVIDERS=groq[,deepinfra] -> provider.order): listed providers are tried first, in order, and models none of them serves fall back to the rest of the pool. The priority composes with ZDR: a listed provider still must qualify as zero-data-retention for the model, or routing skips it.""" @@ -90,9 +88,9 @@ def openrouter_extra_body() -> dict: def client_kwargs(base_url: str, key: str | None = None, reasoning: bool = False) -> dict: """ChatOpenAI connection kwargs for an endpoint. OpenRouter gets the hardcoded ZDR provider - preference (plus the optional OPENROUTER_PROVIDERS allowlist); any other OpenAI-compatible - endpoint (Fireworks/Together direct, local vLLM/Ollama) gets no provider preferences and a - placeholder api_key if none is set (the client requires one). reasoning=True pins OpenRouter's + preference (plus the optional OPENROUTER_PROVIDERS priority); a local OpenAI-compatible + endpoint such as vLLM or Ollama gets no provider preferences and a placeholder api_key if none + is set (the client requires one). reasoning=True pins OpenRouter's unified reasoning parameter on rather than relying on per-model defaults; local endpoints ignore it (thinking is a server-side setting there).""" key = key if key is not None else api_key() @@ -173,6 +171,10 @@ def preflight_provider_pins() -> list[str]: conflict = provider_conflict(model, pins) if conflict: problems.append(f" {role}={model}: {conflict}") + unavailable = [problem for problem in problems if "has no endpoints on OpenRouter" in problem] + if unavailable: + raise SystemExit("OpenRouter model configuration is unavailable:\n" + + "\n".join(unavailable)) if problems: print("warning: some roles are not covered by OPENROUTER_PROVIDERS and will fall back " "to the open ZDR pool:\n" + "\n".join(problems), file=sys.stderr) diff --git a/optimize/ab.py b/optimize/ab.py index 78b1cfd..e2bfb23 100644 --- a/optimize/ab.py +++ b/optimize/ab.py @@ -608,7 +608,7 @@ def build_parser() -> argparse.ArgumentParser: help="quality pass over bundled scripts/ files, greedy one file at a " "time; requires execution-grounded holdout checks") ap.add_argument("--budget", type=int, default=None, - help=f"--description only: max GEPA metric calls for the routing pass " + help=f"--description only: max router case evaluations for the routing pass " f"(default {DEFAULT_ROUTING_BUDGET})") ap.add_argument("--skip-search", action="store_true", help="debug: A/B champion vs itself") ap.add_argument("--challenger-file", help="reuse a checkpointed candidate, skip to the A/B") diff --git a/optimize/promote.py b/optimize/promote.py index 88f403b..7b9b930 100644 --- a/optimize/promote.py +++ b/optimize/promote.py @@ -9,7 +9,8 @@ import uuid from pathlib import Path -from mcp_server.registry import SLUG_RE, load_skills, skill_revision, write_components +from mcp_server.registry import (SLUG_RE, load_skills, read_components, skill_revision, + write_components) RUNS_DIR = Path(__file__).resolve().parent.parent / "runs" PENDING_DIR = RUNS_DIR / "pending" @@ -162,6 +163,11 @@ def list_revisions(skill: str) -> list[dict]: return [{"revision": r["revision"], "created": r["created"]} for r in records] +def load_snapshot_components(skill: str, revision: str) -> dict[str, str]: + """Read one validated rollback snapshot for the read-only version explorer.""" + return read_components(_rollback_source(skill, revision)) + + def list_snapshotted_skills() -> list[str]: """Skills with at least one snapshot. Reading the snapshot store directly keeps the history view off the skill-library hash scan that the skills listing already pays for.""" @@ -269,11 +275,9 @@ def _write_all(fd: int, data: bytes) -> None: written += os.write(fd, data[written:]) -def _audit(action: str, skill: str, revision: str, actor: str = "local-operator") -> None: - """Append a minimal approval trail without recording skill bodies or credentials. - - `actor` is `local-operator` for every UI action: the local UI has no identity or - authentication, so the trail records that a local operator approved, not who.""" +def _audit(action: str, skill: str, revision: str, actor: str = "local-operator", + reason: str = "") -> None: + """Append transition metadata without recording skill bodies or credentials.""" audit_file = audit_path() audit_file.parent.mkdir(parents=True, exist_ok=True, mode=0o700) os.chmod(audit_file.parent, 0o700) @@ -281,15 +285,18 @@ def _audit(action: str, skill: str, revision: str, actor: str = "local-operator" try: record = {"schema_version": 1, "ts": int(time.time()), "action": action, "skill": skill, "revision": revision, "actor": actor} + if reason: + record["reason"] = reason _write_all(fd, (json.dumps(record, separators=(",", ":")) + "\n").encode()) finally: os.close(fd) -def _audit_best_effort(action: str, skill: str, revision: str, actor: str) -> None: +def _audit_best_effort(action: str, skill: str, revision: str, actor: str, + reason: str = "") -> None: """Record a committed transition without changing its successful outcome.""" try: - _audit(action, skill, revision, actor) + _audit(action, skill, revision, actor, reason) except Exception: logger.warning( "Committed %s for skill %r at revision %s, but the audit write failed", diff --git a/optimize/rollout.py b/optimize/rollout.py index 141ac57..92be2bf 100644 --- a/optimize/rollout.py +++ b/optimize/rollout.py @@ -15,8 +15,7 @@ MODEL = agent_model() # The teacher LM (the skill *author*): a stronger model than the serving agent, per the # teacher/student split, where rollouts and judging stay on AGENT_MODEL (the model the skill will -# serve). This role is shared by SkillOpt reflection, eval drafting, and the optional GEPA-based -# description pass, so its name describes the role rather than one optimization algorithm. +# serve). This role is shared by SkillOpt body and description reflection plus eval drafting. SKILLOPT_MODEL = skillopt_model() # Length penalty: the body re-enters context on every agent step, and a completeness-hungry judge @@ -35,10 +34,8 @@ def length_penalty(body: str) -> float: # can never optimize against different instructions than the A/B measures. "agent": the full # deepagents scaffold (file tools and all) per rollout, which reproduces scaffold-driven failures # the direct mode can't see (e.g. writing code to a scratch file instead of answering), at roughly -# A/B-call cost per rollout. Set GEPA_ROLLOUTS=agent to opt in. -# GEPA_ROLLOUTS is a legacy env name: it predates the removal of the GEPA body loop and now selects -# the rollout mode of the best-of-N candidate search. It is kept because it is a documented .env key. -GEPA_ROLLOUTS = os.environ.get("GEPA_ROLLOUTS", "direct") +# A/B-call cost per rollout. Set SKILLOPT_ROLLOUTS=agent to opt in. +SKILLOPT_ROLLOUTS = os.environ.get("SKILLOPT_ROLLOUTS", "direct") from . import (SERVE_TEMPLATE, client_kwargs, is_openrouter, model_api_key, # noqa: E402 model_base_url, openrouter_extra_body, teacher_base_url) @@ -79,7 +76,7 @@ def serve(self, candidate: dict[str, str]) -> str: return SERVE_TEMPLATE.format(body=assemble({**self._frozen, **candidate})) def _rollout(self, system, ex): - if GEPA_ROLLOUTS == "agent": + if SKILLOPT_ROLLOUTS == "agent": answer = self._agent_rollout(system, ex["task"]) else: msg = invoke_retry(self._client(), [("system", system), ("user", ex["task"])]) @@ -113,8 +110,7 @@ def make_reflection_lm(): """Teacher callable: (str | messages) -> str. Our own litellm call instead of a framework's model-string plumbing, so the ZDR provider preference rides on every OpenRouter request, and a local OPENROUTER_BASE_URL (vLLM/Ollama) is honored through litellm's generic openai provider. - Shared by the body pass's candidate authors and the description pass's GEPA reflection, whose - LanguageModel protocol this signature satisfies.""" + Shared by SkillOpt's body and description passes.""" import litellm litellm.success_callback = [_track_reflection] base = teacher_base_url() diff --git a/optimize/routing.py b/optimize/routing.py index 532b190..0874e19 100644 --- a/optimize/routing.py +++ b/optimize/routing.py @@ -1,18 +1,16 @@ -"""Routing-objective GEPA pass over a skill's `description` (component-pass spec, pass 2). - -The inner loop never calls an LLM: each candidate description is scored by the real embedding -router against the skill's `routing:` cases (top-1 and recall@3 on expected matches, precision on -expected-no-route negatives). GEPA's reflection (the only model calls; ZDR like everything else) -turns per-case routing failures into better trigger phrasing. Gate: no regression on any routing -metric vs the champion, at least one strict improvement, no collision with another skill's -description, then the same quarantined pending record and human approval as the body pass.""" +"""SkillOpt routing pass over a skill's `description`. + +Each candidate description is scored by the real embedding router against the skill's `routing:` +cases. SkillOpt reflects on the per-case failures, proposes bounded edits, and keeps only strict +improvements. The promotion gate requires no routing regression, at least one strict improvement, +and no collision with another skill description.""" +import os import time +from dataclasses import dataclass from dataclasses import replace from pathlib import Path -import gepa import yaml -from gepa import EvaluationBatch from mcp_server.registry import SKILLS_DIR, load_skills, optimizable_components, skill_revision from mcp_server.router import Router @@ -22,6 +20,7 @@ from .evidence import RoutingRun, build_routing_evidence, recorded_path, write_evidence from .promote import save_pending from .rollout import make_reflection_lm +from . import skillopt_bridge as sk EVIDENCE_DIR = Path(__file__).resolve().parent.parent / "runs" / "evidence" @@ -31,11 +30,15 @@ "instructions.") -class RoutingAdapter: - """gepa.GEPAAdapter over {'description'}: batch items are routing cases, scored by the real - embedding router, 1.0 for the expected outcome, 0.5 for a top-3 near-miss, 0.0 otherwise.""" +@dataclass +class RoutingBatch: + outputs: list + scores: list[float] + trajectories: list[dict] | None - propose_new_texts = None # gepa probes this optional hook; None -> use its default reflection + +class RoutingAdapter: + """Score a description with the real router: exact is 1, top-three is 0.5, miss is 0.""" def __init__(self, skill: str, router_factory=None): self._skill = skill @@ -73,14 +76,67 @@ def evaluate(self, batch, candidate, capture_traces=False): outputs.append(match) scores.append(score) trajectories.append({"task": case["task"], "output": str(match), "feedback": feedback}) - return EvaluationBatch(outputs=outputs, scores=scores, - trajectories=trajectories if capture_traces else None) - - def make_reflective_dataset(self, candidate, eval_batch, components_to_update): - records = [{"Inputs": t["task"], "Generated Outputs": t["output"], - "Feedback": t["feedback"], "Diagnosis": _DIAGNOSIS} - for t in (eval_batch.trajectories or [])] - return {comp: records for comp in components_to_update} + return RoutingBatch(outputs, scores, trajectories if capture_traces else None) + + +def _score(batch: RoutingBatch) -> float: + return sum(batch.scores) / len(batch.scores) if batch.scores else 0.0 + + +def _hard_score(batch: RoutingBatch) -> float: + return (sum(score == 1.0 for score in batch.scores) / len(batch.scores) + if batch.scores else 0.0) + + +def optimize_description(skill: str, seed: str, cases: list[dict], budget: int, + reflection_lm=None, adapter=None) -> tuple[str, float, float]: + """Train a routing description with bounded SkillOpt edits and a strict router-score gate. + + `budget` caps router case evaluations. The seed consumes one full evaluation, and each proposed + revision consumes another. Rejected edits and routing failures are fed into the next reflection. + """ + if cases and budget < len(cases): + raise ValueError(f"routing budget {budget} is smaller than the {len(cases)}-case suite; " + "one complete suite evaluation is required") + adapter = adapter or RoutingAdapter(skill) + reflection_lm = reflection_lm or make_reflection_lm() + max_edits = int(os.environ.get("SKILLOPT_MAX_EDITS", "3")) + current = best = seed + current_batch = adapter.evaluate(cases, {"description": current}, capture_traces=True) + seed_score = best_score = _score(current_batch) + remaining = max(0, budget - len(cases)) + buffer: list[str] = [] + + while remaining >= len(cases): + failures = [trajectory for trajectory, score in + zip(current_batch.trajectories or [], current_batch.scores) if score < 1.0] + if not failures: + break + context = _DIAGNOSIS + if buffer: + context += "\n\nRejected edits from prior steps:\n" + "\n".join(buffer[-4:]) + edits, _summary = sk.reflect_edits(current, failures, context, max_edits, reflection_lm) + if not edits: + break + edit_budget = sk.decide_edit_budget(current, edits, _hard_score(current_batch), best_score, + len(cases), + context, reflection_lm, max_edits) + edits = sk.rank_edits(current, edits, edit_budget, reflection_lm) if edit_budget else [] + candidate, _report = sk.apply_edits(current, edits) + if not edits or candidate.strip() == current.strip(): + break + candidate_batch = adapter.evaluate(cases, {"description": candidate}, capture_traces=True) + remaining -= len(cases) + candidate_score = _score(candidate_batch) + if candidate_score > best_score: + current = best = candidate.strip() + current_batch = candidate_batch + best_score = candidate_score + else: + buffer.extend( + f"{edit.get('op', 'edit')} target={edit.get('target', '')[:80]!r}" + for edit in edits) + return best, seed_score, best_score def routing_gate(skill: str, metrics: dict, challenger: dict) -> tuple[bool, list[str]]: @@ -118,16 +174,12 @@ def run_routing(skill: str, budget: int = 60, log=print) -> dict: TASKS_DIR, log=log) log(f"[routing] optimizing '{skill}' description against {len(cases)} routing cases " - f"(budget {budget} metric calls; inner loop is embedding-only, no LLM rollouts)…") - result = gepa.optimize(seed_candidate={"description": champion["description"]}, - trainset=cases, adapter=RoutingAdapter(skill), - reflection_lm=make_reflection_lm(), max_metric_calls=budget, - display_progress_bar=True, raise_on_exception=False) - seed_score = result.val_aggregate_scores[0] - best_score = result.val_aggregate_scores[result.best_idx] + f"(budget {budget} router case evaluations; SkillOpt reflection, no LLM rollouts)…") + description, seed_score, best_score = optimize_description( + skill, champion["description"], cases, budget) log(f"[routing] inner-loop score: seed {seed_score:.3f} -> best {best_score:.3f}") - challenger = {**champion, "description": result.best_candidate["description"]} + challenger = {**champion, "description": description} if challenger["description"].strip() == champion["description"].strip(): log("[routing] no better description found, champion holds.") return {"skill": skill, "improved": False} diff --git a/requirements.txt b/requirements.txt index 9c146e4..94ab86e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ langfuse==4.14.0 skillopt==0.2.0 # body-pass optimizer: SkillOpt's held-out gate, patch-edit application, and # update-mode helpers (microsoft/SkillOpt, MIT). Pinned exact, the loop matches # this release's API; bump deliberately (see optimize/skillopt_bridge.py). -gepa==0.1.1 # the description (routing) pass's reflective loop litellm==1.93.0 # teacher-model plumbing (openrouter/... model strings) fastapi==0.139.2 uvicorn==0.51.0 diff --git a/tests/conftest.py b/tests/conftest.py index 004a6a7..2bce5cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def _isolated_local_skills_root(tmp_path_factory, monkeypatch): class FakeIdp: """Forge RS256 ID tokens against an in-memory JWKS, the layer-2 harness for OIDC validation - tests (docs/superpowers/specs/2026-07-19-sso-rbac-design.md). No IdP, no network: one keypair, + tests. No IdP, no network: one keypair, mint tokens with any claims/overrides (expiry, aud, iss, kid, or a different signing key), and expose the matching JWKS to feed `ui.oidc.verify_id_token`.""" diff --git a/tests/test_embedding.py b/tests/test_embedding.py index d0a2173..9c04c00 100644 --- a/tests/test_embedding.py +++ b/tests/test_embedding.py @@ -1,5 +1,6 @@ """Embedding backends: model selection, and the Qwen ONNX pooling/prefix contract with a stubbed session (no model download, no onnxruntime inference).""" +import sys from types import SimpleNamespace import numpy as np @@ -14,6 +15,58 @@ def test_backend_selection_by_model_name(): assert not E.is_qwen_onnx("Qwen/Qwen3-Embedding-0.6B") # torch checkpoint, not the ONNX export +def test_baked_models_use_the_image_cache_but_runtime_overrides_can_download(monkeypatch): + monkeypatch.setenv("BAKED_EMBED_MODEL", "onnx-community/Qwen3-Embedding-0.6B-ONNX") + monkeypatch.setenv("BAKED_EMBED_ONNX_FILE", "onnx/model_q4.onnx") + monkeypatch.setenv("BAKED_FALLBACK_EMBED_MODEL", "BAAI/bge-small-en-v1.5") + + assert E._baked_qwen("onnx-community/Qwen3-Embedding-0.6B-ONNX", "onnx/model_q4.onnx") + assert not E._baked_qwen("someone/Qwen3-Embedding-4B-ONNX", "onnx/model_q4.onnx") + assert E._baked_fastembed("BAAI/bge-small-en-v1.5") + assert not E._baked_fastembed("sentence-transformers/all-MiniLM-L6-v2") + + +def test_baked_qwen_loader_forces_cache_only(monkeypatch): + downloads = [] + + def fetch(model, filename, **kwargs): + downloads.append((model, filename, kwargs)) + return filename + + class Session: + def __init__(self, path, providers): + self.path = path + + def get_inputs(self): + return [] + + def get_outputs(self): + return [SimpleNamespace(name="last_hidden_state")] + + fake_tokenizer = SimpleNamespace(from_file=lambda path: path) + monkeypatch.setitem(sys.modules, "onnxruntime", SimpleNamespace(InferenceSession=Session)) + monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(hf_hub_download=fetch)) + monkeypatch.setitem(sys.modules, "tokenizers", SimpleNamespace(Tokenizer=fake_tokenizer)) + monkeypatch.setenv("BAKED_EMBED_MODEL", "repo/qwen3-embedding-onnx") + monkeypatch.setenv("BAKED_EMBED_ONNX_FILE", "onnx/model_q4.onnx") + + E.QwenOnnxEmbedding("repo/qwen3-embedding-onnx", "onnx/model_q4.onnx") + + assert [call[2]["local_files_only"] for call in downloads] == [True, True] + + +def test_fastembed_loader_uses_baked_cache_only(monkeypatch): + calls = [] + monkeypatch.setitem( + sys.modules, "fastembed", + SimpleNamespace(TextEmbedding=lambda **kwargs: calls.append(kwargs) or object())) + monkeypatch.setenv("BAKED_FALLBACK_EMBED_MODEL", "BAAI/bge-small-en-v1.5") + + E.FastembedEmbedding("BAAI/bge-small-en-v1.5") + + assert calls == [{"model_name": "BAAI/bge-small-en-v1.5", "local_files_only": True}] + + class _Enc(SimpleNamespace): pass diff --git a/tests/test_env_check.py b/tests/test_env_check.py index 5fffc73..b4e0f90 100644 --- a/tests/test_env_check.py +++ b/tests/test_env_check.py @@ -79,9 +79,9 @@ def test_provider_priority_composes_with_zdr(monkeypatch): from optimize import ZDR_PROVIDER, client_kwargs, openrouter_extra_body monkeypatch.delenv("OPENROUTER_PROVIDERS", raising=False) assert openrouter_extra_body() == ZDR_PROVIDER # default: ZDR only, no pin - monkeypatch.setenv("OPENROUTER_PROVIDERS", "fireworks, groq") + monkeypatch.setenv("OPENROUTER_PROVIDERS", "groq, deepinfra") body = openrouter_extra_body() - assert body["provider"]["order"] == ["fireworks", "groq"] # priority, in given order + assert body["provider"]["order"] == ["groq", "deepinfra"] # priority, in given order assert body["provider"]["zdr"] is True # priority never relaxes ZDR assert "order" not in ZDR_PROVIDER["provider"] # constant not mutated monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test") @@ -108,8 +108,8 @@ def test_provider_conflict_loose_name_matching(monkeypatch): import urllib.request from optimize import provider_conflict monkeypatch.setattr(urllib.request, "urlopen", - lambda url, timeout=10: _FakeEndpoints(["DeepInfra", "Io Net", "Fireworks"])) - assert provider_conflict("qwen/x", ["fireworks"]) is None # display-name vs slug + lambda url, timeout=10: _FakeEndpoints(["DeepInfra", "Io Net"])) + assert provider_conflict("qwen/x", ["deep-infra"]) is None # display-name vs slug assert provider_conflict("qwen/x", ["io-net"]) is None # punctuation-insensitive msg = provider_conflict("qwen/x", ["groq"]) assert "no provider that serves 'qwen/x'" in msg and "DeepInfra" in msg @@ -120,11 +120,11 @@ def test_provider_conflict_unknown_model_and_network_failure(monkeypatch): import urllib.request from optimize import provider_conflict monkeypatch.setattr(urllib.request, "urlopen", lambda url, timeout=10: _FakeEndpoints([])) - assert "no endpoints on OpenRouter" in provider_conflict("qwen/typo-27b", ["fireworks"]) + assert "no endpoints on OpenRouter" in provider_conflict("qwen/typo-27b", ["groq"]) def boom(url, timeout=10): raise OSError("offline") monkeypatch.setattr(urllib.request, "urlopen", boom) - assert provider_conflict("qwen/x", ["fireworks"]) is None # fail open offline + assert provider_conflict("qwen/x", ["groq"]) is None # fail open offline def test_preflight_no_pins_makes_no_network_calls(monkeypatch): @@ -139,7 +139,7 @@ def forbidden(url, timeout=10): def test_preflight_warns_on_every_uncovered_role(monkeypatch): import optimize - monkeypatch.setenv("OPENROUTER_PROVIDERS", "fireworks") + monkeypatch.setenv("OPENROUTER_PROVIDERS", "groq") monkeypatch.delenv("MODEL_BASE_URL", raising=False) monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False) monkeypatch.setattr(optimize, "provider_conflict", @@ -151,7 +151,7 @@ def test_preflight_warns_on_every_uncovered_role(monkeypatch): def test_preflight_reports_agent_model_alias_value(monkeypatch): # the pin check must validate the model the agent will actually use, whichever alias set it import optimize - monkeypatch.setenv("OPENROUTER_PROVIDERS", "fireworks") + monkeypatch.setenv("OPENROUTER_PROVIDERS", "groq") for var in ("MODEL_BASE_URL", "BASE_URL", "OPENROUTER_BASE_URL", "MODEL"): monkeypatch.delenv(var, raising=False) monkeypatch.setenv("AGENT_MODEL", "brand/new-model") @@ -165,7 +165,7 @@ def test_agent_model_resolution(monkeypatch): from optimize import agent_model monkeypatch.delenv("AGENT_MODEL", raising=False) monkeypatch.delenv("MODEL", raising=False) - assert agent_model() == "qwen/qwen3.6-27b" + assert agent_model() == "qwen/qwen3-32b" monkeypatch.setenv("MODEL", "legacy/model") assert agent_model() == "legacy/model" monkeypatch.setenv("AGENT_MODEL", "new/model") @@ -209,7 +209,7 @@ def test_judge_warns_when_skillopt_model_is_the_grader(): def test_preflight_checks_strong_model_only_when_explicitly_set(monkeypatch): import optimize - monkeypatch.setenv("OPENROUTER_PROVIDERS", "fireworks") + monkeypatch.setenv("OPENROUTER_PROVIDERS", "groq") monkeypatch.delenv("MODEL_BASE_URL", raising=False) monkeypatch.delenv("OPENROUTER_BASE_URL", raising=False) monkeypatch.setattr(optimize, "provider_conflict", @@ -223,7 +223,7 @@ def test_preflight_checks_strong_model_only_when_explicitly_set(monkeypatch): def test_preflight_skips_roles_on_local_endpoints(monkeypatch): import optimize - monkeypatch.setenv("OPENROUTER_PROVIDERS", "fireworks") + monkeypatch.setenv("OPENROUTER_PROVIDERS", "groq") monkeypatch.setenv("OPENROUTER_BASE_URL", "http://localhost:11434/v1") # fully local monkeypatch.setattr(optimize, "provider_conflict", lambda model, pins: f"nope for {model}") @@ -245,10 +245,10 @@ def invoke(self, messages): judge_mod.invoke_retry(Doomed(), []) assert Doomed.calls == 1 # no retries, no sleeps assert "OpenRouter cannot route this request" in str(exc.value) - monkeypatch.setenv("OPENROUTER_PROVIDERS", "fireworks") + monkeypatch.setenv("OPENROUTER_PROVIDERS", "groq") with pytest.raises(SystemExit) as exc: judge_mod.invoke_retry(Doomed(), []) - assert "OPENROUTER_PROVIDERS=fireworks" in str(exc.value) + assert "OPENROUTER_PROVIDERS=groq" in str(exc.value) def test_invoke_retry_still_retries_transient_errors(monkeypatch): @@ -274,28 +274,27 @@ def test_generic_base_url_and_api_key_win_with_legacy_fallback(monkeypatch): monkeypatch.delenv("API_KEY", raising=False) monkeypatch.delenv("MODEL_API_KEY", raising=False) assert api_key() == "sk-or-legacy" # legacy fallback - monkeypatch.setenv("API_KEY", "fw_generic") - assert api_key() == "fw_generic" # generic wins - assert model_api_key() == "fw_generic" # serving role falls back to shared - monkeypatch.setenv("MODEL_API_KEY", "fw_model_role") - assert model_api_key() == "fw_model_role" - monkeypatch.setenv("BASE_URL", "https://api.fireworks.ai/inference/v1") - assert teacher_base_url() == "https://api.fireworks.ai/inference/v1" + monkeypatch.setenv("API_KEY", "shared_key") + assert api_key() == "shared_key" # generic wins + assert model_api_key() == "shared_key" # serving role falls back to shared + monkeypatch.setenv("MODEL_API_KEY", "model_role_key") + assert model_api_key() == "model_role_key" + monkeypatch.setenv("BASE_URL", "http://ollama:11434/v1") + assert teacher_base_url() == "http://ollama:11434/v1" def test_hosted_https_endpoint_requires_a_key(monkeypatch): from optimize import openrouter_key_missing for var in ("API_KEY", "OPENROUTER_API_KEY", "MODEL_API_KEY", "MODEL_BASE_URL"): monkeypatch.delenv(var, raising=False) - monkeypatch.setenv("BASE_URL", "https://api.fireworks.ai/inference/v1") + monkeypatch.setenv("BASE_URL", "https://openrouter.ai/api/v1") assert openrouter_key_missing() is True # hosted endpoint, no key - monkeypatch.setenv("API_KEY", "fw_x") + monkeypatch.setenv("API_KEY", "sk-or-test") assert openrouter_key_missing() is False -def test_fireworks_direct_gets_clean_openai_request(monkeypatch): +def test_local_endpoint_gets_clean_openai_request(monkeypatch): from optimize import client_kwargs - monkeypatch.setenv("API_KEY", "fw_x") - kw = client_kwargs("https://api.fireworks.ai/inference/v1") - assert kw == {"base_url": "https://api.fireworks.ai/inference/v1", "api_key": "fw_x", + kw = client_kwargs("http://ollama:11434/v1") + assert kw == {"base_url": "http://ollama:11434/v1", "api_key": "local", "extra_body": {}} # no OpenRouter provider prefs diff --git a/tests/test_optimize.py b/tests/test_optimize.py index a46bb33..5064afc 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -300,7 +300,7 @@ class Msg: def test_agent_rollout_mode_routes_through_the_scaffold(monkeypatch): from optimize import rollout as R - monkeypatch.setattr(R, "GEPA_ROLLOUTS", "agent") + monkeypatch.setattr(R, "SKILLOPT_ROLLOUTS", "agent") monkeypatch.setattr(R, "judge", lambda t, r, a, reference="", check=None, deliverable=None: {"score": 0.5, "feedback": "f", "dimensions": {}}) @@ -318,17 +318,11 @@ def fake_agent_rollout(self, system, task): assert seen["system_has_contract"] -def test_removed_gepa_body_loop_leaves_one_candidate_search(): - """The sequential GEPA body loop is gone: best-of-N is the only candidate search, and an - inherited OPTIMIZE_STRATEGY must say so rather than silently mean nothing.""" - from optimize import rollout as R - assert not hasattr(R, "run_gepa") - with pytest.raises(ImportError): - import optimize.gepa_loop # noqa: F401 +def test_skillopt_is_the_only_body_candidate_search(): assert "strategy" not in inspect.signature(ab_mod.run_ab).parameters -@pytest.mark.parametrize("flag", ["--gepa", "--skip-gepa", "--strategy", "--candidates"]) +@pytest.mark.parametrize("flag", ["--strategy", "--candidates"]) def test_cli_rejects_flags_for_passes_that_do_not_exist(flag): """A flag naming a removed or never-shipped pass must fail the parse, not be quietly ignored. Asserting on the parser rather than the source text is what proves the refusal.""" diff --git a/tests/test_routing.py b/tests/test_routing.py index 9a7298f..8fd7f4b 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -46,13 +46,32 @@ def test_adapter_scores_expected_top3_miss_and_no_route(): assert "triggers too broadly" in feedback[4] -def test_adapter_reflective_dataset_carries_routing_diagnosis(): +def test_skillopt_description_search_keeps_only_a_strict_router_improvement(monkeypatch): cases = [{"task": "merge pdfs", "expected": "pdf"}] - batch = _evaluate({"merge pdfs": ("docx", [])}, cases) - adapter = R.RoutingAdapter("pdf", router_factory=lambda d: None) - records = adapter.make_reflective_dataset({"description": "d"}, batch, ["description"]) - assert "routing trigger" in records["description"][0]["Diagnosis"] - assert records["description"][0]["Inputs"] == "merge pdfs" + scripts = { + "old trigger": {"merge pdfs": ("docx", [])}, + "new trigger": {"merge pdfs": ("pdf", [])}, + } + adapter = R.RoutingAdapter( + "pdf", router_factory=lambda description: _ScriptedRouter(scripts[description])) + monkeypatch.setattr(R.sk, "reflect_edits", + lambda *args: ([{"op": "replace", "target": "old", "content": "new"}], [])) + monkeypatch.setattr(R.sk, "decide_edit_budget", lambda *args: 1) + monkeypatch.setattr(R.sk, "rank_edits", lambda skill, edits, budget, lm: edits[:budget]) + result = R.optimize_description("pdf", "old trigger", cases, budget=2, + reflection_lm=lambda messages: "", adapter=adapter) + assert result == ("new trigger", 0.0, 1.0) + + +def test_skillopt_description_search_requires_budget_for_the_full_suite(): + class NeverCalled: + def evaluate(self, *args, **kwargs): + raise AssertionError("the router must not run with an undersized budget") + + cases = [{"task": "one", "expected": "pdf"}, {"task": "two", "expected": "pdf"}] + with pytest.raises(ValueError, match="smaller than the 2-case suite"): + R.optimize_description("pdf", "trigger", cases, budget=1, + reflection_lm=lambda messages: "", adapter=NeverCalled()) def _metrics(champ, chall, parity=None): @@ -92,7 +111,7 @@ def test_gate_blocks_collision_and_parity(monkeypatch): def test_run_routing_auto_drafts_missing_cases(monkeypatch, tmp_path): - """No routing: block -> the drafter is invoked (sentinel) before any GEPA work.""" + """No routing block means the drafter runs before SkillOpt description training.""" skill = tmp_path / "skills" / "sk" skill.mkdir(parents=True) (skill / "SKILL.md").write_text("---\nname: sk\ndescription: d.\n---\nbody\n") @@ -129,13 +148,8 @@ def test_run_routing_writes_an_evidence_bundle_and_records_relative_paths(monkey monkeypatch.setattr(R, "EVIDENCE_DIR", evidence_root) monkeypatch.setattr(P, "PENDING_DIR", tmp_path / "pending") - class _Result: - best_candidate = {"description": "new trigger."} - val_aggregate_scores = [0.5, 0.9] - best_idx = 1 - - monkeypatch.setattr(R.gepa, "optimize", lambda **kwargs: _Result()) - monkeypatch.setattr(R, "make_reflection_lm", lambda: None) + monkeypatch.setattr(R, "optimize_description", + lambda skill, seed, cases, budget: ("new trigger.", 0.5, 0.9)) monkeypatch.setattr(R, "_description_shadows", lambda skill, desc: ("", 0.0)) from optimize import ab as A monkeypatch.setattr(A, "_routing_metrics", lambda skill, champ, chall: { diff --git a/tests/test_ui.py b/tests/test_ui.py index 2e33950..42b7aee 100644 --- a/tests/test_ui.py +++ b/tests/test_ui.py @@ -55,7 +55,7 @@ def test_optimize_without_key_is_friendly_400(client, monkeypatch): monkeypatch.delenv("OPENROUTER_API_KEY") r = client.post("/api/optimize/pdf") assert r.status_code == 400 - assert "OPENROUTER_API_KEY" in r.json()["detail"] + assert "API_KEY" in r.json()["detail"] assert ".env" in r.json()["detail"] @@ -119,6 +119,23 @@ def test_reject_records_a_reject_audit_entry(client): assert trail[0]["skill"] == "pdf" and trail[0]["revision"] == "abc123" +def test_reject_records_an_optional_normalized_reason(client): + P.save_pending("pdf", {"skill": "pdf", "champion_components": {}, + "challenger_components": {}}) + r = client.post("/api/reject/pdf", json={"reason": " deleted required checks\nby mistake "}) + assert r.status_code == 200 + record = client.get("/api/history").json()["audit"]["records"][0] + assert record["action"] == "reject" + assert record["reason"] == "deleted required checks by mistake" + + +def test_reject_reason_is_limited_before_pending_is_consumed(client): + P.save_pending("pdf", {"skill": "pdf", "champion_components": {}, + "challenger_components": {}}) + assert client.post("/api/reject/pdf", json={"reason": "x" * 501}).status_code == 422 + assert P.load_pending("pdf") is not None + + def test_reject_audits_an_empty_revision_when_none_is_recorded(client): P.save_pending("pdf", {"skill": "pdf", "champion_components": {}, "challenger_components": {}}) assert client.post("/api/reject/pdf").status_code == 200 @@ -158,6 +175,16 @@ def test_auth_me_is_a_200_unauthenticated_shape_in_password_mode(client): assert r.json() == {"authenticated": False, "email": "", "name": "", "role": ""} +def test_auth_me_surfaces_the_password_user(client, monkeypatch): + monkeypatch.setenv("AUTH_MODE", "password") + monkeypatch.setenv("AUTH_USER", "reviewer") + monkeypatch.setenv("AUTH_PASSWORD", "secret") + r = client.get("/auth/me", auth=("reviewer", "secret")) + assert r.status_code == 200 + assert r.json() == {"authenticated": True, "email": "", "name": "reviewer", + "role": "admin"} + + def test_compose_mcp_service_mounts_the_runs_directory(): """The mcp service records skill usage into runs/skill_usage.json; without the runs mount that write stays inside the ephemeral container and the board always reads 0 uses.""" @@ -195,6 +222,62 @@ def test_pending_renders_component_diff_and_warnings(client): assert p["evidence"]["markdown"].endswith("EVIDENCE.md") +def test_pending_reports_large_body_change_risk_and_side_by_side_content(client): + before = "\n".join(f"line {number}" for number in range(1, 9)) + P.save_pending("pdf", { + "skill": "pdf", "changed_components": ["body"], + "champion_components": {"description": "d", "body": before}, + "challenger_components": {"description": "d", "body": "line 1"}, + }) + p = client.get("/api/pending/pdf").json() + assert p["risk"] == {"added_lines": 0, "removed_lines": 7, "changed_pct": 87.5, + "size_delta_pct": p["risk"]["size_delta_pct"], "high_risk": True} + assert p["comparison"] == [{"component": "SKILL.md (body)", "before": before, + "after": "line 1"}] + + +def test_skill_version_explorer_reads_active_pending_and_snapshot(client, tmp_path, monkeypatch): + root = tmp_path / "skills" + active = root / "pdf" + active.mkdir(parents=True) + (active / "SKILL.md").write_text( + "---\nname: pdf\ndescription: Active description.\n---\nactive body\n") + (active / "notes.md").write_text("active notes") + monkeypatch.setenv("SKILL_ROUTER_PATHS", str(root)) + + pending = {"description": "Pending description.", "body": "pending body", + "file:notes.md": "pending notes"} + P.save_pending("pdf", {"skill": "pdf", "created": 123, + "champion_components": {"description": "Active description.", + "body": "active body"}, + "challenger_components": pending}) + + snapshot = P.REVISIONS_DIR / "pdf" / "abc123" + snapshot.mkdir(parents=True) + (snapshot / "SKILL.md").write_text( + "---\nname: pdf\ndescription: Snapshot description.\n---\nsnapshot body\n") + (snapshot / "notes.md").write_text("snapshot notes") + + versions = client.get("/api/skills/pdf/versions").json()["versions"] + assert [version["kind"] for version in versions] == ["active", "pending", "snapshot"] + assert client.get("/api/skills/pdf/versions/active").json()["body"] == "active body" + pending_payload = client.get("/api/skills/pdf/versions/pending").json() + assert pending_payload["description"] == "Pending description." + assert pending_payload["files"] == [{"path": "notes.md", "content": "pending notes"}] + snapshot_payload = client.get("/api/skills/pdf/versions/abc123").json() + assert snapshot_payload["body"] == "snapshot body" + assert snapshot_payload["files"] == [{"path": "notes.md", "content": "snapshot notes"}] + + +def test_skill_version_explorer_refuses_unknown_versions(client, tmp_path, monkeypatch): + root = tmp_path / "skills" + skill = root / "pdf" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: pdf\ndescription: PDF.\n---\nbody\n") + monkeypatch.setenv("SKILL_ROUTER_PATHS", str(root)) + assert client.get("/api/skills/pdf/versions/missing").status_code == 404 + + def test_pending_exposes_model_and_judge_for_the_comparison_panel(client): P.save_pending("pdf", { "skill": "pdf", "model": "qwen/qwen3-32b", "judge": "google/gemini-2.5-flash", @@ -220,6 +303,45 @@ def test_comparison_panel_controls_are_in_the_page(client): assert "cmp-overlay" in layout.ancestors["cmp-confirm"] +def test_review_panel_ships_risk_side_diff_and_confirmed_rejection(client): + html = client.get("/").text + layout = _Layout(html) + for element_id in ("risk-summary", "side-diff", "side-diff-body", "reject-overlay", + "reject-reason", "reject-confirm"): + assert element_id in layout.ancestors, f"review panel missing #{element_id}" + assert "pending-card" in layout.ancestors["risk-summary"] + assert "pending-card" in layout.ancestors["side-diff"] + assert "reject-overlay" in layout.ancestors["reject-confirm"] + assert "renderRisk(p.risk);" in html + assert "renderSideDiff(p.comparison);" in html + assert 'JSON.stringify({reason: $("#reject-reason").value})' in html + + +def test_skill_list_ships_search_filters_version_explorer_and_live_updates(client): + html = client.get("/").text + layout = _Layout(html) + for element_id in ("skill-filter", "skill-filter-count", "skill-overlay", + "skill-version", "skill-file", "board-announcer"): + assert element_id in layout.ancestors, f"skill explorer missing #{element_id}" + assert 'id="skill-search"' in html # input is a void element, so _Layout does not record it + assert 'aria-live="polite"' in html + assert "signature !== boardSignature" in html + assert "skillInventory.filter" in html + assert "/api/skills/${encodeURIComponent(skill)}/versions" in html + + +def test_comparison_panel_orders_tokens_and_tables_numbered_task_scores(client): + html = client.get("/").text + compare = html[html.index("function buildCompare(p)"):html.index("function openCompare()")] + + assert compare.index("input") < compare.index("output") + assert "beforeafterΔ" in compare + assert "Task ${i + 1}" in compare + task_count = "Math.max(beforeScores.length, afterScores.length)" + assert f"const scoreRows = Array.from({{length: {task_count}}}" in compare + assert 'class="cmp-pertask"' not in compare + + def test_api_skills_rows_carry_a_load_count(client, monkeypatch): """Every active skill row exposes `uses` so the UI can render the load-counter chip.""" import ui.app as ui_app @@ -233,24 +355,6 @@ class _Skill: assert active and active[0]["uses"] == 7 -def test_pending_reads_legacy_gepa_scores(client): - """Review slots written before the GEPA body loop was removed store the candidate-search - scores under `gepa`; an existing queue must still render.""" - P.save_pending("pdf", { - "skill": "pdf", "dataset": "pdf-holdout", - "gepa": {"seed_score": 0.1, "best_score": 0.9, "budget": 30}, - "ab": {"champion": {"run": 1, "mean": 0.2, "scores": [0.2], "tokens": {}}, - "challenger": {"run": 2, "mean": 0.8, "scores": [0.8], "tokens": {}}}, - "gate": {"promotable": True, "blocked": [], "warnings": []}, - "changed_components": ["body"], - "champion_components": {"description": "d", "body": "old line"}, - "challenger_components": {"description": "d", "body": "new line"}, - }) - p = client.get("/api/pending/pdf").json() - assert p["inner_loop"] == {"seed_score": 0.1, "best_score": 0.9, "budget": 30} - assert "gepa" not in p - - def test_pending_without_search_scores_still_renders(client): P.save_pending("pdf", { "skill": "pdf", "champion_components": {"description": "d", "body": "a"}, @@ -332,7 +436,7 @@ def test_index_ships_eval_chips_and_disabled_candidate_run(client): assert "no evals" in html # chip for skills without an eval task set assert "has_tasks" in html # rendering keys off the API flag assert "auto-drafts" in html # the disabled generate button explains how to get evals - assert "Generate candidate" in html # optimization is offered as an optional experiment + assert "Optimize with SkillOpt" in html # optimization is a first-class, human-gated workflow def test_index_leads_with_review_before_candidate_generation(client): @@ -513,7 +617,7 @@ def _promotable_pending() -> None: def test_approval_and_rollback_are_refused_while_one_is_in_flight(client, tmp_path, monkeypatch): """Promotion and rollback each snapshot, stage, and swap directories over several steps, and the endpoints run on a thread pool. A second action is refused with 409 rather than allowed to - interleave those steps, the way a second candidate run is.""" + interleave those steps, the way a second SkillOpt run is.""" import ui.app as U skill, replaced = _promoted_skill(tmp_path, monkeypatch) _promotable_pending() @@ -715,9 +819,10 @@ def test_index_preserves_a_chosen_revision_across_refreshes(client): def test_index_reports_action_failures(client): html = client.get("/").text - assert 'act("#pending-msg"' in html # approve and reject + assert 'act("#cmp-msg"' in html # approve + assert 'act("#reject-msg"' in html # reject assert 'act("#history-msg"' in html # rollback - assert 'act("#skills-msg"' in html # candidate generation + assert 'act("#skills-msg"' in html # SkillOpt optimization assert "could not load history" in html # a failed poll degrades per section assert "Promise.allSettled" in html diff --git a/ui/app.py b/ui/app.py index f14c97d..ad3746e 100644 --- a/ui/app.py +++ b/ui/app.py @@ -1,7 +1,7 @@ """Change-control UI: the review surface for quarantined instruction changes. -Reviewers see the evidence and the promotion decision first; candidate generation (the optional -generator) is a secondary action that only ever writes to the pending queue. Promotion and +Reviewers see the evidence and the promotion decision first. SkillOpt optimization is a core +workflow that only ever writes to the pending queue. Promotion and rollback both go through `optimize.promote`, which snapshots the displaced revision and swaps directories atomically. """ @@ -15,13 +15,14 @@ from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.responses import FileResponse, RedirectResponse +from pydantic import BaseModel, Field -from mcp_server.registry import SLUG_RE, load_skills +from mcp_server.registry import SLUG_RE, load_skills, read_components, skill_revision from optimize.ab import TASKS_DIR, run_ab from ui.auth import (auth_mode, current_actor, require_auth, require_role, using_default_password) from optimize.promote import (_audit_best_effort, approve_pending, list_revisions, - list_snapshotted_skills, load_pending, pending_path, read_audit, - rollback, stale_evidence_reason) + list_snapshotted_skills, load_pending, load_snapshot_components, + pending_path, read_audit, rollback, stale_evidence_reason) logger = logging.getLogger(__name__) REPO_ROOT = Path(__file__).resolve().parent.parent @@ -39,7 +40,7 @@ def _check(skill: str) -> str: def same_origin(request: Request): """CSRF guard on state-changing endpoints: a cross-site page can POST to localhost (a paid - candidate run, a silent promotion, a rollback) without being able to read the response. Require + SkillOpt run, a silent promotion, or a rollback) without being able to read the response. Require the request to originate from this app's own origin.""" origin = request.headers.get("origin") if origin is None: # non-browser client (curl, the demo's own scripts), no ambient cookies to abuse @@ -50,8 +51,8 @@ def same_origin(request: Request): app = FastAPI(title="ingot change control", description="Review evidence for quarantined skill changes, promote them " "atomically, and roll back a promoted revision.", - # LAN-grade gate: no-op when no users file exists (local default stays open); - # HTTP Basic against runs/auth.json once a user is added (see ui/auth.py). + # Compose uses a LAN-grade Basic-auth gate; a bare process with no credentials + # infers open mode. Additional users live in runs/auth.json (see ui/auth.py). dependencies=[Depends(require_auth)]) if using_default_password(): @@ -73,12 +74,13 @@ def same_origin(request: Request): **oidc_cookie_kwargs()) app.include_router(oidc_router) else: - # Password/open mode carries no session identity, but the frontend polls /auth/me on every - # load: answer with a stable unauthenticated shape instead of a 404 (the OIDC router owns the - # real endpoint when that profile is on). + # Password mode can surface the Basic username in the board. Open mode has no signed-in user. + # The OIDC router owns the richer session-backed endpoint when that profile is active. @app.get("/auth/me") - def auth_me(): - return {"authenticated": False, "email": "", "name": "", "role": ""} + def auth_me(actor: str = Depends(current_actor)): + password = auth_mode() == "password" + return {"authenticated": password, "email": "", "name": actor if password else "", + "role": "admin" if password else ""} RUNS: dict[str, dict] = {} # skill -> {"status": running|done|error, "log": [lines]} @@ -141,10 +143,75 @@ def skills(): ] +def _active_skill(skill: str): + match = next((item for item in _cached_load_skills() if item.name == skill), None) + if match is None: + raise HTTPException(404, f"no active skill named '{skill}'") + return match + + +def _pending_revision(skill, pending: dict) -> str: + recorded = _challenger_revision(pending) + if recorded: + return recorded + try: + return skill_revision(Path(skill.root), pending["challenger_components"]) + except (KeyError, OSError, TypeError, ValueError): + return "" + + +def _version_payload(skill: str, kind: str, revision: str, created: int | None, + components: dict[str, str]) -> dict: + files = [{"path": key[len("file:"):], "content": value} + for key, value in sorted(components.items()) if key.startswith("file:")] + return {"skill": skill, "kind": kind, "revision": revision, "created": created, + "description": components.get("description", ""), + "body": components.get("body", ""), "files": files} + + +@app.get("/api/skills/{skill}/versions") +def skill_versions(skill: str): + """The live, pending, and snapshotted versions available to inspect for one active skill.""" + active = _active_skill(_check(skill)) + versions = [{"key": "active", "kind": "active", "revision": active.revision, + "created": None}] + pending = load_pending(skill) + if pending and isinstance(pending.get("challenger_components"), dict): + versions.append({"key": "pending", "kind": "pending", + "revision": _pending_revision(active, pending), + "created": pending.get("created")}) + versions.extend({"key": item["revision"], "kind": "snapshot", **item} + for item in list_revisions(skill)) + return {"skill": skill, "description": active.description, "versions": versions} + + +@app.get("/api/skills/{skill}/versions/{version}") +def skill_version(skill: str, version: str): + """Read one version's instructions and bundled text files without changing active state.""" + active = _active_skill(_check(skill)) + if version == "active": + return _version_payload(skill, "active", active.revision, None, + read_components(Path(active.root))) + if version == "pending": + pending = load_pending(skill) + components = pending.get("challenger_components") if pending else None + if not isinstance(components, dict): + raise HTTPException(404, f"no pending version for '{skill}'") + return _version_payload(skill, "pending", _pending_revision(active, pending), + pending.get("created"), components) + try: + components = load_snapshot_components(skill, version) + except ValueError as exc: + raise HTTPException(404, str(exc)) + created = next((item["created"] for item in list_revisions(skill) + if item["revision"] == version), None) + return _version_payload(skill, "snapshot", version, created, components) + + @app.post("/api/optimize/{skill}", dependencies=[Depends(same_origin), Depends(require_role("proposer"))]) def optimize(skill: str): - """Start the optional candidate generator for one skill. It never activates anything: the + """Start a SkillOpt optimization for one skill. It never activates anything: the result is a quarantined pending record for review.""" _check(skill) _preflight_optimize(skill) @@ -163,18 +230,18 @@ def _preflight_optimize(skill: str) -> None: _preflight_provider() if not (TASKS_DIR / f"{skill}.yaml").exists(): raise HTTPException(404, f"no eval task set for '{skill}'") - # one candidate run at a time: the token ledger is process-global, and concurrent runs + # one SkillOpt run at a time: the token ledger is process-global, and concurrent runs # would also contend for the same OpenRouter budget if any(s.get("status") == "running" for s in RUNS.values()): - raise HTTPException(409, "a candidate run is already in progress") + raise HTTPException(409, "a SkillOpt run is already in progress") def _preflight_provider() -> None: from optimize import openrouter_key_missing, preflight_provider_pins if openrouter_key_missing(): - raise HTTPException(400, "OPENROUTER_API_KEY is not set, copy .env.example to .env, " + raise HTTPException(400, "API_KEY is not set, copy .env.example to .env, " "add your key (https://openrouter.ai/keys), and restart the stack " - "(or point MODEL_BASE_URL/OPENROUTER_BASE_URL at a local endpoint)") + "(or point BASE_URL/MODEL_BASE_URL at a local endpoint)") try: preflight_provider_pins() except SystemExit as e: # pin/model conflict, surface the explanation, don't start a run @@ -203,9 +270,26 @@ def _label(component: str) -> str: def _inner_loop(record: dict) -> dict | None: - """Candidate-search scores. Records written before the GEPA body loop was removed store these - under `gepa`; read either so an existing review slot still renders.""" - return record.get("inner_loop") or record.get("gepa") + """Candidate-search scores recorded by current SkillOpt optimization runs.""" + return record.get("inner_loop") + + +def _review_risk(champion: dict, challenger: dict) -> dict: + """Line and body-size risk metrics shown before a reviewer opens the full diff.""" + before = str(champion.get("body", "")).splitlines() + after = str(challenger.get("body", "")).splitlines() + removed = added = 0 + for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, before, after).get_opcodes(): + if tag != "equal": + removed += i2 - i1 + added += j2 - j1 + changed_pct = round(100 * max(added, removed) / max(len(before), len(after), 1), 1) + before_chars = len(str(champion.get("body", ""))) + after_chars = len(str(challenger.get("body", ""))) + size_delta_pct = round(100 * (after_chars - before_chars) / max(before_chars, 1), 1) + return {"added_lines": added, "removed_lines": removed, "changed_pct": changed_pct, + "size_delta_pct": size_delta_pct, + "high_risk": changed_pct >= 50 or size_delta_pct <= -50} @app.get("/api/pending/{skill}") @@ -215,17 +299,24 @@ def pending(skill: str): raise HTTPException(404, f"no pending change for '{skill}'") champ, chall = p["champion_components"], p["challenger_components"] blocks = [] - for comp in p.get("changed_components") or [k for k in champ if champ[k] != chall.get(k, "")]: + changed_components = (p.get("changed_components") or + [key for key in champ if champ[key] != chall.get(key, "")]) + for comp in changed_components: label = _label(comp) blocks.append("\n".join(difflib.unified_diff( champ[comp].splitlines(), chall.get(comp, "").splitlines(), fromfile=f"{label} (champion)", tofile=f"{label} (challenger)", lineterm=""))) + comparison = [{"component": _label(component), "before": str(champ.get(component, "")), + "after": str(chall.get(component, ""))} + for component in changed_components] return {"skill": skill, "kind": p.get("kind", "quality"), "inner_loop": _inner_loop(p), "ab": p.get("ab"), "routing": p.get("routing"), "dataset": p.get("dataset"), "evidence": p.get("evidence_paths"), "stale": _stale_reason(skill, p), "model": p.get("model"), "judge": p.get("judge"), "gate": p.get("gate", {"promotable": True, "blocked": []}), - "changed": [_label(c) for c in p.get("changed_components", [])], "diff": "\n\n".join(blocks)} + "risk": _review_risk(champ, chall), "comparison": comparison, + "changed": [_label(c) for c in changed_components], + "diff": "\n\n".join(blocks)} def _stale_reason(skill: str, pending: dict) -> str | None: @@ -339,9 +430,14 @@ def _challenger_revision(pending: dict) -> str: return revision if isinstance(revision, str) else "" +class RejectRequest(BaseModel): + reason: str = Field(default="", max_length=500) + + @app.post("/api/reject/{skill}", dependencies=[Depends(same_origin), Depends(require_role("approver"))]) -def reject(skill: str, actor: str = Depends(current_actor)): +def reject(skill: str, payload: RejectRequest | None = None, + actor: str = Depends(current_actor)): _check(skill) # Load, validate, delete, and audit all under the lock: checking existence first and deleting # later would let a second reject pass the check, then re-delete and double-audit after the @@ -352,7 +448,8 @@ def reject(skill: str, actor: str = Depends(current_actor)): raise HTTPException(404, f"no pending change for '{skill}'") revision = _challenger_revision(pending) pending_path(skill).unlink(missing_ok=True) - _audit_best_effort("reject", skill, revision, actor) + reason = " ".join((payload.reason if payload else "").split()) + _audit_best_effort("reject", skill, revision, actor, reason=reason) return {"result": f"rejected the pending change for '{skill}'"} diff --git a/ui/auth.py b/ui/auth.py index 334dd6e..7ecc567 100644 --- a/ui/auth.py +++ b/ui/auth.py @@ -5,8 +5,8 @@ the audit-trail `actor`, which is the whole point, on a shared server an approval has to be attributable to a person, not to `local-operator`. -Opt-in: with no users file the UI stays open (the zero-config local default is unchanged). Create -the first user to turn auth on: +Compose supplies the local demo login `admin` / `ingot`, so its UI is gated by default. A bare +process with no credentials or users file infers open mode. Add another local user with: python -m ui.auth add alice # prompts for a password, writes runs/auth.json (mode 0600) @@ -86,7 +86,7 @@ def using_default_password() -> bool: def _actor_from(request: Request) -> str | None: - """The authenticated username, `_ANON` when auth is off, or None when creds are missing/invalid.""" + """The authenticated username, `_ANON` in open mode, or None for invalid credentials.""" if not auth_enabled(): return _ANON header = request.headers.get("Authorization", "") @@ -106,8 +106,9 @@ def _challenge() -> HTTPException: # --- Auth mode selection --------------------------------------------------------------------- # Three modes coexist: `oidc` (Sign in with Google, ui/oidc_flow.py), `password` (LAN Basic, above), -# and `open` (the zero-config local default). Explicit AUTH_MODE wins; otherwise infer password when -# credentials exist, else open. OIDC is opt-in via AUTH_MODE=oidc because it needs the OIDC_* config. +# and `open` (explicit in Compose, inferred for a bare process with no credentials). Explicit +# AUTH_MODE wins; otherwise infer password when credentials exist, else open. OIDC is opt-in via +# AUTH_MODE=oidc because it needs the OIDC_* config. _OIDC_BOOTSTRAP = ("/auth/login", "/auth/callback", "/auth/logout") _REQUIRED_OIDC_ENV = ("OIDC_CLIENT_ID", "OIDC_REDIRECT_URL", "SESSION_SECRET") diff --git a/ui/oidc.py b/ui/oidc.py index 2aae88f..c33194c 100644 --- a/ui/oidc.py +++ b/ui/oidc.py @@ -1,4 +1,4 @@ -"""OIDC ID-token validation primitive (see docs/superpowers/specs/2026-07-19-sso-rbac-design.md). +"""OIDC ID-token validation primitive (see docs/sso.md). Provider-agnostic and network-free: given a JWKS (discovery + fetch + caching happen in the implementation phase), validate an ID token's signature, `iss`, `aud`, `exp`/`iat`, and `nonce`, and diff --git a/ui/oidc_flow.py b/ui/oidc_flow.py index d8afc7c..9967fc6 100644 --- a/ui/oidc_flow.py +++ b/ui/oidc_flow.py @@ -1,7 +1,7 @@ """Sign-in-with-Google browser flow (OIDC Authorization Code + PKCE) for the change-control UI. Google is an ordinary OIDC provider, so this is a generic auth-code+PKCE flow pointed at Google's -issuer (see docs/sso.md and docs/superpowers/specs/2026-07-19-sso-rbac-design.md). `ui/oidc.py` stays +issuer (see docs/sso.md). `ui/oidc.py` stays network-free and owns ID-token validation; this module owns the parts that talk to the provider: discovery, JWKS fetch/caching, the login redirect, and the callback that redeems the code and establishes the session. Validation still goes through `ui.oidc.verify_id_token` and the session diff --git a/ui/rbac.py b/ui/rbac.py index 09474e8..9cf6293 100644 --- a/ui/rbac.py +++ b/ui/rbac.py @@ -1,4 +1,4 @@ -"""Role-based authorization for the change-control UI (see docs/superpowers/specs/2026-07-19-sso-rbac-design.md). +"""Role-based authorization for the change-control UI (see docs/sso.md). Pure, provider-agnostic authorization logic: map an authenticated identity's OIDC claims to one app role, and gate actions by role. The *authentication* source (LAN password today, OIDC later) supplies diff --git a/ui/static/index.html b/ui/static/index.html index 637b70c..ad2ba84 100644 --- a/ui/static/index.html +++ b/ui/static/index.html @@ -194,10 +194,32 @@ color: var(--ink-2); } .cmp-tbl .win { color: var(--pass); font-weight: 700; } .cmp-tbl .lose { color: var(--fail); font-weight: 700; } - .cmp-pertask { font-family: var(--mono); font-size: .72rem; color: var(--ink-2); text-align: left !important; } .cmp-actions { display: flex; align-items: center; gap: .7rem; margin-top: 1.1rem; flex-wrap: wrap; } + /* Read-only skill and revision explorer */ + .skill-modal { width: min(900px, 96vw); } + .version-toolbar { display: grid; grid-template-columns: minmax(14rem, 1fr) minmax(12rem, 1fr); + gap: .8rem; margin-bottom: 1rem; } + .version-field { display: flex; flex-direction: column; gap: .35rem; } + .version-field label { font: 600 .66rem/1 var(--mono); letter-spacing: .06em; + text-transform: uppercase; color: var(--ink-2); } + .version-field select { width: 100%; font: .75rem/1.4 var(--mono); padding: .48rem .55rem; + border-radius: 8px; border: 1px solid var(--line-2); background: var(--paper); color: var(--ink); } + .version-meta { display: flex; flex-wrap: wrap; gap: .45rem; margin: 0 0 .7rem; } + .version-desc { color: var(--ink-2); font-size: .84rem; margin: 0 0 .8rem; text-wrap: pretty; } + .version-pre { font-family: var(--mono); font-size: .74rem; line-height: 1.55; margin: 0; + background: var(--paper); border: 1px solid var(--line); border-radius: 12px; padding: .85rem 1rem; + overflow: auto; min-height: 16rem; max-height: 56vh; white-space: pre; color: var(--ink); } + @media (max-width: 620px) { .version-toolbar { grid-template-columns: 1fr; } } + /* ---- skills list ---- */ + .skill-controls { display: flex; align-items: end; gap: .75rem; flex-wrap: wrap; margin-top: 1rem; } + .skill-controls .version-field { min-width: min(18rem, 100%); flex: 1 1 18rem; } + .skill-controls .filter-field { flex: 0 1 13rem; min-width: 10rem; } + .skill-controls input, .skill-controls select, .reject-reason { width: 100%; font: .78rem/1.4 var(--mono); + padding: .52rem .62rem; border-radius: 8px; border: 1px solid var(--line-2); + background: var(--paper-2); color: var(--ink); } + .filter-count { font: .68rem/1.4 var(--mono); color: var(--ink-3); align-self: center; } .skilllist { margin-top: 1.1rem; border-top: 1px solid var(--line); } .srow { display: flex; flex-wrap: wrap; align-items: baseline; gap: .5rem 1rem; padding: .9rem .45rem; border-bottom: 1px solid var(--line); transition: background .15s ease-out; } @@ -205,6 +227,8 @@ .srow:hover { background: var(--paper-2); } .srow.pend:hover { background: var(--rust-bg); } .srow .main { flex: 1 1 26rem; min-width: 0; } + .skill-open { all: unset; display: block; width: 100%; cursor: pointer; border-radius: 8px; } + .skill-open:hover .sname { color: var(--rust); } .srow .nameline { display: flex; align-items: baseline; gap: .6rem; flex-wrap: wrap; } .srow .sname { font-family: var(--mono); font-size: .86rem; font-weight: 600; color: var(--ink); } .srow .sdesc { font-size: .82rem; color: var(--ink-2); margin: .28rem 0 0; text-wrap: pretty; @@ -276,6 +300,15 @@ .review-actions { display: flex; align-items: center; gap: .7rem; margin-top: 1.2rem; flex-wrap: wrap; } .review-actions .msg { font-family: var(--mono); font-size: .74rem; color: var(--pass); } .review-actions .msg.err { color: var(--fail); font-weight: 600; } + .risk-summary { display: grid; grid-template-columns: repeat(auto-fit, minmax(135px, 1fr)); + gap: .55rem; margin-top: 1rem; padding: .8rem; border: 1px solid var(--line); + border-radius: 12px; background: var(--paper); } + .risk-summary.high { border-color: var(--warn); background: var(--warn-bg); } + .risk-stat { display: flex; flex-direction: column; gap: .08rem; } + .risk-stat b { font: 600 1rem/1.2 var(--mono); color: var(--ink); } + .risk-stat span { font: .64rem/1.3 var(--mono); letter-spacing: .05em; text-transform: uppercase; + color: var(--ink-3); } + .risk-warning { grid-column: 1 / -1; font: 600 .74rem/1.5 var(--mono); color: var(--warn); } .difflabel { font-family: var(--mono); font-size: .66rem; letter-spacing: .09em; text-transform: uppercase; color: var(--rust); margin: 1.4rem 0 .4rem; } pre.diff { font-family: var(--mono); font-size: .72rem; line-height: 1.55; margin: 0; @@ -283,6 +316,26 @@ overflow: auto; max-height: 28rem; white-space: pre; color: var(--ink-2); } pre.diff .add { color: var(--pass); } pre.diff .del { color: var(--fail); } pre.diff .hunk { color: var(--rust); } pre.diff .meta { color: var(--ink-3); } + .side-diff { margin-top: .75rem; border: 1px solid var(--line); border-radius: 12px; + background: var(--paper-2); overflow: hidden; } + .side-diff summary { cursor: pointer; padding: .65rem .8rem; font: 600 .7rem/1.3 var(--mono); + color: var(--rust); letter-spacing: .05em; text-transform: uppercase; } + .side-component { border-top: 1px solid var(--line); } + .side-component h4 { margin: 0; padding: .5rem .75rem; font: 600 .7rem/1.3 var(--mono); + color: var(--ink-2); background: var(--paper-3); } + .side-columns { display: grid; grid-template-columns: 1fr 1fr; } + .side-column + .side-column { border-left: 1px solid var(--line); } + .side-column .side-label { display: block; padding: .35rem .65rem; font: 600 .64rem/1.3 var(--mono); + color: var(--ink-3); text-transform: uppercase; border-bottom: 1px solid var(--line); } + .side-column pre { margin: 0; padding: .7rem; max-height: 24rem; overflow: auto; + font: .7rem/1.5 var(--mono); white-space: pre; color: var(--ink-2); } + .reject-help { color: var(--ink-2); font-size: .82rem; margin: 0 0 .8rem; } + .reject-reason { min-height: 6rem; resize: vertical; } + @media (max-width: 720px) { .side-columns { grid-template-columns: 1fr; } + .side-column + .side-column { border-left: 0; border-top: 1px solid var(--line); } } + + .sr-only { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; + overflow: hidden; clip: rect(0, 0, 0, 0); white-space: nowrap; border: 0; } .footer { display: flex; align-items: baseline; justify-content: space-between; gap: 1rem; margin-top: 3rem; padding-top: .6rem; border-top: 1px solid var(--ink); font-family: var(--mono); font-size: .68rem; @@ -315,7 +368,9 @@ .beam { display: block; } } - button:focus-visible, .btn:focus-visible, input:focus-visible, a:focus-visible { + button:focus-visible, .btn:focus-visible, input:focus-visible, select:focus-visible, + textarea:focus-visible, summary:focus-visible, + .skill-open:focus-visible, a:focus-visible { outline: 2px solid var(--rust); outline-offset: 2px; border-radius: 2px; } @media (prefers-reduced-motion: reduce) { html { scroll-behavior: auto; } * { transition: none !important; animation: none !important; } @@ -345,21 +400,22 @@ Langfuse … +
+
/ CHANGE CONTROL ingot
@@ -380,8 +436,13 @@

Pending change

+ + + +
/ HISTORY snapshots and audit trail

Revisions and audit trail

Every promotion snapshots the revision it displaced, so a change is reversible. Restoring one is atomic and is itself recorded. The trail holds metadata only (action, skill, - revision, actor, timestamp), never skill text.

+ revision, actor, timestamp, and an optional rejection reason), never skill text.

@@ -428,19 +523,31 @@

Revisions and audit trail

/ SKILLS

Skills under control

-

Every skill the router serves, with the revision it is serving. Candidate generation - is optional and activates nothing: it authors rewrites in parallel, races them on the train tasks, - A/Bs the winner on held-out tasks, and leaves the result quarantined for review. Skills without an - eval task set can't be measured yet: a CLI run auto-drafts one.

+

Every skill the router serves, with the revision it is serving. SkillOpt trains + bounded instruction edits, measures them on train and held-out tasks, and leaves each result + quarantined for review. Skills without an eval task set can't be measured yet: a CLI run + auto-drafts one.

+
+
+
+
+
+ +
@@ -454,6 +561,7 @@

Candidate generation log

const $ = s => document.querySelector(s); let langfuseUrl = "#", currentPending = null, currentPendingData = null; let booted = false; // entrance animations run on first paint only; 3s refreshes stay still +let boardSignature = null, skillInventory = []; const REDUCED = matchMedia("(prefers-reduced-motion: reduce)").matches; // count-up on KPI values (structure: Magic UI number-ticker, vanilla rAF instead of a spring) @@ -523,7 +631,7 @@

Candidate generation log

// revision must shorten to nonsense rather than throw and blank the history section const shortRev = r => String(r || "").slice(0, 12); const statusChip = s => s.pending ? `change awaiting review` - : s.status === "running" ? `generating candidate` + : s.status === "running" ? `SkillOpt running` : !s.has_tasks ? `no evals` : `evals`; @@ -558,18 +666,30 @@

Candidate generation log

const alerts = []; if (pending) alerts.push(alert("act", "REVIEW", `${pending} change${pending > 1 ? "s" : ""} quarantined and awaiting your decision.`)); - if (running) alerts.push(alert("info", "GENERATING", - "a background candidate run is in progress; watch the log.")); + if (running) alerts.push(alert("info", "OPTIMIZING", + "a SkillOpt run is in progress; watch the log.")); if (!pending && !running) alerts.push(alert("ok", "CLEAR", "nothing quarantined; every skill is serving its approved revision.")); if (!withTasks.length) alerts.push(alert("info", "NO EVALS", - `${skills.length} skill${skills.length === 1 ? "" : "s"} tracked, none with an eval task set, which candidate generation needs.`)); + `${skills.length} skill${skills.length === 1 ? "" : "s"} tracked, none with an eval task set, which SkillOpt optimization needs.`)); $("#attention").innerHTML = alerts.join(""); const pill = $("#status-pill"); if (pending) { pill.className = "pill act"; pill.textContent = `${pending} to review`; } else if (running) { pill.className = "pill run"; pill.textContent = "generating"; } else { pill.className = "pill idle"; pill.textContent = "idle"; } + + const snapshots = history + ? Object.values(history.revisions).reduce((n, revisions) => n + revisions.length, 0) : null; + const decisions = history ? history.audit.total : null; + const signature = JSON.stringify({pending, running, skills: skills.length, snapshots, decisions}); + if (signature !== boardSignature) { + const parts = [`${skills.length} skills tracked`, `${pending} awaiting review`, + `${running} SkillOpt runs active`]; + if (snapshots != null) parts.push(`${snapshots} rollback points`, `${decisions} recorded decisions`); + $("#board-announcer").textContent = `Board updated: ${parts.join(", ")}.`; + boardSignature = signature; + } } // The board re-polls every 3s. Rebuilding the history rows on every poll reset each revision @@ -600,7 +720,7 @@

Candidate generation log

const trail = h.audit.records.map(r => `
${esc(r.action || "?")} ${esc(r.skill || "?")} @ ${esc(shortRev(r.revision))} · ${esc(stamp(r.ts))} - · ${esc(r.actor || "unknown")}
`).join(""); + · ${esc(r.actor || "unknown")}${r.reason ? ` · reason: ${esc(r.reason)}` : ""}`).join(""); $("#history").innerHTML = (rows ? `
${rows}
` : `
No snapshots yet.
@@ -628,34 +748,116 @@

Candidate generation log

function renderSkills(skills) { const signature = JSON.stringify(skills); if (signature === skillsSignature) return; - if (!skills.length) { + skillInventory = skills; + skillsSignature = signature; + renderSkillRows(); +} + +function renderSkillRows() { + if (!skillInventory.length) { $("#skills-list").innerHTML = `
No skills registered.
Fetch or create skills under skills/<name>/SKILL.md and they will appear here.
`; - skillsSignature = signature; + $("#skill-filter-count").textContent = "0 skills"; + return; + } + const query = $("#skill-search").value.trim().toLowerCase(); + const filter = $("#skill-filter").value; + const visible = skillInventory.filter(s => { + const matchesText = !query || `${s.name} ${s.description}`.toLowerCase().includes(query); + const matchesFilter = filter === "all" || (filter === "pending" && s.pending) || + (filter === "evals" && s.has_tasks) || (filter === "no-evals" && !s.has_tasks); + return matchesText && matchesFilter; + }); + $("#skill-filter-count").textContent = `${visible.length} of ${skillInventory.length}`; + if (!visible.length) { + $("#skills-list").innerHTML = + `
No skills match.
+
Change the search text or filter to see more skills.
`; return; } - const ordered = [...skills].sort((a, b) => (b.pending - a.pending) || (b.has_tasks - a.has_tasks)); + const ordered = [...visible].sort((a, b) => (b.pending - a.pending) || (b.has_tasks - a.has_tasks)); $("#skills-list").innerHTML = ordered.map(s => `
-
+
+
${s.pending ? `` : ""} + ${s.status === "running" ? "optimizing…" : "Optimize with SkillOpt"}
`).join(""); - skillsSignature = signature; } +$("#skill-search").oninput = renderSkillRows; +$("#skill-filter").onchange = renderSkillRows; + +let skillVersionFiles = []; +const versionLabel = v => v.kind === "active" ? `Active · ${shortRev(v.revision)}` + : v.kind === "pending" ? `Pending challenger · ${shortRev(v.revision) || "unversioned"}` + : `Snapshot · ${shortRev(v.revision)} · ${stamp(v.created)}`; + +async function openSkillVersions(skill) { + $("#skill-detail-name").textContent = skill; + $("#skill-version").innerHTML = ""; + $("#skill-file").innerHTML = ""; + $("#skill-version-content").textContent = ""; + $("#skill-version-desc").textContent = ""; + $("#skill-version-meta").innerHTML = ""; + say("#skill-version-msg", "Loading versions…", false); + $("#skill-overlay").hidden = false; + try { + const summary = await j(`/api/skills/${encodeURIComponent(skill)}/versions`); + $("#skill-version").innerHTML = summary.versions.map(v => + ``).join(""); + await loadSkillVersion(); + } catch (e) { + say("#skill-version-msg", `⛔ ${e.message}`, true); + } +} + +async function loadSkillVersion() { + const skill = $("#skill-detail-name").textContent; + const version = $("#skill-version").value; + if (!skill || !version) return; + say("#skill-version-msg", "Loading version…", false); + try { + const detail = await j(`/api/skills/${encodeURIComponent(skill)}/versions/${encodeURIComponent(version)}`); + const skillMd = `---\nname: ${detail.skill}\ndescription: ${JSON.stringify(detail.description)}\n---\n\n${detail.body}`; + skillVersionFiles = [{path: "SKILL.md", content: skillMd}, ...(detail.files || [])]; + $("#skill-file").innerHTML = skillVersionFiles.map((file, i) => + ``).join(""); + $("#skill-version-meta").innerHTML = + `${esc(detail.kind)}` + + `rev ${esc(shortRev(detail.revision) || "unrecorded")}` + + (detail.created ? `${esc(stamp(detail.created))}` : ""); + $("#skill-version-desc").textContent = detail.description; + renderSkillVersionFile(); + say("#skill-version-msg", "", false); + } catch (e) { + say("#skill-version-msg", `⛔ ${e.message}`, true); + } +} + +function renderSkillVersionFile() { + const file = skillVersionFiles[Number($("#skill-file").value) || 0]; + $("#skill-version-content").textContent = file ? file.content : ""; +} + +function closeSkillVersions() { $("#skill-overlay").hidden = true; } +$("#skill-close").onclick = closeSkillVersions; +$("#skill-overlay").onclick = e => { if (e.target === $("#skill-overlay")) closeSkillVersions(); }; +$("#skill-version").onchange = loadSkillVersion; +$("#skill-file").onchange = renderSkillVersionFile; + // One failing endpoint must not blank the board: each section renders from what it got, and says // so when its own data is missing, instead of the whole poll rejecting and leaving stale or empty // panels with no explanation. @@ -703,10 +905,10 @@

Candidate generation log

} } -// the optional background experiment: it only ever writes a quarantined candidate +// SkillOpt only ever writes a quarantined candidate. async function generateCandidate(skill) { await act("#skills-msg", async () => - `started a candidate run for '${(await j(`/api/optimize/${skill}`, {method: "POST"})).started}'`); + `started SkillOpt for '${(await j(`/api/optimize/${skill}`, {method: "POST"})).started}'`); } function colorizeDiff(diff) { @@ -723,6 +925,7 @@

Candidate generation log

function showNoPending() { currentPending = null; + currentPendingData = null; $("#pending-card").hidden = true; $("#review-empty").hidden = false; $("#pending-heading").textContent = "No pending changes"; @@ -743,6 +946,31 @@

Candidate generation log

} } +function renderRisk(risk) { + const r = risk || {added_lines: 0, removed_lines: 0, changed_pct: 0, size_delta_pct: 0}; + const signed = n => `${Number(n) >= 0 ? "+" : ""}${Number(n).toFixed(1)}%`; + const warning = r.high_risk + ? `
⚠ High-risk rewrite: inspect the full body before deciding.
` : ""; + const el = $("#risk-summary"); + el.classList.toggle("high", Boolean(r.high_risk)); + el.innerHTML = + `
+${r.added_lines}lines added
` + + `
−${r.removed_lines}lines removed
` + + `
${Number(r.changed_pct).toFixed(1)}%body changed
` + + `
${signed(r.size_delta_pct)}body size
` + warning; +} + +function renderSideDiff(comparison) { + const rows = (comparison || []).map(part => + `

${esc(part.component)}

` + + `
Before
${esc(part.before)}
` + + `
After
${esc(part.after)}
` + + `
`).join(""); + $("#side-diff-body").innerHTML = rows || + `
No component comparison is available.
`; + $("#side-diff").open = false; +} + // `keepMessage` is set when the poll switched cards by itself: a reviewer who just approved a // change has to keep reading "Promoted ...", while opening a card by hand clears the previous one. async function showPending(skill, {keepMessage = false} = {}) { @@ -764,8 +992,7 @@

Candidate generation log

// stale evidence describes a champion that has since moved on disk: promotion would refuse it, // so the review card refuses it first rather than after the click $("#approve").disabled = Boolean((p.gate && !p.gate.promotable) || p.stale); - // records written before the GEPA body loop was removed carry these scores under `gepa`; - // the API normalizes them, and a record with neither simply renders without the line + // A record without candidate-search scores simply renders without the line. const search = p.inner_loop; let detail; if (p.ab) { // quality pass: judge scores + token shift through the full agent @@ -788,11 +1015,12 @@

Candidate generation log

const dOut = ch.tokens.mean_output - c.tokens.mean_output; const dIn = ch.tokens.mean_input - c.tokens.mean_input; const regress = dOut > 0.1 * c.tokens.mean_output; // output tokens are the cost that matters - tok = `
output tokens/task ` + + tok = `
input tokens/task ` + + `${c.tokens.mean_input.toFixed(0)} → ${ch.tokens.mean_input.toFixed(0)} ` + + `(${dIn >= 0 ? "+" : ""}${dIn.toFixed(0)})` + + ` · output ` + `${c.tokens.mean_output.toFixed(0)} → ${ch.tokens.mean_output.toFixed(0)} ` + - `(${dOut >= 0 ? "+" : ""}${dOut.toFixed(0)})${regress ? " ⚠ regression" : ""}` + - ` · input ${c.tokens.mean_input.toFixed(0)} → ${ch.tokens.mean_input.toFixed(0)} ` + - `(${dIn >= 0 ? "+" : ""}${dIn.toFixed(0)})
`; + `(${dOut >= 0 ? "+" : ""}${dOut.toFixed(0)})${regress ? " ⚠ regression" : ""}
`; } detail = duel + meta + tok; } else { // routing pass: embedding-router metrics, no LLM involved @@ -823,7 +1051,9 @@

Candidate generation log

: ""; $("#pending-detail").innerHTML = detail + changed + gate + warn + stale; + renderRisk(p.risk); $("#diff").innerHTML = colorizeDiff(p.diff); + renderSideDiff(p.comparison); if (!keepMessage) say("#pending-msg", "", false); $("#evidence-label").hidden = $("#evidence").hidden = false; showEvidence(skill); @@ -842,18 +1072,22 @@

Candidate generation log

function buildCompare(p) { const ab = p.ab || {}, c = ab.champion || {}, ch = ab.challenger || {}; const ct = c.tokens || {}, cht = ch.tokens || {}; + const beforeScores = c.scores || [], afterScores = ch.scores || []; + const scoreRows = Array.from({length: Math.max(beforeScores.length, afterScores.length)}, (_, i) => + `Task ${i + 1}${sc(beforeScores[i])}${sc(afterScores[i])}` + + `${delta(beforeScores[i], afterScores[i], false)}`).join(""); const models = `

Model breakdown

Serving model (runs the skill)${esc(p.model || ",")}
Judge (LLM-as-a-judge)${esc(p.judge || ",")}
`; - const scores = `

LLM-as-a-judge score · ${(c.scores || []).length} held-out tasks

- + const scores = `

LLM-as-a-judge score · ${beforeScores.length} held-out tasks

+
championchallengerΔ
- + ${scoreRows}
beforeafterΔ
mean${sc(c.mean)}${sc(ch.mean)}${delta(c.mean, ch.mean, false)}
per-task${(c.scores || []).map(sc).join(" ")}
→ ${(ch.scores || []).map(sc).join(" ")}
`; const tokens = `

Token usage / task, before → after

- +
beforeafterΔ
output${num(ct.mean_output)}${num(cht.mean_output)}${delta(ct.mean_output, cht.mean_output, true)}
input${num(ct.mean_input)}${num(cht.mean_input)}${delta(ct.mean_input, cht.mean_input, true)}
output${num(ct.mean_output)}${num(cht.mean_output)}${delta(ct.mean_output, cht.mean_output, true)}
`; return models + scores + tokens; } @@ -882,8 +1116,37 @@

Candidate generation log

say("#pending-msg", r, false); return r; }); -$("#reject").onclick = () => act("#pending-msg", async () => - (await j(`/api/reject/${currentPending}`, {method: "POST"})).result); + +function openReject() { + if (!currentPending) return; + $("#reject-skill").textContent = currentPending; + $("#reject-reason").value = ""; + say("#reject-msg", "", false); + $("#reject-overlay").hidden = false; + $("#reject-reason").focus(); +} +function closeReject() { $("#reject-overlay").hidden = true; } +$("#reject").onclick = openReject; +$("#reject-close").onclick = closeReject; +$("#reject-cancel").onclick = closeReject; +$("#reject-overlay").onclick = e => { if (e.target === $("#reject-overlay")) closeReject(); }; +$("#reject-confirm").onclick = () => act("#reject-msg", async () => { + const skill = currentPending; + const result = (await j(`/api/reject/${skill}`, { + method: "POST", headers: {"Content-Type": "application/json"}, + body: JSON.stringify({reason: $("#reject-reason").value}) + })).result; + closeReject(); + say("#pending-msg", result, false); + return result; +}); + +document.addEventListener("keydown", e => { + if (e.key !== "Escape") return; + if (!$("#reject-overlay").hidden) closeReject(); + else if (!$("#skill-overlay").hidden) closeSkillVersions(); + else if (!$("#cmp-overlay").hidden) closeCompare(); +}); j("/api/config").then(c => { langfuseUrl = c.langfuse_url;