From 305fea571d885c9ce363f4b97a904fc0a9c7d7d7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 8 Jul 2026 02:53:33 -0600 Subject: [PATCH 01/44] fix(bench): SWE-bench scores from repo STATE, not the model printed text Root-fix a scoring bug proven live: on django-12419 @ glm-4.6 the agent made the EXACT gold fix but scored 0 because the harness extracted the patch by parsing the model reply for a fenced diff block, and the model wrote prose instead. Two changes, standard SWE-bench practice (SWE-agent/OpenHands read the diff from repo state and pre-stage the checkout): - boxSetup(task): harness clones the instance repo at base_commit into a fixed /work BEFORE the agent shot (same session) so the agent only edits. A stochastic model cannot be trusted to clone to an exact path (observed cannot-change-to-/work failures). Adapter-agnostic seam on BenchmarkAdapter. - boxExtract(): git diff of the agent edits in /work (test files excluded; the judge applies the gold test_patch itself), preferred over the event-stream parse which stays the fallback. Prompt updated: repo pre-staged, agent only edits. Typecheck clean; calibration still green. Live glm-5.2 verification blocked on transient sandbox box-provisioning failures (box unavailable before start on a green health) - re-run when the platform host-agent recovers. --- bench/src/benchmarks/swe-bench.ts | 51 +++++++++++++++--- bench/src/benchmarks/types.ts | 14 +++++ bench/src/run-benchmarks.ts | 88 ++++++++++++++++++++++++++++++- 3 files changed, 144 insertions(+), 9 deletions(-) diff --git a/bench/src/benchmarks/swe-bench.ts b/bench/src/benchmarks/swe-bench.ts index 9a7e4b98..d60db8a8 100644 --- a/bench/src/benchmarks/swe-bench.ts +++ b/bench/src/benchmarks/swe-bench.ts @@ -25,11 +25,21 @@ import { import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' /** - * The SWE deliverable, extracted from the agent's event STREAM (not the box FS). - * `runLoop`'s `OutputAdapter` only sees events, so the agent prints its unified - * diff in a fenced block and this pulls the last one out — the seam that lets the - * SWE benchmark run through the gate runner (`runGate` / `runBenchmark`) - * like any other. + * Fixed in-box path the agent clones the instance repo into. It is the SINGLE + * source of truth shared by the prompt template (which tells the agent to clone + * here) and `boxExtract` (which runs `git diff` here after the shot) — so the + * harness always knows exactly where the agent's edits live, for any instance. + */ +const SWE_REPO_DIR = '/work' + +/** + * The SWE deliverable's FALLBACK parser, from the agent's event STREAM. + * + * The PRIMARY deliverable is `boxExtract` below: a `git diff` of the agent's + * actual edits, read from the cloned repo's STATE inside the box (standard + * SWE-bench practice). This event-stream parse only runs when that diff is empty + * — a model that edited the source correctly but never printed a fenced diff (the + * exact failure this replaces) still scores off its real changes, not its prose. */ export const swePatchOutput: OutputAdapter = { parse(events) { @@ -59,6 +69,28 @@ export function createSweBenchAdapter(): BenchmarkAdapter { name: 'swe-bench-verified', output: swePatchOutput, + // Extract the patch from repo STATE, not printed text: stage every edit the + // agent made in the cloned repo and diff it against the checked-out + // base_commit (`HEAD`). Test files are excluded — the judge applies the gold + // `test_patch` itself, so an agent edit to a test would collide on apply. The + // paths come out `a/` (cwd = repo root), matching the gold + // patch format the swebench judge's `git apply` expects. + // Pre-stage: clone the instance repo at base_commit into SWE_REPO_DIR so the + // agent only edits (the harness owns the checkout — a stochastic model can't be + // trusted to clone to an exact path). `--quiet` keeps the exec output small. + boxSetup(task) { + const repo = String(task.metadata?.repo ?? '') + const base = String(task.metadata?.base_commit ?? '') + return { + command: `rm -rf ${SWE_REPO_DIR} && git clone --quiet https://github.com/${repo} ${SWE_REPO_DIR} && git -C ${SWE_REPO_DIR} checkout --quiet ${base}`, + } + }, + boxExtract() { + return { + command: `git -C ${SWE_REPO_DIR} add -A && git -C ${SWE_REPO_DIR} diff --cached -- . ':(exclude)*/test*'`, + } + }, + async preflight() { await preflightVenvImports({ modules: ['swebench'], @@ -101,8 +133,10 @@ print(json.dumps(out)) prompt: [ `Repository: ${r.repo} @ ${r.base_commit}`, '', - 'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests, so editing tests does not count. Keep the change minimal.', - 'When done, END your reply with the COMPLETE unified git diff as the LAST thing, fenced exactly as ```diff … ``` (nothing after the closing fence). That fenced diff is the only deliverable.', + `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`, + '', + 'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.', + 'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.', '', '--- Issue ---', String(r.problem_statement ?? ''), @@ -121,6 +155,9 @@ print(json.dumps(out)) const runId = safeRunId('bench', task.id) return runStagedJudge({ tmpPrefix: 'swebench-', + // Debug: retain the staged dir (holds swebench's per-instance apply/run + // logs) for post-mortem when SWEBENCH_KEEP_TMP is set. Off by default. + ...(process.env.SWEBENCH_KEEP_TMP ? { keepTmp: true } : {}), async stage(dir) { await stageFile( join(dir, 'preds.json'), diff --git a/bench/src/benchmarks/types.ts b/bench/src/benchmarks/types.ts index abb9f81a..2329c3e3 100644 --- a/bench/src/benchmarks/types.ts +++ b/bench/src/benchmarks/types.ts @@ -49,6 +49,20 @@ export interface BenchmarkAdapter { * so the gate runner (`runGate` / `runBenchmark`) needs no * per-benchmark branching. */ output?: OutputAdapter + /** Post-shot deliverable extraction from the box FILESYSTEM, not the event stream. + * When set, the shot runner execs `command` in the STILL-ALIVE box after the agent + * turn drains and uses its stdout as the judged artifact — the durable way to capture + * a git diff of the agent's in-box edits (standard SWE-bench practice: SWE-agent / + * OpenHands read the diff from repo STATE), instead of hoping the model printed a + * fenced diff in its reply. Empty stdout ⇒ the runner falls back to `output` (the + * event-stream parse). `cwd` defaults to the box root. */ + boxExtract?(task: BenchTask): { command: string; cwd?: string } + /** Optional workspace pre-stage run in the box BEFORE the agent shot (same + * session as `boxExtract`). For repo-state benchmarks (SWE-bench) this clones + * the instance repo at `base_commit` into a fixed path so the agent only edits + * — the harness owns the checkout, not the (stochastic) model. A non-zero exit + * fails the shot loud rather than letting the agent run against an empty box. */ + boxSetup?(task: BenchTask): { command: string; cwd?: string } /** Benchmark-owned worker leaf. Set when the benchmark's native protocol IS the * worker (e.g. AppWorld's interactive ReAct episode runs inside the engine, * not as a chat completion) — the experiment uses this instead of the diff --git a/bench/src/run-benchmarks.ts b/bench/src/run-benchmarks.ts index a0a8be49..98a73980 100644 --- a/bench/src/run-benchmarks.ts +++ b/bench/src/run-benchmarks.ts @@ -203,12 +203,96 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB deliverable, ) try { + // Pre-stage the workspace BEFORE the agent runs: for benchmarks whose artifact + // is repo state (SWE-bench), the harness clones the instance repo at base_commit + // into a fixed path so the agent only edits — a stochastic model can't be relied + // on to clone to an exact path itself (the flaky "agent cloned to a random dir" + // failure this removes). Runs under run.sessionId so it lands in the SAME remapped + // workspace the agent writes to and boxExtract reads from. + if (adapter.boxSetup) { + const setup = adapter.boxSetup(task) + const sres = await run.box.exec(setup.command, { + timeoutMs: 300_000, + sessionId: run.sessionId, + ...(setup.cwd ? { cwd: setup.cwd } : {}), + }) + if (sres.exitCode !== 0) + throw new Error(`boxSetup failed (exit ${sres.exitCode}): ${(sres.stderr ?? '').slice(0, 200)}`) + } const turn = await run.start(prompt ?? task.prompt) - const artifact = (turn.out ?? '').trim() + // Event-stream deliverable (adapter.output ?? finalText) — the FALLBACK. + let artifact = (turn.out ?? '').trim() + let boxExtractError: string | undefined + // Primary deliverable for benchmarks whose real artifact lives in the box FS + // (SWE-bench: a git diff of the agent's edits). Run the adapter's extraction + // command in the STILL-ALIVE box (valid until run.close() below) and prefer its + // stdout; the event-stream parse remains the fallback when the box yields nothing. + if (adapter.boxExtract) { + try { + const ex = adapter.boxExtract(task) + // The agent runs under a DRIVER SESSION whose workspace is a remapped + // virtual root; an exec WITHOUT that sessionId lands on the host FS and + // cannot see the agent's edits. Thread run.sessionId so the extraction + // runs in the SAME workspace the agent wrote to. + const res = await run.box.exec(ex.command, { + timeoutMs: 120_000, + sessionId: run.sessionId, + ...(ex.cwd ? { cwd: ex.cwd } : {}), + }) + const boxArtifact = (res.stdout ?? '').trim() + if (boxArtifact.length > 0) artifact = boxArtifact + else if (res.exitCode !== 0) + boxExtractError = `exit ${res.exitCode}: ${(res.stderr ?? '').slice(0, 160)}` + if (process.env.BENCH_ARTIFACT_DIR) { + try { + const { mkdirSync, writeFileSync } = await import('node:fs') + mkdirSync(process.env.BENCH_ARTIFACT_DIR, { recursive: true }) + const safe = `${adapter.name}_${task.id}_${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_') + const map = await run.box + .exec( + 'echo "PWD:"; pwd; echo "LS:"; ls -la; echo "GITROOTS:"; find / -maxdepth 5 -type d -name .git 2>/dev/null; echo "SETTINGS:"; find / -maxdepth 8 -name global_settings.py -path "*conf*" 2>/dev/null', + { timeoutMs: 60_000, sessionId: run.sessionId }, + ) + .catch((e: unknown) => ({ exitCode: -1, stdout: '', stderr: String(e) })) + writeFileSync( + `${process.env.BENCH_ARTIFACT_DIR}/${safe}.exec.json`, + JSON.stringify( + { sessionId: run.sessionId, extract: res, map: { exitCode: map.exitCode, stdout: map.stdout, stderr: map.stderr } }, + null, + 2, + ), + ) + } catch { + // debug-only + } + } + } catch (err) { + boxExtractError = err instanceof Error ? err.message.slice(0, 160) : String(err) + } + } + const detail = + turn.readError !== undefined + ? `read: ${turn.readError.slice(0, 160)}` + : boxExtractError !== undefined + ? `boxExtract: ${boxExtractError}` + : undefined + // Debug affordance: dump the judged artifact (the exact model_patch the judge + // will score) so a scoring failure can be diagnosed off the real bytes without + // re-running the agent. Off by default; set BENCH_ARTIFACT_DIR to enable. + if (process.env.BENCH_ARTIFACT_DIR) { + try { + const { mkdirSync, writeFileSync } = await import('node:fs') + mkdirSync(process.env.BENCH_ARTIFACT_DIR, { recursive: true }) + const safe = `${adapter.name}_${task.id}_${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_') + writeFileSync(`${process.env.BENCH_ARTIFACT_DIR}/${safe}.patch`, artifact) + } catch { + // debug-only; never fail the shot on a dump error + } + } return { artifact, ok: artifact.length > 0, - ...(turn.readError ? { detail: `read: ${turn.readError.slice(0, 160)}` } : {}), + ...(detail ? { detail } : {}), } } finally { if (timer) clearTimeout(timer) From 92a8c17c1ef879546b7378a4b74d64c04f8ad3de Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 8 Jul 2026 09:46:26 -0600 Subject: [PATCH 02/44] feat(improvement): raw-trace filesystem context for the code-surface proposer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code-surface proposer fed the coding agent the ~400-char DISTILLED findings (generationFailureDistiller), not the raw traces. The meta-harness edge (yoonholee.com/meta-harness) is raw-trace filesystem context: the coding agent greps/cats the FULL run traces of failed candidates to diagnose, rather than reading a pre-summary. Add rawTraceDistiller() — an additive analyzeGeneration producer that, instead of summarizing, points the proposer at the prior generation's real run traces already durable on disk under runDir (per-cell spans.jsonl event logs + cached-result.json scores + artifacts) with a grep/cat-to-diagnose instruction. It emits AnalystFinding[] with ABSOLUTE paths (the harness runs from a worktree cwd) so it drops into the same analyzeGeneration slot and renders through the same agenticGenerator prompt path. The existing distiller stays the default. improve() gains a one-line enable: opts.rawTraceContext = true wires rawTraceDistiller() when the caller has not supplied their own analyzeGeneration (explicit analyzeGeneration still wins; null still disables). Self-test builds a real tmp runDir with fake candidate + trace files and asserts the findings reference the actual absolute trace-file paths + the grep/cat instruction, plus worst-candidate-first ordering and the clean-generation fallback. tsc clean, biome clean, 3/3 new tests + 7/7 existing improve tests pass. --- src/improvement/improve.ts | 16 +- src/improvement/index.ts | 4 + src/improvement/raw-trace-distiller.test.ts | 195 ++++++++++++ src/improvement/raw-trace-distiller.ts | 319 ++++++++++++++++++++ 4 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 src/improvement/raw-trace-distiller.test.ts create mode 100644 src/improvement/raw-trace-distiller.ts diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index a6718fba..312b08de 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -48,6 +48,7 @@ import type { LocalHarness } from '../mcp/local-harness' import { assertModelAllowed } from '../runtime/supervise/model-policy' import { agenticGenerator, type Verifier } from './agentic-generator' import { type CandidateGenerator, improvementDriver } from './improvement-driver' +import { rawTraceDistiller } from './raw-trace-distiller' /** The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law * profile levers; `code` is the implementation-tier surface. */ @@ -94,6 +95,16 @@ export interface ImproveOptions { * trace-analyst over the runDir's traces) to replace it; pass `null` to disable * and keep the static `findings` all the way through. */ analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null + /** META-HARNESS mode: instead of the ~400-char distilled findings, feed the + * proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's + * real run traces under `runDir` (per-cell `spans.jsonl` event logs + + * `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose + * instruction — so the coding agent reads the actual failures itself rather than + * a pre-summary. Requires a REAL `runDir` (that is where the traces live). + * Ignored when `analyzeGeneration` is set explicitly (that wins) or is `null` + * (disabled). Equivalent to `analyzeGeneration: rawTraceDistiller()`; this flag + * is the one-line enable. Default `false` (the distiller stays the default). */ + rawTraceContext?: boolean /** CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a * repo, and the facade assembles the whole candidate pipeline — git worktrees * (`gitWorktreeAdapter`) driven by `improvementDriver` with the full agentic @@ -384,7 +395,10 @@ export async function improve( ? {} : { analyzeGeneration: - opts.analyzeGeneration ?? generationFailureDistiller(findings), + opts.analyzeGeneration ?? + (opts.rawTraceContext + ? rawTraceDistiller({ fallbackFindings: findings }) + : generationFailureDistiller(findings)), }), }) diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 89846962..28f56598 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -33,4 +33,8 @@ export { improvementDriver, } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' +export { + type RawTraceDistillerOptions, + rawTraceDistiller, +} from './raw-trace-distiller' export { type ReflectiveGeneratorOptions, reflectiveGenerator } from './reflective-generator' diff --git a/src/improvement/raw-trace-distiller.test.ts b/src/improvement/raw-trace-distiller.test.ts new file mode 100644 index 00000000..1785fd7c --- /dev/null +++ b/src/improvement/raw-trace-distiller.test.ts @@ -0,0 +1,195 @@ +/** + * `rawTraceDistiller` proof — the meta-harness edge. + * + * The guarantee this guards: the raw-trace `analyzeGeneration` producer does NOT + * pre-summarize a generation's failures — it points the coding-agent proposer at + * the ACTUAL run-trace files on disk (per-cell spans.jsonl + cached-result.json + + * artifacts under the durable runDir) with a grep/cat-to-diagnose instruction. We + * build a real tmp runDir with fake candidate + trace files, call the factory, and + * assert the emitted findings reference those real absolute paths + the + * grep/cat instruction (so the coding harness, running from a worktree cwd, can + * open them). + */ + +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { rawTraceDistiller } from './raw-trace-distiller' + +/** Minimal shape the factory reads — cast once to the analyzeGeneration input. */ +type AnalyzeInput = Parameters>[0] + +/** Write `//{spans.jsonl,cached-result.json,extra}` + * the way agent-eval's campaign executor does, and return the file paths. */ +function writeCellTrace( + campaignDir: string, + cellId: string, + opts: { spans: string; result: unknown; extraArtifact?: { name: string; body: string } }, +): { cellDir: string; spansPath: string; resultPath: string; artifactPath?: string } { + const cellDir = join(campaignDir, cellId.replace(/[^a-zA-Z0-9_-]/g, '_')) + mkdirSync(cellDir, { recursive: true }) + const spansPath = join(cellDir, 'spans.jsonl') + const resultPath = join(cellDir, 'cached-result.json') + writeFileSync(spansPath, opts.spans) + writeFileSync(resultPath, JSON.stringify(opts.result)) + let artifactPath: string | undefined + if (opts.extraArtifact) { + artifactPath = join(cellDir, opts.extraArtifact.name) + writeFileSync(artifactPath, opts.extraArtifact.body) + } + return { cellDir, spansPath, resultPath, artifactPath } +} + +describe('rawTraceDistiller', () => { + let root: string + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'raw-trace-')) + }) + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + it('points the proposer at the real trace-file paths + a grep/cat instruction', async () => { + const genRoot = join(root, 'gen-0') + const cand0Dir = join(genRoot, 'candidate-0') + const cand1Dir = join(genRoot, 'candidate-1') + + // Candidate 0: the worst — a failing cell with a distinctive error in its trace. + const c0 = writeCellTrace(cand0Dir, 'crack-7z-hash:0', { + spans: '{"name":"agent.run","error":"exit code 1: hashcat not found"}\n', + result: { cellId: 'crack-7z-hash:0', composite: 0 }, + extraArtifact: { name: 'agent-output.txt', body: 'command failed\n' }, + }) + // Candidate 1: a milder failure. + const c1 = writeCellTrace(cand1Dir, 'build-project:0', { + spans: '{"name":"agent.run","note":"partial credit"}\n', + result: { cellId: 'build-project:0', composite: 0.4 }, + }) + + const input: AnalyzeInput = { + generation: 0, + runDir: genRoot, + candidates: [ + { + surfaceHash: 'cand0hash', + composite: 0.0, + campaign: { + runDir: cand0Dir, + cells: [ + { + cellId: 'crack-7z-hash:0', + scenarioId: 'crack-7z-hash', + error: 'exit code 1: hashcat not found', + judgeScores: { j: { composite: 0 } }, + }, + ], + artifactsByPath: { 'crack-7z-hash:0/agent-output.txt': c0.artifactPath! }, + }, + }, + { + surfaceHash: 'cand1hash', + composite: 0.4, + campaign: { + runDir: cand1Dir, + cells: [ + { + cellId: 'build-project:0', + scenarioId: 'build-project', + judgeScores: { j: { composite: 0.4 } }, + }, + ], + }, + }, + ], + history: [], + } as unknown as AnalyzeInput + + const findings = await rawTraceDistiller()(input) + const blob = JSON.stringify(findings) + + // 1. Non-empty structured findings, each renderable through the proposer prompt + // (defaultBuildPrompt reads severity + claim + recommended_action). + expect(Array.isArray(findings)).toBe(true) + expect(findings.length).toBeGreaterThanOrEqual(2) + for (const f of findings as Array>) { + expect(typeof f.claim).toBe('string') + expect(typeof f.recommended_action).toBe('string') + expect(typeof f.severity).toBe('string') + } + + // 2. The findings reference the ACTUAL absolute trace-file paths on disk. + expect(blob).toContain(resolve(c0.spansPath)) + expect(blob).toContain(resolve(c0.resultPath)) + expect(blob).toContain(resolve(c0.artifactPath!)) + expect(blob).toContain(resolve(c1.spansPath)) + // …and the candidate/gen directories, so the agent can ls/grep -r them. + expect(blob).toContain(resolve(cand0Dir)) + expect(blob).toContain(resolve(genRoot)) + + // 3. The grep/cat-to-diagnose instruction — the meta-harness recipe, not a digest. + expect(blob.toLowerCase()).toContain('grep') + expect(blob.toLowerCase()).toContain('cat') + const leader = findings[0] as Record + expect(String(leader.recommended_action)).toMatch(/do not rely on a pre-summarized finding/i) + + // 4. Worst candidate first (composite 0 before 0.4), and paths are absolute. + const cand0Finding = (findings as Array>).find( + (f) => f.subject === 'cand0hash', + )! + expect(cand0Finding).toBeDefined() + const meta = cand0Finding.metadata as { cells: Array<{ files: string[] }> } + expect(meta.cells[0]!.files.every((p) => p.startsWith('/'))).toBe(true) + expect(meta.cells[0]!.files).toContain(resolve(c0.spansPath)) + }) + + it('uses input.runDir when no override is given and resolves relative dirs to absolute', async () => { + const genRoot = join(root, 'gen-3') + const candDir = join(genRoot, 'candidate-0') + const c = writeCellTrace(candDir, 's:0', { + spans: '{"error":"boom"}\n', + result: { composite: 0 }, + }) + const input = { + generation: 3, + runDir: genRoot, + candidates: [ + { + surfaceHash: 'h', + composite: 0, + campaign: { + runDir: candDir, + cells: [{ cellId: 's:0', scenarioId: 's', judgeScores: { j: { composite: 0 } } }], + }, + }, + ], + history: [], + } as unknown as AnalyzeInput + + const findings = await rawTraceDistiller()(input) + expect(JSON.stringify(findings)).toContain(resolve(c.spansPath)) + }) + + it('falls back to the seed findings when a generation has no failing cells', async () => { + const seed = [{ seed: 'keep-me' }] + const input = { + generation: 0, + runDir: join(root, 'gen-0'), + candidates: [ + { + surfaceHash: 'perfect', + composite: 1, + campaign: { + runDir: join(root, 'gen-0', 'candidate-0'), + cells: [{ cellId: 's:0', scenarioId: 's', judgeScores: { j: { composite: 1 } } }], + }, + }, + ], + history: [], + } as unknown as AnalyzeInput + + const findings = await rawTraceDistiller({ fallbackFindings: seed })(input) + expect(findings).toEqual(seed) + }) +}) diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts new file mode 100644 index 00000000..0dbfdfe8 --- /dev/null +++ b/src/improvement/raw-trace-distiller.ts @@ -0,0 +1,319 @@ +/** + * + * `rawTraceDistiller` — the meta-harness `analyzeGeneration` producer. + * + * The default `generationFailureDistiller` (in `improve.ts`) COMPRESSES each + * generation's failing cells into ~400-char structured findings before the next + * proposal round. That is the ACE-style recipe: a small summary is the proposer's + * whole view of what went wrong. This producer does the opposite — the + * meta-harness recipe (yoonholee.com/meta-harness): it does NOT summarize. It + * points the coding-agent proposer at the generation's RAW run traces already on + * disk under `runDir` — the durable per-cell `spans.jsonl` event logs, + * `cached-result.json` scores, and any artifacts the substrate persisted — and + * instructs the agent to `grep`/`cat`/`ls` them to diagnose the failures itself + * (up to the harness's full context, ~millions of tokens, vs a ~400-char digest). + * + * It emits `AnalystFinding[]` so it drops into the SAME `opts.analyzeGeneration` + * slot the default distiller uses, and renders through the same + * `agenticGenerator` prompt path (`claim` + `recommended_action`). The findings + * carry ABSOLUTE paths — the coding harness runs with `cwd` = a candidate + * worktree, so a relative `runDir` would be uncattable from there. + * + * Runtime layout it reads (written by agent-eval's optimization loop): + * + * /gen-/ ← the generation dir (input.runDir) + * candidate-/ ← one candidate campaign (campaign.runDir) + * / ← one scenario×rep cell + * spans.jsonl ← the raw trace (event/span log) + * cached-result.json ← the cell's score + artifact ref + * ← whatever the dispatch wrote + * + * @experimental + */ + +import { existsSync, readdirSync } from 'node:fs' +import { basename, join, resolve } from 'node:path' +import { type AnalystFinding, makeFinding } from '@tangle-network/agent-eval' +import type { Scenario, SelfImproveOptions } from '@tangle-network/agent-eval/contract' + +const ANALYST_ID = 'raw-trace-distiller' +/** A cell counts as "failing" below this mean composite (matches the default + * distiller's near-perfect threshold) or when it recorded an `error`. */ +const PASS_THRESHOLD = 0.999 + +export interface RawTraceDistillerOptions { + /** Anchor the emitted paths at this run root instead of the generation `runDir` + * the loop passes in. Normally unset — each call points at that generation's + * own directory (`input.runDir`). Pass an absolute path when you construct the + * producer ahead of the loop and want a fixed anchor (e.g. a test fixture). */ + runDir?: string + /** Max candidates to surface trace paths for, worst-scoring first. Default 12. */ + maxCandidates?: number + /** Max failing cells to enumerate per candidate before collapsing the rest into + * an "ls the candidate dir" pointer. Default 8. */ + maxCellsPerCandidate?: number + /** Max concrete file paths to list per cell (the agent can always `ls` the dir + * for the rest). Default 24. */ + maxFilesPerCell?: number + /** Findings to fall back to when the generation had NO failing cells, so a + * clean round never wipes the proposer's steering context. Mirrors the default + * distiller's static-seed fallback. Default: a single instruction finding. */ + fallbackFindings?: unknown[] +} + +interface CellTrace { + scenarioId: string + composite: number + error?: string + cellDir: string + files: string[] + truncatedFiles: boolean +} + +/** + * Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE + * FILESYSTEM CONTEXT — paths into the prior generation's real run traces plus a + * grep/cat-to-diagnose instruction — instead of a pre-summarized digest. + * + * Drop-in for `opts.analyzeGeneration` on `improve()` / `selfImprove()`: + * + * await improve(profile, seedFindings, { + * surface: 'code', + * code: { repoRoot }, + * runDir: '/abs/run', // MUST be a real path — the traces live here + * analyzeGeneration: rawTraceDistiller(), + * scenarios, judge, agent, + * }) + */ +export function rawTraceDistiller( + options: RawTraceDistillerOptions = {}, +): NonNullable['analyzeGeneration']> { + const maxCandidates = options.maxCandidates ?? 12 + const maxCellsPerCandidate = options.maxCellsPerCandidate ?? 8 + const maxFilesPerCell = options.maxFilesPerCell ?? 24 + + return async (input) => { + const genRoot = absoluteRunDir(options.runDir ?? input.runDir) + const durable = isDurable(genRoot) + + // Rank candidates worst-first; the worst failures are the highest-signal + // context for the next edit. Stable sort keeps equal-composite order. + const ranked = [...input.candidates] + .map((c) => ({ + surfaceHash: c.surfaceHash, + composite: c.composite, + campaignDir: absoluteRunDir(c.campaign.runDir), + cells: failingCells(c.campaign, maxCellsPerCandidate, maxFilesPerCell), + })) + .sort((a, b) => a.composite - b.composite) + .slice(0, maxCandidates) + + const totalFailingCells = ranked.reduce((n, c) => n + c.cells.length, 0) + + // A clean generation: keep the proposer's steering context rather than + // wiping it (parity with the default distiller's static-seed fallback). + if (totalFailingCells === 0) { + return ( + options.fallbackFindings ?? [ + makeFinding({ + analyst_id: ANALYST_ID, + severity: 'info', + area: 'raw-trace-context', + confidence: 1, + claim: `Generation ${input.generation} had no failing cells. The full raw run traces are on disk under ${genRoot}.`, + recommended_action: `To keep improving, grep/cat the raw traces under ${genRoot} (per-cell spans.jsonl + cached-result.json) to find the weakest passing runs, then make a targeted harness-code edit.`, + evidence_refs: [{ kind: 'artifact', uri: genRoot }], + metadata: { generation: input.generation, runDir: genRoot, failingCells: 0 }, + }), + ] + ) + } + + const findings: AnalystFinding[] = [] + + // 1. The meta-harness instruction: diagnose from the RAW traces, not a digest. + findings.push( + makeFinding({ + analyst_id: ANALYST_ID, + severity: 'high', + area: 'raw-trace-context', + confidence: 1, + claim: `Generation ${input.generation} produced ${totalFailingCells} failing/low-scoring cell(s) across ${ranked.length} candidate(s). Their FULL RAW run traces are on disk under ${genRoot} — the actual event logs (spans.jsonl), scores (cached-result.json), and artifacts, not a summary.${ + durable + ? '' + : ' (WARNING: this run root does not exist on disk — it looks like an in-memory run; pass a real runDir to improve() to get raw-trace context.)' + }`, + recommended_action: `Do NOT rely on a pre-summarized finding. Before editing, DIAGNOSE from the raw traces: run \`grep\`/\`cat\`/\`ls\` over the trace files and directories named in the following findings to see exactly what each failing run did and why it scored low, then make the smallest harness-code edit that fixes the dominant failure. Start with \`grep -rIn "error" ${genRoot}\` then \`cat\` the spans.jsonl of the worst cell.`, + evidence_refs: [{ kind: 'artifact', uri: genRoot }], + metadata: { + generation: input.generation, + runDir: genRoot, + failingCells: totalFailingCells, + candidates: ranked.length, + }, + }), + ) + + // 2. One finding per failing candidate: its campaign dir + the concrete raw + // trace files to grep/cat. + for (const cand of ranked) { + if (cand.cells.length === 0) continue + const scenarioList = cand.cells.map((c) => c.scenarioId).join(', ') + const fileLines = cand.cells + .map((c) => { + const header = ` cell ${c.scenarioId} (composite ${c.composite.toFixed(3)}${ + c.error ? `, error: ${truncate(c.error, 160)}` : '' + }) — dir ${c.cellDir}` + const files = c.files.map((f) => ` - ${f}`).join('\n') + const more = c.truncatedFiles ? `\n - …(ls ${c.cellDir} for the rest)` : '' + return c.files.length > 0 ? `${header}\n${files}${more}` : header + }) + .join('\n') + + findings.push( + makeFinding({ + analyst_id: ANALYST_ID, + severity: cand.composite < 0.5 ? 'critical' : 'high', + area: 'raw-trace-context', + confidence: 1, + subject: cand.surfaceHash, + claim: `Candidate ${cand.surfaceHash} scored composite ${cand.composite.toFixed(3)} with ${cand.cells.length} failing cell(s) [${scenarioList}]. Its raw traces are under ${cand.campaignDir}.`, + recommended_action: `grep/cat these raw trace files to diagnose WHY this candidate failed before editing:\n${fileLines}\nOr scan the whole candidate at once: \`grep -rIn . ${cand.campaignDir}\` and \`ls -R ${cand.campaignDir}\`.`, + evidence_refs: [ + { kind: 'artifact', uri: cand.campaignDir }, + ...cand.cells.flatMap((c) => + c.files.map((f) => ({ kind: 'artifact' as const, uri: f })), + ), + ], + metadata: { + surfaceHash: cand.surfaceHash, + composite: cand.composite, + campaignDir: cand.campaignDir, + cells: cand.cells.map((c) => ({ + scenarioId: c.scenarioId, + composite: c.composite, + cellDir: c.cellDir, + files: c.files, + ...(c.error ? { error: c.error } : {}), + })), + }, + }), + ) + } + + return findings + } +} + +/** The failing cells of a candidate campaign, each with its on-disk trace files. + * Mirrors the default distiller's per-cell composite (mean of judge composites, + * 0 when a cell produced no judge score) and its failing predicate. */ +function failingCells( + campaign: { + runDir: string + cells: ReadonlyArray<{ + cellId: string + scenarioId: string + error?: string + judgeScores: Record + }> + artifactsByPath?: Record + }, + maxCells: number, + maxFiles: number, +): CellTrace[] { + const campaignDir = absoluteRunDir(campaign.runDir) + const durable = isDurable(campaignDir) + const out: CellTrace[] = [] + for (const cell of campaign.cells) { + const scores = Object.values(cell.judgeScores ?? {}) + const composite = + scores.length === 0 + ? 0 + : scores.reduce((sum, s) => sum + (s.composite ?? 0), 0) / scores.length + if (!cell.error && composite >= PASS_THRESHOLD) continue + + const cellDir = join(campaignDir, sanitizeCellId(cell.cellId)) + const artifactPaths = artifactPathsForCell(campaign.artifactsByPath, cell.cellId) + const discovered = durable ? listTraceFiles(cellDir) : [] + // Canonical anchors the substrate always writes, kept even when a mem:// run + // never flushed them to disk (so the agent still learns the expected path). + const canonical = [join(cellDir, 'spans.jsonl'), join(cellDir, 'cached-result.json')] + const files = dedupeSorted([...discovered, ...artifactPaths, ...canonical]) + + out.push({ + scenarioId: cell.scenarioId, + composite: Number(composite.toFixed(3)), + ...(cell.error ? { error: cell.error } : {}), + cellDir, + files: files.slice(0, maxFiles), + truncatedFiles: files.length > maxFiles, + }) + if (out.length >= maxCells) break + } + return out +} + +/** Absolute paths of artifacts the campaign recorded for a cell. `artifactsByPath` + * is keyed `${cellId}/${relPath}` → absolute path. */ +function artifactPathsForCell( + artifactsByPath: Record | undefined, + cellId: string, +): string[] { + if (!artifactsByPath) return [] + const prefix = `${cellId}/` + return Object.entries(artifactsByPath) + .filter(([key]) => key.startsWith(prefix)) + .map(([, absPath]) => resolve(absPath)) +} + +/** Real files directly under `dir` and one level of sub-directories (artifacts + * are sometimes nested). Absolute paths, sorted. `[]` when the dir is absent — + * an expected state for an in-memory or not-yet-flushed run, not an error. A + * readdir fault on a dir that DOES exist is genuinely broken and throws loud. */ +function listTraceFiles(dir: string): string[] { + if (!existsSync(dir)) return [] + const out: string[] = [] + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name) + if (entry.isFile()) { + out.push(full) + } else if (entry.isDirectory()) { + for (const sub of readdirSync(full, { withFileTypes: true })) { + if (sub.isFile()) out.push(join(full, sub.name)) + } + } + } + return out +} + +/** Substrate cell-dir sanitization — must match agent-eval's + * `cellId.replace(/[^a-zA-Z0-9_-]/g, '_')` so the computed dir matches disk. */ +function sanitizeCellId(cellId: string): string { + return cellId.replace(/[^a-zA-Z0-9_-]/g, '_') +} + +/** A run root is durable (has real files) when it is not an in-memory sentinel + * and exists on disk. `mem://` runs keep everything in-process — no traces. */ +function isDurable(runDir: string): boolean { + return !runDir.startsWith('mem://') && existsSync(runDir) +} + +/** Resolve a run dir to absolute (the coding harness runs from a worktree cwd, so + * relative paths are uncattable there). `mem://` sentinels pass through untouched. */ +function absoluteRunDir(runDir: string): string { + return runDir.startsWith('mem://') ? runDir : resolve(runDir) +} + +function dedupeSorted(paths: string[]): string[] { + return [...new Set(paths)].sort((a, b) => { + // Group by directory then filename for a stable, readable listing. + const da = a.slice(0, a.length - basename(a).length) + const db = b.slice(0, b.length - basename(b).length) + return da === db ? basename(a).localeCompare(basename(b)) : da.localeCompare(db) + }) +} + +function truncate(s: string, n: number): string { + return s.length <= n ? s : `${s.slice(0, n - 1)}…` +} From 6bf0e5480fd208e6aab2d619b92538b24b3f2f8b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 8 Jul 2026 13:28:52 -0600 Subject: [PATCH 03/44] feat(improvement): wire profile stack through runtime primitives --- bench/src/benchmarks/swe-bench.ts | 33 ++- bench/src/run-benchmarks.test.mts | 50 ++++ bench/src/run-benchmarks.ts | 59 ++-- docs/api/index.md | 185 ++++++++++--- docs/api/primitive-catalog.md | 32 ++- docs/api/runtime.md | 252 +++++++++++++----- .../open-source-skill-lifecycle.ts | 128 +++++++++ package.json | 6 +- pnpm-lock.yaml | 81 +++--- scripts/gen-primitive-catalog.mjs | 37 ++- src/improvement/agentic-generator.ts | 88 +++++- src/improvement/raw-trace-distiller.test.ts | 26 ++ src/improvement/raw-trace-distiller.ts | 23 +- src/runtime/index.ts | 2 + src/runtime/sandbox-run.ts | 18 ++ src/runtime/strategy.ts | 49 +++- tests/agentic-generator.test.ts | 98 +++++++ tests/loops/strategy-suite.test.ts | 98 +++++++ tests/profile-improvement-stack.test.ts | 238 +++++++++++++++++ tests/runtime/sandbox-run.test.ts | 41 +++ 20 files changed, 1340 insertions(+), 204 deletions(-) create mode 100644 examples/open-source-skill-lifecycle/open-source-skill-lifecycle.ts create mode 100644 tests/profile-improvement-stack.test.ts diff --git a/bench/src/benchmarks/swe-bench.ts b/bench/src/benchmarks/swe-bench.ts index d60db8a8..75371416 100644 --- a/bench/src/benchmarks/swe-bench.ts +++ b/bench/src/benchmarks/swe-bench.ts @@ -58,12 +58,38 @@ export const swePatchOutput: OutputAdapter = { } const DATASET = 'princeton-nlp/SWE-bench_Verified' +const TEST_FILE_EXCLUDES = [ + "':(exclude,glob)**/tests/**'", + "':(exclude,glob)**/test/**'", + "':(exclude,glob)test_*.py'", + "':(exclude,glob)**/test_*.py'", + "':(exclude,glob)*_test.py'", + "':(exclude,glob)**/*_test.py'", + "':(exclude,glob)conftest.py'", + "':(exclude,glob)**/conftest.py'", +].join(' ') interface SweReport { resolved_instances?: number resolved_ids?: string[] } +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'\\''`)}'` +} + +function sweMetadata(task: BenchTask): { repo: string; base: string } { + const repo = String(task.metadata?.repo ?? '') + const base = String(task.metadata?.base_commit ?? '') + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo)) { + throw new Error(`swe-bench: invalid repo metadata for ${task.id}: ${repo}`) + } + if (!/^[0-9a-f]{7,40}$/i.test(base)) { + throw new Error(`swe-bench: invalid base_commit metadata for ${task.id}: ${base}`) + } + return { repo, base } +} + export function createSweBenchAdapter(): BenchmarkAdapter { return { name: 'swe-bench-verified', @@ -79,15 +105,14 @@ export function createSweBenchAdapter(): BenchmarkAdapter { // agent only edits (the harness owns the checkout — a stochastic model can't be // trusted to clone to an exact path). `--quiet` keeps the exec output small. boxSetup(task) { - const repo = String(task.metadata?.repo ?? '') - const base = String(task.metadata?.base_commit ?? '') + const { repo, base } = sweMetadata(task) return { - command: `rm -rf ${SWE_REPO_DIR} && git clone --quiet https://github.com/${repo} ${SWE_REPO_DIR} && git -C ${SWE_REPO_DIR} checkout --quiet ${base}`, + command: `rm -rf ${shellQuote(SWE_REPO_DIR)} && git clone --quiet ${shellQuote(`https://github.com/${repo}`)} ${shellQuote(SWE_REPO_DIR)} && git -C ${shellQuote(SWE_REPO_DIR)} checkout --quiet ${shellQuote(base)}`, } }, boxExtract() { return { - command: `git -C ${SWE_REPO_DIR} add -A && git -C ${SWE_REPO_DIR} diff --cached -- . ':(exclude)*/test*'`, + command: `git -C ${shellQuote(SWE_REPO_DIR)} add -A && git -C ${shellQuote(SWE_REPO_DIR)} diff --cached -- . ${TEST_FILE_EXCLUDES}`, } }, diff --git a/bench/src/run-benchmarks.test.mts b/bench/src/run-benchmarks.test.mts index dbc631c6..0291ee47 100644 --- a/bench/src/run-benchmarks.test.mts +++ b/bench/src/run-benchmarks.test.mts @@ -137,6 +137,56 @@ async function main(): Promise { assert.equal(leakyPrompts[1]!.includes(leakyGold), false, 'retry prompt redacts hidden gold fields') assert.match(leakyPrompts[1]!, /publicHint/) + const runtime = await import('@tangle-network/agent-runtime/loops') + if (runtime.openSandboxRun.toString().includes('beforeStart')) { + // The default shot path supports benchmark-owned box setup/extract without real sandbox infra. + const order: string[] = [] + const fakeClient = { + async create() { + return { + id: 'box-default-shot', + async exec(command: string, options?: { sessionId?: string }) { + order.push(`exec:${command}:streams=${order.filter((x) => x.startsWith('stream:')).length}:session=${options?.sessionId ? 'yes' : 'no'}`) + return { exitCode: 0, stdout: command === 'extract-patch' ? 'PATCH' : '', stderr: '' } + }, + async *streamPrompt(_prompt: string, options?: { sessionId?: string }) { + order.push(`stream:session=${options?.sessionId ? 'yes' : 'no'}`) + yield { type: 'result', data: { finalText: 'fallback text' } } + }, + async delete() { + order.push('delete') + }, + } + }, + async criuStatus() { + return { available: false } + }, + } + const boxAdapter: BenchmarkAdapter = { + name: 'boxy', + preflight: async () => {}, + loadTasks: async () => [{ id: 'boxy-0', prompt: 'edit repo', metadata: {} }], + judge: async (_task, artifact) => ({ resolved: artifact === 'PATCH', score: artifact === 'PATCH' ? 1 : 0 }), + goldArtifact: async () => 'PATCH', + boxSetup: () => ({ command: 'setup-repo' }), + boxExtract: () => ({ command: 'extract-patch' }), + } + const boxy = await runBenchmarks({ + benchmarks: ['boxy'], + cells: [{ label: 'default-shot', model: 'm', backend: 'sandbox' }], + routerBaseUrl: 'x', + routerKey: 'x', + resolveAdapter: () => boxAdapter, + resolveClient: () => fakeClient as never, + }) + assert.equal(boxy.rows[0]!.resolveRate, 1, 'boxExtract artifact is judged instead of fallback text') + assert.deepEqual( + order.slice(0, 3), + ['exec:setup-repo:streams=0:session=yes', 'stream:session=yes', 'exec:extract-patch:streams=1:session=yes'], + 'setup runs before the prompt stream, extract runs after the prompt stream, both in the same session', + ) + } + // An unavailable benchmark (preflight throws) is skipped, not fatal; the sweep still runs the rest. const flaky: Record = { ok: stubAdapter('ok', 2), diff --git a/bench/src/run-benchmarks.ts b/bench/src/run-benchmarks.ts index 98a73980..e55e5c3a 100644 --- a/bench/src/run-benchmarks.ts +++ b/bench/src/run-benchmarks.ts @@ -28,7 +28,13 @@ * }) */ -import type { AgentProfile, AgentRunSpec, Deliverable } from '@tangle-network/agent-runtime/loops' +import { mkdirSync, writeFileSync } from 'node:fs' +import type { + AgentProfile, + AgentRunSpec, + Deliverable, + OpenSandboxRunOptions, +} from '@tangle-network/agent-runtime/loops' import { openSandboxRun } from '@tangle-network/agent-runtime/loops' import type { SandboxEvent } from '@tangle-network/sandbox' import { resolveAdapter } from './adapters' @@ -70,6 +76,7 @@ export type BenchShot = (input: { readonly bridgeBearer?: string readonly sandboxBaseUrl?: string readonly timeoutMs?: number + readonly resolveClient?: typeof resolveBenchClient }) => Promise<{ artifact: string; ok: boolean; detail?: string }> export interface RunBenchmarksOptions { @@ -92,6 +99,8 @@ export interface RunBenchmarksOptions { readonly concurrency?: number /** Per-shot wall-clock (ms). */ readonly timeoutMs?: number + /** Test seam: resolve the runtime transport. Defaults to `resolveBenchClient`. */ + readonly resolveClient?: typeof resolveBenchClient /** Max attempts per (benchmark × cell × task). Default 1. Attempts after the first receive * non-answer checker feedback and the previous artifacts; the loop stops early on pass. */ readonly loopAttempts?: number @@ -164,8 +173,8 @@ function finalText(events: readonly SandboxEvent[]): string { /** The default real-agent shot: one `openSandboxRun` over the cell's harness+model, deliverable * extracted by the adapter's parser (or final text), abortable on `timeoutMs`. */ -const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs }) => { - const client = resolveBenchClient({ +const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerBaseUrl, routerKey, bridgeUrl, bridgeBearer, sandboxBaseUrl, timeoutMs, resolveClient }) => { + const client = (resolveClient ?? resolveBenchClient)({ backend: cell.backend ?? 'router', routerBaseUrl, routerKey, @@ -197,28 +206,29 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB } const controller = new AbortController() const timer = timeoutMs ? setTimeout(() => controller.abort(), timeoutMs) : undefined - const run = await openSandboxRun( - client, - { agentRun, signal: controller.signal, runId: `bench:${adapter.name}:${task.id}:${uniq}`, scenarioId: task.id }, - deliverable, - ) - try { - // Pre-stage the workspace BEFORE the agent runs: for benchmarks whose artifact - // is repo state (SWE-bench), the harness clones the instance repo at base_commit - // into a fixed path so the agent only edits — a stochastic model can't be relied - // on to clone to an exact path itself (the flaky "agent cloned to a random dir" - // failure this removes). Runs under run.sessionId so it lands in the SAME remapped - // workspace the agent writes to and boxExtract reads from. - if (adapter.boxSetup) { - const setup = adapter.boxSetup(task) - const sres = await run.box.exec(setup.command, { + const runOptions: OpenSandboxRunOptions = { + agentRun, + signal: controller.signal, + runId: `bench:${adapter.name}:${task.id}:${uniq}`, + scenarioId: task.id, + } + const boxSetup = adapter.boxSetup + if (boxSetup) { + runOptions.beforeStart = async ({ box, sessionId }) => { + const setup = boxSetup(task) + const sres = await box.exec(setup.command, { timeoutMs: 300_000, - sessionId: run.sessionId, + sessionId, ...(setup.cwd ? { cwd: setup.cwd } : {}), }) if (sres.exitCode !== 0) - throw new Error(`boxSetup failed (exit ${sres.exitCode}): ${(sres.stderr ?? '').slice(0, 200)}`) + throw new Error( + `boxSetup failed (exit ${sres.exitCode}): ${(sres.stderr ?? '').slice(0, 200)}`, + ) } + } + const run = await openSandboxRun(client, runOptions, deliverable) + try { const turn = await run.start(prompt ?? task.prompt) // Event-stream deliverable (adapter.output ?? finalText) — the FALLBACK. let artifact = (turn.out ?? '').trim() @@ -240,12 +250,11 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB ...(ex.cwd ? { cwd: ex.cwd } : {}), }) const boxArtifact = (res.stdout ?? '').trim() - if (boxArtifact.length > 0) artifact = boxArtifact - else if (res.exitCode !== 0) + if (res.exitCode !== 0) boxExtractError = `exit ${res.exitCode}: ${(res.stderr ?? '').slice(0, 160)}` + else if (boxArtifact.length > 0) artifact = boxArtifact if (process.env.BENCH_ARTIFACT_DIR) { try { - const { mkdirSync, writeFileSync } = await import('node:fs') mkdirSync(process.env.BENCH_ARTIFACT_DIR, { recursive: true }) const safe = `${adapter.name}_${task.id}_${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_') const map = await run.box @@ -281,7 +290,6 @@ const openSandboxShot: BenchShot = async ({ adapter, task, cell, prompt, routerB // re-running the agent. Off by default; set BENCH_ARTIFACT_DIR to enable. if (process.env.BENCH_ARTIFACT_DIR) { try { - const { mkdirSync, writeFileSync } = await import('node:fs') mkdirSync(process.env.BENCH_ARTIFACT_DIR, { recursive: true }) const safe = `${adapter.name}_${task.id}_${uniq}`.replace(/[^a-zA-Z0-9_.-]/g, '_') writeFileSync(`${process.env.BENCH_ARTIFACT_DIR}/${safe}.patch`, artifact) @@ -491,6 +499,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise 1 ? await loopedShot(shotInput, shot, loopAttempts) : await shot(shotInput) const score: BenchScore = await job.adapter.judge(job.task, out.artifact) @@ -539,7 +548,7 @@ export async function runBenchmarks(opts: RunBenchmarksOptions): Promise() for (const r of perTask) { - const key = `${r.benchmark}${r.cell}` + const key = `${r.benchmark}\u0000${r.cell}` const e = byKey.get(key) ?? { benchmark: r.benchmark, cell: r.cell, n: 0, resolved: 0, errored: 0, scoreSum: 0 } e.n += 1 if (!r.ok) e.errored += 1 diff --git a/docs/api/index.md b/docs/api/index.md index 287d6842..98938e64 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -2395,7 +2395,7 @@ Content type for the response. ### VerifyResult -Defined in: [improvement/agentic-generator.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L42) +Defined in: [improvement/agentic-generator.ts:48](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L48) Outcome of verifying a candidate worktree. `feedback` (compiler errors, failing test output) is fed into the next shot when `ok` is false. @@ -2406,19 +2406,19 @@ Outcome of verifying a candidate worktree. `feedback` (compiler errors, > **ok**: `boolean` -Defined in: [improvement/agentic-generator.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L43) +Defined in: [improvement/agentic-generator.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L49) ##### feedback? > `optional` **feedback?**: `string` -Defined in: [improvement/agentic-generator.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L44) +Defined in: [improvement/agentic-generator.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L50) *** ### AgenticGeneratorOptions -Defined in: [improvement/agentic-generator.ts:52](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L52) +Defined in: [improvement/agentic-generator.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L58) `@tangle-network/agent-runtime` improvement — the CODE-surface proposer for agent-eval's improvement loop. @@ -2439,7 +2439,7 @@ mutates a git worktree via a pluggable `CandidateGenerator`: > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [improvement/agentic-generator.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L54) +Defined in: [improvement/agentic-generator.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L60) Local coding harness to run in the worktree. Default `claude`. @@ -2447,7 +2447,7 @@ Local coding harness to run in the worktree. Default `claude`. > `optional` **timeoutMs?**: `number` -Defined in: [improvement/agentic-generator.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L56) +Defined in: [improvement/agentic-generator.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L62) Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). @@ -2455,7 +2455,7 @@ Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). > `optional` **buildPrompt?**: (`args`) => `string` -Defined in: [improvement/agentic-generator.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L59) +Defined in: [improvement/agentic-generator.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L65) Build the harness task prompt from the report + findings. Override for domain phrasing; the default turns findings into a concrete coder task. @@ -2480,7 +2480,7 @@ Build the harness task prompt from the report + findings. Override for > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L65) +Defined in: [improvement/agentic-generator.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L71) Verify the worktree after each dirtying shot. When set, a candidate that fails verification is NOT returned — the failure feeds the next shot @@ -2492,7 +2492,7 @@ Verify the worktree after each dirtying shot. When set, a candidate that > `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> -Defined in: [improvement/agentic-generator.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L67) +Defined in: [improvement/agentic-generator.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L73) Test seam — inject the harness runner (defaults to `runLocalHarness`). @@ -2528,7 +2528,7 @@ Does NOT throw when: > `optional` **isDirty?**: (`worktreePath`) => `boolean` -Defined in: [improvement/agentic-generator.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L69) +Defined in: [improvement/agentic-generator.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L75) Test seam — inject the worktree-dirty check (defaults to `git status`). @@ -2546,7 +2546,7 @@ Test seam — inject the worktree-dirty check (defaults to `git status`). ### ImproveOptions -Defined in: [improvement/improve.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L56) +Defined in: [improvement/improve.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L57) #### Type Parameters @@ -2564,7 +2564,7 @@ Defined in: [improvement/improve.ts:56](https://github.com/tangle-network/agent- > `optional` **surface?**: [`ImproveSurface`](#improvesurface) -Defined in: [improvement/improve.ts:59](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L59) +Defined in: [improvement/improve.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L60) Which profile lever to optimize. Default `'prompt'`. Selects the default generator + the baseline-surface extraction shape. @@ -2573,7 +2573,7 @@ Which profile lever to optimize. Default `'prompt'`. Selects the default > `optional` **generator?**: `SurfaceProposer`\<`unknown`\> -Defined in: [improvement/improve.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L63) +Defined in: [improvement/improve.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L64) The `SurfaceProposer` that mutates the surface. When unset, the facade picks the default for `surface` (`gepaProposer` for prompt, `skillOptProposer` @@ -2583,7 +2583,7 @@ The `SurfaceProposer` that mutates the surface. When unset, the facade > `optional` **gate?**: `"none"` \| `"holdout"` -Defined in: [improvement/improve.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L66) +Defined in: [improvement/improve.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L67) Gate mode. `'holdout'` (default) runs the held-out promotion gate; `'none'` is a baseline-only run (`budget.generations = 0`). @@ -2592,7 +2592,7 @@ Gate mode. `'holdout'` (default) runs the held-out promotion gate; > **scenarios**: `TScenario`[] -Defined in: [improvement/improve.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L68) +Defined in: [improvement/improve.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L69) Scenarios to evaluate against. Passthrough to `selfImprove`. @@ -2600,7 +2600,7 @@ Scenarios to evaluate against. Passthrough to `selfImprove`. > **judge**: `JudgeConfig`\<`TArtifact`, `TScenario`\> -Defined in: [improvement/improve.ts:70](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L70) +Defined in: [improvement/improve.ts:71](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L71) Judge that scores artifacts. Passthrough to `selfImprove`. @@ -2608,7 +2608,7 @@ Judge that scores artifacts. Passthrough to `selfImprove`. > **agent**: (`surface`, `scenario`, `ctx`) => `Promise`\<`TArtifact`\> -Defined in: [improvement/improve.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L73) +Defined in: [improvement/improve.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L74) The agent under improvement — same shape as `selfImprove.agent`: it takes the current surface + scenario + ctx and returns the artifact to judge. @@ -2635,7 +2635,7 @@ The agent under improvement — same shape as `selfImprove.agent`: it takes > `optional` **budget?**: `SelfImproveBudget` -Defined in: [improvement/improve.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L75) +Defined in: [improvement/improve.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L76) Budget + loop shape. Passthrough; `gate: 'none'` forces `generations = 0`. @@ -2643,7 +2643,7 @@ Budget + loop shape. Passthrough; `gate: 'none'` forces `generations = 0`. > `optional` **llm?**: `SelfImproveLlm` -Defined in: [improvement/improve.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L78) +Defined in: [improvement/improve.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L79) LLM config. Passthrough to `selfImprove` AND used to construct the default reflective proposer (`gepaProposer`/`skillOptProposer`) when `generator` is unset. @@ -2652,7 +2652,7 @@ LLM config. Passthrough to `selfImprove` AND used to construct the default > `optional` **allowedModels?**: readonly `string`[] -Defined in: [improvement/improve.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L82) +Defined in: [improvement/improve.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L83) Restrict the run to this subset of models. When set, the reflection model (`llm.model`, or the default when unset) must be a member, or `improve()` throws @@ -2662,7 +2662,7 @@ Restrict the run to this subset of models. When set, the reflection model > `optional` **runDir?**: `string` -Defined in: [improvement/improve.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L88) +Defined in: [improvement/improve.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L89) Run directory passthrough to `selfImprove`. Pass a REAL path to make the loop durable: campaign cells + the loop provenance record land on the filesystem as @@ -2674,7 +2674,7 @@ Run directory passthrough to `selfImprove`. Pass a REAL path to make the loop > `optional` **analyzeGeneration?**: ((`input`) => `Promise`\<`unknown`[]\>) \| `null` -Defined in: [improvement/improve.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L96) +Defined in: [improvement/improve.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L97) Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). DEFAULT: the built-in failure distiller — after each generation it turns the @@ -2684,11 +2684,27 @@ Per-generation findings producer passthrough (see selfImprove.analyzeGeneration) trace-analyst over the runDir's traces) to replace it; pass `null` to disable and keep the static `findings` all the way through. +##### rawTraceContext? + +> `optional` **rawTraceContext?**: `boolean` + +Defined in: [improvement/improve.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L107) + +META-HARNESS mode: instead of the ~400-char distilled findings, feed the + proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's + real run traces under `runDir` (per-cell `spans.jsonl` event logs + + `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose + instruction — so the coding agent reads the actual failures itself rather than + a pre-summary. Requires a REAL `runDir` (that is where the traces live). + Ignored when `analyzeGeneration` is set explicitly (that wins) or is `null` + (disabled). Equivalent to `analyzeGeneration: rawTraceDistiller()`; this flag + is the one-line enable. Default `false` (the distiller stays the default). + ##### code? > `optional` **code?**: `ImproveCodeOptions` -Defined in: [improvement/improve.ts:104](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L104) +Defined in: [improvement/improve.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L115) CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a repo, and the facade assembles the whole candidate pipeline — git worktrees @@ -2702,7 +2718,7 @@ CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a > `optional` **skills?**: `ImproveSkillsOptions` -Defined in: [improvement/improve.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L111) +Defined in: [improvement/improve.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L122) SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, `surface: 'skills'` optimizes the profile's skills REFS array (file pointers) @@ -2715,7 +2731,7 @@ SKILLS-surface wiring for real skill-DOCUMENT optimization. Without this, > `optional` **storage?**: `CampaignStorage` -Defined in: [improvement/improve.ts:113](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L113) +Defined in: [improvement/improve.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L124) Storage passthrough to `selfImprove`; overrides the default chosen from `runDir`. @@ -2723,7 +2739,7 @@ Storage passthrough to `selfImprove`; overrides the default chosen from `runDir` ### ImproveResult -Defined in: [improvement/improve.ts:144](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L144) +Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L155) #### Type Parameters @@ -2741,7 +2757,7 @@ Defined in: [improvement/improve.ts:144](https://github.com/tangle-network/agent > **profile**: `AgentProfile` -Defined in: [improvement/improve.ts:147](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L147) +Defined in: [improvement/improve.ts:158](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L158) The profile after improvement: the winner surface applied back into the matching field when the gate shipped, else the input profile unchanged. @@ -2750,7 +2766,7 @@ The profile after improvement: the winner surface applied back into the > **shipped**: `boolean` -Defined in: [improvement/improve.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L149) +Defined in: [improvement/improve.ts:160](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L160) True when `gateDecision === 'ship'`. @@ -2758,7 +2774,7 @@ True when `gateDecision === 'ship'`. > **lift**: `number` -Defined in: [improvement/improve.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L151) +Defined in: [improvement/improve.ts:162](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L162) Held-out lift (`winner − baseline` composite). @@ -2766,7 +2782,7 @@ Held-out lift (`winner − baseline` composite). > **gateDecision**: `"ship"` \| `"hold"` \| `"need_more_work"` \| `"model_ceiling"` \| `"arch_ceiling"` -Defined in: [improvement/improve.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L153) +Defined in: [improvement/improve.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L164) The five-valued gate verdict from `selfImprove`. @@ -2774,7 +2790,7 @@ The five-valued gate verdict from `selfImprove`. > **raw**: `SelfImproveResult`\<`TScenario`, `TArtifact`\> -Defined in: [improvement/improve.ts:155](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L155) +Defined in: [improvement/improve.ts:166](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L166) Full `selfImprove` result for advanced inspection. @@ -2924,6 +2940,61 @@ Minimum tools the server must expose to pass. Default 1. *** +### RawTraceDistillerOptions + +Defined in: [improvement/raw-trace-distiller.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L44) + +#### Properties + +##### runDir? + +> `optional` **runDir?**: `string` + +Defined in: [improvement/raw-trace-distiller.ts:49](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L49) + +Anchor the emitted paths at this run root instead of the generation `runDir` + the loop passes in. Normally unset — each call points at that generation's + own directory (`input.runDir`). Pass an absolute path when you construct the + producer ahead of the loop and want a fixed anchor (e.g. a test fixture). + +##### maxCandidates? + +> `optional` **maxCandidates?**: `number` + +Defined in: [improvement/raw-trace-distiller.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L51) + +Max candidates to surface trace paths for, worst-scoring first. Default 12. + +##### maxCellsPerCandidate? + +> `optional` **maxCellsPerCandidate?**: `number` + +Defined in: [improvement/raw-trace-distiller.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L54) + +Max failing cells to enumerate per candidate before collapsing the rest into + an "ls the candidate dir" pointer. Default 8. + +##### maxFilesPerCell? + +> `optional` **maxFilesPerCell?**: `number` + +Defined in: [improvement/raw-trace-distiller.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L57) + +Max concrete file paths to list per cell (the agent can always `ls` the dir + for the rest). Default 24. + +##### fallbackFindings? + +> `optional` **fallbackFindings?**: `unknown`[] + +Defined in: [improvement/raw-trace-distiller.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L61) + +Findings to fall back to when the generation had NO failing cells, so a + clean round never wipes the proposer's steering context. Mirrors the default + distiller's static-seed fallback. Default: a single instruction finding. + +*** + ### ReflectiveGeneratorOptions Defined in: [improvement/reflective-generator.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/reflective-generator.ts#L21) @@ -6491,7 +6562,7 @@ Defined in: [conversation/types.ts:237](https://github.com/tangle-network/agent- > **Verifier** = (`worktreePath`) => `Promise`\<[`VerifyResult`](#verifyresult)\> \| [`VerifyResult`](#verifyresult) -Defined in: [improvement/agentic-generator.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L50) +Defined in: [improvement/agentic-generator.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L56) Verifies the edited worktree. Sync or async; throws only on a setup fault (a candidate that fails verification returns `{ok:false}`, it does not @@ -6513,7 +6584,7 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault > **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"code"` -Defined in: [improvement/improve.ts:54](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L54) +Defined in: [improvement/improve.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L55) The agent-profile lever `improve` optimizes. Mirrors the AgentProfile-law profile levers; `code` is the implementation-tier surface. @@ -7833,7 +7904,7 @@ Wire integration: > **agenticGenerator**(`opts?`): [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/agentic-generator.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L73) +Defined in: [improvement/agentic-generator.ts:79](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L79) Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. @@ -7853,7 +7924,7 @@ Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a rea > **commandVerifier**(`command`, `args?`, `timeoutMs?`): [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L161) +Defined in: [improvement/agentic-generator.ts:237](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L237) A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other exit ⇒ failed with stdout+stderr as feedback. The common case — verify by @@ -7926,7 +7997,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:347](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L347) +Defined in: [improvement/improve.ts:358](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L358) Run the held-out-gated self-improvement loop on ONE profile surface. @@ -8014,6 +8085,48 @@ Build a `Verifier` that boots a generated MCP server over stdio and checks it ex *** +### rawTraceDistiller() + +> **rawTraceDistiller**\<`TScenario`, `TArtifact`\>(`options?`): (`input`) => `Promise`\<`unknown`[]\> + +Defined in: [improvement/raw-trace-distiller.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/raw-trace-distiller.ts#L88) + +Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE +FILESYSTEM CONTEXT — paths into the prior generation's real run traces plus a +grep/cat-to-diagnose instruction — instead of a pre-summarized digest. + +Drop-in for `opts.analyzeGeneration` on `improve()` / `selfImprove()`: + + await improve(profile, seedFindings, { + surface: 'code', + code: { repoRoot }, + runDir: '/abs/run', // MUST be a real path — the traces live here + analyzeGeneration: rawTraceDistiller(), + scenarios, judge, agent, + }) + +#### Type Parameters + +##### TScenario + +`TScenario` *extends* `Scenario` = `Scenario` + +##### TArtifact + +`TArtifact` = `unknown` + +#### Parameters + +##### options? + +[`RawTraceDistillerOptions`](#rawtracedistilleroptions) = `{}` + +#### Returns + +(`input`) => `Promise`\<`unknown`[]\> + +*** + ### reflectiveGenerator() > **reflectiveGenerator**(`opts`): [`CandidateGenerator`](#candidategenerator) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5dedff73..17216676 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.89.0` and `@tangle-network/agent-eval@0.106.2` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.89.0` and `@tangle-network/agent-eval@0.108.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 211 exports. +Import from `@tangle-network/agent-runtime` — 213 exports. | Symbol | Kind | Summary | |---|---|---| @@ -57,6 +57,7 @@ Import from `@tangle-network/agent-runtime` — 211 exports. | `notifyRuntimeDecisionPoint` | function | Fire `hooks.onDecisionPoint`, swallowing sync throws and surfacing async failures to `onError`. | | `notifyRuntimeHookEvent` | function | Fire `hooks.onEvent`, swallowing sync throws and surfacing async failures to `onError`. | | `parseLoopRunnerArgv` | function | Parse `--mode X --config Y` from an argv tail (`process.argv.slice(2)`). | +| `rawTraceDistiller` | function | Build an `analyzeGeneration` producer that feeds the proposer RAW-TRACE | | `readDepth` | function | Read the depth counter off an inbound request. Missing → 0 (caller is the | | `readinessServerSentEvent` | function | Serialize a `KnowledgeReadinessReport` as a Server-Sent Event string. | | `reflectiveGenerator` | function | Cheap no-sandbox `CandidateGenerator` (the `shots=1` setting): draft surface edits via the improvement adapter and apply them as one coherent candidate. | @@ -120,6 +121,7 @@ Import from `@tangle-network/agent-runtime` — 211 exports. | `ModelInfo` | interface | A model entry as returned by the Tangle Router `/v1/models` endpoint. | | `OpenAIChatTool` | interface | OpenAI Chat Completions tool descriptor. The shape mirrors the | | `OtelExportConfig` | interface | OTEL span exporter — streams LoopTraceEvents to an OTLP/HTTP collector. | +| `RawTraceDistillerOptions` | interface | `rawTraceDistiller` — the meta-harness `analyzeGeneration` producer. | | `ReflectiveGeneratorOptions` | interface | `reflectiveGenerator` — the cheap, no-sandbox `CandidateGenerator`. It drafts | | `RouterEnv` | interface | Env keys the router base URL is resolved from. | | `RunRecord` | interface | Mandatory paper-grade fields for a single evaluation run. Optional | @@ -260,7 +262,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. ### Recursive atom + loop kernel (alias of ./runtime) -Import from `@tangle-network/agent-runtime/loops` — 430 exports. +Import from `@tangle-network/agent-runtime/loops` — 432 exports. | Symbol | Kind | Summary | |---|---|---| @@ -481,6 +483,7 @@ Import from `@tangle-network/agent-runtime/loops` — 430 exports. | `MountManifestEntry` | interface | One mounted resource recorded during box preparation — a pure provenance | | `NaiveDriverOptions` | interface | Options for {@link naiveDriver}. | | `ObserveInput` | interface | The third-person observer — the connective tissue that closes the loop. | +| `OpenSandboxRunBeforeStartContext` | interface | Context available after the box/session exists and before the first prompt is | | `OutputAdapter` | interface | Stream of `SandboxEvent`s → typed `Output`. | | `PairwiseVerdict` | interface | One profile pair compared on the scenarios they BOTH ran — the "who actually beat whom" verdict. | | `PanelJudge` | interface | One judge in a panel — a labeled persona-derived judge child. Content (the rubric) lives in | @@ -591,7 +594,7 @@ Import from `@tangle-network/agent-runtime/loops` — 430 exports. | `WinnerStrategy` | type | Built-in valid-only winner strategies for `selectValidWinner` (selector≠judge): best gated-valid | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge @@ -963,7 +966,7 @@ Import from `@tangle-network/agent-eval` — 49 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 229 exports. +Import from `@tangle-network/agent-eval/campaign` — 261 exports. | Symbol | Kind | Summary | |---|---|---| @@ -972,6 +975,7 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `buildAnalystSurfaceDispatch` | function | Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop | | `buildEvidenceVector` | function | The Evidence Bus. For each objective, pair candidate vs baseline by full | | `buildLoopProvenanceRecord` | function | Build the durable provenance record from a completed loop result. | +| `callbackGovernor` | function | The LLM-supervisor slot: a governor whose `decide` defers to a caller-supplied | | `campaignBreakdown` | function | Per-candidate evidence a reflective/patch proposer grounds its next proposal | | `campaignMeanComposite` | function | Mean composite across a campaign: per cell, the mean of its judges' | | `compareProposers` | function | Run a head-to-head lift benchmark across surface proposers on a shared holdout, returning per-proposer lift CIs and pairwise "who wins" verdicts. | @@ -990,6 +994,7 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `fapoEscalationEntry` | function | Build a `ProposerEntry` that runs the full FAPO escalation policy (prompt → parameter → structural) as a single comparable optimizer entry. | | `fapoProposer` | function | Build a FAPO policy proposer from level-specific candidate generators. | | `fsCampaignStorage` | function | Node-filesystem storage — the default. Lazily requires `node:fs` so the | +| `fsLineageStore` | function | JSONL-file store: append-only durability, snapshot via rewrite, `load` parses | | `gepaParetoEntry` | function | GEPA with the Pareto frontier + combine-complementary-lessons. | | `gepaProposer` | function | GEPA reflective proposer: each generation reflects on the weakest scenarios and dimensions to produce targeted prompt rewrites, optionally combining Pareto-frontier parents. | | `gepaReflectionEntry` | function | GEPA, reflection-only (single-parent, no Pareto combine). | @@ -997,15 +1002,20 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `haloProposer` | function | Wrap the real halo-engine CLI as a SurfaceProposer (prompt-tier). | | `heldOutGate` | function | Composable held-out gate: ships only when the PAIRED bootstrap CI lower bound | | `heldoutSignificance` | function | Significance of the held-out composite lift: ship only when the paired | +| `heuristicGovernor` | function | The reference deterministic policy an agent {@link Governor} can replace. | | `inMemoryCampaignStorage` | function | In-memory storage for filesystem-less runtimes. Artifacts + trace spans | | `isProposedCandidate` | function | Type guard: a proposal carrying its rationale vs a bare | | `labelTrustRank` | function | Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). | +| `lineageNodeId` | function | Deterministic node id: a hash of the node's lineage + content + proposer. | | `llmJudge` | function | Build a campaign-shaped `JudgeConfig` whose `score()` makes ONE LLM call | | `loadEvalFixture` | function | Load ONE fixture by name: reads `PROMPT.md` (plus `EVAL.ts`/`EVAL.tsx` and `package.json` under | | `loadEvalFixtureScenarios` | function | Load fixtures (all discovered, or just `names`) as campaign `Scenario`s tagged `eval-fixture`. | | `loopProvenanceSpans` | function | Build the loop's OTLP-ingestable spans from a provenance record. One root | | `makePlaybackDispatch` | function | Adapt a `PlaybackDriver` into a `runProfileMatrix` dispatch. The artifact the | +| `memLineageStore` | function | In-memory store (default; for tests and ephemeral runs). | | `memoryCurationProposer` | function | Build the CURATOR proposer. | +| `neutralizationGate` | function | Composable placebo gate: ships only when the candidate's held-out lift is NOT | +| `neutralizeText` | function | Blank every non-whitespace character to a 1-byte filler while preserving all | | `openAutoPr` | function | Open a GitHub PR for a gate-approved surface promotion, attaching the manifest hash, gate verdict, and diff as the PR body. | | `pairHoldout` | function | Pair candidate vs baseline holdout observations by FULL cellId. `select` | | `parameterSweepProposer` | function | Config/parameter-level proposer for FAPO's middle escalation level. | @@ -1024,11 +1034,15 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `runCampaign` | function | Core campaign orchestrator: fan scenarios through dispatch, score with judges, aggregate bootstrap CIs, and persist reproducible `CampaignResult` records. | | `runEval` | function | Simplest evaluation preset: run scenarios through dispatch, score with judges, and return a `CampaignResult` — no optimizer, no gate, no PR. | | `runImprovementLoop` | function | Gated-promotion shell over `runOptimization`: scores the winner against the baseline on a holdout set, runs the release gate, and optionally opens a PR. | +| `runLineage` | function | Drive a multi-track improvement DAG under an agent-managed governor. Seeds each | +| `runLineageLoop` | function | Wire the {@link runLineage} DAG's `step`/`merge` seams to a real | | `runOptimization` | function | Improvement loop body: N generations of propose → campaign → rank, maintaining a Pareto frontier and promoting the top-scoring candidates to the next generation. | | `runProfileMatrix` | function | Profile × scenario matrix runner: fan N agent profiles across M scenarios, project each cell to a validated `RunRecord` with real token usage, and enforce the backend-integrity guard before returning. | | `runSkillOpt` | function | SkillOpt sequential hill-climb: each epoch reflects on train-scenario weaknesses, proposes bounded patches, accepts the first patch that strictly improves the held-out composite, and anneals the edit | | `scoreboardSummary` | function | Roll the per-requirement rows up into the launch headline counts. | +| `scoreDiscrimination` | function | Rank scenarios by how well they DISCRIMINATE candidates. | | `scoreUserStory` | function | Score one story's produced state against its requirements. Thin wrapper over | +| `selectDiscriminative` | function | Select the top-`k` most discriminative scenario ids for a holdout, EXCLUDING | | `sequentialDecide` | function | `SurfaceProposer.decide` adapter — stops the optimization loop the moment | | `sequentialPairedGate` | function | Anytime-valid sequential paired gate. Conforms to the existing `Gate` | | `skillOptEntry` | function | SkillOpt patch-mode hill-climb. Runs findings-BLIND: `runSkillOpt` owns its | @@ -1041,6 +1055,7 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `paretoPolicy` | const | The default strategy: symmetric multi-objective Pareto significance. Ship iff | | `FsLabeledScenarioStore` | class | Filesystem `LabeledScenarioStore`: appends one JSONL file per source with provenance and | | `LabeledScenarioStoreError` | class | Typed rejection from a labeled-scenario store (bad provenance, rate limit, invalid sample args) — carries a stable string `code`. | +| `Lineage` | class | _(no summary — add a TSDoc line at the declaration)_ | | `ProfileMatrixError` | class | Thrown when the matrix is misconfigured (no profiles, a profile whose model | | `SkillPatchParseError` | class | Parse + validate the patch response. Throws `SkillPatchParseError` when the | | `WorktreeAdapterError` | class | Typed failure from a `WorktreeAdapter` operation (create/finalize/discard) — wraps the underlying git error as `cause`. | @@ -1064,6 +1079,7 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `JudgeConfig` | interface | Pluggable dimensional scorer. `score` is the contract: | | `JudgeScore` | interface | The canonical judge verdict shape — one declaration, shared by campaign | | `LabeledScenarioWrite` | interface | Required-provenance write. The store rejects writes that | +| `LineageNode` | interface | Lineage DAG — a git-graph of improvement candidates. | | `LoopProvenanceRecord` | interface | The durable provenance record. Aligns to the hosted `EvalRunEvent` path but | | `MemoryCurationProposerOptions` | interface | `memoryCurationProposer` — a CURATOR `SurfaceProposer`, the complement to the | | `Mutator` | interface | Stateless surface mutation — given findings + current | @@ -1082,14 +1098,17 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `RejectedEdit` | interface | A patch that was tried and not accepted — fed back to the model so it does | | `RunCampaignOptions` | interface | `runCampaign` — Pass A substrate primitive. ONE function that orchestrates | | `RunEvalOptions` | interface | `runEval` — the simplest preset over `runCampaign`. No optimizer, no | +| `RunLineageLoopSeed` | interface | A seed track: the initial surface + track identity. Unlike | | `RunSkillOptOptions` | interface | `runSkillOpt` — the SkillOpt epoch hill-climb (Microsoft, arXiv:2605.23904). | | `Scenario` | interface | Stable identifier + kind tag for any scenario. Consumers | +| `ScenarioSignal` | interface | Per-scenario observation: the composite scores each candidate earned on it. | | `ScoreboardRow` | interface | One row of the launch scoreboard — story × requirement → PASS/FAIL. | | `ScoreboardSummary` | interface | Launch-readiness headline counts rolled up from the per-requirement rows. | | `SessionScript` | interface | One session within a multi-session journey. Dispatch is | | `SkillOptEvidence` | interface | Evidence the optimizer reflects on: where the current surface is weakest. | | `SkillPatch` | interface | A named, attributable bundle of ops the optimizer proposes as one edit. | | `SurfaceProposer` | interface | A surface-improvement strategy. Given the current best | +| `SurfaceScore` | interface | The measured fitness of one surface — the value recorded on a DAG node. | | `TraceAnalystProposerOptions` | interface | `traceAnalystProposer` — wraps agent-eval's OWN trace-analyst engine | | `UserStory` | interface | A user story = a runnable product journey plus the requirements that define | | `UserStoryVerdict` | interface | A scored user story — the completion verdict plus its human title. | @@ -1101,6 +1120,7 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `GateDecision` | type | Five-valued verdict taxonomy (MOSS-paper alignment). | | `LabeledScenarioSource` | type | Source tag — required on every store write. Used by the | | `LabelTrust` | type | How much a label can be trusted to evaluate against — the gold-admission | +| `LineageNodeInput` | type | Input to {@link Lineage.addNode}: everything but the derived `id`/`seq` and the | | `LlmJudgeDimension` | type | A rubric dimension as a bare key or the full `{ key, description }` shape. A | | `MutableSurface` | type | The mutable surface a proposer changes. Tiers (see | | `ObjectiveSource` | type | Where an objective's per-cell scalar comes from. `composite` reads the | @@ -1111,7 +1131,7 @@ Import from `@tangle-network/agent-eval/campaign` — 229 exports. | `SequentialDecision` | type | Anytime-valid sequential promotion gate — an e-process (betting | | `SkillPatchOp` | type | A single bounded edit against a skill surface. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CompareProposersOptions`, `DimensionRegression`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `OpenAutoPrResult`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `RunImprovementLoopResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `WorktreeAdapter`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `JsonPrimitive`, `JsonValue`, `RedactionStatus`, `RunOptimizationOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CompareProposersOptions`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `Governor`, `GovernorContext`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `HeuristicGovernorOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LineageEdge`, `LineageGraph`, `LineageStore`, `LlmJudgeOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `LoopProvenanceCandidate`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `RunImprovementLoopResult`, `RunLineageLoopOptions`, `RunLineageLoopResult`, `RunLineageOptions`, `RunLineageResult`, `RunLineageSeed`, `RunLineageStepResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `WorktreeAdapter`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `GovernorOp`, `JsonPrimitive`, `JsonValue`, `RedactionStatus`, `RunOptimizationOptions`. ### TOKEN / USAGE — usage extraction + run-record usage types diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 830b5e11..ffce8b7b 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -6448,9 +6448,39 @@ Defined in: [runtime/sandbox-run.ts:101](https://github.com/tangle-network/agent *** +### OpenSandboxRunBeforeStartContext + +Defined in: [runtime/sandbox-run.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L116) + +Context available after the box/session exists and before the first prompt is +drained. Intended for benchmark-owned workspace setup such as cloning a repo +into a fixed path. + +#### Properties + +##### box + +> `readonly` **box**: `SandboxInstance` + +Defined in: [runtime/sandbox-run.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L117) + +##### sessionId + +> `readonly` **sessionId**: `string` + +Defined in: [runtime/sandbox-run.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L118) + +##### signal + +> `readonly` **signal**: `AbortSignal` + +Defined in: [runtime/sandbox-run.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L119) + +*** + ### OpenSandboxRunOptions -Defined in: [runtime/sandbox-run.ts:114](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L114) +Defined in: [runtime/sandbox-run.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L123) **`Experimental`** @@ -6460,7 +6490,7 @@ Defined in: [runtime/sandbox-run.ts:114](https://github.com/tangle-network/agent > **agentRun**: [`AgentRunSpec`](#agentrunspec)\<`string`\> -Defined in: [runtime/sandbox-run.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L116) +Defined in: [runtime/sandbox-run.ts:125](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L125) **`Experimental`** @@ -6470,7 +6500,7 @@ Profile + sandbox env/overrides. `sandboxOverrides.backend.type` is the harness. > **signal**: `AbortSignal` -Defined in: [runtime/sandbox-run.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L117) +Defined in: [runtime/sandbox-run.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L126) **`Experimental`** @@ -6478,7 +6508,7 @@ Defined in: [runtime/sandbox-run.ts:117](https://github.com/tangle-network/agent > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [runtime/sandbox-run.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L119) +Defined in: [runtime/sandbox-run.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L128) **`Experimental`** @@ -6488,7 +6518,7 @@ Optional execution-scoped observers. Hook failures never fail the run. > `optional` **runId?**: `string` -Defined in: [runtime/sandbox-run.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L121) +Defined in: [runtime/sandbox-run.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L130) **`Experimental`** @@ -6498,7 +6528,7 @@ Stable run id for trace joins. Defaults to a short runtime-minted id. > `optional` **scenarioId?**: `string` -Defined in: [runtime/sandbox-run.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L123) +Defined in: [runtime/sandbox-run.ts:132](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L132) **`Experimental`** @@ -6508,18 +6538,40 @@ Optional benchmark/scenario id carried into emitted hook events. > `optional` **promptOptions?**: [`OpenSandboxRunPromptOptions`](#opensandboxrunpromptoptions) -Defined in: [runtime/sandbox-run.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L126) +Defined in: [runtime/sandbox-run.ts:135](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L135) **`Experimental`** Per-prompt sandbox SDK options forwarded to both `start()` and `resume()`. The runtime still owns the session id and abort signal for each turn. +##### beforeStart? + +> `optional` **beforeStart?**: (`ctx`) => `void` \| `Promise`\<`void`\> + +Defined in: [runtime/sandbox-run.ts:139](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L139) + +**`Experimental`** + +Optional pre-start workspace setup. Runs after `lineage.start()` creates the +box/session and before the first prompt stream is consumed. A thrown error +fails the turn before the agent spends tokens. + +###### Parameters + +###### ctx + +[`OpenSandboxRunBeforeStartContext`](#opensandboxrunbeforestartcontext) + +###### Returns + +`void` \| `Promise`\<`void`\> + ##### now? > `optional` **now?**: () => `number` -Defined in: [runtime/sandbox-run.ts:128](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L128) +Defined in: [runtime/sandbox-run.ts:141](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L141) **`Experimental`** @@ -6533,7 +6585,7 @@ Test seam for deterministic hook timestamps. Defaults to `Date.now`. > `optional` **maxConcurrency?**: `number` -Defined in: [runtime/sandbox-run.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L130) +Defined in: [runtime/sandbox-run.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L143) **`Experimental`** @@ -6543,7 +6595,7 @@ Bounds box-creation bursts inside lineage fanout. Default from lineage. > `optional` **readRetryDelayMs?**: `number` -Defined in: [runtime/sandbox-run.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L133) +Defined in: [runtime/sandbox-run.ts:146](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L146) **`Experimental`** @@ -7744,8 +7796,8 @@ The critic's model — lets the analyst be a stronger (or cheaper) model than th Defined in: [runtime/strategy.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L110) Across-run learning: when set, the analyst's observe() pass appends trace-derived - facts here (the flywheel write side). Priming (the read side) is the caller's move — - query the corpus and fold facts into the task's systemPrompt before runAgentic. + facts here (the flywheel write side). Read-back is opt-in via `corpusReadback` + because unconditional priming can pollute context on some domains. ##### corpusTags? @@ -7755,11 +7807,60 @@ Defined in: [runtime/strategy.ts:112](https://github.com/tangle-network/agent-ru Tags written onto learned facts (and used by the caller's priming query). +##### corpusReadback? + +> `optional` **corpusReadback?**: [`CorpusReadbackOptions`](#corpusreadbackoptions) + +Defined in: [runtime/strategy.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L115) + +In-context learning: when set, query `corpus` before each depth shot and inject + the top trace-derived facts as guidance for the active run. No corpus means no read-back. + +*** + +### CorpusReadbackOptions + +Defined in: [runtime/strategy.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L118) + +#### Properties + +##### minConfidence? + +> `optional` **minConfidence?**: `number` + +Defined in: [runtime/strategy.ts:120](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L120) + +Minimum confidence for a fact to be injected. Default 0.7. + +##### tags? + +> `optional` **tags?**: readonly `string`[] + +Defined in: [runtime/strategy.ts:122](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L122) + +Extra tags a fact must carry, in addition to `corpusTags`. + +##### maxFacts? + +> `optional` **maxFacts?**: `number` + +Defined in: [runtime/strategy.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L124) + +Max facts injected per shot. Default 3. + +##### includeOperatorFacts? + +> `optional` **includeOperatorFacts?**: `boolean` + +Defined in: [runtime/strategy.ts:126](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L126) + +Default false: only facts tagged `audience:agent` are injected into the worker. + *** ### AgenticRunResult -Defined in: [runtime/strategy.ts:554](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L554) +Defined in: [runtime/strategy.ts:594](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L594) #### Properties @@ -7767,7 +7868,7 @@ Defined in: [runtime/strategy.ts:554](https://github.com/tangle-network/agent-ru > **mode**: `string` -Defined in: [runtime/strategy.ts:556](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L556) +Defined in: [runtime/strategy.ts:596](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L596) The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). @@ -7775,25 +7876,25 @@ The strategy name (built-in 'depth'/'breadth' or a custom strategy's name). > **score**: `number` -Defined in: [runtime/strategy.ts:557](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L557) +Defined in: [runtime/strategy.ts:597](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L597) ##### resolved > **resolved**: `boolean` -Defined in: [runtime/strategy.ts:558](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L558) +Defined in: [runtime/strategy.ts:598](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L598) ##### completions > **completions**: `number` -Defined in: [runtime/strategy.ts:559](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L559) +Defined in: [runtime/strategy.ts:599](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L599) ##### progression > **progression**: `number`[] -Defined in: [runtime/strategy.ts:561](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L561) +Defined in: [runtime/strategy.ts:601](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L601) DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-so-far per rollout. @@ -7801,13 +7902,13 @@ DEPTH: score after each shot — the progress-over-rounds curve. BREADTH: best-s > **shots**: `number` -Defined in: [runtime/strategy.ts:562](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L562) +Defined in: [runtime/strategy.ts:602](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L602) ##### usd > **usd**: `number` -Defined in: [runtime/strategy.ts:565](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L565) +Defined in: [runtime/strategy.ts:605](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L605) The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: real router tokens, priced usd (0 when the model is unpriced — never fabricated), wall ms. @@ -7816,13 +7917,13 @@ The cost vector, stamped by `runAgentic` from the Supervisor's conserved pool: r > **ms**: `number` -Defined in: [runtime/strategy.ts:566](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L566) +Defined in: [runtime/strategy.ts:606](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L606) ##### tokens > **tokens**: `object` -Defined in: [runtime/strategy.ts:567](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L567) +Defined in: [runtime/strategy.ts:607](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L607) ###### input @@ -7836,7 +7937,7 @@ Defined in: [runtime/strategy.ts:567](https://github.com/tangle-network/agent-ru ### Strategy -Defined in: [runtime/strategy.ts:701](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L701) +Defined in: [runtime/strategy.ts:744](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L744) A Strategy is HOW you spend the compute budget to beat the Environment's check — it builds the driver `Agent` the Supervisor runs. This is the OPEN extension point: a dev @@ -7853,7 +7954,7 @@ the reference implementations to copy: > `readonly` **name**: `string` -Defined in: [runtime/strategy.ts:702](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L702) +Defined in: [runtime/strategy.ts:745](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L745) #### Methods @@ -7861,7 +7962,7 @@ Defined in: [runtime/strategy.ts:702](https://github.com/tangle-network/agent-ru > **driver**(`surface`, `task`, `opts`, `budget`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:703](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L703) +Defined in: [runtime/strategy.ts:746](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L746) ###### Parameters @@ -7889,7 +7990,7 @@ Defined in: [runtime/strategy.ts:703](https://github.com/tangle-network/agent-ru ### ShotPersona -Defined in: [runtime/strategy.ts:733](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L733) +Defined in: [runtime/strategy.ts:776](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L776) A role for one shot — multi-agent loops (researcher + engineer, a panel of k researchers) give each shot its own system prompt and optionally its own model. @@ -7900,7 +8001,7 @@ A role for one shot — multi-agent loops (researcher + engineer, a panel of k > `optional` **systemPrompt?**: `string` -Defined in: [runtime/strategy.ts:736](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L736) +Defined in: [runtime/strategy.ts:779](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L779) Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it is injected as a hand-off message (the transcript's earlier roles stay intact). @@ -7909,7 +8010,7 @@ Replaces the task's systemPrompt for a FRESH shot; on a carried conversation it > `optional` **model?**: `string` -Defined in: [runtime/strategy.ts:738](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L738) +Defined in: [runtime/strategy.ts:781](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L781) Per-shot model override (e.g. a stronger model for the engineer shot). @@ -7917,7 +8018,7 @@ Per-shot model override (e.g. a stronger model for the engineer shot). ### ShotSpec -Defined in: [runtime/strategy.ts:741](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L741) +Defined in: [runtime/strategy.ts:784](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L784) #### Properties @@ -7925,7 +8026,7 @@ Defined in: [runtime/strategy.ts:741](https://github.com/tangle-network/agent-ru > `optional` **handle?**: [`ArtifactHandle`](#artifacthandle) -Defined in: [runtime/strategy.ts:743](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L743) +Defined in: [runtime/strategy.ts:786](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L786) present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh one (sample/restart). @@ -7933,25 +8034,25 @@ present ⇒ continue this artifact (depth); absent ⇒ the shot opens a fresh on > `optional` **messages?**: `Msg`[] -Defined in: [runtime/strategy.ts:744](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L744) +Defined in: [runtime/strategy.ts:787](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L787) ##### steer? > `optional` **steer?**: `string` -Defined in: [runtime/strategy.ts:745](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L745) +Defined in: [runtime/strategy.ts:788](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L788) ##### persona? > `optional` **persona?**: [`ShotPersona`](#shotpersona) -Defined in: [runtime/strategy.ts:746](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L746) +Defined in: [runtime/strategy.ts:789](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L789) ##### tools? > `optional` **tools?**: `string`[] -Defined in: [runtime/strategy.ts:749](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L749) +Defined in: [runtime/strategy.ts:792](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L792) Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot on the relevant capabilities. Restriction-only; unknown names throw. Omitted ⇒ all. @@ -7960,7 +8061,7 @@ Restrict THIS shot to a subset of the domain's tools (by name) — focus a shot ### StrategyResult -Defined in: [runtime/strategy.ts:751](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L751) +Defined in: [runtime/strategy.ts:794](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L794) #### Properties @@ -7968,37 +8069,37 @@ Defined in: [runtime/strategy.ts:751](https://github.com/tangle-network/agent-ru > **score**: `number` -Defined in: [runtime/strategy.ts:752](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L752) +Defined in: [runtime/strategy.ts:795](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L795) ##### resolved > **resolved**: `boolean` -Defined in: [runtime/strategy.ts:753](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L753) +Defined in: [runtime/strategy.ts:796](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L796) ##### completions > **completions**: `number` -Defined in: [runtime/strategy.ts:754](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L754) +Defined in: [runtime/strategy.ts:797](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L797) ##### progression > **progression**: `number`[] -Defined in: [runtime/strategy.ts:755](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L755) +Defined in: [runtime/strategy.ts:798](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L798) ##### shots > **shots**: `number` -Defined in: [runtime/strategy.ts:756](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L756) +Defined in: [runtime/strategy.ts:799](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L799) *** ### StrategyCtx -Defined in: [runtime/strategy.ts:768](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L768) +Defined in: [runtime/strategy.ts:811](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L811) What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. @@ -8008,7 +8109,7 @@ What a strategy body composes with: the artifact lifecycle, the budget, and the > `readonly` **surface**: `StrategyArtifacts` -Defined in: [runtime/strategy.ts:770](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L770) +Defined in: [runtime/strategy.ts:813](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L813) Open/close artifacts the body manages itself (e.g. one persistent handle for depth). @@ -8016,25 +8117,25 @@ Open/close artifacts the body manages itself (e.g. one persistent handle for dep > `readonly` **task**: [`AgenticTask`](#agentictask) -Defined in: [runtime/strategy.ts:771](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L771) +Defined in: [runtime/strategy.ts:814](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L814) ##### opts > `readonly` **opts**: [`AgenticOptions`](#agenticoptions) -Defined in: [runtime/strategy.ts:772](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L772) +Defined in: [runtime/strategy.ts:815](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L815) ##### budget > `readonly` **budget**: `number` -Defined in: [runtime/strategy.ts:773](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L773) +Defined in: [runtime/strategy.ts:816](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L816) ##### scope > `readonly` **scope**: [`Scope`](#scope-1)\<[`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:774](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L774) +Defined in: [runtime/strategy.ts:817](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L817) #### Methods @@ -8042,7 +8143,7 @@ Defined in: [runtime/strategy.ts:774](https://github.com/tangle-network/agent-ru > **shot**(`spec?`): `Promise`\<`ShotResult` \| `null`\> -Defined in: [runtime/strategy.ts:776](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L776) +Defined in: [runtime/strategy.ts:819](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L819) Run ONE worker shot; its harness-scored result, or null if it went down. @@ -8060,7 +8161,7 @@ Run ONE worker shot; its harness-scored result, or null if it went down. > **critique**(`messages`): `Promise`\<`string` \| `null`\> -Defined in: [runtime/strategy.ts:778](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L778) +Defined in: [runtime/strategy.ts:821](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L821) The firewalled critic reads the trajectory → a steer string, or null on COMPLETE/down. @@ -8078,7 +8179,7 @@ The firewalled critic reads the trajectory → a steer string, or null on COMPLE > **consult**(`messages`, `instruction`): `Promise`\<`string` \| `null`\> -Defined in: [runtime/strategy.ts:783](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L783) +Defined in: [runtime/strategy.ts:826](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L826) The RAW analyst channel: the firewalled critic answers `instruction` over the trajectory verbatim — no findings extraction, so verdict-shaped formats @@ -8103,7 +8204,7 @@ The RAW analyst channel: the firewalled critic answers `instruction` over the > **listTools**(`handle`): `Promise`\<`object`[]\> -Defined in: [runtime/strategy.ts:787](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L787) +Defined in: [runtime/strategy.ts:830](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L830) The tools THIS artifact's task actually offers (names + descriptions only — never the implementations). Tool sets vary per task on heterogeneous domains; a strategy @@ -8123,7 +8224,7 @@ The tools THIS artifact's task actually offers (names + descriptions only — ne ### RunAgenticOptions -Defined in: [runtime/strategy.ts:1016](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1016) +Defined in: [runtime/strategy.ts:1059](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1059) #### Extends @@ -8255,8 +8356,8 @@ The critic's model — lets the analyst be a stronger (or cheaper) model than th Defined in: [runtime/strategy.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L110) Across-run learning: when set, the analyst's observe() pass appends trace-derived - facts here (the flywheel write side). Priming (the read side) is the caller's move — - query the corpus and fold facts into the task's systemPrompt before runAgentic. + facts here (the flywheel write side). Read-back is opt-in via `corpusReadback` + because unconditional priming can pollute context on some domains. ###### Inherited from @@ -8274,23 +8375,36 @@ Tags written onto learned facts (and used by the caller's priming query). [`AgenticOptions`](#agenticoptions).[`corpusTags`](#corpustags) +##### corpusReadback? + +> `optional` **corpusReadback?**: [`CorpusReadbackOptions`](#corpusreadbackoptions) + +Defined in: [runtime/strategy.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L115) + +In-context learning: when set, query `corpus` before each depth shot and inject + the top trace-derived facts as guidance for the active run. No corpus means no read-back. + +###### Inherited from + +[`AgenticOptions`](#agenticoptions).[`corpusReadback`](#corpusreadback) + ##### surface > **surface**: [`AgenticSurface`](#agenticsurface) -Defined in: [runtime/strategy.ts:1017](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1017) +Defined in: [runtime/strategy.ts:1060](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1060) ##### task > **task**: [`AgenticTask`](#agentictask) -Defined in: [runtime/strategy.ts:1018](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1018) +Defined in: [runtime/strategy.ts:1061](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1061) ##### hooks? > `optional` **hooks?**: [`RuntimeHooks`](index.md#runtimehooks) -Defined in: [runtime/strategy.ts:1021](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1021) +Defined in: [runtime/strategy.ts:1064](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1064) Lifecycle observability — every spawn/settle (shots, analysts) streams here live. The seam online watchdogs/route-auditors subscribe to. @@ -8299,7 +8413,7 @@ Lifecycle observability — every spawn/settle (shots, analysts) streams here li > `optional` **strategy?**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:1023](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1023) +Defined in: [runtime/strategy.ts:1066](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1066) A Strategy (the open way) — author/pass your own. Overrides `mode` when present. @@ -8307,7 +8421,7 @@ A Strategy (the open way) — author/pass your own. Overrides `mode` when presen > `optional` **mode?**: `"depth"` \| `"breadth"` -Defined in: [runtime/strategy.ts:1025](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1025) +Defined in: [runtime/strategy.ts:1068](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1068) Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. @@ -8315,7 +8429,7 @@ Built-in shorthand: 'depth'→refine, 'breadth'→sample. Default 'depth'. > **budget**: `number` -Defined in: [runtime/strategy.ts:1027](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1027) +Defined in: [runtime/strategy.ts:1070](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1070) budget: refine→max shots; sample→rollout width. @@ -8323,7 +8437,7 @@ budget: refine→max shots; sample→rollout width. > `optional` **rootBudget?**: [`Budget`](#budget-12) -Defined in: [runtime/strategy.ts:1028](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1028) +Defined in: [runtime/strategy.ts:1071](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1071) *** @@ -15382,7 +15496,7 @@ The compressed consumable a skill carries: everything an author needs to emit a > `const` **sample**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:712](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L712) +Defined in: [runtime/strategy.ts:755](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L755) Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). @@ -15392,7 +15506,7 @@ Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N > `const` **refine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:717](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L717) +Defined in: [runtime/strategy.ts:760](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L760) Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next attempt → repeat (deepen one lineage). @@ -15402,7 +15516,7 @@ Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next > `const` **adaptiveRefine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:917](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L917) +Defined in: [runtime/strategy.ts:960](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L960) A NEW strategy, authored from the steps (~20 lines): refine, but when a steered shot fails to improve the score it ABANDONS that line and restarts fresh (branch-when-stuck) @@ -15416,7 +15530,7 @@ A NEW strategy, authored from the steps (~20 lines): refine, but when a steered > `const` **sampleThenRefine**: [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:960](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L960) +Defined in: [runtime/strategy.ts:1003](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1003) The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), then refine the best-verifying line with the remaining budget. Sample's basin escape + @@ -17373,7 +17487,7 @@ Run provenance recorder forwarded to every `prepareBox` the lineage runs > **openSandboxRun**\<`Out`\>(`client`, `options`, `deliverable`): `Promise`\<[`SandboxRun`](#sandboxrun)\<`Out`\>\> -Defined in: [runtime/sandbox-run.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L143) +Defined in: [runtime/sandbox-run.ts:156](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/sandbox-run.ts#L156) **`Experimental`** @@ -17653,7 +17767,7 @@ Multi-generation strategy search: author candidates from tournament losses, play > **depthStrategy**(`surface`, `task`, `opts`, `cfg`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:576](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L576) +Defined in: [runtime/strategy.ts:616](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L616) DEPTH: one persistent artifact, carried across analyst-steered shots. @@ -17687,7 +17801,7 @@ DEPTH: one persistent artifact, carried across analyst-steered shots. > **breadthStrategy**(`_surface`, `task`, `opts`, `cfg`): [`Agent`](#agent)\<`unknown`, [`Outcome`](#outcome-1)\<`unknown`\>\> -Defined in: [runtime/strategy.ts:644](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L644) +Defined in: [runtime/strategy.ts:687](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L687) BREADTH: K independent rollouts (each own artifact), verifier picks the best. @@ -17721,7 +17835,7 @@ BREADTH: K independent rollouts (each own artifact), verifier picks the best. > **defineStrategy**(`name`, `run`): [`Strategy`](#strategy-3) -Defined in: [runtime/strategy.ts:791](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L791) +Defined in: [runtime/strategy.ts:834](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L834) Author a Strategy from the composable steps — the open, compact way. @@ -17745,7 +17859,7 @@ Author a Strategy from the composable steps — the open, compact way. > **runAgentic**(`opts`): `Promise`\<[`AgenticRunResult`](#agenticrunresult)\> -Defined in: [runtime/strategy.ts:1032](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1032) +Defined in: [runtime/strategy.ts:1075](https://github.com/tangle-network/agent-runtime/blob/main/src/runtime/strategy.ts#L1075) Run a Strategy through the keystone Supervisor — `Agent.act` over a conserved-budget Scope. diff --git a/examples/open-source-skill-lifecycle/open-source-skill-lifecycle.ts b/examples/open-source-skill-lifecycle/open-source-skill-lifecycle.ts new file mode 100644 index 00000000..79537046 --- /dev/null +++ b/examples/open-source-skill-lifecycle/open-source-skill-lifecycle.ts @@ -0,0 +1,128 @@ +/** + * Import a real open-source `SKILL.md` into the AgentProfile lifecycle. + * + * This is intentionally not a new skill system. The remote skill becomes a + * `SkillDraft`, then the existing lifecycle does the work: + * + * skillGenerator -> runLifecycle -> thresholdPromotionGate -> composeProfile + * + * Default source: + * anthropics/skills: skills/frontend-design/SKILL.md + * + * Run: + * pnpm tsx examples/open-source-skill-lifecycle/open-source-skill-lifecycle.ts + * + * Or pass another raw SKILL.md URL: + * SKILL_URL=https://raw.githubusercontent.com/.../SKILL.md pnpm tsx examples/open-source-skill-lifecycle/open-source-skill-lifecycle.ts + */ + +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + composeProfile, + type EvalResult, + runLifecycle, + type SkillDraft, + skillGenerator, + thresholdPromotionGate, +} from '@tangle-network/agent-runtime/lifecycle' + +const defaultSkillUrl = + 'https://raw.githubusercontent.com/anthropics/skills/main/skills/frontend-design/SKILL.md' + +async function main(): Promise { + const url = process.env.SKILL_URL ?? process.argv[2] ?? defaultSkillUrl + const imported = await fetchSkillDraft(url) + const baseline: AgentProfile = { name: 'open-source-skill-import', resources: { skills: [] } } + const requiredNeedle = firstUsefulTerm(imported) + + const result = await runLifecycle({ + baseline, + domain: 'open-source-skill-import-smoke', + generators: [ + skillGenerator({ + distill: () => [imported], + }), + ], + evalRunner: (profile) => heldBackSkillPresenceCheck(profile, requiredNeedle), + gate: thresholdPromotionGate(), + traces: { sourceUrl: url }, + }) + + const composed = composeProfile(result.registry, baseline, { kind: 'skill', k: 1 }) + const composedSkills = composed.resources?.skills ?? [] + const summary = { + sourceUrl: url, + importedSkill: imported.name, + requiredNeedle, + candidates: result.outcomes.length, + promoted: result.promoted.length, + baselineScore: result.baselineResult.composite, + composedScore: (await heldBackSkillPresenceCheck(composed, requiredNeedle)).composite, + composedSkills: composedSkills.length, + } + console.log(JSON.stringify(summary, null, 2)) +} + +async function fetchSkillDraft(url: string): Promise { + const response = await fetch(url) + if (!response.ok) { + throw new Error(`fetchSkillDraft: ${response.status} ${response.statusText} for ${url}`) + } + const content = await response.text() + const metadata = parseSkillFrontmatter(content) + return { + name: metadata.name ?? nameFromUrl(url), + description: metadata.description, + content, + } +} + +function parseSkillFrontmatter(content: string): { name?: string; description?: string } { + if (!content.startsWith('---\n')) return {} + const end = content.indexOf('\n---', 4) + if (end === -1) return {} + const metadata: { name?: string; description?: string } = {} + for (const line of content.slice(4, end).split('\n')) { + const match = /^([A-Za-z0-9_-]+):\s*(.*)$/.exec(line.trim()) + if (!match) continue + const key = match[1] + const raw = match[2] + if (!key || raw === undefined) continue + const value = raw.replace(/^['"]|['"]$/g, '').trim() + if (key === 'name') metadata.name = value + if (key === 'description') metadata.description = value + } + return metadata +} + +function nameFromUrl(url: string): string { + const parts = new URL(url).pathname.split('/').filter(Boolean) + const file = parts.at(-1) ?? 'imported-skill' + if (file.toLowerCase() !== 'skill.md') return file.replace(/\.md$/i, '') + return parts.at(-2) ?? 'imported-skill' +} + +function firstUsefulTerm(skill: SkillDraft): string { + const haystack = `${skill.name}\n${skill.description ?? ''}\n${skill.content}`.toLowerCase() + for (const term of ['frontend', 'design', 'testing', 'mcp', 'skill']) { + if (haystack.includes(term)) return term + } + return skill.name.toLowerCase() +} + +async function heldBackSkillPresenceCheck( + profile: AgentProfile, + requiredNeedle: string, +): Promise { + const skills = profile.resources?.skills ?? [] + const matched = skills.some( + (skill) => + skill.kind === 'inline' && skill.content.toLowerCase().includes(requiredNeedle.toLowerCase()), + ) + return { composite: matched ? 1 : 0, costUsd: skills.length * 0.001 } +} + +main().catch((error) => { + console.error(error) + process.exit(1) +}) diff --git a/package.json b/package.json index e5eea4cd..a45756e3 100644 --- a/package.json +++ b/package.json @@ -94,9 +94,9 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "^0.106.2", - "@tangle-network/agent-interface": ">=0.14.0 <1.0.0", - "@tangle-network/sandbox": ">=0.8.0 <1.0.0", + "@tangle-network/agent-eval": "^0.108.0", + "@tangle-network/agent-interface": "^0.19.0", + "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.9.3", "playwright": "^1.61.0", "tsup": "^8.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f165fdc..116e34b7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,14 +12,14 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: ^0.106.2 - version: 0.106.2(typescript@5.9.3) + specifier: ^0.108.0 + version: 0.108.0(typescript@5.9.3) '@tangle-network/agent-interface': - specifier: '>=0.14.0 <1.0.0' - version: 0.14.0 + specifier: ^0.19.0 + version: 0.19.0 '@tangle-network/sandbox': - specifier: '>=0.8.0 <1.0.0' - version: 0.9.5(viem@2.54.2(typescript@5.9.3)(zod@4.4.3)) + specifier: ^0.9.7 + version: 0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) '@types/node': specifier: ^25.9.3 version: 25.9.3 @@ -633,11 +633,11 @@ packages: '@shikijs/vscode-textmate@10.0.2': resolution: {integrity: sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==} - '@tangle-network/agent-core@0.3.4': - resolution: {integrity: sha512-Hvz3ABRouNtBmRvGqPxifAO2yuILneJMylWH5jW/jeS2F03RvqkGYuXyGXWWLqosYbb3hVAvSEe4Ykm2FMGEDQ==} + '@tangle-network/agent-core@0.3.8': + resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} - '@tangle-network/agent-eval@0.106.2': - resolution: {integrity: sha512-wA3OEH2iFYSfcMPrdIqFkxrhcEsbkXOrljbTzMh4JZbgHf75Dsg2/WCpFoNm3y4yL94APgI9XmIredGTlkB1DQ==} + '@tangle-network/agent-eval@0.108.0': + resolution: {integrity: sha512-z0Q6gn+eIcXv+4HV4bIAC3+XDA5D4kdXKPqy2RyPImwVBAYTpP58Ykei1A2ZaT+hgvKeC86XOcTnxIKcF9xL5g==} engines: {node: '>=20'} hasBin: true @@ -647,11 +647,14 @@ packages: '@tangle-network/agent-interface@0.13.0': resolution: {integrity: sha512-CeTPGRLoXqpt0h+BCyFgZPkfU1zyRpWmqfD+85i/uk+uvbqxkfI+JprfKVf3tBsQuCgJPSjPt5qjdW8n3h2BVg==} - '@tangle-network/agent-interface@0.14.0': - resolution: {integrity: sha512-9CyGhIpl90E7v4MTm3b1ti3Bp7BfPigk2Nafgi21Lg0U+QxlNB656F2JmVpUuSbOo9aGZPtg5nXu5EBTlV5a1g==} + '@tangle-network/agent-interface@0.17.1': + resolution: {integrity: sha512-B7dRJTo0HSUtgBCB1VMwkTFYkLUaRr/4BcRglrQuGhGUwOzKv1RYyMejOVh5M3a5AagY9N79f7GYbjcA3UmnIA==} - '@tangle-network/sandbox@0.9.5': - resolution: {integrity: sha512-yvX2OX6uISBVnMQ+v6Upkesa3u8yj6BHxsfcS6p8Vze+M4WBpyhkwA+onzFHuo9rti557ItZn8yDu4a/klljvQ==} + '@tangle-network/agent-interface@0.19.0': + resolution: {integrity: sha512-WCN1j9dVmPlCDAU9I1R6OvGdZ9uvKrxz+3gsh61DbKoLvnxxRQEUWuZ/JByzSZGq4J84zdzryr6aET7KTZ6z+w==} + + '@tangle-network/sandbox@0.9.7': + resolution: {integrity: sha512-9pCwJ5MlF7RUpp0AQKQDFyR0yu+E0udEhWkqhrlb/RuoJxlt72zVPuzO4FnMb1MZTkfjStmomC3k5xQyqi1YSA==} peerDependencies: '@mastra/core': ^1.36.0 '@modelcontextprotocol/sdk': ^1.29.0 @@ -862,8 +865,8 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] - hono@4.12.27: - resolution: {integrity: sha512-1yrb/+w6HWQJrUCLkJ2IF5jNIPvvFkblV5RNOYl6bV+OA6p9GLcMpHFFGTosSvHvcAUibuUukRqhlYI4z32C7Q==} + hono@4.12.28: + resolution: {integrity: sha512-YwUvVpSF7m1yOblFPrU3Hbo8XhPheBoiyfGuII6z19LnOr6JpDnyyp7LFNrfV56wS8tpvtBFGRISHN02pDdLOA==} engines: {node: '>=16.9.0'} isows@1.0.7: @@ -933,8 +936,8 @@ packages: openapi3-ts@4.6.0: resolution: {integrity: sha512-a4sfn6L2sIShhtzJqmjGrARvxAW/3F2BJDdyRVvNF9VhAsZSh5hSyI3a9TNvmzBxXmq66nY5LNT5bQcBxYAZZg==} - ox@0.14.29: - resolution: {integrity: sha512-M5j87Ec4V99MQdRct/g09eWXW60g6zhHTUs1lr4deUtrPDnezBdCJTgKd7pxqTpSZBFveV0ALi9jMMuT1qKyNg==} + ox@0.14.30: + resolution: {integrity: sha512-LI11uu+8iiM1B3CLckgd++YF1a0A2k5wDoM9ZeQMiL21BOzQs6L//BLS6hb1HSEKCyycdDIQLsVQx9MjpcC0hA==} peerDependencies: typescript: '>=5.4.0' peerDependenciesMeta: @@ -1123,8 +1126,8 @@ packages: undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} - viem@2.54.2: - resolution: {integrity: sha512-o0+5dEAUekBMTbixXy2mKbSDPnwsCJ+8+mOeMBDjkuS9iM4fcr3yKUWb2zlOy2NKInkg3anl1W11sxYspLiXig==} + viem@2.54.6: + resolution: {integrity: sha512-OfybECKJYVmhiNqz+SHhed+O2h6niQ+0Wjg9J0b4bV+/QrvLgjxhfKO7hZqsuK1YtZ/0BErBKy708Zp+cU5T0Q==} peerDependencies: typescript: '>=5.0.4' peerDependenciesMeta: @@ -1444,9 +1447,9 @@ snapshots: '@shikijs/types': 3.23.0 '@shikijs/vscode-textmate': 10.0.2 - '@hono/node-server@2.0.8(hono@4.12.27)': + '@hono/node-server@2.0.8(hono@4.12.28)': dependencies: - hono: 4.12.27 + hono: 4.12.28 '@jridgewell/gen-mapping@0.3.13': dependencies: @@ -1599,19 +1602,19 @@ snapshots: '@shikijs/vscode-textmate@10.0.2': {} - '@tangle-network/agent-core@0.3.4': + '@tangle-network/agent-core@0.3.8': dependencies: - '@tangle-network/agent-interface': 0.14.0 + '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 - '@tangle-network/agent-eval@0.106.2(typescript@5.9.3)': + '@tangle-network/agent-eval@0.108.0(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 19.0.45(zod@4.4.3) - '@hono/node-server': 2.0.8(hono@4.12.27) + '@hono/node-server': 2.0.8(hono@4.12.28) '@tangle-network/agent-interface': 0.10.1 '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) - hono: 4.12.27 + hono: 4.12.28 zod: 4.4.3 transitivePeerDependencies: - '@mastra/core' @@ -1630,16 +1633,20 @@ snapshots: dependencies: zod: 4.4.3 - '@tangle-network/agent-interface@0.14.0': + '@tangle-network/agent-interface@0.17.1': + dependencies: + zod: 4.4.3 + + '@tangle-network/agent-interface@0.19.0': dependencies: zod: 4.4.3 - '@tangle-network/sandbox@0.9.5(viem@2.54.2(typescript@5.9.3)(zod@4.4.3))': + '@tangle-network/sandbox@0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3))': dependencies: - '@tangle-network/agent-core': 0.3.4 + '@tangle-network/agent-core': 0.3.8 '@tangle-network/agent-interface': 0.13.0 optionalDependencies: - viem: 2.54.2(typescript@5.9.3)(zod@4.4.3) + viem: 2.54.6(typescript@5.9.3)(zod@4.4.3) '@tangle-network/tcloud-attestation@0.1.1': {} @@ -1647,10 +1654,10 @@ snapshots: dependencies: '@scure/bip32': 2.2.0 '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.9.5(viem@2.54.2(typescript@5.9.3)(zod@4.4.3)) + '@tangle-network/sandbox': 0.9.7(viem@2.54.6(typescript@5.9.3)(zod@4.4.3)) '@tangle-network/tcloud-attestation': 0.1.1 commander: 14.0.3 - viem: 2.54.2(typescript@5.9.3)(zod@4.4.3) + viem: 2.54.6(typescript@5.9.3)(zod@4.4.3) transitivePeerDependencies: - '@mastra/core' - '@modelcontextprotocol/sdk' @@ -1864,7 +1871,7 @@ snapshots: fsevents@2.3.3: optional: true - hono@4.12.27: {} + hono@4.12.28: {} isows@1.0.7(ws@8.21.0): dependencies: @@ -1930,7 +1937,7 @@ snapshots: dependencies: yaml: 2.9.0 - ox@0.14.29(typescript@5.9.3)(zod@4.4.3): + ox@0.14.30(typescript@5.9.3)(zod@4.4.3): dependencies: '@adraffy/ens-normalize': 1.11.1 '@noble/ciphers': 1.3.0 @@ -2126,7 +2133,7 @@ snapshots: undici-types@7.24.6: {} - viem@2.54.2(typescript@5.9.3)(zod@4.4.3): + viem@2.54.6(typescript@5.9.3)(zod@4.4.3): dependencies: '@noble/curves': 1.9.1 '@noble/hashes': 1.8.0 @@ -2134,7 +2141,7 @@ snapshots: '@scure/bip39': 1.6.0 abitype: 1.2.3(typescript@5.9.3)(zod@4.4.3) isows: 1.0.7(ws@8.21.0) - ox: 0.14.29(typescript@5.9.3)(zod@4.4.3) + ox: 0.14.30(typescript@5.9.3)(zod@4.4.3) ws: 8.21.0 optionalDependencies: typescript: 5.9.3 diff --git a/scripts/gen-primitive-catalog.mjs b/scripts/gen-primitive-catalog.mjs index 6ed9f83a..1cec390a 100644 --- a/scripts/gen-primitive-catalog.mjs +++ b/scripts/gen-primitive-catalog.mjs @@ -265,13 +265,14 @@ const bySpecifier = new Map() for (let i = 0; i < allModules.length; i++) bySpecifier.set(allModules[i].specifier, extracted[i]) // ───────────────────────────────────────────────────────────────────────────── -// Ratchet: undocumented callables may only DECREASE. Undocumented interface/type +// Ratchet: undocumented OWN callables may only DECREASE. Undocumented interface/type // entries collapse out of the table (see renderSection), but a blank function/class/ // const row is a reach-for primitive with no summary — visible shame, capped here. -// The ceiling is the exact current count; when a backfill lowers the real number, -// lower the constant to match. Exceeding it (a new undocumented callable) exits 1. +// The ceiling is the exact current count for this package's exports; when a backfill lowers +// the real number, lower the constant to match. Substrate callables from agent-eval are still +// rendered and reported, but cannot fail this repo's docs when an external release drifts. -const maxUndocumentedCallables = 0 +const maxUndocumentedOwnCallables = 0 const ratchetKinds = new Set(['function', 'class', 'const']) // ───────────────────────────────────────────────────────────────────────────── @@ -322,21 +323,34 @@ function renderSection(mod, importLabel) { const importLabelOf = (basePackage, subpath) => subpath === '.' ? basePackage : `${basePackage}/${subpath.replace(/^\.\//, '')}` -const undocumentedCallables = [] +const undocumentedOwnCallables = [] +const undocumentedSubstrateCallables = [] +const ownSpecifiers = new Set(ownModules.map((m) => m.specifier)) for (const mod of allModules) { for (const e of entriesFor(mod)) { - if (!e.summary && ratchetKinds.has(e.kind)) undocumentedCallables.push(`${e.name} (${e.kind}, ${mod.specifier})`) + if (!e.summary && ratchetKinds.has(e.kind)) { + const finding = `${e.name} (${e.kind}, ${mod.specifier})` + if (ownSpecifiers.has(mod.specifier)) undocumentedOwnCallables.push(finding) + else undocumentedSubstrateCallables.push(finding) + } } } -if (undocumentedCallables.length > maxUndocumentedCallables) { +if (undocumentedOwnCallables.length > maxUndocumentedOwnCallables) { console.error( - `primitive-catalog: ${undocumentedCallables.length} undocumented function/class/const exports exceed the ` + - `ratchet ceiling of ${maxUndocumentedCallables}. Add a TSDoc summary line at each new declaration ` + + `primitive-catalog: ${undocumentedOwnCallables.length} undocumented own function/class/const exports exceed the ` + + `ratchet ceiling of ${maxUndocumentedOwnCallables}. Add a TSDoc summary line at each new declaration ` + '(summary BEFORE any @tag — a tag-first block reads as blank). Undocumented callables:\n ' + - undocumentedCallables.join('\n '), + undocumentedOwnCallables.join('\n '), ) process.exit(1) } +if (undocumentedSubstrateCallables.length) { + console.error( + `primitive-catalog: ${undocumentedSubstrateCallables.length} undocumented substrate function/class/const exports ` + + 'were rendered but did not fail this package ratchet:\n ' + + undocumentedSubstrateCallables.join('\n '), + ) +} const out = [] out.push(' + + + + + + + stream driver + local · per instance + + + + + + repro author + glm-5.2 · HTTP zai + + + + + + k=4 patch attempts + worker · HTTP zai + ≤12 tool turns each + + + + + + score in jail + docker · --network none + apply patch → run repro + + + + + argmax + + + + + ARM F · raw-failure repair + worker retries on error text + + + + + ARM L · supervisor + glm-5.2 · HTTP zai + fires on verified failure only + + F + L + + + + HIDDEN JUDGE (official SWE-bench) + docker · runs AFTER arm decisions lock + + + + +
+ HTTP → zai (glm-5.2) + worker calls (HTTP zai) + local docker jail (no network) + supervisor (arm L only) +
+

Every model call goes to one endpoint — api.z.ai/api/coding/paas/v4 — over HTTPS; every grader runs locally in a network-isolated docker container. Arms F and L share the same repro instrument and the same k=4 candidates; they diverge only at the repair step (raw-failure vs supervisor plan). The hidden judge never touches any model prompt.

+ + +

Supervisor gate — when arm L fires

+
+ + + + + + + +
candidate outcomeseverityarm L actionwhy
bug fixed (repro passes)0HOLD · already-passingnothing to repair
applied but bug persists1FIRE · wrong-fixverified wrong fix
repro timed out2HOLD · ambiguousnot a clean signal
patch failed to apply3FIRE · apply-failedobjective; re-anchor guidance
empty diff (no attempt)4FIRE · empty-diffplan helps most here
+

The supervisor's plan is grounded ONLY in execution-verified evidence (the failing diff + the jailed failure output) — never the worker's self-report — so the credulity hazard measured in the arena cannot occur. Every plan is stripped of code and audited for hidden-test leaks.

+
+ +
+
+

Live run

+
+
+
+
+
+

Prior full run (day-1, n=20) — reference

+
+
+

Day-1 arm L = memory recall only (no supervisor). L−F = −1 (noise). The live run adds the supervisor, testing whether execution-grounded advice moves the curve.

+
+
+
+ +

L − F cumulative resolved rate — day-1 run (memory-only L)

+
+ +
+ Frozen (F) + Learning (L) + deadline drop (both arms) +
+
+ +

Per-instance — day-1 run

+
+
#instanceFLrecall injected
+
+ + + From bb9828892664e3cf50aaeefb4331641ab0359cad Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 12 Jul 2026 21:25:36 -0600 Subject: [PATCH 37/44] =?UTF-8?q?fix(bench):=20stream-observe=20diagram=20?= =?UTF-8?q?=E2=80=94=20time-ordered=20execution=20sequence=20with=20labele?= =?UTF-8?q?d=20nodes=20(role/model/harness/transport),=20and=20correct=20t?= =?UTF-8?q?he=20mental=20model=20(best-of-k=20+=20supervised-repair,=20NOT?= =?UTF-8?q?=20supervisor-spawns-workers)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bench/src/stream-observe.tpl.html | 132 ++++++++++++++++++------------ 1 file changed, 80 insertions(+), 52 deletions(-) diff --git a/bench/src/stream-observe.tpl.html b/bench/src/stream-observe.tpl.html index a93c2697..5e56e804 100644 --- a/bench/src/stream-observe.tpl.html +++ b/bench/src/stream-observe.tpl.html @@ -56,78 +56,106 @@

Stream loop — run topology & telemetry

The two-arm SWE-bench stream: frozen (F) vs supervised-learning (L), same instance in the same slot. This snapshots the request graph, the model routing, and the measured curve. Live run = glm-5.2 self-supervision · CONC=1

-

Request topology — one instance

+
+

NOT a delegation tree. This experiment does not have a supervisor spawn workers a/b/c. The driver samples k=4 workers independently and in parallel (no supervisor involved), scores them, keeps the best — and only if it failed does the supervisor write one advice plan for a retry. The supervisor is a reviewer at the end, not a dispatcher at the start. (The delegation-tree topology — supervisor → workers — exists in the codebase via supervise()/coordination-MCP, but this run doesn't use it.)

+
+ +

Execution sequence — one instance, left→right in time

- - - - - - + + + + + t0 · setupt1 · sample k=4t2 · score + t3 · selectt4 · repairt5 · grade + + + - - stream driver - local · per instance + + stream driver + local · swe-stream.mts + no model - - - - repro author - glm-5.2 · HTTP zai + + + repro author + glm-5.2 · HTTPS zai + no tools · writes 1 test - + - - - k=4 patch attempts - worker · HTTP zai - ≤12 tool turns each + + sample + + + + + + + + + worker aworker bworker cworker d + + + glm-5.2 · ≤12 tool turnsglm-5.2 · ≤12 tool turns + glm-5.2 · ≤12 tool turnsglm-5.2 · ≤12 tool turns + - + independent · parallel · emit-patch harness (read/edit/run tools) + - - - score in jail - docker · --network none - apply patch → run repro + + + + + + score in jail + docker · net=none + apply patch → run + the repro → severity + (local, no model) - + - - argmax + + + argmax + pick best - + + + - - ARM F · raw-failure repair - worker retries on error text + + ARM F · repair + worker retries on raw + error text (the control) - - - ARM L · supervisor - glm-5.2 · HTTP zai - fires on verified failure only + + ARM L · supervisor + glm-5.2 · HTTPS zai + no-tools · writes 1 plan + → worker retries w/ plan - F - L - + + + - - HIDDEN JUDGE (official SWE-bench) - docker · runs AFTER arm decisions lock + + HIDDEN JUDGE · official SWE-bench tests + docker · runs AFTER both arms lock · never in any prompt - -
- HTTP → zai (glm-5.2) - worker calls (HTTP zai) - local docker jail (no network) - supervisor (arm L only) + repro author (glm-5.2, no tools) + worker (glm-5.2, emit-patch harness) + supervisor (glm-5.2, no tools, arm L only) + docker jail (local, no network)
-

Every model call goes to one endpoint — api.z.ai/api/coding/paas/v4 — over HTTPS; every grader runs locally in a network-isolated docker container. Arms F and L share the same repro instrument and the same k=4 candidates; they diverge only at the repair step (raw-failure vs supervisor plan). The hidden judge never touches any model prompt.

+

Every model box calls one endpoint — api.z.ai/api/coding/paas/v4 over HTTPS; every docker box runs locally, network-isolated. In THIS run all three model roles are the same model (glm-5.2 = self-supervision); in the cross-strength config the worker is a cheaper model and only the supervisor is glm-5.2. Arms F and L share t0–t3 (same repro, same 4 candidates); they diverge only at t4 (raw-failure retry vs supervisor-plan retry). Timings per phase are in the ledger (imagePullMs, wallMs, judgeMs); per-model-call timestamps are not yet emitted — see the note below.

Supervisor gate — when arm L fires

From 16f98ae2f817fe87e5215e0b9fef3cdd1095a395 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 12 Jul 2026 22:48:02 -0600 Subject: [PATCH 38/44] feat(lifecycle): sweEvalRunner closes the self-improve loop over the SWE Docker judge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fills the EvalRunner hole (marginal-lift.ts): drive each pinned Verified instance with the multi-turn atom (runAgentic + refine) under the profile's prompt, capture the git diff, grade it with the swebench Docker judge held outside the agent, and return { composite: mean(resolved), costUsd }. The environment/tasks/judge are injected (bench owns them; importing bench from src would invert the package dependency). The driven system prompt folds profile.prompt.instructions in after systemPrompt — the surface applyArtifact mutates for prompt artifacts — so a prompt candidate measurably changes the worker instead of scoring a fake zero delta. All-judge-failures throws rather than reporting a fabricated composite. bench/src/self-improve-swe.mts wires it end-to-end: promptGenerator (one deterministic candidate via the injected refine seam) -> runLifecycle -> thresholdPromotionGate(0) -> composeProfile, with an SMOKE=1 import-only path. --- bench/.gitignore | 2 +- bench/src/self-improve-swe.mts | 124 +++++++++++++++++++ bench/tsconfig.json | 1 + docs/api/primitive-catalog.md | 7 +- src/lifecycle/index.ts | 6 + src/lifecycle/swe-eval-runner.ts | 200 +++++++++++++++++++++++++++++++ 6 files changed, 337 insertions(+), 3 deletions(-) create mode 100644 bench/src/self-improve-swe.mts create mode 100644 src/lifecycle/swe-eval-runner.ts diff --git a/bench/.gitignore b/bench/.gitignore index 877f3d04..886ffc82 100644 --- a/bench/.gitignore +++ b/bench/.gitignore @@ -1,4 +1,4 @@ -.venv/ +.venv .venv-*/ node_modules/ *.log diff --git a/bench/src/self-improve-swe.mts b/bench/src/self-improve-swe.mts new file mode 100644 index 00000000..a146280a --- /dev/null +++ b/bench/src/self-improve-swe.mts @@ -0,0 +1,124 @@ +/** + * PHASE-1 closed self-improve loop over the REAL SWE Docker judge: + * generate → drive → grade → promote → compose, in ONE `runLifecycle` call. + * + * The one previously-missing piece is `sweEvalRunner` (the lifecycle's + * `EvalRunner` for the SWE domain): it drives each pinned Verified instance + * with the multi-turn atom (`runAgentic` + refine) under the profile's prompt, + * captures the `git diff`, and grades it with the swebench Docker judge held + * OUTSIDE the agent. This driver wires it to `runLifecycle` with the shipped + * `promptGenerator` (one deterministic candidate instruction injected through + * its `refine` seam) and `thresholdPromotionGate(0)`, then prints the baseline + * composite, each candidate's measured delta, and the composed profile. + * + * IN (env): HOLDOUT_IDS=, + * WORKER_MODEL, MAX_TOKENS, ROUTER_BASE, TANGLE_API_KEY, + * INNER_TURNS, BUDGET, RUN_TOOL + * OUT (fd1): baseline composite, per-candidate scoreDelta/promoted, the + * composed profile's prompt surface + * diagnostics go to STDERR. + * + * SMOKE=1 → import/wiring check only: the module graph (this file + + * swe-bench-env + lifecycle incl. sweEvalRunner) loaded; print READY on stderr + * and exit 0 WITHOUT a clone / model call / dataset read. + */ +import type { AgentProfile } from '@tangle-network/agent-interface' +import { + composeProfile, + promptGenerator, + runLifecycle, + sweEvalRunner, + thresholdPromotionGate, +} from '@tangle-network/agent-runtime/lifecycle' +import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env' + +async function main(): Promise { + const smoke = ['1', 'true', 'yes'].includes((process.env.SMOKE ?? '').toLowerCase()) + if (smoke) { + console.error('SMOKE ok: self-improve-swe + swe-bench-env + lifecycle (sweEvalRunner) import graph loaded') + return + } + + const routerKey = process.env.TANGLE_API_KEY + if (!routerKey) throw new Error('TANGLE_API_KEY required (the worker calls the router)') + const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' + const model = process.env.WORKER_MODEL ?? 'google/gemini-2.5-flash-lite' + const ids = (process.env.HOLDOUT_IDS ?? '').split(',').map((s) => s.trim()).filter(Boolean) + if (ids.length === 0) throw new Error('HOLDOUT_IDS required (comma-separated Verified instance ids)') + const innerTurns = Number(process.env.INNER_TURNS ?? 12) + const maxTokens = Number(process.env.MAX_TOKENS ?? 12000) + const budget = Number(process.env.BUDGET ?? 1) + const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase()) + + const { environment, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun }) + const pool = await adapter.loadTasks({ ids, split: 'test' }) + const missing = ids.filter((id) => !pool.some((t) => t.id === id)) + if (missing.length) throw new Error(`self-improve-swe: not in Verified: ${missing.join(', ')}`) + + const seedPrompt = enableRun ? SWE_SEED_PROMPT_WITH_RUN : SWE_SEED_PROMPT + const evalRunner = sweEvalRunner({ + environment, + tasks: pool, + judge: (task, patch) => adapter.judge(task, patch), + seedPrompt, + routerBaseUrl, + routerKey, + model, + maxTokens, + innerTurns, + budget, + }) + + const baseline: AgentProfile = { name: 'swe-baseline', prompt: { systemPrompt: seedPrompt } } + + // ONE deterministic candidate through the shipped generator's injected-refine + // seam — the smallest real population that exercises the whole loop. + const generator = promptGenerator({ + refine: () => [ + { + instruction: + 'Before your first edit, state a one-line root-cause hypothesis naming the exact file and ' + + 'function you believe is at fault; if a later read_file contradicts it, revise the hypothesis ' + + 'before editing further.', + label: 'root-cause-hypothesis-first', + rationale: + 'Weak workers patch the first plausible site they read; forcing a named hypothesis before the ' + + 'first edit targets the wrong-file/wrong-function failure mode.', + }, + ], + }) + + const t0 = Date.now() + const out = await runLifecycle({ + baseline, + domain: 'swe', + generators: [generator], + evalRunner, + gate: thresholdPromotionGate(0), + }) + + console.log( + `baseline composite=${out.baselineResult.composite.toFixed(4)} ` + + `costUsd=${out.baselineResult.costUsd.toFixed(4)} (n=${pool.length} instances)`, + ) + for (const o of out.outcomes) { + console.log( + `candidate "${o.artifact.name}" [${o.artifact.id}] kind=${o.kind} ` + + `scoreDelta=${o.scoreDelta.toFixed(4)} costDelta=${o.costDelta.toFixed(4)} ` + + `promoted=${o.promoted} (${o.verdict.reason})`, + ) + } + console.log(`promoted=[${out.promoted.join(', ')}]`) + + const composed = composeProfile(out.registry, baseline, { kind: 'prompt' }) + console.log( + `composed profile: systemPrompt=${(composed.prompt?.systemPrompt ?? '').length}b ` + + `instructions=${JSON.stringify(composed.prompt?.instructions ?? [])}`, + ) + console.error(`[self-improve-swe] done in ${Math.round((Date.now() - t0) / 1000)}s`) +} + +main().catch((e) => { + console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) + process.exit(1) +}) diff --git a/bench/tsconfig.json b/bench/tsconfig.json index 7501a77c..055fdec1 100644 --- a/bench/tsconfig.json +++ b/bench/tsconfig.json @@ -13,6 +13,7 @@ "resolveJsonModule": true, "paths": { "@tangle-network/agent-runtime/loops": ["../src/runtime/index.ts"], + "@tangle-network/agent-runtime/lifecycle": ["../src/lifecycle/index.ts"], "@tangle-network/sandbox": ["../node_modules/@tangle-network/sandbox"] } }, diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b49576b6..5837fe3f 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -670,7 +670,7 @@ Import from `@tangle-network/agent-runtime/analyst-loop` — 15 exports. ### Artifact lifecycle — generate → measure → promote → compose -Import from `@tangle-network/agent-runtime/lifecycle` — 59 exports. +Import from `@tangle-network/agent-runtime/lifecycle` — 63 exports. | Symbol | Kind | Summary | |---|---|---| @@ -689,6 +689,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 59 exports. | `routerSeedAuthor` | function | A router-backed `AuthorDiverseSeeds`: one structured LLM call that authors | | `runLifecycle` | function | Run ONE generation of the artifact lifecycle. | | `skillGenerator` | function | Build a `CandidateGenerator` for the skill surface that distills new skills | +| `sweEvalRunner` | function | Build the `EvalRunner` closed over one fixed SWE exam. `runLifecycle` calls | | `thresholdPromotionGate` | function | The simplest honest gate: promote iff the candidate's marginal lift on the | | `worktreeBuildCandidate` | function | Build the production per-candidate seam for `buildableGenerator`. Each call to | | `lifecycleReasonKey` | const | The metadata key under which the registry records WHY an artifact left the | @@ -713,6 +714,8 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 59 exports. | `PromptDraft` | interface | A proposed prompt instruction line plus the WHY behind it. The `rationale` | | `RunLifecycleOptions` | interface | `runLifecycle` — the ONE closed-loop orchestrator: generate → measure → | | `SkillDraft` | interface | A distilled skill draft: a name + the `SKILL.md` body. | +| `SweEvalTask` | interface | The minimal shape of a held-out SWE instance the runner needs. The bench | +| `SweEvalTaskResult` | interface | Per-instance audit row surfaced through `EvalResult.details`. | | `WorktreeBuildOptions` | interface | `worktreeBuildCandidate` — the PRODUCTION `BuildCandidate`: one fan-out leaf | | `ArtifactInput` | type | The input to `register` — everything on `ProfileArtifact` except the | | `ArtifactKind` | type | The profile levers an artifact can target. One-to-one with the §1.5 profile | @@ -725,7 +728,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 59 exports. | `RefinePrompt` | type | REFINE — incumbent-grounded rewrites. Given the lifecycle context, return | | `RefineSkill` | type | REFINE — improve ONE distilled draft (wording, structure, examples). The | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BuildableGeneratorOptions`, `DedupeResult`, `DriftWatchResult`, `HeldOutPromotionGateOptions`, `MeasureMarginalLiftOptions`, `ProductionPromptGeneratorOptions`, `PromptGeneratorOptions`, `RunLifecycleResult`, `SkillGeneratorOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BuildableGeneratorOptions`, `DedupeResult`, `DriftWatchResult`, `HeldOutPromotionGateOptions`, `MeasureMarginalLiftOptions`, `ProductionPromptGeneratorOptions`, `PromptGeneratorOptions`, `RunLifecycleResult`, `SkillGeneratorOptions`, `SweEvalRunnerOptions`. ### Knowledge orchestration — supervised KB updates diff --git a/src/lifecycle/index.ts b/src/lifecycle/index.ts index a6507268..f224689a 100644 --- a/src/lifecycle/index.ts +++ b/src/lifecycle/index.ts @@ -91,6 +91,12 @@ export { type SkillGeneratorOptions, skillGenerator, } from './skill-generator' +export { + type SweEvalRunnerOptions, + type SweEvalTask, + type SweEvalTaskResult, + sweEvalRunner, +} from './swe-eval-runner' export { type WorktreeBuildOptions, worktreeBuildCandidate } from './tool-build' export { type BuildableGeneratorOptions, diff --git a/src/lifecycle/swe-eval-runner.ts b/src/lifecycle/swe-eval-runner.ts new file mode 100644 index 00000000..bde7236c --- /dev/null +++ b/src/lifecycle/swe-eval-runner.ts @@ -0,0 +1,200 @@ +/** + * `sweEvalRunner` — the SWE-domain `EvalRunner`: the first production filler of + * the lifecycle's scoring hole (`marginal-lift.ts`'s `EvalRunner` type). + * + * It scores an `AgentProfile` by DRIVING a real coding agent over pinned + * SWE-bench instances and grading each resulting patch with the official + * Docker judge. Three invariants make the number real: + * + * 1. MULTI-TURN DRIVE — each instance is driven with the atom + * (`runAgentic` + `refine`), not a single-shot prompt, so the profile is + * scored on the same loop production runs use. + * 2. JUDGE OUTSIDE THE AGENT — the grader is an injected `judge` (the + * swebench Docker harness); the driven surface's own `score()` is replaced + * by a patch-presence probe, so the scaffold can never grade its own axis + * and the judge runs exactly once per (profile, task). + * 3. PROMPT BRIDGE — `applyArtifact` lands `prompt` candidates on + * `profile.prompt.instructions`, so the driven system prompt MUST fold + * `instructions` in after `systemPrompt` (seed fallback when empty). + * Reading `systemPrompt` alone would score every prompt candidate as a + * fake zero delta and the loop could never promote anything. + * + * The SWE environment, task rows, and judge are INJECTED: they live in the + * bench package (`createSweBenchEnvironment`, the swebench adapter), which + * depends on this package — importing them here would invert the dependency. + * + * Phase-1 scope: returns the scalar `composite` (mean resolved) + `costUsd` + * only — no per-task `RunRecord`s — so it pairs with `thresholdPromotionGate`, + * not `heldOutPromotionGate`. Tasks run sequentially on purpose: the patch + * capture closes over one mutable cell per drive. + */ + +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { ValidationError } from '../errors' +import { + type AgenticSurface, + type ArtifactHandle, + refine, + runAgentic, + type SurfaceScore, +} from '../runtime' +import type { EvalResult, EvalRunner } from './marginal-lift' + +const exec = promisify(execFile) + +/** The minimal shape of a held-out SWE instance the runner needs. The bench + * adapter's `BenchTask` satisfies it structurally; the generic `T` lets the + * injected `judge` keep its own richer task type. */ +export interface SweEvalTask { + /** SWE-bench instance id (must be in the environment's pinned pool). */ + id: string + /** The adapter-rendered issue prompt (the user message for the worker). */ + prompt: string +} + +/** Per-instance audit row surfaced through `EvalResult.details`. */ +export interface SweEvalTaskResult { + id: string + resolved: boolean + patchBytes: number + usd: number + shots: number + tokens: { input: number; output: number } + /** Set when the Docker judge threw for this instance (scored unresolved). */ + judgeError?: string +} + +export interface SweEvalRunnerOptions { + /** The SWE `AgenticSurface` (bench's `createSweBenchEnvironment`), pinned to + * the held-out ids so `open(task)` can find each instance. */ + environment: AgenticSurface + /** The held-out instances EVERY profile is scored on (same set for the + * baseline and each candidate arm — the ablation holds the exam constant). */ + tasks: readonly T[] + /** The grader, held OUTSIDE the agent (bench `adapter.judge` → the official + * swebench Docker harness): patch in, resolved verdict out. */ + judge: (task: T, patch: string) => Promise<{ resolved: boolean }> + /** Fallback system prompt when the profile carries none (the SWE seed). */ + seedPrompt: string + routerBaseUrl: string + routerKey: string + /** The worker model the refine strategy drives. */ + model: string + /** Completion cap per worker turn (required for thinking models). */ + maxTokens?: number + /** Worker turns per shot before the driver's analyst intervenes. */ + innerTurns?: number + /** Max refine shots per task. Default 1. */ + budget?: number +} + +/** + * Build the `EvalRunner` closed over one fixed SWE exam. `runLifecycle` calls + * it once for the baseline profile and once per candidate; each call drives + * every pinned instance and returns `{ composite: mean(resolved), costUsd }`. + */ +export function sweEvalRunner( + opts: SweEvalRunnerOptions, +): EvalRunner { + if (opts.tasks.length === 0) { + throw new ValidationError('sweEvalRunner: at least one held-out task is required') + } + + return async (profile: AgentProfile, signal?: AbortSignal): Promise => { + const systemPrompt = renderSystemPrompt(profile, opts.seedPrompt) + const rows: SweEvalTaskResult[] = [] + let costUsd = 0 + let judgeCalls = 0 + let judgeErrors = 0 + + for (const task of opts.tasks) { + if (signal?.aborted) throw new Error('sweEvalRunner: aborted') + + // Capture the LATEST non-empty diff from inside score() — the refine loop + // calls it before the surface closes and rms the checkout, and a later + // empty read must never clobber a real patch (the swe-emit-patch pattern). + let capturedPatch = '' + const proxy: AgenticSurface = { + ...opts.environment, + async score(_t, handle: ArtifactHandle): Promise { + try { + const d = await exec('git', ['-C', handle.id, 'diff'], { + maxBuffer: 40_000_000, + timeout: 60_000, + }) + if (d.stdout.trim()) capturedPatch = d.stdout + } catch { + /* workspace gone or git error → keep whatever we already captured */ + } + return { passes: capturedPatch.trim() ? 1 : 0, total: 1, errored: 0 } + }, + } + + const run = await runAgentic({ + surface: proxy, + task: { id: task.id, systemPrompt, userPrompt: task.prompt, meta: { instanceId: task.id } }, + strategy: refine, + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, + model: opts.model, + maxTokens: opts.maxTokens, + innerTurns: opts.innerTurns, + budget: opts.budget ?? 1, + }) + costUsd += run.usd + + let resolved = false + let judgeError: string | undefined + if (capturedPatch.trim()) { + judgeCalls += 1 + try { + resolved = (await opts.judge(task, capturedPatch)).resolved + } catch (e) { + judgeErrors += 1 + judgeError = e instanceof Error ? e.message : String(e) + } + } + + rows.push({ + id: task.id, + resolved, + patchBytes: capturedPatch.length, + usd: run.usd, + shots: run.shots, + tokens: run.tokens, + ...(judgeError === undefined ? {} : { judgeError }), + }) + console.error( + `[swe-eval] ${task.id} resolved=${resolved} patch=${capturedPatch.length}b ` + + `shots=${run.shots} tok=in:${run.tokens.input}/out:${run.tokens.output} usd=${run.usd.toFixed(4)}` + + (judgeError ? ` judgeError=${judgeError.slice(0, 200)}` : ''), + ) + } + + // Every judge invocation failing is broken grading infrastructure (docker + // down, venv missing) — reporting composite=0 from it would fabricate a + // score, so fail loud instead. + if (judgeErrors > 0 && judgeErrors === judgeCalls) { + throw new Error( + `sweEvalRunner: all ${judgeErrors} judge invocation(s) failed — grading infra is down, refusing to report a fabricated composite`, + ) + } + + return { + composite: rows.filter((r) => r.resolved).length / rows.length, + costUsd, + details: { systemPrompt, rows }, + } + } +} + +/** The profile→prompt bridge: base `systemPrompt` (seed fallback) followed by + * every `instructions` line — the surface `applyArtifact` mutates for `prompt` + * artifacts, so candidates measurably change the driven worker. */ +function renderSystemPrompt(profile: AgentProfile, seedPrompt: string): string { + const base = profile.prompt?.systemPrompt?.trim() || seedPrompt + const instructions = (profile.prompt?.instructions ?? []).filter((s) => s.trim().length > 0) + return [base, ...instructions].join('\n\n') +} From 7132f9ae5a69e96f463841e50228238330d7b9ee Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 12 Jul 2026 23:05:09 -0600 Subject: [PATCH 39/44] feat(improvement): senior scientific-method optimizer prompt + driver-loop tool builder Replace the thin builder/author prompts with one shared senior doctrine (src/improvement/optimizer-prompt.ts): diagnose the dominant failure mode, state a hypothesis with a predicted lift, decompose into checkable sub-goals, design to isolate the mechanism, generalize past the shown findings, preserve what works, verify for real, then reflect. Seeded from GEPA's REFLECTION_SYSTEM and the /evolve, /pursue, self-improving-loop skill docs; embedded by toolBuildPrompt / mcpBuildPrompt / defaultBuildPrompt and by authorStrategy's authoring instruction. Add driverLoopGenerator: the driver->worker CandidateGenerator on the shipped atom (runBrainLoop + ToolLoopChat, the same seam driverAgent runs on). A driver LLM authors each worker instruction, observes the session's diff / files / verifier output, rates it, and decides refine / re-scope / decompose - replacing agenticGenerator's canned EMPTY_TREE_NOTE/failureNote respawn. Workers stay runLocalHarness in the candidate worktree; agenticGenerator is kept intact as the offline path. worktreeBuildCandidate wires the driver loop as the tool/mcp build default whenever a driver brain is configured. Completion oracle stays code-owned: after the driver stops, ground truth (dirty tree + raw-trace evidence + verifier exit) decides applied, never the driver's prose. Scripted-brain unit tests prove instruction threading, refine on red verifier, fail-closed gating, and the session cap. --- docs/api/primitive-catalog.md | 9 +- src/improvement/agentic-generator.ts | 38 ++- src/improvement/build-prompts.ts | 54 +++- src/improvement/driver-loop-generator.test.ts | 220 +++++++++++++ src/improvement/driver-loop-generator.ts | 295 ++++++++++++++++++ src/improvement/index.ts | 12 +- src/improvement/optimizer-prompt.ts | 123 ++++++++ src/lifecycle/tool-build.ts | 29 +- src/runtime/strategy-author.ts | 7 +- 9 files changed, 749 insertions(+), 38 deletions(-) create mode 100644 src/improvement/driver-loop-generator.test.ts create mode 100644 src/improvement/driver-loop-generator.ts create mode 100644 src/improvement/optimizer-prompt.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 5837fe3f..1f47c799 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 233 exports. +Import from `@tangle-network/agent-runtime` — 240 exports. | Symbol | Kind | Summary | |---|---|---| @@ -40,11 +40,14 @@ Import from `@tangle-network/agent-runtime` — 233 exports. | `createSupervisedKnowledgeUpdater` | function | Create an `improveKnowledgeBase` update callback backed by runtime supervision. | | `d1ToSqlAdapter` | function | Adapt a Cloudflare D1 binding to the SqlAdapter shape. Lives here so D1 | | `decideKnowledgeReadiness` | function | Map a `KnowledgeReadinessReport` to a three-state branch (`ready` / `blocked` / `caveat`) the runtime, route handlers, and UI shells all switch on. | +| `defaultBuildPrompt` | function | Turn the analyst's findings (+ optional report) into a concrete coder task — | | `defineConversation` | function | Declarative constructor for a multi-agent `Conversation`. Validates inputs | | `defineRuntimeHooks` | function | Identity helper that types a {@link RuntimeHooks} literal so the fields are inferred. | | `deriveExecutionId` | function | Derive a stable executionId from the run identity. The same | +| `driverLoopGenerator` | function | Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budge | | `enumerateNeighborPolicies` | function | All bounded single-dial neighbors of `policy`, in a fixed priority order: k | | `exportEvalRuns` | function | Ship self-improvement eval-run events to Tangle Intelligence. Unlike the | +| `findingLines` | function | Render findings as the ranked-evidence block every build prompt ends with. | | `getModels` | function | Fetch the model catalog from the router's `/v1/models`. Throws on a non-2xx | | `handleChatTurn` | function | Run one chat turn. Returns immediately with a `ReadableStream` body; | | `improve` | function | Run the held-out-gated self-improvement loop on ONE profile surface. | @@ -97,15 +100,18 @@ Import from `@tangle-network/agent-runtime` — 233 exports. | `turnId` | function | Deterministic turn identifier. Stable across retries of the same logical | | `validateChatModelId` | function | Validate a caller-supplied chat-model id. Rejects non-strings, malformed | | `worktreeLoopRunner` | function | `code` mode on the GENERIC recursive path: author one `AgentProfile` per harness, run them as a | +| `buildDriverSystem` | const | The driver's stance for `driverLoopGenerator` — the build-domain instance of | | `DEFAULT_MAX_DEPTH` | const | Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. | | `DEFAULT_ROUTER_BASE_URL` | const | Default Tangle Router base URL used when no env override is set. | | `defaultIsRetryable` | const | Default retryable classification — network/timeout class errors. Errors | | `DELEGATED_LOOP_MODES` | const | All valid delegated-loop mode names — used for validation and CLI surfaces. | | `FORWARD_HEADERS` | const | Standard names — lowercased so Headers maps interop on every runtime. | | `INTELLIGENCE_WIRE_VERSION` | const | Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). | +| `optimizerMethod` | const | The shared method block every build/author prompt embeds. Domain framing | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | | `ROLLOUT_POLICY_BOUNDS` | const | Proposal bounds per dial. These are the SEARCH bounds (what the proposer may | | `ROLLOUT_POLICY_EXTENSION` | const | The profile extensions namespace the policy persists under. | +| `strategyAuthorMethod` | const | The senior authoring process for `authorStrategy` — the same method, shaped | | `AgentEvalError` | class | Base class for every contract error this package throws — carries the stable | | `BackendTransportError` | class | A backend transport call (HTTP, gRPC, sidecar IPC) failed with a non-success | | `CircuitBreakerState` | class | Live circuit-breaker state — one instance per (participant, conversation run). | @@ -129,6 +135,7 @@ Import from `@tangle-network/agent-runtime` — 233 exports. | `CircuitBreakerConfig` | interface | Circuit-breaker tuning. `failuresToOpen` consecutive failures opens it; closed only after `cooldownMs`. | | `ConversationJournalEntry` | interface | Durable conversation transcript — survives a driver process crash mid-run. | | `D1DatabaseLike` | interface | Structural type matching the surface of `D1Database` we depend on, so the | +| `DriverLoopGeneratorOptions` | interface | `driverLoopGenerator` — the driver→worker `CandidateGenerator`: the build | | `LoopSpanNode` | interface | Sink-neutral node in a reconstructed loop span tree. The root node's | | `McpServeSpec` | interface | `mcpServeVerifier` — the intrinsic verifier for a built MCP server: the | | `ModelInfo` | interface | A model entry as returned by the Tangle Router `/v1/models` endpoint. | diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 2f9577ed..7b6a6e9c 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -38,6 +38,7 @@ import { join } from 'node:path' import type { AnalystFinding } from '@tangle-network/agent-eval' import { type LocalHarness, runLocalHarness } from '../mcp/local-harness' import type { CandidateGenerator } from './improvement-driver' +import { optimizerMethod } from './optimizer-prompt' const RAW_TRACE_ANALYST_ID = 'raw-trace-distiller' const RAW_TRACE_AREA = 'raw-trace-context' @@ -119,11 +120,11 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG // Dirty: with no verifier the diff IS the candidate (we trust the diff, // not the harness's stdout). With a verifier the candidate must pass it. if (!verify) { - return { applied: true, summary: summarize(findings) } + return { applied: true, summary: summarizeFindings(findings) } } const result = await verify(worktreePath) if (result.ok) { - return { applied: true, summary: summarize(findings) } + return { applied: true, summary: summarizeFindings(findings) } } // Dirty but failing — resume next shot atop these edits with the error. attemptNote = failureNote(result.feedback) @@ -135,14 +136,24 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG } } -/** Turn the analyst's findings (+ optional report) into a concrete coder task. */ -function defaultBuildPrompt(args: { report: unknown; findings: AnalystFinding[] }): string { +/** Turn the analyst's findings (+ optional report) into a concrete coder task — + * the senior scientific-method framing shared with the tool/MCP build prompts. */ +export function defaultBuildPrompt(args: { report: unknown; findings: AnalystFinding[] }): string { const lines: string[] = [ - 'You are improving this codebase based on an evaluation analysis.', - 'Make the smallest set of edits that addresses the findings below, then stop.', - 'Do not change unrelated code. Do not commit — leave changes in the working tree.', + 'You are improving this codebase based on an evaluation analysis: real runs failed, an', + 'analyst distilled the findings below, and your change will be measured on held-out tasks', + 'against the unchanged baseline — only a real lift promotes it.', '', - 'Findings:', + optimizerMethod, + '', + 'THE SURFACE — what a deliverable change looks like here:', + '- edit this codebase in place: the smallest coherent change set that fully tests your', + ' hypothesis about the dominant failure mode (see the method above — no unrelated edits,', + ' they confound the measurement),', + '- keep the diff reviewable: a reviewer should be able to trace every hunk back to a finding,', + '- do not commit — leave changes in the working tree.', + '', + 'FINDINGS — ranked evidence from real failed runs:', ] for (const f of args.findings) { const where = f.subject ? ` [${f.subject}]` : '' @@ -177,7 +188,10 @@ function failureNote(feedback?: string): string { ].join('\n') } -function rawTraceEvidenceProblem(worktreePath: string, findings: AnalystFinding[]): string | null { +export function rawTraceEvidenceProblem( + worktreePath: string, + findings: AnalystFinding[], +): string | null { const changedPaths = worktreeChangedPaths(worktreePath) const substantive = changedPaths.filter((path) => path !== RAW_TRACE_DIAGNOSIS_PATH) if (substantive.length === 0) { @@ -207,7 +221,7 @@ function rawTraceEvidenceProblem(worktreePath: string, findings: AnalystFinding[ return null } -function requiresRawTraceEvidence(findings: AnalystFinding[]): boolean { +export function requiresRawTraceEvidence(findings: AnalystFinding[]): boolean { return findings.some((finding) => { const f = finding as unknown as Record return f.analyst_id === RAW_TRACE_ANALYST_ID || f.area === RAW_TRACE_AREA @@ -267,7 +281,7 @@ export function commandVerifier( } /** A one-line summary for the commit message, derived from the findings. */ -function summarize(findings: AnalystFinding[]): string { +export function summarizeFindings(findings: AnalystFinding[]): string { if (findings.length === 0) return 'agentic improvement' if (findings.length === 1) return `agentic: ${truncate(findings[0]!.claim, 64)}` return `agentic: ${findings.length} findings addressed` @@ -286,7 +300,7 @@ function worktreeDirty(worktreePath: string): boolean { return worktreeChangedPaths(worktreePath).length > 0 } -function worktreeChangedPaths(worktreePath: string): string[] { +export function worktreeChangedPaths(worktreePath: string): string[] { const result = spawnSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: worktreePath, encoding: 'utf-8', diff --git a/src/improvement/build-prompts.ts b/src/improvement/build-prompts.ts index c83e2815..9220744b 100644 --- a/src/improvement/build-prompts.ts +++ b/src/improvement/build-prompts.ts @@ -16,10 +16,12 @@ */ import type { AnalystFinding } from '@tangle-network/agent-eval' +import { optimizerMethod } from './optimizer-prompt' type FindingsArg = { report: unknown; findings: AnalystFinding[] } -function findingLines(findings: AnalystFinding[]): string[] { +/** Render findings as the ranked-evidence block every build prompt ends with. */ +export function findingLines(findings: AnalystFinding[]): string[] { return findings.map((f) => { const where = f.subject ? ` [${f.subject}]` : '' const action = f.recommended_action ? ` → ${f.recommended_action}` : '' @@ -30,13 +32,25 @@ function findingLines(findings: AnalystFinding[]): string[] { /** Build the starting instruction for a coder agent tasked with implementing a new tool. */ export function toolBuildPrompt(args: FindingsArg): string { return [ - 'You are building a new TOOL for this codebase to address the gaps below.', - 'Write the tool as a small, self-contained module PLUS tests that exercise it.', - 'The tool must compile and its tests must pass — they will be run automatically;', - 'if verification fails you will get the error and another attempt. Do not commit;', - 'leave the changes in the working tree.', + 'You are building a new TOOL for this codebase — a capability the agent measurably lacks,', + 'evidenced by the failure findings at the bottom. The tool is an experiment: after it is', + 'built and verified, its marginal lift is measured on held-out tasks, and only a real lift', + 'promotes it.', '', - 'Gaps the tool should close:', + optimizerMethod, + '', + 'THE SURFACE — what a deliverable tool looks like here:', + '- ONE small, self-contained module PLUS tests that exercise its contract (what callers rely', + ' on), not its internals. The tests are the experiment for sub-goal correctness — write the', + ' test that would fail if your hypothesis about the gap were wrong.', + '- It must compile and its tests must pass — they run automatically; on failure you get the', + ' verifier output and another attempt, resuming on top of your own edits (fix in place, do', + ' not start over).', + '- Match the codebase grain: reuse its existing helpers, style, and test framework; a tool', + ' that fights the codebase is the wrong tool even if it passes.', + '- Do not commit; leave the changes in the working tree.', + '', + 'FINDINGS — ranked evidence from real failed runs (the gaps the tool must close):', ...findingLines(args.findings), ].join('\n') } @@ -44,17 +58,25 @@ export function toolBuildPrompt(args: FindingsArg): string { /** Build the starting instruction for a coder agent tasked with implementing a new MCP server. */ export function mcpBuildPrompt(args: FindingsArg): string { return [ - 'You are building a new MCP SERVER (Model Context Protocol) that exposes', - 'tool(s) addressing the gaps below, so any harness can mount it.', - 'Requirements that WILL be checked by booting the server:', + 'You are building a new MCP SERVER (Model Context Protocol) exposing tool(s) that close the', + 'capability gaps evidenced by the failure findings at the bottom, so any harness can mount', + 'them. The server is an experiment: after it is built and boot-verified, its marginal lift is', + 'measured on held-out tasks, and only a real lift promotes it.', + '', + optimizerMethod, + '', + 'THE SURFACE — what a deliverable MCP server looks like here (checked by BOOTING it):', '- it starts over stdio and answers the MCP `initialize` handshake,', - '- `tools/list` returns at least one tool with a valid input schema.', - 'Newline-delimited JSON-RPC 2.0, protocol version 2024-11-05. Include a start', - 'command (e.g. a package.json `start` script or a clear entrypoint). If the', - 'boot-and-probe fails you will get the error and another attempt. Do not', - 'commit; leave the changes in the working tree.', + '- `tools/list` returns at least one tool with a valid input schema,', + '- newline-delimited JSON-RPC 2.0, protocol version 2024-11-05,', + '- a clear start command (a package.json `start` script or an obvious entrypoint).', + 'Design the tool surface for the FINDINGS, not for generality: each exposed tool should map to', + 'a named failure mechanism, with a description that tells the agent when to reach for it (a', + 'tool the agent never calls measures zero). If the boot-and-probe fails you get the error and', + 'another attempt, resuming on top of your own edits. Do not commit; leave the changes in the', + 'working tree.', '', - 'Capabilities the server should provide:', + 'FINDINGS — ranked evidence from real failed runs (the capabilities the server must provide):', ...findingLines(args.findings), ].join('\n') } diff --git a/src/improvement/driver-loop-generator.test.ts b/src/improvement/driver-loop-generator.test.ts new file mode 100644 index 00000000..08edac20 --- /dev/null +++ b/src/improvement/driver-loop-generator.test.ts @@ -0,0 +1,220 @@ +/** + * `driverLoopGenerator` proof — the driver→worker atom over a scripted brain. + * + * All process seams are injected (brain, harness runner, git readers, verifier) + * so the test proves the LOOP, deterministically and offline: + * 1. the driver's authored instruction reaches the worker verbatim (the atom: + * an LLM authors the goal, not a canned template), + * 2. verifier feedback flows back to the driver, which REFINES with a second + * authored session, and the candidate lands once the check passes, + * 3. the completion oracle is code-owned: a driver that claims success over a + * clean tree (or a failing verifier) produces `applied:false`, + * 4. the worker-session budget fails closed with an explanatory tool result, + * 5. without a verifier the legacy contract holds (dirty tree = candidate). + */ + +import type { AnalystFinding } from '@tangle-network/agent-eval' +import { describe, expect, it } from 'vitest' +import type { LocalHarnessResult, RunLocalHarnessOptions } from '../mcp/local-harness' +import type { ToolLoopChat } from '../runtime/tool-loop' +import type { VerifyResult } from './agentic-generator' +import { driverLoopGenerator } from './driver-loop-generator' + +const finding = (claim: string): AnalystFinding => + ({ + schema_version: '1.0.0', + finding_id: `f-${claim}`, + analyst_id: 'test-analyst', + produced_at: new Date(0).toISOString(), + severity: 'high', + area: 'tool-use', + claim, + evidence_refs: [], + confidence: 0.9, + }) as unknown as AnalystFinding + +/** One scripted driver turn: the tool calls the "LLM" emits (arguments as JSON strings). */ +type ScriptedTurn = { calls?: Array<{ name: string; args: Record }>; say?: string } + +/** A `ToolLoopChat` that replays a fixed script and records every conversation it saw. */ +function scriptedBrain(turns: ScriptedTurn[]) { + const seen: Array>> = [] + let i = 0 + const chat: ToolLoopChat = async (messages) => { + seen.push(messages.map((m) => ({ ...m }))) + const turn = turns[i] ?? {} + i += 1 + return { + content: turn.say ?? '', + toolCalls: (turn.calls ?? []).map((c, j) => ({ + id: `call-${i}-${j}`, + name: c.name, + arguments: JSON.stringify(c.args), + })), + } + } + return { chat, seen } +} + +function harnessStub(onRun?: (opts: RunLocalHarnessOptions) => void) { + const prompts: string[] = [] + const run = async (opts: RunLocalHarnessOptions): Promise => { + prompts.push(opts.taskPrompt) + onRun?.(opts) + return { + exitCode: 0, + stdout: 'worker done', + stderr: '', + killedBySignal: null, + durationMs: 10, + timedOut: false, + } + } + return { run, prompts } +} + +const generateArgs = (findings: AnalystFinding[], maxShots: number) => ({ + worktreePath: '/wt/cand0', + report: undefined, + findings, + maxShots, + signal: new AbortController().signal, +}) + +describe('driverLoopGenerator — the driver→worker build atom', () => { + it('authors the worker instruction, refines on verifier feedback, and lands the candidate', async () => { + const findings = [finding('the agent lacks a JSON schema validator')] + // Worktree turns dirty after the first worker session. + let sessions = 0 + const worker = harnessStub(() => { + sessions += 1 + }) + // Verifier: red after session 1, green after session 2 (and for the final gate). + const verdicts: VerifyResult[] = [] + const verify = (): VerifyResult => { + const v: VerifyResult = + sessions < 2 + ? { ok: false, feedback: 'tests failed: missing export validateJson' } + : { ok: true } + verdicts.push(v) + return v + } + const { chat, seen } = scriptedBrain([ + { + calls: [ + { + name: 'run_worker', + args: { + instruction: + 'Build src/validate.ts exporting validateJson(schema, value) with tests in validate.test.ts; run pnpm test until green.', + }, + }, + ], + }, + { calls: [{ name: 'run_verifier', args: {} }] }, + { + calls: [ + { + name: 'run_worker', + args: { + instruction: + 'Continue from the current tree; pnpm test fails because validateJson is not exported — export it from src/validate.ts and re-run the tests.', + }, + }, + ], + }, + { calls: [{ name: 'run_verifier', args: {} }] }, + { + say: 'Predicted the missing-validator gap; verifier green after one refine; next I would tighten schema errors.', + }, + ]) + const generator = driverLoopGenerator({ + brain: chat, + runHarness: worker.run, + verify, + changedPaths: () => (sessions > 0 ? ['src/validate.ts', 'src/validate.test.ts'] : []), + readDiff: () => '+ export function validateJson()', + }) + expect(generator.kind).toBe('driver-loop:claude') + + const result = await generator.generate(generateArgs(findings, 3)) + + expect(result.applied).toBe(true) + expect(result.summary).toContain('JSON schema validator') + // The atom: BOTH worker goals are the driver's own authored text, refine included. + expect(worker.prompts).toEqual([ + 'Build src/validate.ts exporting validateJson(schema, value) with tests in validate.test.ts; run pnpm test until green.', + 'Continue from the current tree; pnpm test fails because validateJson is not exported — export it from src/validate.ts and re-run the tests.', + ]) + // Verifier ran for the driver twice + once more as the code-owned final gate. + expect(verdicts.map((v) => v.ok)).toEqual([false, true, true]) + // The red verdict's feedback reached the driver as a tool message (what it refined from). + const flat = seen.flat() + expect( + flat.some( + (m) => m.role === 'tool' && String(m.content).includes('missing export validateJson'), + ), + ).toBe(true) + }) + + it('fails closed: a driver claiming success over a clean tree is not a candidate', async () => { + let verifyCalls = 0 + const { chat } = scriptedBrain([{ say: 'All done, the tool is perfect.' }]) + const generator = driverLoopGenerator({ + brain: chat, + runHarness: harnessStub().run, + verify: () => { + verifyCalls += 1 + return { ok: true } + }, + changedPaths: () => [], + readDiff: () => '', + }) + const result = await generator.generate(generateArgs([finding('gap')], 2)) + expect(result.applied).toBe(false) + // Clean tree short-circuits before the verifier — the claim never even reached it. + expect(verifyCalls).toBe(0) + }) + + it('fails closed: a dirty tree with a red verifier is discarded regardless of driver prose', async () => { + const { chat } = scriptedBrain([ + { calls: [{ name: 'run_worker', args: { instruction: 'do the change' } }] }, + { say: 'Success! Everything works.' }, + ]) + const generator = driverLoopGenerator({ + brain: chat, + runHarness: harnessStub().run, + verify: () => ({ ok: false, feedback: 'boot-and-probe: initialize handshake timed out' }), + changedPaths: () => ['server.mjs'], + readDiff: () => '+ server', + }) + const result = await generator.generate(generateArgs([finding('gap')], 1)) + expect(result.applied).toBe(false) + expect(result.summary).toBe('') + }) + + it('caps worker sessions at maxShots and tells the driver, without spawning', async () => { + const worker = harnessStub() + const { chat, seen } = scriptedBrain([ + { calls: [{ name: 'run_worker', args: { instruction: 'first session' } }] }, + { calls: [{ name: 'run_worker', args: { instruction: 'second session (over budget)' } }] }, + { say: 'stopping' }, + ]) + const generator = driverLoopGenerator({ + brain: chat, + runHarness: worker.run, + changedPaths: () => ['a.ts'], + readDiff: () => '+a', + }) + const result = await generator.generate(generateArgs([finding('gap')], 1)) + expect(worker.prompts).toEqual(['first session']) + const flat = seen.flat() + expect( + flat.some( + (m) => m.role === 'tool' && String(m.content).includes('worker-session budget exhausted'), + ), + ).toBe(true) + // No verifier configured: dirty tree IS the candidate (legacy contract). + expect(result.applied).toBe(true) + }) +}) diff --git a/src/improvement/driver-loop-generator.ts b/src/improvement/driver-loop-generator.ts new file mode 100644 index 00000000..eae58010 --- /dev/null +++ b/src/improvement/driver-loop-generator.ts @@ -0,0 +1,295 @@ +/** + * `driverLoopGenerator` — the driver→worker `CandidateGenerator`: the build + * loop run by the ATOM instead of the canned respawn. + * + * `agenticGenerator` steers with three hardcoded conditions picking a canned + * note (`EMPTY_TREE_NOTE` / `failureNote`) and respawns. This generator swaps + * that respawn brain for a real driver: an LLM on the canonical tool-loop seam + * (`runBrainLoop` + `ToolLoopChat` — the exact loop `driverAgent` runs its + * brain on) that AUTHORS each worker instruction, OBSERVES what the session + * actually produced (diff, files, verifier output), RATES it, and DECIDES + * refine / re-scope / decompose — prompted with the senior scientific-method + * doctrine (`buildDriverSystem`). + * + * The worker stays the proven primitive: `runLocalHarness` in the candidate + * worktree, same as `agenticGenerator` — only the brain between sessions + * changes. The worktree machinery (`worktreeBuildCandidate`) and verifiers + * (`commandVerifier` / `mcpServeVerifier`) are reused verbatim. + * + * Completion-oracle invariant (the supervisor doctrine, kept): the driver's + * prose NEVER decides the outcome. After the loop, code re-checks ground + * truth — tree dirty, raw-trace evidence present, verifier green — and only + * that decides `applied`. A driver that claims success over a failing verifier + * produces a discarded candidate, not a shipped one. + * + * @experimental + */ + +import { spawnSync } from 'node:child_process' +import { readFileSync, statSync } from 'node:fs' +import { resolve, sep } from 'node:path' +import type { AnalystFinding } from '@tangle-network/agent-eval' +import { type LocalHarness, runLocalHarness } from '../mcp/local-harness' +import { runBrainLoop, type ToolLoopChat } from '../runtime/tool-loop' +import { + defaultBuildPrompt, + rawTraceEvidenceProblem, + requiresRawTraceEvidence, + summarizeFindings, + type Verifier, + worktreeChangedPaths, +} from './agentic-generator' +import type { CandidateGenerator } from './improvement-driver' +import { buildDriverSystem } from './optimizer-prompt' + +export interface DriverLoopGeneratorOptions { + /** The driver-LLM seam — ONE inference turn over the conversation + tool specs (the canonical + * `ToolLoopChat`, same seam as `driverAgent`): `routerBrain(cfg)` in production, a scripted + * mock in tests. */ + brain: ToolLoopChat + /** Local coding harness the driver's worker sessions run in the worktree. Default `claude`. */ + harness?: LocalHarness + /** Per-worker-session wall-clock timeout (ms). Default = `runLocalHarness` default (5m). */ + timeoutMs?: number + /** Build the driver's task briefing (domain framing + method + findings) — the same senior + * prompt the worker path uses (`toolBuildPrompt` / `mcpBuildPrompt`). The driver reads it and + * folds what each worker needs into its instruction. Default `defaultBuildPrompt`. */ + buildPrompt?: (args: { report: unknown; findings: AnalystFinding[] }) => string + /** Verify the worktree (the intrinsic check). Exposed to the driver as `run_verifier` AND + * re-run by code as the final keep/discard gate. Omitted ⇒ the final gate is dirty-tree only + * (legacy `agenticGenerator` behavior sans verifier). */ + verify?: Verifier + /** Max driver inference turns. Default `max(8, 2 + maxShots * 3)` — room for one + * observe/rate/decide cycle per worker session plus orientation. */ + maxTurns?: number + /** Test seam — inject the harness runner (defaults to `runLocalHarness`). */ + runHarness?: typeof runLocalHarness + /** Test seam — inject the worktree diff reader (defaults to `git diff` in the worktree). */ + readDiff?: (worktreePath: string) => string + /** Test seam — inject the changed-paths reader (defaults to `git status --porcelain`). */ + changedPaths?: (worktreePath: string) => string[] +} + +const workerOutputTailChars = 2_000 +const diffMaxChars = 6_000 +const readFileDefaultBytes = 8_192 + +/** Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budget is spent. */ +export function driverLoopGenerator(opts: DriverLoopGeneratorOptions): CandidateGenerator { + const harness = opts.harness ?? 'claude' + const buildPrompt = opts.buildPrompt ?? defaultBuildPrompt + const run = opts.runHarness ?? runLocalHarness + const changed = opts.changedPaths ?? worktreeChangedPaths + const readDiff = opts.readDiff ?? worktreeDiff + const verify = opts.verify + + return { + kind: `driver-loop:${harness}`, + async generate({ worktreePath, report, findings, maxShots, signal }) { + const briefing = buildPrompt({ report, findings }) + const needsRawTraceEvidence = requiresRawTraceEvidence(findings) + const sessionCap = Math.max(1, maxShots) + let sessionsUsed = 0 + + // Ground-truth verification shared by the driver's `run_verifier` tool and the final + // code-owned gate — ONE definition of "delivered" so the driver can never see a different + // check than the one that decides the keep. + const groundVerify = async (): Promise<{ ok: boolean; feedback?: string }> => { + if (changed(worktreePath).length === 0) { + return { ok: false, feedback: 'the working tree has no changes — nothing to verify' } + } + if (needsRawTraceEvidence) { + const problem = rawTraceEvidenceProblem(worktreePath, findings) + if (problem) return { ok: false, feedback: problem } + } + if (!verify) { + return { ok: true, feedback: 'no verifier configured: a dirty tree is the candidate' } + } + return await verify(worktreePath) + } + + const execute = async (name: string, args: Record): Promise => { + switch (name) { + case 'run_worker': { + const instruction = typeof args.instruction === 'string' ? args.instruction.trim() : '' + if (instruction.length === 0) { + return 'error: run_worker requires a non-empty `instruction`' + } + if (sessionsUsed >= sessionCap) { + return `error: worker-session budget exhausted (${sessionsUsed}/${sessionCap} used). Inspect and verify what exists, then stop with your final assessment.` + } + sessionsUsed += 1 + const result = await run({ + harness, + cwd: worktreePath, + taskPrompt: instruction, + ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), + signal, + }) + return JSON.stringify({ + session: `${sessionsUsed}/${sessionCap}`, + exitCode: result.exitCode, + timedOut: result.timedOut, + killedBySignal: result.killedBySignal, + durationMs: result.durationMs, + changedPaths: changed(worktreePath), + stdoutTail: tail(result.stdout, workerOutputTailChars), + stderrTail: tail(result.stderr, workerOutputTailChars), + }) + } + case 'inspect_worktree': { + const paths = changed(worktreePath) + const diff = truncate(readDiff(worktreePath), diffMaxChars) + return JSON.stringify({ + changedPaths: paths, + diff: + diff.length > 0 + ? diff + : '(no tracked-file diff — new files are untracked; read_file them)', + }) + } + case 'read_file': + return readWorktreeFile(worktreePath, args) + case 'run_verifier': { + const result = await groundVerify() + return JSON.stringify({ + ok: result.ok, + feedback: truncate(result.feedback ?? '', 4_000), + }) + } + default: + return `error: unknown tool: ${name}` + } + } + + await runBrainLoop({ + chat: opts.brain, + tools: driverToolSpecs, + execute, + initialMessages: [ + { role: 'system', content: buildDriverSystem }, + { + role: 'user', + content: [ + `THE BUILD BRIEF (the contract your workers must satisfy — fold what each needs into its instruction; workers never see this brief):`, + '', + briefing, + '', + `Worker-session budget: ${sessionCap}. The worktree is a fresh checkout at ${worktreePath}.`, + ].join('\n'), + }, + ], + maxTurns: opts.maxTurns ?? Math.max(8, 2 + sessionCap * 3), + hooks: { stopBefore: () => signal.aborted }, + }) + + // The completion oracle: the driver stopped (or ran out of turns) — ground truth decides. + const verdict = await groundVerify() + if (!verdict.ok) return { applied: false, summary: '' } + return { applied: true, summary: summarizeFindings(findings) } + }, + } +} + +const driverToolSpecs = [ + { + type: 'function' as const, + function: { + name: 'run_worker', + description: + 'Run ONE coding-harness session in the worktree with your instruction as its entire goal. The worktree persists between sessions. Sessions are capped — author each instruction richly (outcome, context, placement, the check it is held to).', + parameters: { + type: 'object', + properties: { + instruction: { + type: 'string', + description: 'The complete, self-contained goal for this worker session.', + }, + }, + required: ['instruction'], + }, + }, + }, + { + type: 'function' as const, + function: { + name: 'inspect_worktree', + description: + 'Current git state of the worktree: changed paths + the tracked-file diff (truncated). New untracked files show in changedPaths only — read_file them.', + parameters: { type: 'object', properties: {} }, + }, + }, + { + type: 'function' as const, + function: { + name: 'read_file', + description: 'Read one file from the worktree (paths are worktree-relative).', + parameters: { + type: 'object', + properties: { + path: { type: 'string', description: 'Worktree-relative file path.' }, + maxBytes: { type: 'number', description: 'Byte cap (default 8192).' }, + }, + required: ['path'], + }, + }, + }, + { + type: 'function' as const, + function: { + name: 'run_verifier', + description: + 'Run the intrinsic check of the surface (compile+tests / boot-and-probe). Its result — not your judgment — decides whether the candidate is kept.', + parameters: { type: 'object', properties: {} }, + }, + }, +] + +/** `git diff` over the worktree (tracked files). Fails loud like `worktreeChangedPaths` — a git + * fault on a fresh checkout is a broken setup, not an empty diff. */ +function worktreeDiff(worktreePath: string): string { + const result = spawnSync('git', ['diff'], { cwd: worktreePath, encoding: 'utf-8' }) + if (result.error) { + throw new Error( + `driverLoopGenerator: git diff failed to spawn in ${worktreePath}: ${result.error.message}`, + ) + } + if (result.status !== 0) { + throw new Error( + `driverLoopGenerator: git diff exited ${result.status} in ${worktreePath}: ${result.stderr.trim()}`, + ) + } + return result.stdout +} + +/** Bounded, worktree-jailed file read for the driver's `read_file`. A path escaping the worktree + * is refused (the driver only rates work in the candidate tree; it has no business elsewhere). */ +function readWorktreeFile(worktreePath: string, args: Record): string { + const rel = typeof args.path === 'string' ? args.path : '' + if (rel.length === 0) return 'error: read_file requires `path`' + const root = resolve(worktreePath) + const target = resolve(root, rel) + if (target !== root && !target.startsWith(root + sep)) { + return `error: path escapes the worktree: ${rel}` + } + const maxBytes = + typeof args.maxBytes === 'number' && args.maxBytes > 0 + ? Math.min(args.maxBytes, 65_536) + : readFileDefaultBytes + try { + const size = statSync(target).size + const body = readFileSync(target, 'utf-8').slice(0, maxBytes) + return size > maxBytes ? `${body}\n… (${size - maxBytes} bytes truncated)` : body + } catch (e) { + return `error: ${e instanceof Error ? e.message : String(e)}` + } +} + +function tail(s: string, n: number): string { + const trimmed = s.trim() + return trimmed.length <= n ? trimmed : `…${trimmed.slice(-n)}` +} + +function truncate(s: string, n: number): string { + return s.length <= n ? s : `${s.slice(0, n - 1)}…` +} diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 321cef8d..770e57e0 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -9,18 +9,25 @@ * cohorting. This module supplies only the one genuinely runtime-specific piece: * a CODE-surface `SurfaceProposer` you pass to `selfImprove` as `proposer`, which * mutates a git worktree via a pluggable `CandidateGenerator`: - * - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches + * - `reflectiveGenerator` — cheap, no sandbox, applies pre-drafted patches * - `agenticGenerator` — full coding harness in the worktree, multi-shot + * - `driverLoopGenerator` — the driver→worker atom: an LLM driver authors, + * observes, rates, and steers the harness sessions (default for tool/mcp) */ export { type AgenticGeneratorOptions, agenticGenerator, commandVerifier, + defaultBuildPrompt, type Verifier, type VerifyResult, } from './agentic-generator' -export { mcpBuildPrompt, toolBuildPrompt } from './build-prompts' +export { findingLines, mcpBuildPrompt, toolBuildPrompt } from './build-prompts' +export { + type DriverLoopGeneratorOptions, + driverLoopGenerator, +} from './driver-loop-generator' export { type ImproveOptions, type ImproveResult, @@ -33,6 +40,7 @@ export { improvementDriver, } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' +export { buildDriverSystem, optimizerMethod, strategyAuthorMethod } from './optimizer-prompt' export { type RawTraceDistillerOptions, rawTraceDistiller, diff --git a/src/improvement/optimizer-prompt.ts b/src/improvement/optimizer-prompt.ts new file mode 100644 index 00000000..33e04fb7 --- /dev/null +++ b/src/improvement/optimizer-prompt.ts @@ -0,0 +1,123 @@ +/** + * The senior scientific-method optimizer doctrine — the ONE substantial prompt + * core shared by every builder/author surface (tool build, MCP build, codebase + * improvement, strategy authoring) and by the driver that steers build workers + * (`driverLoopGenerator`). + * + * Seeded from the proven senior prompts rather than invented: GEPA's + * `REFLECTION_SYSTEM` (localize → diagnose → minimal generalizable fix → + * preserve what works), the /evolve loop (one hypothesis with a mechanism and a + * falsifiable prediction; attack the largest measured gap first), /pursue (one + * coherent change set, no partial scaffolding), and the self-improving-loop / + * supervisor doctrine (a keep is decided by a real check, never by the author; + * observe → rate → decide). Generalized from "mutate a prompt string" to + * "build a code surface a held-out measurement will grade". + */ + +/** + * The shared method block every build/author prompt embeds. Domain framing + * (what a tool/MCP/codebase-edit deliverable looks like) wraps around it; this + * is the process itself. + */ +export const optimizerMethod = [ + 'THE METHOD — you are a senior engineer-scientist improving a measured system, not a code', + 'generator. Your change is an experiment: it exists to move a real, externally graded number,', + 'and it will be measured against a baseline on held-out tasks you cannot see. Work in this order:', + '', + '1. DIAGNOSE FIRST. Read every finding before touching anything — findings are ranked evidence', + ' from real failed runs. Name the DOMINANT failure mode (the single mechanism behind the', + ' largest share of failures) in one sentence. Attack that first; leave long-tail noise until', + ' the dominant mode is closed. A fix aimed at the wrong mechanism measures zero however clean', + ' the code is.', + '2. STATE A HYPOTHESIS WITH A PREDICTED LIFT. Before designing, write down: "failures like X', + ' happen because MECHANISM; this change interrupts that mechanism; I predict it addresses', + ' roughly N of the M findings shown." A change you cannot connect to a mechanism is a guess,', + ' not an experiment.', + '3. DECOMPOSE INTO SUB-GOALS. Break the work into steps that are each independently checkable', + ' (it compiles, a test passes, the server answers). Sequence them so the riskiest assumption', + ' is tested first — if the hypothesis is wrong, find out on step 1, not step 5.', + '4. DESIGN TO ISOLATE THE MECHANISM. Make the smallest COHERENT change that fully tests the', + ' hypothesis: small enough that a measured lift is attributable to this change alone, complete', + ' enough that it actually fires on the real execution path (a lever that exists but never', + ' fires measures zero). No drive-by refactors, no unrelated cleanup, no speculative scope —', + ' anything changed alongside confounds the measurement.', + '5. GENERALIZE, NEVER MEMORIZE. Fix the failure CLASS, not the shown instances: encode rules and', + ' logic that transfer to unseen tasks. A patch memorized to the quoted examples will not', + ' survive the held-out measurement — that is overfitting, and the gate will catch it.', + '6. PRESERVE WHAT WORKS. The baseline already passes tasks; do not delete or weaken the behavior', + ' those passes depend on. A fix that trades one failure class for a new one measures as noise.', + '7. VERIFY FOR REAL, THEN REFLECT. Run the verification you were given and make it genuinely', + ' pass — never weaken a check, stub the thing it exercises, or special-case its inputs; a', + ' gamed check delivers nothing because promotion is decided by a separate measurement you', + ' never see. Then record briefly: what you predicted, what the verifier actually showed, and', + ' what you would try next if the measured lift comes back null.', +].join('\n') + +/** + * The driver's stance for `driverLoopGenerator` — the build-domain instance of + * the supervisor doctrine (observe → rate → decide; refine / re-scope / + * decompose; the check decides delivery, never the driver's prose). + */ +export const buildDriverSystem = [ + 'You are the DRIVER of a build loop: a senior engineering lead steering a coding WORKER inside', + 'an isolated git worktree toward a verified artifact. You never edit files yourself — your only', + 'levers are the tools below. Your intelligence goes into three places: how you AUTHOR each', + 'worker instruction, how you OBSERVE and RATE what a session actually produced, and what you', + 'DECIDE next.', + '', + 'TOOLS', + '- run_worker{instruction}: one full coding-harness session in the worktree, with your', + ' instruction as its entire goal. Sessions are expensive and capped — author each one well.', + ' The worktree PERSISTS between sessions: a later worker resumes on top of earlier edits.', + '- inspect_worktree{}: current git status + diff — what has actually changed so far.', + '- read_file{path,maxBytes?}: read one file from the worktree (new untracked files do not show', + ' in the diff — read them to rate the work).', + '- run_verifier{}: the intrinsic check of the surface (compile+tests for a tool, boot-and-probe', + ' for an MCP server). Its exit decides what counts as delivered — your opinion does not.', + '', + 'AUTHOR RICHLY. A worker handed a one-line label will flail. Each instruction must carry: the', + 'outcome in concrete terms; the hypothesis and sub-goal it serves; what already exists in the', + 'tree that it must build on, not duplicate; where the deliverable must land; and the exact check', + 'it will be held to. The worker sees NOTHING you were given (no findings, no method, no prior', + 'session context) unless you fold it into the instruction.', + '', + 'THE LOOP — every turn: observe, rate, decide.', + '- OBSERVE: after each session, inspect the worktree and run the verifier. Read WHAT failed and', + ' WHY — "it failed" alone tells you nothing.', + '- RATE: judge with a reason. Verified and complete → stop. Close, one correctable fault →', + ' REFINE: author "continue from the current tree; the check fails because X; fix X" (depth,', + ' not a fresh start — never let a worker revert its own near-miss). Empty tree, or it solved a', + ' different problem → RE-SCOPE: re-author narrower and more concrete; a second identical', + ' instruction fails identically. Too big for one session → DECOMPOSE: author the first', + ' self-contained slice, verify it, then author the next on top.', + '- DECIDE: exactly one move per turn, with the reason stated in one line.', + '', + 'STOP when the verifier passes — reply with no tool call and a short reflection: what you', + 'predicted, what the verifier showed, what you would try next if the measured lift comes back', + 'null. If the session budget runs out first, say plainly what remains and why. An honest', + 'no-winner is a real result; a claimed success is not — the final keep/discard decision is made', + 'by code from the verifier exit and the tree state, never from your words.', +].join('\n') + +/** + * The senior authoring process for `authorStrategy` — the same method, shaped + * to the strategy contract (author-blind, conserved budget, one module out). + */ +export const strategyAuthorMethod = [ + 'Work as a senior researcher, in this order:', + '1. DIAGNOSE: read the per-task losses above and name the DOMINANT failure mode in one sentence', + ' — the single mechanism behind the largest share of lost score (e.g. first attempts near-miss', + ' and never get corrected; fresh retries discard progress; one persona plateaus).', + '2. HYPOTHESIS + PREDICTED LIFT: state "these losses happen because MECHANISM; the composition', + ' below interrupts it; I predict roughly +N on this environment at the same budget."', + '3. DESIGN TO ISOLATE THE MECHANISM: change ONE coordination mechanism relative to the baselines', + ' (carry vs fresh, where the critique lands, a persona split, a tool restriction) so any', + ' measured lift is attributable to it. Do not stack three clever ideas — a tangled win teaches', + ' nothing and a tangled loss cannot be debugged.', + '4. DECOMPOSE THE BUDGET: plan how the shots divide across explore / attempt / critique / repair', + ' before writing code, and spend the whole budget — an early stop on a mid score is a loss.', + '5. GENERALIZE: the strategy runs on unseen tasks from this environment. Read tools via', + ' listTools(handle), never hardcode task specifics from the losses shown.', + '6. PREDICT, THEN REFLECT: put the hypothesis, the mechanism, and the predicted lift in a', + ' comment at the top of the module — the holdout verdict will be read against it.', +].join('\n') diff --git a/src/lifecycle/tool-build.ts b/src/lifecycle/tool-build.ts index ac4e5f70..e15e5af8 100644 --- a/src/lifecycle/tool-build.ts +++ b/src/lifecycle/tool-build.ts @@ -31,8 +31,10 @@ import { import { ValidationError } from '../errors' import { agenticGenerator, commandVerifier } from '../improvement/agentic-generator' import { mcpBuildPrompt, toolBuildPrompt } from '../improvement/build-prompts' +import { driverLoopGenerator } from '../improvement/driver-loop-generator' import { type McpServeSpec, mcpServeVerifier } from '../improvement/mcp-serve-verifier' import type { LocalHarness } from '../mcp/local-harness' +import type { ToolLoopChat } from '../runtime/tool-loop' import type { GenerateContext } from './generator' import type { BuildableKind, BuildCandidate, BuiltCandidate } from './tool-generator' @@ -61,6 +63,15 @@ export interface WorktreeBuildOptions { * `cwd` defaults to the candidate worktree. */ mcp?: McpServeSpec + /** + * The driver-LLM seam (the canonical `ToolLoopChat`, e.g. `routerBrain(cfg)`). + * When set — the default composition for tool/mcp builds — the build runs the + * driver→worker atom (`driverLoopGenerator`): a driver LLM authors each worker + * instruction, observes the session's diff + verifier output, rates it, and + * decides refine / re-scope / decompose. Unset ⇒ the plain multi-shot + * `agenticGenerator` (canned resume notes, no LLM driver) — the offline path. + */ + driver?: { brain: ToolLoopChat; maxTurns?: number } } /** @@ -78,15 +89,23 @@ export function worktreeBuildCandidate(opts: WorktreeBuildOptions): BuildCandida branchPrefix: `build-${opts.kind}`, }) - // The harness generator: surface-specific build-prompt + surface-specific - // verifier, everything else shared. Identical to the documented composition - // in build-prompts.ts — no per-kind wrapper, just the pieces wired by data. - const generator = agenticGenerator({ + // The build generator: surface-specific build-prompt + surface-specific + // verifier, everything else shared. With a driver brain configured the + // driver→worker atom steers the sessions (the default composition for + // tool/mcp); without one, the offline multi-shot respawn loop runs. + const shared = { harness, ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), buildPrompt: opts.kind === 'mcp' ? mcpBuildPrompt : toolBuildPrompt, verify: buildVerifier(opts), - }) + } + const generator = opts.driver + ? driverLoopGenerator({ + ...shared, + brain: opts.driver.brain, + ...(opts.driver.maxTurns !== undefined ? { maxTurns: opts.driver.maxTurns } : {}), + }) + : agenticGenerator(shared) return async ( ctx: GenerateContext, diff --git a/src/runtime/strategy-author.ts b/src/runtime/strategy-author.ts index 71e8a410..6bc10e4a 100644 --- a/src/runtime/strategy-author.ts +++ b/src/runtime/strategy-author.ts @@ -15,6 +15,7 @@ import { mkdirSync, writeFileSync } from 'node:fs' import { join } from 'node:path' import type { ChatClient } from '@tangle-network/agent-eval' +import { strategyAuthorMethod } from '../improvement/optimizer-prompt' import type { Strategy } from './strategy' /** The compressed consumable a skill carries: everything an author needs to emit a loop. */ @@ -155,11 +156,13 @@ async function requestAuthoredCode( { role: 'system', content: - 'You are a senior engineer authoring optimization strategies for agent loops. Output exactly one fenced ```ts code block and nothing else.', + 'You are a senior researcher authoring optimization strategies for agent loops: you read ' + + 'per-task losses like experimental data, form a mechanism-level hypothesis, and author the ' + + 'one composition that tests it. Output exactly one fenced ```ts code block and nothing else.', }, { role: 'user', - content: `${opts.contract ?? strategyAuthorContract}\n\nBASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}):\n${opts.lossesJson}\n\nAuthor ONE new strategy that you expect to beat the baselines on THIS environment at the same budget. Use the losses to target the observed failure mode. Output only the module code block.`, + content: `${opts.contract ?? strategyAuthorContract}\n\nBASELINE RESULTS on the "${opts.environmentName}" environment (budget=${opts.budget}) — the per-task losses are your gradient:\n${opts.lossesJson}\n\nAuthor ONE new strategy that you expect to beat the baselines on THIS environment at the same budget.\n${strategyAuthorMethod}\n\nOutput only the module code block.`, }, ], }, From 24cfced213ee5e6a12cbd8c82e80f8c84591550b Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 12 Jul 2026 23:26:41 -0600 Subject: [PATCH 40/44] feat(runtime): same-host stdio MCP sandbox client + wire buildable MCP into the scorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 of the self-improve loop: a worktree-BUILT MCP server (stdio, cwd on the host) was unreachable by every shipped backend — none spawned profile.mcp stdio servers same-host, so a built tool could never be scored live. - connectStdioMcp (runtime/stdio-mcp-client): the ONE persistent newline-JSON-RPC spawn+handshake, extracted from mcpServeVerifier's probe; the verifier is rebased on it so 'verified it serves' and 'served while scored' can never drift. Spawn faults stay McpSpawnFault (setup bug, thrown); crash/timeout stays a failed candidate with the stderr tail. - materializeLocalMcp: spawn every enabled stdio server in profile.mcp, namespace tools __ (provider-safe schemas via the shared sanitizeMcpToolSchema), fail-closed when a declared server cannot boot. - localSandboxClient + resolveSandboxClient backend:'local': the same-host pseudo-box — router-brain tool loop (runBrainLoop) with the profile's MCP tools live; profile arrives per-create on backend.profile; delete() kills the children. Event protocol matches inlineSandboxClient. - sweEvalRunner opts.materializeMcp: overlay the live MCP tools onto the driven surface (tools()/call()) for the eval's duration, close in finally. Default stays prompt-only. - bench self-improve-swe.mts SURFACE=mcp: buildableGenerator over worktreeBuildCandidate (driver-loop brain via routerBrain), ranked/gated by the SAME eval runner with materializeMcp on. Verified: tsc green (root+examples); vitest 1318 passed 0 failed; chain smoke mcpServeVerifier -> applyArtifact -> materializeLocalMcp round-trips a live cwd-bound tools/call. --- bench/src/self-improve-swe.mts | 138 +++++++-- docs/api/primitive-catalog.md | 12 +- src/improvement/mcp-serve-verifier.ts | 146 +++------ src/lifecycle/swe-eval-runner.ts | 209 ++++++++----- src/runtime/index.ts | 17 ++ src/runtime/local-sandbox-client.ts | 111 +++++++ src/runtime/mcp-environment.ts | 8 +- src/runtime/resolve-sandbox-client.test.ts | 24 ++ src/runtime/resolve-sandbox-client.ts | 19 +- src/runtime/stdio-mcp-client.test.ts | 105 +++++++ src/runtime/stdio-mcp-client.ts | 329 +++++++++++++++++++++ 11 files changed, 891 insertions(+), 227 deletions(-) create mode 100644 src/runtime/local-sandbox-client.ts create mode 100644 src/runtime/stdio-mcp-client.test.ts create mode 100644 src/runtime/stdio-mcp-client.ts diff --git a/bench/src/self-improve-swe.mts b/bench/src/self-improve-swe.mts index a146280a..d4820ba2 100644 --- a/bench/src/self-improve-swe.mts +++ b/bench/src/self-improve-swe.mts @@ -6,36 +6,56 @@ * `EvalRunner` for the SWE domain): it drives each pinned Verified instance * with the multi-turn atom (`runAgentic` + refine) under the profile's prompt, * captures the `git diff`, and grades it with the swebench Docker judge held - * OUTSIDE the agent. This driver wires it to `runLifecycle` with the shipped - * `promptGenerator` (one deterministic candidate instruction injected through - * its `refine` seam) and `thresholdPromotionGate(0)`, then prints the baseline - * composite, each candidate's measured delta, and the composed profile. + * OUTSIDE the agent. This driver wires it to `runLifecycle` with + * `thresholdPromotionGate(0)`, then prints the baseline composite, each + * candidate's measured delta, and the composed profile. + * + * SURFACE picks the candidate surface: + * - `prompt` (default): the shipped `promptGenerator` with one deterministic + * candidate instruction through its injected-refine seam. + * - `mcp` (Phase 3): `buildableGenerator` fans out `worktreeBuildCandidate` + * builds of a real MCP server (driver→worker atom via `routerBrain`), each + * verified by boot-and-probe (`mcpServeVerifier` inside the build), then + * ranked/gated by the SAME `sweEvalRunner` — whose `materializeMcp` path + * spawns the built server same-host so its tools are LIVE while scored. * * IN (env): HOLDOUT_IDS=, * WORKER_MODEL, MAX_TOKENS, ROUTER_BASE, TANGLE_API_KEY, - * INNER_TURNS, BUDGET, RUN_TOOL + * INNER_TURNS, BUDGET, RUN_TOOL, SURFACE=prompt|mcp + * SURFACE=mcp also reads: BUILD_REPO_ROOT (required), + * MCP_SERVE_COMMAND (required), MCP_SERVE_ARGS, BUILD_BASE_REF, + * BUILD_HARNESS, BUILD_FANOUT, BUILD_MAX_SHOTS, DRIVER_MODEL * OUT (fd1): baseline composite, per-candidate scoreDelta/promoted, the - * composed profile's prompt surface + * composed profile's surface * diagnostics go to STDERR. * * SMOKE=1 → import/wiring check only: the module graph (this file + - * swe-bench-env + lifecycle incl. sweEvalRunner) loaded; print READY on stderr - * and exit 0 WITHOUT a clone / model call / dataset read. + * swe-bench-env + lifecycle incl. sweEvalRunner + the same-host MCP client) + * loaded; print READY on stderr and exit 0 WITHOUT a clone / model call / + * dataset read. */ import type { AgentProfile } from '@tangle-network/agent-interface' import { + buildableGenerator, + type CandidateGenerator, composeProfile, + type EvalRunner, promptGenerator, runLifecycle, sweEvalRunner, thresholdPromotionGate, + worktreeBuildCandidate, + type WorktreeBuildOptions, } from '@tangle-network/agent-runtime/lifecycle' +import { routerBrain } from '@tangle-network/agent-runtime/loops' import { createSweBenchEnvironment, SWE_SEED_PROMPT, SWE_SEED_PROMPT_WITH_RUN } from './swe-bench-env' async function main(): Promise { const smoke = ['1', 'true', 'yes'].includes((process.env.SMOKE ?? '').toLowerCase()) if (smoke) { - console.error('SMOKE ok: self-improve-swe + swe-bench-env + lifecycle (sweEvalRunner) import graph loaded') + console.error( + 'SMOKE ok: self-improve-swe + swe-bench-env + lifecycle (sweEvalRunner, buildableGenerator, same-host MCP) import graph loaded', + ) return } @@ -49,6 +69,11 @@ async function main(): Promise { const maxTokens = Number(process.env.MAX_TOKENS ?? 12000) const budget = Number(process.env.BUDGET ?? 1) const enableRun = ['1', 'true', 'yes'].includes((process.env.RUN_TOOL ?? '').toLowerCase()) + const surfaceRaw = (process.env.SURFACE ?? 'prompt').toLowerCase() + if (surfaceRaw !== 'prompt' && surfaceRaw !== 'mcp') { + throw new Error(`SURFACE must be 'prompt' or 'mcp' (got '${surfaceRaw}')`) + } + const surface = surfaceRaw as 'prompt' | 'mcp' const { environment, adapter } = await createSweBenchEnvironment(ids.length, { ids, enableRun }) const pool = await adapter.loadTasks({ ids, split: 'test' }) @@ -67,26 +92,32 @@ async function main(): Promise { maxTokens, innerTurns, budget, + // SURFACE=mcp: the candidate profile carries a worktree-built stdio server; + // spawn it same-host so its tools are live while the profile is scored. + materializeMcp: surface === 'mcp', }) const baseline: AgentProfile = { name: 'swe-baseline', prompt: { systemPrompt: seedPrompt } } - // ONE deterministic candidate through the shipped generator's injected-refine - // seam — the smallest real population that exercises the whole loop. - const generator = promptGenerator({ - refine: () => [ - { - instruction: - 'Before your first edit, state a one-line root-cause hypothesis naming the exact file and ' + - 'function you believe is at fault; if a later read_file contradicts it, revise the hypothesis ' + - 'before editing further.', - label: 'root-cause-hypothesis-first', - rationale: - 'Weak workers patch the first plausible site they read; forcing a named hypothesis before the ' + - 'first edit targets the wrong-file/wrong-function failure mode.', - }, - ], - }) + const generator: CandidateGenerator = + surface === 'mcp' + ? mcpBuildGenerator(evalRunner, { routerBaseUrl, routerKey, model }) + : // ONE deterministic candidate through the shipped generator's injected-refine + // seam — the smallest real population that exercises the whole loop. + promptGenerator({ + refine: () => [ + { + instruction: + 'Before your first edit, state a one-line root-cause hypothesis naming the exact file and ' + + 'function you believe is at fault; if a later read_file contradicts it, revise the hypothesis ' + + 'before editing further.', + label: 'root-cause-hypothesis-first', + rationale: + 'Weak workers patch the first plausible site they read; forcing a named hypothesis before the ' + + 'first edit targets the wrong-file/wrong-function failure mode.', + }, + ], + }) const t0 = Date.now() const out = await runLifecycle({ @@ -110,14 +141,61 @@ async function main(): Promise { } console.log(`promoted=[${out.promoted.join(', ')}]`) - const composed = composeProfile(out.registry, baseline, { kind: 'prompt' }) - console.log( - `composed profile: systemPrompt=${(composed.prompt?.systemPrompt ?? '').length}b ` + - `instructions=${JSON.stringify(composed.prompt?.instructions ?? [])}`, - ) + const composed = composeProfile(out.registry, baseline, { kind: surface }) + if (surface === 'mcp') { + const servers = Object.entries(composed.mcp ?? {}).map( + ([key, s]) => `${key} (${s.transport} ${s.command ?? s.url ?? '?'} cwd=${s.cwd ?? '-'})`, + ) + console.log(`composed profile: mcp=[${servers.join('; ')}]`) + } else { + console.log( + `composed profile: systemPrompt=${(composed.prompt?.systemPrompt ?? '').length}b ` + + `instructions=${JSON.stringify(composed.prompt?.instructions ?? [])}`, + ) + } console.error(`[self-improve-swe] done in ${Math.round((Date.now() - t0) / 1000)}s`) } +/** The Phase-3 buildable surface: fan-out worktree builds of a real MCP server, + * driver→worker atom steering each build, boot-and-probe verification, then + * rank/gate on the injected SWE eval runner (materializeMcp path). */ +function mcpBuildGenerator( + evalRunner: EvalRunner, + router: { routerBaseUrl: string; routerKey: string; model: string }, +): CandidateGenerator { + const repoRoot = process.env.BUILD_REPO_ROOT + if (!repoRoot) { + throw new Error('SURFACE=mcp requires BUILD_REPO_ROOT (the checkout candidate worktrees fork from)') + } + const command = process.env.MCP_SERVE_COMMAND + if (!command) { + throw new Error( + 'SURFACE=mcp requires MCP_SERVE_COMMAND (boots the built server in its worktree, stdio transport)', + ) + } + const args = (process.env.MCP_SERVE_ARGS ?? '').split(' ').filter(Boolean) + return buildableGenerator({ + kind: 'mcp', + fanout: Number(process.env.BUILD_FANOUT ?? 2), + evalRunner, + buildCandidate: worktreeBuildCandidate({ + kind: 'mcp', + repoRoot, + baseRef: process.env.BUILD_BASE_REF ?? 'main', + harness: (process.env.BUILD_HARNESS ?? 'claude') as NonNullable, + maxShots: Number(process.env.BUILD_MAX_SHOTS ?? 3), + mcp: { command, ...(args.length ? { args } : {}) }, + driver: { + brain: routerBrain({ + routerBaseUrl: router.routerBaseUrl, + routerKey: router.routerKey, + model: process.env.DRIVER_MODEL ?? router.model, + }), + }, + }), + }) +} + main().catch((e) => { console.error(e instanceof Error ? (e.stack ?? e.message) : String(e)) process.exit(1) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 1f47c799..a64c6906 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -282,7 +282,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. ### Recursive atom + loop kernel (alias of ./runtime) -Import from `@tangle-network/agent-runtime/loops` — 456 exports. +Import from `@tangle-network/agent-runtime/loops` — 467 exports. | Symbol | Kind | Summary | |---|---|---| @@ -304,6 +304,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `completionAuthorizes` | function | Decide whether a `CompletionVerdict` may end the node under the policy: authority scales with the verdict's determinism, and probabilistic verdicts must clear `minConfidence`. | | `composeCheckSources` | function | Concatenate check sources (official first by convention — ordering does not affect | | `computeFindingId` | function | Compute the stable finding_id from the identity-defining fields. | +| `connectStdioMcp` | function | Spawn a stdio MCP server, complete the handshake, and return the LIVE connection. | | `contentAddress` | function | Mint the content-addressed `outRef` for a result artifact: `sha256:` over a | | `createAgentEnvironmentProviderRegistry` | function | Create a registry that resolves provider names to concrete provider instances. | | `createBudgetPool` | function | Create a conserved reservation pool from a root `Budget`. `now()` is injected so the | @@ -350,6 +351,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `inProcessSandboxClient` | function | Adapt a single `onPrompt(prompt, ctx)` callback into a `SandboxClient` for | | `jjWorkspace` | function | A jj-backed `Workspace` (Jujutsu, colocated with git for the durable remote). | | `leaderboard` | function | Aggregate a fleet of records into the ranked, multi-axis report. Pure — no IO, deterministic. | +| `localSandboxClient` | function | A `SandboxClient` that runs the worker same-host with the profile's stdio MCP servers live. | | `localShell` | function | Host-process `Shell`: run a command via `execFile`, resolving `{ stdout, stderr, code }` (never throws on non-zero exit). | | `loopCampaignDispatch` | function | Adapter for plain `runCampaign` scenarios. This is the runtime-side pair for | | `loopDispatch` | function | Adapter for `runProfileMatrix` (profile is an axis). Returns a | @@ -357,6 +359,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `makeFinding` | function | Convenience factory: produce a fully-formed AnalystFinding with the | | `mapSandboxEvent` | function | Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary, | | `mapSandboxToolEvent` | function | Project one `SandboxEvent` onto the `tool_call` / `tool_result` variants of | +| `materializeLocalMcp` | function | Spawn every enabled stdio server in `profile.mcp` as a same-host child and | | `modelAuthoredChecks` | function | Default authored-check source: one metered LLM call per task, before sampling, | | `naiveDriver` | function | `naiveDriver` — the no-signal steering control. | | `observe` | function | The third-person trace analyst: read a worker's trace and produce steer findings for the next attempt plus durable `learned` facts for the cross-run corpus. | @@ -399,6 +402,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `sandboxCheckRunner` | function | Default CheckRunner backend: pipes the check program into `python3` over the sandbox | | `sandboxClientAsProvider` | function | Adapt a `SandboxClient` into the shared `AgentEnvironmentProvider` contract. | | `sandboxSessionTraceSource` | function | The SANDBOX / fleet trace source: read a box session's message parts and decode the harness's tool | +| `sanitizeMcpToolSchema` | function | Coerce an MCP inputSchema to an OpenAI-tool-valid top-level object schema. | | `selectBestIndex` | function | Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero | | `selectChampion` | function | Search-side champion selection over a tournament report. | | `selectValidWinner` | function | The single content-free valid-only winner selector. Among the gated-VALID children only | @@ -438,6 +442,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `InMemoryCorpus` | class | In-memory `Corpus`. Keyed by record `id`; `append` validates the record, is idempotent on an | | `InMemoryResultBlobStore` | class | In-memory `ResultBlobStore`. Content-addressed: `put` verifies the supplied | | `InMemorySpawnJournal` | class | In-memory `SpawnJournal`. Appends are observed-committed only; the impl enforces | +| `McpSpawnFault` | class | A missing start binary / spawn fault: a SETUP bug, never a failed candidate. | | `SandboxInstance` | class | A sandbox instance with methods for interaction. | | `SandboxRunAbortError` | class | Thrown when a turn is aborted/timed-out mid-settle. Carries the events drained | | `Agent` | interface | One self-similar atom. A leaf is an `Agent` that never calls `scope.spawn`; a driver | @@ -509,6 +514,8 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `LeaderboardScenario` | interface | The campaign scenario a case is wrapped into: the case rides along so | | `LeaderboardScore` | interface | Structured per-case verdict a `score` function may return (a bare number is | | `LeaderboardSpec` | interface | The declarative leaderboard spec. `TArtifact` is the artifact channel the | +| `LocalMcpMaterialization` | interface | The live same-host materialization of a profile's `mcp` surface. | +| `LocalSandboxClientOptions` | interface | `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose | | `LoopCampaignDispatchOptions` | interface | Options for adapting plain agent-eval campaign scenarios into runtime `runLoop` cells. | | `LoopIterationDispatchPayload` | interface | Where the iteration's worker was placed. `sibling` = a fresh sandbox the | | `LoopLineageOptions` | interface | Opt-in box-lineage controls for `runLoop`. Default OFF — with both flags | @@ -564,6 +571,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `ShapeRegistry` | interface | The open shape registry — the extension point that makes a new loop-shape ONE file + one | | `ShotPersona` | interface | A role for one shot — multi-agent loops (researcher + engineer, a panel of k | | `Spend` | interface | Conserved spend, reconciled from the normalized `UsageEvent` stream. Tokens and usd | +| `StdioMcpServerSpec` | interface | Same-host stdio MCP: the ONE persistent newline-delimited JSON-RPC 2.0 | | `SteerContext` | interface | How a combinator's `act` consumes findings to steer — the SINGLE firewalled steer surface a | | `Strategy` | interface | A Strategy is HOW you spend the compute budget to beat the Environment's check — it | | `StrategyCtx` | interface | What a strategy body composes with: the artifact lifecycle, the budget, and the two steps. | @@ -635,7 +643,7 @@ Import from `@tangle-network/agent-runtime/loops` — 456 exports. | `WinnerStrategy` | type | Built-in valid-only winner strategies for `selectValidWinner` (selector≠judge): best gated-valid | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `McpEnvironmentOptions`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgenticOptions`, `AgenticRunResult`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `ArtifactHandle`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `DefinedLeaderboard`, `Driver`, `EventBus`, `EvolutionArchiveNode`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `Observation`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ResourceRequest`, `RouterChatResult`, `RouterChatToolsResult`, `RouterToolLoopResult`, `RunAgenticOptions`, `SandboxRun`, `ShotSpec`, `StdioMcpConnection`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SurfaceScore`, `ToolSpec`, `TraceSource`, `ValidationCtx`, `Validator`, `WaterfallCollector`, `WaterfallReport`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge diff --git a/src/improvement/mcp-serve-verifier.ts b/src/improvement/mcp-serve-verifier.ts index d23a36ae..f61eb1e5 100644 --- a/src/improvement/mcp-serve-verifier.ts +++ b/src/improvement/mcp-serve-verifier.ts @@ -6,21 +6,24 @@ * handshake: `initialize` → `notifications/initialized` → `tools/list`, and * asserts the server answers with at least `minTools` tools. * + * The spawn + handshake is the SHARED same-host stdio connection + * (`connectStdioMcp`) — the same code path that later serves the built server + * LIVE to a scored run (`materializeLocalMcp`), so "verified it serves" and + * "served while scored" can never drift apart. + * * Outcomes follow the `Verifier` contract: a server that fails to start, exits * early, errors the handshake, times out, or exposes no tools is a FAILED * candidate (`{ok:false}`, fed back into the next generation shot); a missing * start binary or spawn fault THROWS (a setup bug, never a silent fallback). - * - * Protocol matches the runtime's own stdio MCP server (src/mcp/server.ts): - * newline-delimited JSON-RPC 2.0, protocol version 2024-11-05. */ -import { spawn } from 'node:child_process' -import { createInterface } from 'node:readline' +import { + connectStdioMcp, + McpSpawnFault, + type StdioMcpConnection, +} from '../runtime/stdio-mcp-client' import type { Verifier, VerifyResult } from './agentic-generator' -const PROTOCOL_VERSION = '2024-11-05' - export interface McpServeSpec { /** Command that starts the built MCP server in the worktree (stdio transport). */ command: string @@ -33,117 +36,38 @@ export interface McpServeSpec { minTools?: number } -interface JsonRpcResponse { - jsonrpc?: string - id?: number | string | null - result?: unknown - error?: { code: number; message: string } -} - /** Build a `Verifier` that boots a generated MCP server over stdio and checks it exposes tools. */ export function mcpServeVerifier(spec: McpServeSpec): Verifier { - const timeoutMs = spec.timeoutMs ?? 30_000 const minTools = spec.minTools ?? 1 - return (worktreePath: string): Promise => - new Promise((resolve, reject) => { - const child = spawn(spec.command, spec.args ?? [], { + return async (worktreePath: string): Promise => { + let conn: StdioMcpConnection + try { + conn = await connectStdioMcp({ + command: spec.command, + ...(spec.args ? { args: spec.args } : {}), cwd: worktreePath, - stdio: ['pipe', 'pipe', 'pipe'], - env: { ...process.env, ...spec.env }, + ...(spec.env ? { env: spec.env } : {}), + ...(spec.timeoutMs !== undefined ? { timeoutMs: spec.timeoutMs } : {}), }) - - const stderr: string[] = [] - let settled = false - let nextId = 1 - const initId = nextId++ - let listId = -1 - - const settle = (fn: () => void) => { - if (settled) return - settled = true - clearTimeout(timer) - rl.close() - child.kill('SIGKILL') - fn() + } catch (err) { + if (err instanceof McpSpawnFault) { + throw new Error(`mcpServeVerifier: ${err.message}`) } - const withStderr = (msg: string) => - stderr.length > 0 ? `${msg}\nstderr:\n${stderr.join('').slice(-2000)}` : msg - const pass = () => settle(() => resolve({ ok: true })) - const failCandidate = (msg: string) => - settle(() => resolve({ ok: false, feedback: withStderr(msg) })) - const setupFault = (err: Error) => settle(() => reject(err)) - - const send = (msg: Record): boolean => { - try { - child.stdin.write(`${JSON.stringify(msg)}\n`) - return true - } catch (err) { - // EPIPE: the server died mid-handshake — a failed candidate, not a fault. - failCandidate(`writing to MCP server stdin failed: ${(err as Error).message}`) - return false + // Crash-on-boot / handshake error / timeout: a failed candidate, with the + // connection's stderr-tailed message as the feedback for the next shot. + return { ok: false, feedback: err instanceof Error ? err.message : String(err) } + } + try { + if (conn.tools.length < minTools) { + return { + ok: false, + feedback: `tools/list returned ${conn.tools.length} tool(s), need >= ${minTools}`, } } - - child.on('error', (err) => { - const code = (err as NodeJS.ErrnoException).code - setupFault( - code === 'ENOENT' - ? new Error( - `mcpServeVerifier: '${spec.command}' not found in PATH (setup bug, not a failed candidate)`, - ) - : new Error(`mcpServeVerifier: '${spec.command}' failed to spawn: ${err.message}`), - ) - }) - child.on('exit', (code, signal) => { - // An exit before the handshake completes is a failed candidate (the - // server crashed on boot); after we settle, our own SIGKILL fires here. - failCandidate(`MCP server exited (code ${code}, signal ${signal}) before serving`) - }) - child.stderr.on('data', (d) => stderr.push(String(d))) - - const rl = createInterface({ input: child.stdout }) - rl.on('line', (line) => { - let msg: JsonRpcResponse | undefined - try { - msg = JSON.parse(line) as JsonRpcResponse - } catch { - return // servers log to stdout too; skip non-JSON lines - } - if (!msg || typeof msg !== 'object') return - - if (msg.id === initId) { - if (msg.error) return failCandidate(`initialize errored: ${JSON.stringify(msg.error)}`) - if (!send({ jsonrpc: '2.0', method: 'notifications/initialized' })) return - listId = nextId++ - send({ jsonrpc: '2.0', id: listId, method: 'tools/list' }) - return - } - if (msg.id === listId) { - if (msg.error) return failCandidate(`tools/list errored: ${JSON.stringify(msg.error)}`) - const tools = (msg.result as { tools?: unknown[] } | undefined)?.tools - if (!Array.isArray(tools)) return failCandidate('tools/list result has no tools array') - if (tools.length < minTools) { - return failCandidate(`tools/list returned ${tools.length} tool(s), need >= ${minTools}`) - } - return pass() - } - }) - - const timer = setTimeout( - () => failCandidate(`MCP server did not complete the handshake within ${timeoutMs}ms`), - timeoutMs, - ) - - send({ - jsonrpc: '2.0', - id: initId, - method: 'initialize', - params: { - protocolVersion: PROTOCOL_VERSION, - capabilities: {}, - clientInfo: { name: 'agent-runtime-mcp-verify', version: '0' }, - }, - }) - }) + return { ok: true } + } finally { + await conn.close() + } + } } diff --git a/src/lifecycle/swe-eval-runner.ts b/src/lifecycle/swe-eval-runner.ts index bde7236c..8b15a7f8 100644 --- a/src/lifecycle/swe-eval-runner.ts +++ b/src/lifecycle/swe-eval-runner.ts @@ -23,10 +23,17 @@ * bench package (`createSweBenchEnvironment`, the swebench adapter), which * depends on this package — importing them here would invert the dependency. * - * Phase-1 scope: returns the scalar `composite` (mean resolved) + `costUsd` - * only — no per-task `RunRecord`s — so it pairs with `thresholdPromotionGate`, - * not `heldOutPromotionGate`. Tasks run sequentially on purpose: the patch - * capture closes over one mutable cell per drive. + * Phase-3 adds the `materializeMcp` path: when set and the profile declares + * `mcp` servers, each enabled stdio server is spawned SAME-HOST + * (`materializeLocalMcp`) for the duration of the eval and its tools are + * overlaid on the domain surface — so a worktree-BUILT MCP candidate is LIVE + * while the profile is scored, and its marginal lift is real. Default off: + * the prompt-only path spawns nothing. + * + * Phase-1 scope otherwise: returns the scalar `composite` (mean resolved) + + * `costUsd` only — no per-task `RunRecord`s — so it pairs with + * `thresholdPromotionGate`, not `heldOutPromotionGate`. Tasks run sequentially + * on purpose: the patch capture closes over one mutable cell per drive. */ import { execFile } from 'node:child_process' @@ -36,6 +43,8 @@ import { ValidationError } from '../errors' import { type AgenticSurface, type ArtifactHandle, + type LocalMcpMaterialization, + materializeLocalMcp, refine, runAgentic, type SurfaceScore, @@ -88,6 +97,13 @@ export interface SweEvalRunnerOptions { innerTurns?: number /** Max refine shots per task. Default 1. */ budget?: number + /** Same-host MCP materialization: when true and the profile declares `mcp` + * servers, spawn each enabled stdio server as a LOCAL child for the eval's + * duration and expose its tools (`__`) to the driven worker + * alongside the domain tools. Default false (prompt-only). A declared + * server that cannot boot THROWS — scoring it silently without its tools + * would fake the with/without ablation. */ + materializeMcp?: boolean } /** @@ -104,89 +120,122 @@ export function sweEvalRunner( return async (profile: AgentProfile, signal?: AbortSignal): Promise => { const systemPrompt = renderSystemPrompt(profile, opts.seedPrompt) - const rows: SweEvalTaskResult[] = [] - let costUsd = 0 - let judgeCalls = 0 - let judgeErrors = 0 - - for (const task of opts.tasks) { - if (signal?.aborted) throw new Error('sweEvalRunner: aborted') - - // Capture the LATEST non-empty diff from inside score() — the refine loop - // calls it before the surface closes and rms the checkout, and a later - // empty read must never clobber a real patch (the swe-emit-patch pattern). - let capturedPatch = '' - const proxy: AgenticSurface = { - ...opts.environment, - async score(_t, handle: ArtifactHandle): Promise { - try { - const d = await exec('git', ['-C', handle.id, 'diff'], { - maxBuffer: 40_000_000, - timeout: 60_000, - }) - if (d.stdout.trim()) capturedPatch = d.stdout - } catch { - /* workspace gone or git error → keep whatever we already captured */ - } - return { passes: capturedPatch.trim() ? 1 : 0, total: 1, errored: 0 } - }, - } + // Same-host materialization of the profile's MCP surface (opt-in): the + // spawned servers live across all tasks of THIS eval and die in finally. + const mcp = opts.materializeMcp ? await materializeLocalMcp(profile) : undefined + const environment = + mcp && mcp.tools.length > 0 ? withLocalMcpTools(opts.environment, mcp) : opts.environment + try { + return await driveAndGrade(environment, mcp, systemPrompt, opts, signal) + } finally { + await mcp?.close() + } + } +} - const run = await runAgentic({ - surface: proxy, - task: { id: task.id, systemPrompt, userPrompt: task.prompt, meta: { instanceId: task.id } }, - strategy: refine, - routerBaseUrl: opts.routerBaseUrl, - routerKey: opts.routerKey, - model: opts.model, - maxTokens: opts.maxTokens, - innerTurns: opts.innerTurns, - budget: opts.budget ?? 1, - }) - costUsd += run.usd - - let resolved = false - let judgeError: string | undefined - if (capturedPatch.trim()) { - judgeCalls += 1 +/** Drive every pinned instance under `environment` and grade the patches — + * the Phase-1 loop body, factored so the MCP materialization wraps it. */ +async function driveAndGrade( + environment: AgenticSurface, + mcp: LocalMcpMaterialization | undefined, + systemPrompt: string, + opts: SweEvalRunnerOptions, + signal?: AbortSignal, +): Promise { + const rows: SweEvalTaskResult[] = [] + let costUsd = 0 + let judgeCalls = 0 + let judgeErrors = 0 + + for (const task of opts.tasks) { + if (signal?.aborted) throw new Error('sweEvalRunner: aborted') + + // Capture the LATEST non-empty diff from inside score() — the refine loop + // calls it before the surface closes and rms the checkout, and a later + // empty read must never clobber a real patch (the swe-emit-patch pattern). + let capturedPatch = '' + const proxy: AgenticSurface = { + ...environment, + async score(_t, handle: ArtifactHandle): Promise { try { - resolved = (await opts.judge(task, capturedPatch)).resolved - } catch (e) { - judgeErrors += 1 - judgeError = e instanceof Error ? e.message : String(e) + const d = await exec('git', ['-C', handle.id, 'diff'], { + maxBuffer: 40_000_000, + timeout: 60_000, + }) + if (d.stdout.trim()) capturedPatch = d.stdout + } catch { + /* workspace gone or git error → keep whatever we already captured */ } - } - - rows.push({ - id: task.id, - resolved, - patchBytes: capturedPatch.length, - usd: run.usd, - shots: run.shots, - tokens: run.tokens, - ...(judgeError === undefined ? {} : { judgeError }), - }) - console.error( - `[swe-eval] ${task.id} resolved=${resolved} patch=${capturedPatch.length}b ` + - `shots=${run.shots} tok=in:${run.tokens.input}/out:${run.tokens.output} usd=${run.usd.toFixed(4)}` + - (judgeError ? ` judgeError=${judgeError.slice(0, 200)}` : ''), - ) + return { passes: capturedPatch.trim() ? 1 : 0, total: 1, errored: 0 } + }, } - // Every judge invocation failing is broken grading infrastructure (docker - // down, venv missing) — reporting composite=0 from it would fabricate a - // score, so fail loud instead. - if (judgeErrors > 0 && judgeErrors === judgeCalls) { - throw new Error( - `sweEvalRunner: all ${judgeErrors} judge invocation(s) failed — grading infra is down, refusing to report a fabricated composite`, - ) - } + const run = await runAgentic({ + surface: proxy, + task: { id: task.id, systemPrompt, userPrompt: task.prompt, meta: { instanceId: task.id } }, + strategy: refine, + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, + model: opts.model, + maxTokens: opts.maxTokens, + innerTurns: opts.innerTurns, + budget: opts.budget ?? 1, + }) + costUsd += run.usd - return { - composite: rows.filter((r) => r.resolved).length / rows.length, - costUsd, - details: { systemPrompt, rows }, + let resolved = false + let judgeError: string | undefined + if (capturedPatch.trim()) { + judgeCalls += 1 + try { + resolved = (await opts.judge(task, capturedPatch)).resolved + } catch (e) { + judgeErrors += 1 + judgeError = e instanceof Error ? e.message : String(e) + } } + + rows.push({ + id: task.id, + resolved, + patchBytes: capturedPatch.length, + usd: run.usd, + shots: run.shots, + tokens: run.tokens, + ...(judgeError === undefined ? {} : { judgeError }), + }) + console.error( + `[swe-eval] ${task.id} resolved=${resolved} patch=${capturedPatch.length}b ` + + `shots=${run.shots} tok=in:${run.tokens.input}/out:${run.tokens.output} usd=${run.usd.toFixed(4)}` + + (judgeError ? ` judgeError=${judgeError.slice(0, 200)}` : ''), + ) + } + + // Every judge invocation failing is broken grading infrastructure (docker + // down, venv missing) — reporting composite=0 from it would fabricate a + // score, so fail loud instead. + if (judgeErrors > 0 && judgeErrors === judgeCalls) { + throw new Error( + `sweEvalRunner: all ${judgeErrors} judge invocation(s) failed — grading infra is down, refusing to report a fabricated composite`, + ) + } + + return { + composite: rows.filter((r) => r.resolved).length / rows.length, + costUsd, + details: { systemPrompt, mcpTools: mcp?.tools.map((t) => t.function.name) ?? [], rows }, + } +} + +/** Overlay the same-host MCP tools onto the domain surface: `tools()` appends + * the namespaced MCP tools, `call()` routes owned names to the live stdio + * children — open/score/close pass through untouched. */ +function withLocalMcpTools(env: AgenticSurface, mcp: LocalMcpMaterialization): AgenticSurface { + return { + ...env, + tools: async (task, handle) => [...(await env.tools(task, handle)), ...mcp.tools], + call: (handle, name, args) => + mcp.owns(name) ? mcp.call(name, args) : env.call(handle, name, args), } } diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 6da24b4b..6c2bbd64 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -153,6 +153,10 @@ export { } from './in-process-sandbox-client' // The one pseudo-box adapter: any non-box Executor → a SandboxClient for runLoop. export { inlineSandboxClient } from './inline-sandbox-client' +// The same-host pseudo-box: a router-brain tool loop with the profile's stdio +// MCP servers spawned as LOCAL children — the one client that can reach an MCP +// server built into a host worktree. +export { type LocalSandboxClientOptions, localSandboxClient } from './local-sandbox-client' export { type LoopCampaignDispatchOptions, type LoopDispatchOptions, @@ -164,6 +168,7 @@ export { createMcpEnvironment, type McpEndpoint, type McpEnvironmentOptions, + sanitizeMcpToolSchema, } from './mcp-environment' // The third-person observer: a worker's trace → trace-grounded findings, an // operator report, and durable corpus facts for the next run (the closed loop). @@ -331,6 +336,18 @@ export { SandboxRunAbortError, type TurnResult, } from './sandbox-run' +// Same-host stdio MCP: the ONE spawn+handshake connection (shared by the serve +// verifier and the live consumers) + the profile.mcp materializer. +export { + connectStdioMcp, + type LocalMcpMaterialization, + type MaterializeLocalMcpOptions, + McpSpawnFault, + type McpToolDescriptor, + materializeLocalMcp, + type StdioMcpConnection, + type StdioMcpServerSpec, +} from './stdio-mcp-client' export { type ApplyContinuation, type DumbDriverOptions, diff --git a/src/runtime/local-sandbox-client.ts b/src/runtime/local-sandbox-client.ts new file mode 100644 index 00000000..c9339b5e --- /dev/null +++ b/src/runtime/local-sandbox-client.ts @@ -0,0 +1,111 @@ +/** + * `localSandboxClient` — the SAME-HOST pseudo-box: a `SandboxClient` whose + * `create()` MATERIALIZES the profile's stdio MCP servers as local child + * processes (`materializeLocalMcp`) and whose `streamPrompt` drives a real + * tool loop (`runBrainLoop` over the router brain) with those live tools. + * + * This is the backend between `inlineSandboxClient` (off-box, ZERO tools) and + * a real sandbox (in-box, remote): a locally-BUILT MCP server (its cwd is a + * host worktree) is unreachable from a remote box and invisible to the + * one-shot inline executors — here it is spawned next to the worker and its + * tools are live for the whole session. `delete()` kills the children. + * + * The profile arrives per-create on `options.backend.profile` — exactly what + * the kernel's `createSandboxForSpec` sets via `buildBackendOptions` — so the + * same client materializes a different candidate profile per box. The + * profile's prompt surface (systemPrompt + instructions) becomes the system + * message, so BOTH optimizable surfaces are live here. + * + * Event protocol matches `inlineSandboxClient`: one `llm_call` metering event + * + one terminal `result` event with finalText/tokenUsage/costUsd. + */ + +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' +import { routerBrain } from './router-client' +import { materializeLocalMcp } from './stdio-mcp-client' +import { runBrainLoop, type ToolLoopChat } from './tool-loop' +import type { SandboxClient } from './types' + +export interface LocalSandboxClientOptions { + /** The worker brain: router chat-completions with tool-calling. All three required. */ + router: { baseUrl: string; key: string; model: string } + /** Tool-loop turns per prompt. Default 8. */ + maxTurns?: number + /** Brain sampling temperature. Default: `routerBrain`'s (0.4). */ + temperature?: number + /** Fallback profile when `create(options)` carries none on `backend.profile`. */ + profile?: AgentProfile +} + +/** A `SandboxClient` that runs the worker same-host with the profile's stdio MCP servers live. */ +export function localSandboxClient(opts: LocalSandboxClientOptions): SandboxClient { + const maxTurns = opts.maxTurns ?? 8 + let seq = 0 + return { + async create(options?: CreateSandboxOptions): Promise { + const profile = + (options?.backend as { profile?: AgentProfile } | undefined)?.profile ?? opts.profile ?? {} + // Materialize NOW: a declared server that cannot boot fails the create, + // not the first prompt — matching the real sandbox backend, where a box + // whose MCP cannot start never comes up. + const mcp = await materializeLocalMcp(profile) + const brain: ToolLoopChat = routerBrain( + { + routerBaseUrl: opts.router.baseUrl, + routerKey: opts.router.key, + model: opts.router.model, + }, + opts.temperature !== undefined ? { temperature: opts.temperature } : {}, + ) + const system = [profile.prompt?.systemPrompt, ...(profile.prompt?.instructions ?? [])] + .filter((s): s is string => typeof s === 'string' && s.trim().length > 0) + .join('\n\n') + const id = `local-${seq++}` + return { + id, + async *streamPrompt( + message: string, + popts?: { signal?: AbortSignal }, + ): AsyncGenerator { + let costUsd = 0 + const chat: ToolLoopChat = async (messages, tools) => { + const r = await brain(messages, tools) + if (r.costUsd) costUsd += r.costUsd + return r + } + const r = await runBrainLoop({ + chat, + tools: mcp.tools, + execute: (name, args) => mcp.call(name, args), + initialMessages: [ + ...(system ? [{ role: 'system', content: system }] : []), + { role: 'user', content: message }, + ], + maxTurns, + hooks: { stopBefore: () => popts?.signal?.aborted === true }, + }) + // Speak the runtime's metering protocol (see inlineSandboxClient): + // a flat `llm_call` event so the kernel never meters a fabricated $0. + if (r.usage.input || r.usage.output || costUsd) { + yield { + type: 'llm_call', + data: { tokensIn: r.usage.input, tokensOut: r.usage.output, costUsd }, + } as unknown as SandboxEvent + } + yield { + type: 'result', + data: { + finalText: r.final, + tokenUsage: { inputTokens: r.usage.input, outputTokens: r.usage.output }, + costUsd, + }, + } as unknown as SandboxEvent + }, + async delete(): Promise { + await mcp.close() + }, + } as unknown as SandboxInstance + }, + } +} diff --git a/src/runtime/mcp-environment.ts b/src/runtime/mcp-environment.ts index 8fa4928e..b4018015 100644 --- a/src/runtime/mcp-environment.ts +++ b/src/runtime/mcp-environment.ts @@ -77,8 +77,10 @@ async function rpc( ) } -/** Coerce an MCP inputSchema to an OpenAI-tool-valid top-level object schema. */ -function sanitizeSchema(s: unknown): Record { +/** Coerce an MCP inputSchema to an OpenAI-tool-valid top-level object schema. + * Shared with the same-host stdio client (`materializeLocalMcp`) — one coercion + * rule for every MCP tool a worker sees, regardless of transport. */ +export function sanitizeMcpToolSchema(s: unknown): Record { const o = s && typeof s === 'object' ? (s as Record) : {} const banned = o.oneOf || o.anyOf || o.allOf || o.not || o.enum if (o.type === 'object' && !banned && o.properties && typeof o.properties === 'object') { @@ -128,7 +130,7 @@ export function createMcpEnvironment(opts: McpEnvironmentOptions): Environment { function: { name: t.name, description: (t.description ?? '').slice(0, 1000), - parameters: sanitizeSchema(t.inputSchema), + parameters: sanitizeMcpToolSchema(t.inputSchema), }, }), ) diff --git a/src/runtime/resolve-sandbox-client.test.ts b/src/runtime/resolve-sandbox-client.test.ts index cf88f6fd..0a6b5561 100644 --- a/src/runtime/resolve-sandbox-client.test.ts +++ b/src/runtime/resolve-sandbox-client.test.ts @@ -10,8 +10,13 @@ const inlineSandboxClient = vi.fn((factory: unknown): SandboxClient => { return { create: async () => ({}) as never, __factory: factory } as unknown as SandboxClient }) +const localSandboxClient = vi.fn((options: unknown): SandboxClient => { + return { create: async () => ({}) as never, __local: options } as unknown as SandboxClient +}) + vi.mock('./supervise/runtime', () => ({ createExecutor })) vi.mock('./inline-sandbox-client', () => ({ inlineSandboxClient })) +vi.mock('./local-sandbox-client', () => ({ localSandboxClient })) const { resolveSandboxClient } = await import('./resolve-sandbox-client') @@ -85,4 +90,23 @@ describe('resolveSandboxClient', () => { /router\.baseUrl, router\.key and router\.model/, ) }) + + it("backend 'local' wires the same-host client with the local options", () => { + const local = { router: { baseUrl: 'https://router.tangle.tools', key: 'sk-l', model: 'gem' } } + const client = resolveSandboxClient({ backend: 'local', local }) + expect(localSandboxClient).toHaveBeenCalledWith(local) + expect((client as unknown as { __local: unknown }).__local).toBe(local) + }) + + it("backend 'local' fails loud when a router field is missing", () => { + expect(() => resolveSandboxClient({ backend: 'local' })).toThrow( + /local\.router\.baseUrl, local\.router\.key and local\.router\.model/, + ) + expect(() => + resolveSandboxClient({ + backend: 'local', + local: { router: { baseUrl: 'x', key: '', model: 'm' } }, + }), + ).toThrow(/local\.router/) + }) }) diff --git a/src/runtime/resolve-sandbox-client.ts b/src/runtime/resolve-sandbox-client.ts index f3973ff3..f339a4c4 100644 --- a/src/runtime/resolve-sandbox-client.ts +++ b/src/runtime/resolve-sandbox-client.ts @@ -17,15 +17,20 @@ * through the resumable `bridgeExecutor`. * - `backend: 'router'` → OFF-BOX: a router chat-completion as the leaf executor, * presented as a `SandboxClient` (no sandbox dependency). + * - `backend: 'local'` → SAME-HOST: a router-brain tool loop with the + * profile's stdio MCP servers spawned as LOCAL child + * processes — the only backend that can reach an MCP + * server built into a host worktree. */ import { inlineSandboxClient } from './inline-sandbox-client' +import { type LocalSandboxClientOptions, localSandboxClient } from './local-sandbox-client' import { createExecutor } from './supervise/runtime' import type { SandboxClient } from './types' export interface ResolveSandboxClientOptions { /** The execution transport for the driven loop. */ - backend: 'sandbox' | 'bridge' | 'router' + backend: 'sandbox' | 'bridge' | 'router' | 'local' /** `sandbox` backend: the caller's real Sandbox-backed client. Required for that backend. */ sandboxClient?: SandboxClient /** `bridge` backend: local cli-bridge transport. `bearer` + `model` required. */ @@ -44,6 +49,9 @@ export interface ResolveSandboxClientOptions { key: string model: string } + /** `local` backend: same-host pseudo-box — the router brain drives a tool loop + * with the profile's stdio MCP servers spawned as local children. */ + local?: LocalSandboxClientOptions } /** @@ -92,5 +100,14 @@ export function resolveSandboxClient(opts: ResolveSandboxClientOptions): Sandbox }), ) } + case 'local': { + const local = opts.local + if (!local?.router?.baseUrl || !local.router.key || !local.router.model) { + throw new Error( + "resolveSandboxClient: backend 'local' requires local.router.baseUrl, local.router.key and local.router.model", + ) + } + return localSandboxClient(local) + } } } diff --git a/src/runtime/stdio-mcp-client.test.ts b/src/runtime/stdio-mcp-client.test.ts new file mode 100644 index 00000000..5653406d --- /dev/null +++ b/src/runtime/stdio-mcp-client.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest' +import { mcpServeVerifier } from '../improvement/mcp-serve-verifier' +import { connectStdioMcp, McpSpawnFault, materializeLocalMcp } from './stdio-mcp-client' + +/** A minimal stdio MCP server (newline JSON-RPC 2.0): initialize / tools/list / + * tools/call, one `hello` tool — the smallest real server the protocol allows. */ +const HELLO_SERVER = ` +const rl = require('node:readline').createInterface({ input: process.stdin }) +rl.on('line', (line) => { + let m + try { m = JSON.parse(line) } catch { return } + if (m.id === undefined) return + const reply = (result) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, result }) + '\\n') + if (m.method === 'initialize') { + reply({ protocolVersion: '2024-11-05', capabilities: {}, serverInfo: { name: 'hello', version: '0' } }) + } else if (m.method === 'tools/list') { + reply({ tools: [{ name: 'hello', description: 'say hello', inputSchema: { type: 'object', properties: { name: { type: 'string' } } } }] }) + } else if (m.method === 'tools/call') { + reply({ content: [{ type: 'text', text: 'hello ' + ((m.params && m.params.arguments && m.params.arguments.name) || 'world') }] }) + } +}) +` + +describe('connectStdioMcp', () => { + it('handshakes, lists tools, round-trips tools/call, closes', async () => { + const conn = await connectStdioMcp({ command: 'node', args: ['-e', HELLO_SERVER] }) + try { + expect(conn.tools.map((t) => t.name)).toEqual(['hello']) + await expect(conn.callTool('hello', { name: 'drew' })).resolves.toBe('hello drew') + } finally { + await conn.close() + } + }) + + it('throws McpSpawnFault for a missing binary (setup bug, not a failed candidate)', async () => { + await expect( + connectStdioMcp({ command: 'definitely-not-a-real-binary-xyz', timeoutMs: 5_000 }), + ).rejects.toBeInstanceOf(McpSpawnFault) + }) + + it('a server that exits before serving fails with a plain Error carrying stderr', async () => { + const err: unknown = await connectStdioMcp({ + command: 'node', + args: ['-e', 'process.stderr.write("boom"); process.exit(3)'], + timeoutMs: 10_000, + }).catch((e) => e) + expect(err).toBeInstanceOf(Error) + expect(err).not.toBeInstanceOf(McpSpawnFault) + expect((err as Error).message).toMatch(/exited/) + expect((err as Error).message).toMatch(/boom/) + }) +}) + +describe('materializeLocalMcp', () => { + it('spawns each enabled stdio server and namespaces its tools __', async () => { + const mat = await materializeLocalMcp({ + mcp: { + greeter: { transport: 'stdio', command: 'node', args: ['-e', HELLO_SERVER], enabled: true }, + off: { transport: 'stdio', command: 'node', args: ['-e', HELLO_SERVER], enabled: false }, + }, + }) + try { + expect(mat.tools.map((t) => t.function.name)).toEqual(['greeter__hello']) + expect(mat.owns('greeter__hello')).toBe(true) + expect(mat.owns('hello')).toBe(false) + await expect(mat.call('greeter__hello', { name: 'phase3' })).resolves.toBe('hello phase3') + } finally { + await mat.close() + } + }) + + it('fails closed on a non-stdio transport (the same-host client cannot fake a remote server)', async () => { + await expect( + materializeLocalMcp({ mcp: { remote: { transport: 'http', url: 'https://example.test' } } }), + ).rejects.toThrow(/transport 'http'/) + }) + + it('a profile with no MCP surface materializes zero tools and closes cleanly', async () => { + const mat = await materializeLocalMcp({}) + expect(mat.tools).toEqual([]) + await mat.close() + }) +}) + +describe('mcpServeVerifier (rebased on connectStdioMcp)', () => { + it('passes a server that boots and lists >= minTools tools', async () => { + const verify = mcpServeVerifier({ command: 'node', args: ['-e', HELLO_SERVER] }) + await expect(verify(process.cwd())).resolves.toEqual({ ok: true }) + }) + + it('fails a candidate that lists fewer tools than minTools', async () => { + const verify = mcpServeVerifier({ command: 'node', args: ['-e', HELLO_SERVER], minTools: 2 }) + const r = await verify(process.cwd()) + expect(r.ok).toBe(false) + expect(r.feedback).toMatch(/1 tool/) + }) + + it('throws on a missing start binary (setup bug, never a silent fallback)', async () => { + const verify = mcpServeVerifier({ + command: 'definitely-not-a-real-binary-xyz', + timeoutMs: 5_000, + }) + await expect(verify(process.cwd())).rejects.toThrow(/not found in PATH/) + }) +}) diff --git a/src/runtime/stdio-mcp-client.ts b/src/runtime/stdio-mcp-client.ts new file mode 100644 index 00000000..d2f9815f --- /dev/null +++ b/src/runtime/stdio-mcp-client.ts @@ -0,0 +1,329 @@ +/** + * Same-host stdio MCP: the ONE persistent newline-delimited JSON-RPC 2.0 + * connection to a spawned MCP server child process. This is the handshake + * `mcpServeVerifier` boots for its probe (`initialize` → + * `notifications/initialized` → `tools/list`), extracted so a LIVE consumer — + * the local sandbox client, the SWE eval runner's `materializeMcp` path — can + * keep the server running and route `tools/call` to it during a driven run. + * A locally-BUILT MCP server (its cwd is a host worktree) is unreachable from + * a remote box; spawning it here, next to the worker, is what makes a built + * tool scoreable at all. + * + * Two layers: + * - `connectStdioMcp` — spawn ONE server at its cwd, run the real MCP + * handshake, return a live connection: the listed tools, `callTool`, `close`. + * - `materializeLocalMcp` — spawn EVERY enabled stdio server in + * `profile.mcp`, namespace each server's tools as `__` so + * they can share a worker's tool list with a domain surface's tools, and + * expose one route/close facade over the set. Fail-CLOSED: a declared + * server that cannot boot throws — silently scoring the profile as if it + * had no MCP surface would fake the with/without ablation. + * + * Failure taxonomy (mirrors `commandVerifier`/`mcpServeVerifier`): a missing + * start binary or spawn fault is a SETUP bug (`McpSpawnFault` — candidate + * graders must rethrow it, never score it); a server that crashes, errors the + * handshake, or times out is an ordinary `Error` carrying the stderr tail; a + * JSON-RPC error on `tools/call` is the AGENT's outcome — returned as an + * `ERROR: …` string, never thrown (the `createMcpEnvironment` convention). + * + * Protocol matches the runtime's own stdio MCP server (src/mcp/server.ts): + * newline-delimited JSON-RPC 2.0, protocol version 2024-11-05. + */ + +import { spawn } from 'node:child_process' +import { createInterface } from 'node:readline' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { ValidationError } from '../errors' +import { sanitizeMcpToolSchema } from './mcp-environment' +import type { AgenticTool } from './strategy' + +const PROTOCOL_VERSION = '2024-11-05' + +export interface StdioMcpServerSpec { + /** Command that starts the MCP server (stdio transport). */ + command: string + args?: string[] + /** Working directory the server starts in (a built candidate's worktree, typically). */ + cwd?: string + /** Extra env for the server process (merged over `process.env`). */ + env?: Record + /** Handshake AND per-request timeout (ms). Default 30s. */ + timeoutMs?: number +} + +/** A missing start binary / spawn fault: a SETUP bug, never a failed candidate. + * Graders (the serve verifier) must rethrow this instead of scoring it. */ +export class McpSpawnFault extends Error {} + +export interface McpToolDescriptor { + name: string + description?: string + inputSchema?: unknown +} + +export interface StdioMcpConnection { + /** The tools the server exposed at connect time (`tools/list`). */ + readonly tools: readonly McpToolDescriptor[] + /** `tools/call` → the result's text content. A JSON-RPC error / `isError` + * result becomes an `ERROR: …` string (the agent's outcome); a dead + * transport or timeout throws (an infra fault). */ + callTool(name: string, args: Record): Promise + /** Kill the server child. Idempotent. */ + close(): Promise +} + +interface JsonRpcResponse { + jsonrpc?: string + id?: number | string | null + result?: unknown + error?: { code: number; message: string } +} + +/** Spawn a stdio MCP server, complete the handshake, and return the LIVE connection. */ +export async function connectStdioMcp(spec: StdioMcpServerSpec): Promise { + const timeoutMs = spec.timeoutMs ?? 30_000 + const child = spawn(spec.command, spec.args ?? [], { + ...(spec.cwd ? { cwd: spec.cwd } : {}), + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, ...spec.env }, + }) + + const stderr: string[] = [] + child.stderr.on('data', (d) => stderr.push(String(d))) + const stderrTail = () => (stderr.length > 0 ? `\nstderr:\n${stderr.join('').slice(-2000)}` : '') + // A write to a dying stdin surfaces as an async 'error' on the stream; the + // child's own 'error'/'close' events already carry the failure, so this + // listener only prevents an unhandled-'error' crash. + child.stdin.on('error', () => {}) + + let nextId = 1 + let spawnFault: McpSpawnFault | undefined + let closed = false + const pending = new Map< + number, + { resolve: (r: JsonRpcResponse) => void; reject: (e: Error) => void; timer: NodeJS.Timeout } + >() + + const failAllPending = (err: Error) => { + for (const p of pending.values()) { + clearTimeout(p.timer) + p.reject(err) + } + pending.clear() + } + + child.on('error', (err) => { + const code = (err as NodeJS.ErrnoException).code + spawnFault = + code === 'ENOENT' + ? new McpSpawnFault( + `'${spec.command}' not found in PATH (setup bug, not a failed candidate)`, + ) + : new McpSpawnFault(`'${spec.command}' failed to spawn: ${err.message}`) + failAllPending(spawnFault) + }) + // 'close' (not 'exit'): it fires after stdio flushes, so the stderr tail in + // the failure message is complete. After our own SIGKILL in close(), pending + // is already empty and this is a no-op. + child.on('close', (code, signal) => { + failAllPending( + spawnFault ?? + new Error( + `MCP server exited (code ${code}, signal ${signal}) before serving${stderrTail()}`, + ), + ) + }) + + const rl = createInterface({ input: child.stdout }) + rl.on('line', (line) => { + let msg: JsonRpcResponse | undefined + try { + msg = JSON.parse(line) as JsonRpcResponse + } catch { + return // servers log to stdout too; skip non-JSON lines + } + if (!msg || typeof msg !== 'object' || typeof msg.id !== 'number') return + const p = pending.get(msg.id) + if (!p) return + pending.delete(msg.id) + clearTimeout(p.timer) + p.resolve(msg) + }) + + const send = (msg: Record): void => { + child.stdin.write(`${JSON.stringify(msg)}\n`) + } + + const request = ( + method: string, + params?: Record, + // The handshake-phase timeout keeps the verifier's long-standing feedback + // wording; live tools/call timeouts name the method instead. + timeoutMessage = `MCP server did not answer '${method}' within ${timeoutMs}ms`, + ): Promise => + new Promise((resolve, reject) => { + if (spawnFault) return reject(spawnFault) + if (closed) return reject(new Error('MCP connection closed')) + const id = nextId++ + const timer = setTimeout(() => { + pending.delete(id) + reject(new Error(`${timeoutMessage}${stderrTail()}`)) + }, timeoutMs) + pending.set(id, { resolve, reject, timer }) + try { + send({ jsonrpc: '2.0', id, method, ...(params ? { params } : {}) }) + } catch (err) { + pending.delete(id) + clearTimeout(timer) + reject( + new Error( + `writing to MCP server stdin failed: ${err instanceof Error ? err.message : String(err)}${stderrTail()}`, + ), + ) + } + }) + + const close = async (): Promise => { + if (closed) return + closed = true + rl.close() + failAllPending(new Error('MCP connection closed')) + child.kill('SIGKILL') + } + + const handshakeTimeout = `MCP server did not complete the handshake within ${timeoutMs}ms` + try { + const init = await request( + 'initialize', + { + protocolVersion: PROTOCOL_VERSION, + capabilities: {}, + clientInfo: { name: 'agent-runtime-local-mcp', version: '0' }, + }, + handshakeTimeout, + ) + if (init.error) + throw new Error(`initialize errored: ${JSON.stringify(init.error)}${stderrTail()}`) + send({ jsonrpc: '2.0', method: 'notifications/initialized' }) + const list = await request('tools/list', undefined, handshakeTimeout) + if (list.error) + throw new Error(`tools/list errored: ${JSON.stringify(list.error)}${stderrTail()}`) + const tools = (list.result as { tools?: McpToolDescriptor[] } | undefined)?.tools + if (!Array.isArray(tools)) + throw new Error(`tools/list result has no tools array${stderrTail()}`) + + const callTool = async (name: string, args: Record): Promise => { + const res = await request('tools/call', { name, arguments: args }) + if (res.error) return `ERROR: ${JSON.stringify(res.error).slice(0, 300)}` + const result = res.result as + | { content?: Array<{ text?: string }>; isError?: boolean } + | undefined + const text = + result?.content?.map((c) => c.text ?? '').join('\n') ?? JSON.stringify(result ?? null) + return result?.isError ? `ERROR: ${text}` : text + } + + return { tools, callTool, close } + } catch (err) { + await close() + throw err + } +} + +export interface MaterializeLocalMcpOptions { + /** Handshake / per-request timeout per server (ms). Default 30s. */ + timeoutMs?: number + /** Cap on a tool result's text fed back to the worker. Default 2000 chars. */ + maxResultChars?: number +} + +/** The live same-host materialization of a profile's `mcp` surface. */ +export interface LocalMcpMaterialization { + /** Worker-facing tool specs: namespaced `__`, provider-safe schemas. */ + tools: AgenticTool[] + /** Whether `name` is one of this materialization's namespaced tools. */ + owns(name: string): boolean + /** Route a namespaced call to its server's live stdio child. */ + call(name: string, args: Record): Promise + /** Kill every spawned server. Idempotent. */ + close(): Promise +} + +/** + * Spawn every enabled stdio server in `profile.mcp` as a same-host child and + * expose their tools under `__` names. A profile with no MCP + * surface materializes zero tools (a valid, cheap no-op). + */ +export async function materializeLocalMcp( + profile: AgentProfile, + opts: MaterializeLocalMcpOptions = {}, +): Promise { + const maxResultChars = opts.maxResultChars ?? 2000 + const connections: StdioMcpConnection[] = [] + const routes = new Map() + const tools: AgenticTool[] = [] + + const close = async (): Promise => { + await Promise.all(connections.map((c) => c.close())) + } + + try { + for (const [key, server] of Object.entries(profile.mcp ?? {})) { + if (server.enabled === false) continue + const transport = server.transport ?? 'stdio' + if (transport !== 'stdio') { + throw new ValidationError( + `materializeLocalMcp: profile.mcp['${key}'] has transport '${transport}' — the same-host client only spawns stdio servers (remote servers materialize in the sandbox backend)`, + ) + } + if (!server.command || server.command.trim().length === 0) { + throw new ValidationError( + `materializeLocalMcp: profile.mcp['${key}'] declares a stdio server with no command`, + ) + } + const conn = await connectStdioMcp({ + command: server.command, + ...(server.args ? { args: server.args } : {}), + ...(server.cwd ? { cwd: server.cwd } : {}), + ...(server.env ? { env: server.env } : {}), + ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), + }) + connections.push(conn) + const prefix = key.replace(/[^a-zA-Z0-9_-]/g, '_') + for (const t of conn.tools) { + const name = `${prefix}__${t.name.replace(/[^a-zA-Z0-9_-]/g, '_')}` + if (routes.has(name)) { + throw new ValidationError( + `materializeLocalMcp: namespaced tool '${name}' collides across servers — rename the profile.mcp keys`, + ) + } + routes.set(name, { conn, tool: t.name }) + tools.push({ + type: 'function', + function: { + name, + description: (t.description ?? '').slice(0, 1000), + parameters: sanitizeMcpToolSchema(t.inputSchema), + }, + }) + } + } + } catch (err) { + // Fail CLOSED: never hand back a partial materialization of a profile + // whose declared server could not boot — that would fake the ablation. + await close() + throw err + } + + return { + tools, + owns: (name) => routes.has(name), + call: async (name, args) => { + const route = routes.get(name) + if (!route) throw new Error(`materializeLocalMcp: unknown tool '${name}'`) + const out = await route.conn.callTool(route.tool, args) + return out.length > maxResultChars ? out.slice(0, maxResultChars) : out + }, + close, + } +} From aee7d6919c69473a127092169c233ab583915b3f Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sun, 12 Jul 2026 23:44:59 -0600 Subject: [PATCH 41/44] feat(improvement): typed trace ingestion + spans->OTLP so trace proposers consume real runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 4 of the self-improve loop: kill the trace-ingestion rot between what the loop RECORDS and what its analysts READ. - campaign-otlp: the missing converter from the campaign trace writer's per-cell spans.jsonl ({name, cellId, startMs, durationMs, ...attrs}) to the OTLP-flat JSONL OtlpFileTraceStore/the trace-analyst registry parse, plus campaignTraceResolver(runDir) — the resolveTraces implementation for traceAnalystProposer/haloProposer: proposing generation g reads gen-(g-1) (baseline for g=0) under the same runDir the loop records into. One trace per cell keyed on the cell's on-disk path (the same cellId recurs across baseline and every candidate); deterministic FNV folds to 32/16-hex ids. - findings: isAnalystFinding + toAnalystFindings replace the blind 'as AnalystFinding[]' cast in improvement-driver — real findings pass by reference, untyped seeds are lifted into makeFinding envelopes (claim = the curator extraction order, original under metadata.raw), garbage dropped without throwing. Build prompts can no longer render undefined claims. - improve(): rawTraceDistiller is now the DEFAULT analyzeGeneration for durable runs (real runDir — where the traces live); the in-memory digest distiller stays for mem:// runs and now emits TYPED AnalystFindings too, so exactly one findings shape crosses the wire. rawTraceContext becomes the explicit override in either direction. Proof: vitest drives the real published runCampaign recording -> resolver -> OtlpFileTraceStore (2 traces indexed, tool names visible) -> traceAnalystProposer returns a candidate, offline via its analyze/fetch seams. Converter lives in this repo because agent-runtime-swe consumes the published agent-eval (0.108.0); lifting it into agent-eval next release is the follow-up. --- docs/api/primitive-catalog.md | 11 +- src/improvement/campaign-otlp.test.ts | 219 +++++++++++++++++++++ src/improvement/campaign-otlp.ts | 265 ++++++++++++++++++++++++++ src/improvement/findings.test.ts | 59 ++++++ src/improvement/findings.ts | 111 +++++++++++ src/improvement/improve.test.ts | 45 +++++ src/improvement/improve.ts | 82 +++++--- src/improvement/improvement-driver.ts | 12 +- src/improvement/index.ts | 13 ++ 9 files changed, 786 insertions(+), 31 deletions(-) create mode 100644 src/improvement/campaign-otlp.test.ts create mode 100644 src/improvement/campaign-otlp.ts create mode 100644 src/improvement/findings.test.ts create mode 100644 src/improvement/findings.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index a64c6906..39bd5d4c 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 240 exports. +Import from `@tangle-network/agent-runtime` — 249 exports. | Symbol | Kind | Summary | |---|---|---| @@ -26,10 +26,13 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `buildForwardHeaders` | function | Build the headers to emit on an outbound participant call, given the | | `buildLoopOtelSpans` | function | Build a nested, real-duration OTLP span tree for ONE loop run from its full | | `buildLoopSpanNodes` | function | Sink-neutral core behind {@link buildLoopOtelSpans}: reconstruct the | +| `campaignCellSpansToOtlp` | function | Convert ONE cell's `spans.jsonl` content to OTLP-flat JSONL lines. | +| `campaignTraceResolver` | function | Build the `resolveTraces` function `traceAnalystProposer`/`haloProposer` | | `cleanModelId` | function | Trim a candidate model id; `undefined` for non-strings and blanks. | | `commandVerifier` | function | A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other | | `composeRuntimeHooks` | function | Merge several {@link RuntimeHooks} into one. Falsy entries are dropped (so you can | | `computeBackoff` | function | Compute the delay before the next attempt. Default: 250ms exponential with jitter. | +| `convertCampaignDirToOtlp` | function | Walk `dir` (a campaign run dir, a generation dir, or a whole `selfImprove` | | `createConversationBackend` | function | Wrap a `Conversation` so it satisfies `AgentExecutionBackend`. The result is | | `createIterableBackend` | function | Wrap any custom async-iterable stream into a typed `AgentExecutionBackend`. | | `createOpenAICompatibleBackend` | function | OpenAI-compat streaming backend. Routes `runAgentTaskStream` through any | @@ -52,6 +55,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `handleChatTurn` | function | Run one chat turn. Returns immediately with a `ReadableStream` body; | | `improve` | function | Run the held-out-gated self-improvement loop on ONE profile surface. | | `improvementDriver` | function | The one reflective/agentic improvement proposer (`SurfaceProposer`): owns the candidate worktree lifecycle and delegates HOW a change is produced to a pluggable `CandidateGenerator`. | +| `isAnalystFinding` | function | Structural guard for the schema-versioned `AnalystFinding` envelope. | | `isDelegatedLoopMode` | function | Type guard — returns true when `value` is a valid `DelegatedLoopMode` string. | | `isDepthExceeded` | function | Refuse further forwarding when the inbound depth has reached the limit. | | `knowledgeReadinessDeliverable` | function | Build the completion check a supervised KB update uses to stop only when the KB is ready. | @@ -96,6 +100,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `startRuntimeRun` | function | Construct a runtime-run handle. The returned handle is mutable across its | | `streamToolLoop` | function | Streaming bounded tool loop: yields each raw turn event (the caller maps + | | `structuralRolloutPolicyFromProfile` | function | Read the persisted policy off the profile. `undefined` when the profile does | +| `toAnalystFindings` | function | Normalize a mixed `unknown[]` findings array to `AnalystFinding[]`: | | `toolBuildPrompt` | function | Build the starting instruction for a coder agent tasked with implementing a new tool. | | `turnId` | function | Deterministic turn identifier. Stable across retries of the same logical | | `validateChatModelId` | function | Validate a caller-supplied chat-model id. Rejects non-strings, malformed | @@ -107,6 +112,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `DELEGATED_LOOP_MODES` | const | All valid delegated-loop mode names — used for validation and CLI surfaces. | | `FORWARD_HEADERS` | const | Standard names — lowercased so Headers maps interop on every runtime. | | `INTELLIGENCE_WIRE_VERSION` | const | Wire version the eval-runs ingest enforces (X-Tangle-Wire-Version + body). | +| `LIFTED_FINDING_ANALYST_ID` | const | Analyst id stamped on findings lifted from untyped seed values. | | `optimizerMethod` | const | The shared method block every build/author prompt embeds. Domain framing | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | | `ROLLOUT_POLICY_BOUNDS` | const | Proposal bounds per dial. These are the SEARCH bounds (what the proposer may | @@ -128,6 +134,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `SqlConversationJournal` | class | SQL-backed ConversationJournal. Two tables — runs (one row per runId, holds | | `ValidationError` | class | Caller passed invalid arguments (out of range, mutually-exclusive options, bad shape). | | `BackendErrorDetail` | interface | Typed transport / backend failure detail. Carried on `backend_error` and | +| `CampaignOtlpOptions` | interface | Campaign `spans.jsonl` → OTLP-flat JSONL — the missing converter between | | `CandidateGenerator` | interface | The byte-producing seam — the ONE thing that differs between the cheap | | `ChatStreamEvent` | interface | The NDJSON line protocol every product chat client already speaks. | | `ChatTurnIdentity` | interface | Identity of a chat turn. `tenantId` is the workspace id for workspace- | @@ -165,7 +172,7 @@ Import from `@tangle-network/agent-runtime` — 240 exports. | `ToolLoopStopReason` | type | Why the loop stopped. `completed` = model finished naturally; `stuck-loop` = | | `Verifier` | type | Verifies the edited worktree. Sync or async; throws only on a setup fault | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `HaltContext`, `HaltSignal`, `ImprovementDriverOptions`, `ImproveOptions`, `ImproveResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToolLoopResult`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentAdapter`, `AgentBackendContext`, `AgentBackendInput`, `AgentExecutionBackend`, `AgenticGeneratorOptions`, `AgentKnowledgeProvider`, `AgentTaskContext`, `AgentTaskRunResult`, `AgentTaskSpec`, `BackendCallPolicy`, `CampaignTraceResolverOptions`, `ChatTurnHooks`, `ChatTurnResult`, `ControlBudget`, `ControlEvalResult`, `ControlRunResult`, `ControlStep`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `D1StmtLike`, `DataAcquisitionPlan`, `DelegatedLoopResult`, `EvalRunEvent`, `EvalRunGeneration`, `EvalRunsExportConfig`, `EvalRunsExportResult`, `HaltContext`, `HaltSignal`, `ImprovementDriverOptions`, `ImproveOptions`, `ImproveResult`, `KnowledgeReadinessCheckInput`, `KnowledgeReadinessReport`, `KnowledgeRequirement`, `LoopRunnerCliArgs`, `LoopRunnerCliResult`, `OtelAttribute`, `OtelExporter`, `OtelSpan`, `PersonaConversationResult`, `ResearchLoopResult`, `ResearchLoopRunnerOptions`, `ResolveAgentBackendOptions`, `ResolvedChatModel`, `RunChatTurnInput`, `RunConversationOptions`, `RunDelegatedLoopOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `RuntimeDecisionEvidenceRef`, `RuntimeDecisionPoint`, `RuntimeEventCollector`, `RuntimeHookContext`, `RuntimeHookErrorContext`, `RuntimeHookEvent`, `RuntimeRunHandle`, `RuntimeRunPersistenceAdapter`, `RuntimeRunRow`, `RuntimeSessionStore`, `RuntimeStreamEventCollector`, `RuntimeTelemetryOptions`, `RunToolLoopOptions`, `SanitizedKnowledgeReadinessReport`, `StreamToolLoopOptions`, `SupervisedKnowledgeUpdateInput`, `SupervisedKnowledgeUpdateOptions`, `SupervisedKnowledgeUpdateResult`, `ToAnalystFindingsOptions`, `ToolLoopResult`, `VetoedFact`, `WorktreeLoopRunnerOptions`, `AgentRuntimeEvent`, `AgentRuntimeEventSink`, `AgentTaskStatus`, `AuthSource`, `ControlDecision`, `ConversationStreamEvent`, `DelegatedLoopMode`, `DelegatedLoopRegistry`, `DelegatedLoopRunner`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `KnowledgeReadinessCheck`, `KnowledgeReadinessCheckResult`, `RuntimeDecisionKind`, `RuntimeHookTarget`, `RuntimeStreamEvent`, `StreamToolLoopYield`, `SupervisedKnowledgeUpdater`, `ToolLoopEvent`, `TurnOrder`. ### Vertical agent — manifest + improvement adapter diff --git a/src/improvement/campaign-otlp.test.ts b/src/improvement/campaign-otlp.test.ts new file mode 100644 index 00000000..45a3626a --- /dev/null +++ b/src/improvement/campaign-otlp.test.ts @@ -0,0 +1,219 @@ +/** + * Campaign spans→OTLP converter + resolver proof, at the REAL seams: + * + * 1. The published `runCampaign` records `spans.jsonl` per cell (nothing + * hand-crafted); `campaignTraceResolver` converts that recording to + * OTLP-flat JSONL the published `OtlpFileTraceStore` indexes — the exact + * store the trace-analyst registry reads. + * 2. The published `traceAnalystProposer` wired with the resolver produces a + * candidate from those real recorded traces (analyze + apply seams + * injected — offline, no LLM). + * + * The regression this guards: the campaign trace format + * (`{name, cellId, startMs, durationMs, ...attrs}`) is NOT the OTLP shape the + * analysts parse, so without the converter every trace-native proposer threw + * "resolveTraces returned no OTLP traces" on the traces the loop itself wrote. + */ + +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { makeFinding } from '@tangle-network/agent-eval' +import { + runCampaign, + type Scenario, + traceAnalystProposer, +} from '@tangle-network/agent-eval/campaign' +import type { DispatchContext } from '@tangle-network/agent-eval/contract' +import { OtlpFileTraceStore } from '@tangle-network/agent-eval/traces' +import { afterAll, describe, expect, it } from 'vitest' +import { + campaignCellSpansToOtlp, + campaignTraceResolver, + convertCampaignDirToOtlp, +} from './campaign-otlp' + +const scenarios: Scenario[] = [ + { id: 'alpha', kind: 'fixture' }, + { id: 'beta', kind: 'fixture' }, +] + +/** Dispatch that reports usage AND records a tool-ish span — the two write + * paths (`ctx.cost.observe` → `cost.` spans, `ctx.trace.span`) the + * real substrate uses to populate `spans.jsonl`. */ +async function tracedDispatch(scenario: Scenario, ctx: DispatchContext): Promise<{ text: string }> { + ctx.cost.observe(0.0002, 'stub') + ctx.cost.observeTokens({ input: 3, output: 5 }) + const span = ctx.trace.span('worker.turn', { 'tool.name': 'grep' }) + span.end({ outcome: 'ok' }) + return { text: `artifact-${scenario.id}` } +} + +const root = mkdtempSync(join(tmpdir(), 'campaign-otlp-')) +afterAll(() => rmSync(root, { recursive: true, force: true })) + +/** One real campaign per loop slot, into the run-optimization layout. */ +async function recordLayout(): Promise { + for (const dir of [join(root, 'baseline'), join(root, 'gen-0', 'candidate-0')]) { + await runCampaign({ + scenarios, + dispatch: tracedDispatch, + runDir: dir, + resumable: false, + }) + } +} + +describe('campaignTraceResolver — real runCampaign recording → OTLP the analysts index', () => { + it('converts the recorded spans.jsonl and OtlpFileTraceStore indexes them', async () => { + await recordLayout() + + // Proposing generation 0 reads the BASELINE campaign's traces. + const resolver = campaignTraceResolver({ runDir: root }) + const otlp = resolver({ generation: 0 }) + expect(otlp.trim()).not.toBe('') + + const lines = otlp.trim().split('\n') + const parsed = lines.map((l) => JSON.parse(l) as Record) + // Every line carries the OTLP identity fields the analysts key on. + for (const p of parsed) { + expect(typeof p.trace_id).toBe('string') + expect(typeof p.span_id).toBe('string') + } + // One trace per cell: 2 scenarios × 1 rep. + expect(new Set(parsed.map((p) => p.trace_id)).size).toBe(2) + const names = parsed.map((p) => p.name as string) + expect(names.some((n) => n.startsWith('cell.'))).toBe(true) + expect(names).toContain('cost.stub') + expect(names).toContain('worker.turn') + // Only baseline traces — gen-0 stays out of generation 0's view. + for (const p of parsed) { + const res = p.resource as { attributes: Record } + expect(String(res.attributes['campaign.cell_dir'])).toContain(join(root, 'baseline')) + } + + // The consumer contract: the published analyst store indexes the output. + const tracePath = join(root, 'converted.jsonl') + writeFileSync(tracePath, otlp) + const store = new OtlpFileTraceStore({ path: tracePath }) + const overview = await store.getOverview() + expect(overview.total_traces).toBe(2) + expect(overview.tool_names).toContain('grep') + expect(overview.errors.span_count).toBe(0) + + // Generation 1 reads gen-0; a generation with no recorded predecessor + // falls back to every trace under the run root. + const gen1 = resolver({ generation: 1 }) + expect(gen1).toContain(join(root, 'gen-0')) + expect(gen1).not.toContain('baseline') + const gen5 = resolver({ generation: 5 }) + const gen5Ids = new Set( + gen5 + .trim() + .split('\n') + .map((l) => (JSON.parse(l) as { trace_id: string }).trace_id), + ) + expect(gen5Ids.size).toBe(4) + }) + + it('traceAnalystProposer consumes the resolver end-to-end and returns a candidate', async () => { + let analyzedTraces = 0 + const proposer = traceAnalystProposer({ + baseUrl: 'https://stub.local/v1', + apiKey: 'stub-key', + model: 'stub-model', + resolveTraces: campaignTraceResolver({ runDir: root }), + // Analyst seam: assert the materialized trace file is REAL converted + // content (indexable by the same store the registry uses), then return + // one finding for the apply step. + analyze: async (tracePath) => { + const store = new OtlpFileTraceStore({ path: tracePath }) + const overview = await store.getOverview() + analyzedTraces = overview.total_traces + return [ + makeFinding({ + analyst_id: 'test-analyst', + severity: 'high', + area: 'tool-use', + confidence: 1, + claim: 'worker.turn spans show grep-only exploration', + recommended_action: 'instruct the agent to read files before editing', + evidence_refs: [], + }), + ] + }, + // Apply seam: the one LLM edit, stubbed offline. + fetchImpl: async () => + new Response( + JSON.stringify({ + choices: [{ message: { content: 'REVISED PROMPT: read before editing' } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + model: 'stub-model', + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ), + }) + + const candidates = await proposer.propose({ + currentSurface: 'baseline prompt', + history: [], + findings: [], + populationSize: 1, + generation: 0, + signal: new AbortController().signal, + }) + + expect(analyzedTraces).toBe(2) + expect(candidates.length).toBeGreaterThanOrEqual(1) + const first = candidates[0] as { surface: unknown } + expect(String(first.surface)).toContain('REVISED PROMPT') + }) +}) + +describe('campaignCellSpansToOtlp — wire-shape unit checks', () => { + it('maps error records to STATUS_CODE_ERROR and skips torn lines', () => { + const content = [ + JSON.stringify({ name: 'ok.step', cellId: 'c', startMs: 1000, durationMs: 5 }), + '{"torn": ', + JSON.stringify({ name: 'bad.step', cellId: 'c', startMs: 2000, error: 'boom' }), + ].join('\n') + const lines = campaignCellSpansToOtlp(content, { cellId: 'c', cellKey: '/tmp/x/c' }) + // Root anchor + 2 records (torn line skipped). + expect(lines).toHaveLength(3) + const spans = lines.map((l) => JSON.parse(l) as Record) + const anchor = spans[0] as { status: { code: string }; parent_span_id: string } + expect(anchor.parent_span_id).toBe('') + expect(anchor.status.code).toBe('STATUS_CODE_ERROR') + const bad = spans.find((s) => s.name === 'bad.step') as { + status: { code: string; message: string } + } + expect(bad.status).toEqual({ code: 'STATUS_CODE_ERROR', message: 'boom' }) + const ok = spans.find((s) => s.name === 'ok.step') as { + status: { code: string } + start_time: string + end_time: string + } + expect(ok.status.code).toBe('STATUS_CODE_OK') + expect(Date.parse(ok.end_time) - Date.parse(ok.start_time)).toBe(5) + }) + + it('distinct cell paths with the same cellId yield distinct traces', () => { + const content = JSON.stringify({ name: 's', cellId: 'cell_a', startMs: 1 }) + const a = JSON.parse( + campaignCellSpansToOtlp(content, { cellId: 'cell_a', cellKey: '/run/baseline/cell_a' })[0]!, + ) as { trace_id: string } + const b = JSON.parse( + campaignCellSpansToOtlp(content, { + cellId: 'cell_a', + cellKey: '/run/gen-0/candidate-0/cell_a', + })[0]!, + ) as { trace_id: string } + expect(a.trace_id).not.toBe(b.trace_id) + expect(a.trace_id).toMatch(/^[0-9a-f]{32}$/) + }) + + it('empty content and an absent dir resolve to no traces, not a throw', () => { + expect(campaignCellSpansToOtlp('', { cellId: 'c' })).toEqual([]) + expect(convertCampaignDirToOtlp('/nonexistent/definitely-not-here')).toBe('') + }) +}) diff --git a/src/improvement/campaign-otlp.ts b/src/improvement/campaign-otlp.ts new file mode 100644 index 00000000..15ff3952 --- /dev/null +++ b/src/improvement/campaign-otlp.ts @@ -0,0 +1,265 @@ +/** + * Campaign `spans.jsonl` → OTLP-flat JSONL — the missing converter between + * what the substrate RECORDS and what its trace analysts READ. + * + * agent-eval's `runCampaign` durably records one `spans.jsonl` per cell + * (`defaultBuildTraceWriter`): flat records + * `{ name, cellId, startMs, durationMs?, ...attributes }` with no trace/span + * ids. Its trace consumers (`OtlpFileTraceStore`, the trace-analyst registry, + * `haloProposer`/`traceAnalystProposer` via `resolveTraces`) read OTLP-flat + * JSONL (`trace_id`/`span_id`/ISO times/status/attributes — the shape + * `projectOtlpFlatLine` parses). Nothing shipped converts between the two, so + * the traces real optimization runs write could never reach the trace-native + * proposers. This module is that wire adapter: + * + * - {@link campaignCellSpansToOtlp} — one cell's `spans.jsonl` content → + * OTLP lines (a per-cell root AGENT anchor span + one child per record). + * - {@link convertCampaignDirToOtlp} — walk any campaign run/generation dir + * for `spans.jsonl` files and concatenate their OTLP lines. + * - {@link campaignTraceResolver} — the `resolveTraces` implementation for + * `traceAnalystProposer`/`haloProposer`: proposing generation g reads the + * traces the loop just recorded (`gen-`, or `baseline` for g = 0) + * under the same `runDir` handed to `improve()`/`selfImprove()`. + * + * Trace identity: one trace per CELL, keyed on the cell's on-disk path — the + * same sanitized `cellId` recurs across the baseline and every candidate + * campaign, so folding the id alone would merge distinct runs into one trace. + * Ids are deterministic FNV-1a folds to OTLP's 32/16-hex width, so re-converts + * are stable and byte-identical. + */ + +import { type Dirent, readdirSync, readFileSync } from 'node:fs' +import { basename, dirname, join, resolve } from 'node:path' +import type { ProposeContext } from '@tangle-network/agent-eval/campaign' +import { OPENINFERENCE_SPAN_KIND } from '@tangle-network/agent-eval/traces' + +/** How deep {@link convertCampaignDirToOtlp} recurses looking for + * `spans.jsonl`. The deepest shipped layout is + * `/gen-/candidate-//spans.jsonl` (3 levels); one spare + * level tolerates a wrapper dir without letting a mis-pointed root scan a + * whole filesystem. */ +const MAX_WALK_DEPTH = 4 + +export interface CampaignOtlpOptions { + /** OTLP `service.name` on every emitted span. Default `'campaign'`. */ + serviceName?: string +} + +interface CampaignSpanRecord { + name: string + cellId: string + startMs: number + durationMs?: number + attributes: Record + error?: string +} + +/** + * Convert ONE cell's `spans.jsonl` content to OTLP-flat JSONL lines. + * `cellKey` is the identity the trace id folds from — pass the cell's on-disk + * path (unique per campaign); `cellId` is the display/attribute label. + * Returns `[]` for empty/recordless content (a dispatch that never touched + * `ctx.trace`/`ctx.cost` writes an empty file — that is data, not an error). + */ +export function campaignCellSpansToOtlp( + content: string, + cell: { cellId: string; cellKey?: string }, + opts: CampaignOtlpOptions = {}, +): string[] { + const records = parseCampaignSpans(content, cell.cellId) + if (records.length === 0) return [] + + const serviceName = opts.serviceName ?? 'campaign' + const key = cell.cellKey ?? cell.cellId + const traceId = foldTo32Hex(key) + const rootSpanId = foldTo16Hex(`${key}::root`) + + const startMs = Math.min(...records.map((r) => r.startMs)) + const endMs = Math.max(...records.map((r) => r.startMs + (r.durationMs ?? 0))) + const anyError = records.some((r) => r.error !== undefined) + + const resource = { + attributes: { + 'service.name': serviceName, + 'campaign.cell_id': cell.cellId, + ...(cell.cellKey ? { 'campaign.cell_dir': cell.cellKey } : {}), + }, + } + + const lines: string[] = [ + JSON.stringify({ + trace_id: traceId, + span_id: rootSpanId, + parent_span_id: '', + name: `cell.${cell.cellId}`, + start_time: msToIso(startMs), + end_time: msToIso(endMs), + status: anyError + ? { code: 'STATUS_CODE_ERROR', message: 'one or more spans errored' } + : { code: 'STATUS_CODE_OK', message: '' }, + resource, + attributes: { + [OPENINFERENCE_SPAN_KIND]: 'AGENT', + 'agent.name': cell.cellId, + }, + }), + ] + + for (let i = 0; i < records.length; i++) { + const r = records[i]! + lines.push( + JSON.stringify({ + trace_id: traceId, + span_id: foldTo16Hex(`${key}::${i}`), + parent_span_id: rootSpanId, + name: r.name, + start_time: msToIso(r.startMs), + end_time: msToIso(r.startMs + (r.durationMs ?? 0)), + status: + r.error === undefined + ? { code: 'STATUS_CODE_OK', message: '' } + : { code: 'STATUS_CODE_ERROR', message: r.error }, + resource, + attributes: r.attributes, + }), + ) + } + return lines +} + +/** + * Walk `dir` (a campaign run dir, a generation dir, or a whole `selfImprove` + * run root) for `spans.jsonl` files and return their concatenated OTLP-flat + * JSONL — the exact string the `resolveTraces` contract expects. `''` when no + * spans exist (the proposers fail loud on empty by design). + */ +export function convertCampaignDirToOtlp(dir: string, opts: CampaignOtlpOptions = {}): string { + const root = resolve(dir) + const files = findSpansFiles(root, MAX_WALK_DEPTH) + const lines: string[] = [] + for (const file of files) { + let content: string + try { + content = readFileSync(file, 'utf8') + } catch { + continue + } + const cellDir = dirname(file) + lines.push( + ...campaignCellSpansToOtlp(content, { cellId: basename(cellDir), cellKey: cellDir }, opts), + ) + } + return lines.length > 0 ? `${lines.join('\n')}\n` : '' +} + +export interface CampaignTraceResolverOptions extends CampaignOtlpOptions { + /** The `selfImprove`/`improve()` run root — the SAME `runDir` the loop + * records under (`/baseline/...`, `/gen-/candidate-/...`). + * Must be a real path; a `mem://` run records nothing to resolve. */ + runDir: string +} + +/** + * Build the `resolveTraces` function `traceAnalystProposer`/`haloProposer` + * take: proposing generation g reads the traces of the campaigns the loop just + * scored — `gen-` (or `baseline` when g = 0), falling back to every trace + * under the run root when that directory has none (e.g. a caller pointing at a + * single campaign dir rather than a loop root). + * + * traceAnalystProposer({ ..., resolveTraces: campaignTraceResolver({ runDir }) }) + */ +export function campaignTraceResolver( + opts: CampaignTraceResolverOptions, +): (ctx: Pick) => string { + return (ctx) => { + const priorDir = + ctx.generation <= 0 + ? join(opts.runDir, 'baseline') + : join(opts.runDir, `gen-${ctx.generation - 1}`) + const prior = convertCampaignDirToOtlp(priorDir, opts) + if (prior) return prior + return convertCampaignDirToOtlp(opts.runDir, opts) + } +} + +/** Parse `spans.jsonl` content into records, splitting the trace writer's + * reserved fields from the free-form attributes. Malformed lines are skipped + * (one torn write must not censor the rest of the cell). */ +function parseCampaignSpans(content: string, cellId: string): CampaignSpanRecord[] { + const out: CampaignSpanRecord[] = [] + for (const line of content.split('\n')) { + const trimmed = line.trim() + if (!trimmed) continue + let raw: unknown + try { + raw = JSON.parse(trimmed) + } catch { + continue + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) continue + const o = raw as Record + if (typeof o.name !== 'string' || typeof o.startMs !== 'number') continue + + const attributes: Record = { 'campaign.cell_id': cellId } + for (const [k, v] of Object.entries(o)) { + if (k === 'name' || k === 'cellId' || k === 'startMs' || k === 'durationMs') continue + attributes[k] = v + } + out.push({ + name: o.name, + cellId, + startMs: o.startMs, + ...(typeof o.durationMs === 'number' ? { durationMs: o.durationMs } : {}), + attributes, + ...(typeof o.error === 'string' ? { error: o.error } : {}), + }) + } + return out +} + +/** All `spans.jsonl` files under `dir`, depth-bounded, symlink-dirs skipped + * (a cycle under a run dir must not hang the proposal round). Sorted for a + * deterministic concatenation order. */ +function findSpansFiles(dir: string, depth: number): string[] { + const out: string[] = [] + let entries: Dirent[] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return out + } + for (const entry of entries) { + const full = join(dir, entry.name) + if (entry.isFile() && entry.name === 'spans.jsonl') { + out.push(full) + } else if (depth > 0 && entry.isDirectory() && !entry.isSymbolicLink()) { + out.push(...findSpansFiles(full, depth - 1)) + } + } + return out.sort() +} + +function msToIso(ms: number): string { + return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : new Date(0).toISOString() +} + +/** Deterministic FNV-1a fold to OTLP's 16-hex span-id width. */ +function foldTo16Hex(s: string): string { + const a = fnv1a(s) + const b = fnv1a(`${s}::salt`) + return a + b +} + +/** Deterministic FNV-1a fold to OTLP's 32-hex trace-id width. */ +function foldTo32Hex(s: string): string { + return foldTo16Hex(s) + foldTo16Hex(`${s}::trace`) +} + +function fnv1a(s: string): string { + let h = 0x811c9dc5 + for (let i = 0; i < s.length; i++) { + h ^= s.charCodeAt(i) + h = Math.imul(h, 0x01000193) >>> 0 + } + return h.toString(16).padStart(8, '0') +} diff --git a/src/improvement/findings.test.ts b/src/improvement/findings.test.ts new file mode 100644 index 00000000..dec293de --- /dev/null +++ b/src/improvement/findings.test.ts @@ -0,0 +1,59 @@ +/** + * Typed-findings accessor proof: conforming `AnalystFinding`s pass through by + * reference, untyped seeds are LIFTED into real envelopes (not blind-cast), + * and garbage is dropped without throwing. Guards the regression where + * `improvement-driver` cast `unknown[]` findings to `AnalystFinding[]` and + * build prompts rendered `(undefined) undefined` lines. + */ + +import { makeFinding } from '@tangle-network/agent-eval' +import { describe, expect, it } from 'vitest' +import { isAnalystFinding, LIFTED_FINDING_ANALYST_ID, toAnalystFindings } from './findings' + +const realFinding = makeFinding({ + analyst_id: 'test', + severity: 'high', + area: 'tool-use', + confidence: 1, + claim: 'the agent never reads before editing', + evidence_refs: [], +}) + +describe('isAnalystFinding', () => { + it('accepts the makeFinding envelope and rejects look-alikes', () => { + expect(isAnalystFinding(realFinding)).toBe(true) + expect(isAnalystFinding({ claim: 'partial', severity: 'high' })).toBe(false) + expect(isAnalystFinding('a string finding')).toBe(false) + expect(isAnalystFinding(null)).toBe(false) + expect(isAnalystFinding({ ...realFinding, severity: 'catastrophic' })).toBe(false) + }) +}) + +describe('toAnalystFindings', () => { + it('passes real findings through by reference and lifts the rest', () => { + const digest = { scenario: 'b', composite: 0.2, notes: 'tour is not a permutation' } + const out = toAnalystFindings([realFinding, 'raw string lesson', digest, 42, null, {}]) + + // Real finding: same object, not re-enveloped. + expect(out[0]).toBe(realFinding) + // String seed: becomes the claim. + expect(out[1]!.claim).toBe('raw string lesson') + expect(out[1]!.analyst_id).toBe(LIFTED_FINDING_ANALYST_ID) + // Digest object: most actionable text field wins; original rides in metadata. + expect(out[2]!.claim).toBe('tour is not a permutation') + expect(out[2]!.metadata).toEqual({ raw: digest }) + // number / null / {} carry no text — dropped, never thrown. + expect(out).toHaveLength(3) + expect(out.every(isAnalystFinding)).toBe(true) + }) + + it('prefers recommended_action over claim, matching the curator extraction order', () => { + const out = toAnalystFindings([{ claim: 'what happened', recommended_action: 'what to do' }]) + expect(out[0]!.claim).toBe('what to do') + }) + + it('falls back to compact JSON for text-less objects so the signal survives', () => { + const out = toAnalystFindings([{ seed: 'static-seed-finding' }]) + expect(out[0]!.claim).toBe('{"seed":"static-seed-finding"}') + }) +}) diff --git a/src/improvement/findings.ts b/src/improvement/findings.ts new file mode 100644 index 00000000..f65f845e --- /dev/null +++ b/src/improvement/findings.ts @@ -0,0 +1,111 @@ +/** + * Typed-findings accessor — the one place `unknown[]` findings become + * `AnalystFinding[]`. + * + * agent-eval's `ProposeContext.findings` is `TFindings[] = unknown[]` on the + * wire: the loop threads whatever the previous `analyzeGeneration` producer (or + * the caller's static seed) returned. Consumers that need the typed envelope + * (`claim`/`severity`/`recommended_action`) were down-casting with a bare + * `as AnalystFinding[]` — a lie at runtime whenever the seed was a raw string + * or an ad-hoc digest, which then rendered `undefined` into build prompts. + * + * `toAnalystFindings` replaces that cast: real findings pass through + * unchanged (structural guard, fail-closed), and non-conforming values are + * LIFTED into a real `AnalystFinding` envelope via `makeFinding` — the most + * actionable text becomes the claim, the original value rides in `metadata.raw` + * — so everything downstream of the accessor handles exactly one shape. + */ + +import { type AnalystFinding, makeFinding } from '@tangle-network/agent-eval' + +const SEVERITIES: ReadonlySet = new Set(['critical', 'high', 'medium', 'low', 'info']) + +/** Analyst id stamped on findings lifted from untyped seed values. */ +export const LIFTED_FINDING_ANALYST_ID = 'lifted-seed' + +/** Structural guard for the schema-versioned `AnalystFinding` envelope. + * Strict on the identity fields `makeFinding` always populates — a partial + * look-alike is lifted (re-enveloped), not trusted. */ +export function isAnalystFinding(value: unknown): value is AnalystFinding { + if (!value || typeof value !== 'object') return false + const o = value as Record + return ( + o.schema_version === '1.0.0' && + typeof o.finding_id === 'string' && + typeof o.analyst_id === 'string' && + typeof o.severity === 'string' && + SEVERITIES.has(o.severity) && + typeof o.area === 'string' && + typeof o.claim === 'string' && + typeof o.confidence === 'number' && + Array.isArray(o.evidence_refs) + ) +} + +/** The most actionable text of an untyped finding-ish value — mirrors the + * extraction order agent-eval's curator proposers use (`recommended_action` > + * `claim` > `lesson` > `notes` > `text` > `message`), so the two ends of the + * wire read the same field first. */ +function liftedClaim(value: unknown): string | null { + if (typeof value === 'string') return value.trim() || null + if (value && typeof value === 'object') { + const o = value as Record + for (const key of ['recommended_action', 'claim', 'lesson', 'notes', 'text', 'message']) { + const v = o[key] + if (typeof v === 'string' && v.trim()) return v.trim() + } + try { + const json = JSON.stringify(value) + if (json && json !== '{}' && json !== '[]') { + return json.length > 400 ? `${json.slice(0, 399)}…` : json + } + } catch { + /* circular / non-serializable → drop below */ + } + } + return null +} + +export interface ToAnalystFindingsOptions { + /** `analyst_id` stamped on lifted (non-conforming) values. + * Default {@link LIFTED_FINDING_ANALYST_ID}. */ + analystId?: string + /** `area` stamped on lifted values. Default `'seed'`. */ + area?: string +} + +/** + * Normalize a mixed `unknown[]` findings array to `AnalystFinding[]`: + * conforming findings pass through by reference; strings and finding-ish + * objects are lifted into envelopes (claim = most actionable text, original + * value under `metadata.raw`); values with no extractable text are dropped. + * Never throws — a malformed seed must not kill a proposal round. + */ +export function toAnalystFindings( + findings: readonly unknown[], + opts: ToAnalystFindingsOptions = {}, +): AnalystFinding[] { + const analystId = opts.analystId ?? LIFTED_FINDING_ANALYST_ID + const area = opts.area ?? 'seed' + const out: AnalystFinding[] = [] + for (const f of findings) { + if (isAnalystFinding(f)) { + out.push(f) + continue + } + const claim = liftedClaim(f) + if (!claim) continue + out.push( + makeFinding({ + analyst_id: analystId, + severity: 'info', + area, + confidence: 0.5, + claim, + evidence_refs: [], + ...(f && typeof f === 'object' ? { metadata: { raw: f } } : {}), + }), + ) + } + return out +} diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 758770fe..13c6b792 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -227,6 +227,51 @@ describe('improve() — default proposer resolution (substrate export drift guar const secondRound = JSON.stringify(findingsSeen[1]) expect(secondRound).toContain('"scenario":"b"') expect(secondRound).toContain('not a permutation') + // The digest is TYPED on the wire now: real AnalystFinding envelopes, not + // ad-hoc {scenario, composite} objects a consumer must down-cast. + const { isAnalystFinding } = await import('./findings') + expect(findingsSeen[1]!.length).toBeGreaterThanOrEqual(1) + expect(findingsSeen[1]!.every(isAnalystFinding)).toBe(true) + }) + + it('a real runDir defaults analyzeGeneration to the RAW-TRACE distiller', async () => { + const { mkdtempSync, rmSync } = await import('node:fs') + const { tmpdir } = await import('node:os') + const { join } = await import('node:path') + const runDir = mkdtempSync(join(tmpdir(), 'improve-rawtrace-')) + const failingJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'failing-judge', + dimensions: [{ key: 'q', description: 'fixture quality' }], + score: () => ({ dimensions: { q: 0 }, composite: 0, notes: 'always failing' }), + } + const findingsSeen: unknown[][] = [] + const stubProposer = { + kind: 'stub-recorder', + async propose(ctx: { findings: unknown[] }) { + findingsSeen.push(ctx.findings) + return [{ surface: `candidate-${findingsSeen.length}`, label: 'stub', rationale: 'stub' }] + }, + } + try { + await improve(promptProfile(), [], { + surface: 'prompt', + scenarios, + judge: failingJudge, + agent: stubAgent, + generator: stubProposer as never, + runDir, + budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, + }) + // The durable-run default is rawTraceDistiller: a later round's findings + // are its raw-trace-context envelopes (paths into the recorded traces), + // NOT the ~400-char failure digest. + const later = findingsSeen.at(-1)! + expect(JSON.stringify(later)).toContain('raw-trace-context') + const { isAnalystFinding } = await import('./findings') + expect(later.every(isAnalystFinding)).toBe(true) + } finally { + rmSync(runDir, { recursive: true, force: true }) + } }) it("surface 'code' + opts.code assembles the worktree pipeline and measures a candidate", async () => { diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index cfcd9b07..5d00ddb0 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -30,6 +30,7 @@ * @experimental */ +import { makeFinding } from '@tangle-network/agent-eval' import { gepaProposer, gitWorktreeAdapter, @@ -109,22 +110,20 @@ export interface ImproveOptions { * in-process). */ runDir?: string /** Per-generation findings producer passthrough (see selfImprove.analyzeGeneration). - * DEFAULT: the built-in failure distiller — after each generation it turns the - * worst-scoring/errored cells into structured findings ({ scenario, composite, - * notes, error }) for the NEXT proposal round, so the proposer reasons over what - * actually failed instead of a static seed. Pass your own producer (e.g. a - * trace-analyst over the runDir's traces) to replace it; pass `null` to disable - * and keep the static `findings` all the way through. */ + * DEFAULT: with a real (non-`mem://`) `runDir`, the raw-trace distiller + * (`rawTraceDistiller`) — typed `AnalystFinding`s pointing the proposer at the + * prior generation's actual on-disk traces; for in-memory runs (no traces on + * disk to point at), the built-in failure distiller — the worst-scoring/errored + * cells distilled into typed `AnalystFinding`s for the NEXT proposal round. + * Pass your own producer to replace either; pass `null` to disable and keep the + * static `findings` all the way through. */ analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null - /** META-HARNESS mode: instead of the ~400-char distilled findings, feed the - * proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's - * real run traces under `runDir` (per-cell `spans.jsonl` event logs + - * `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose - * instruction — so the coding agent reads the actual failures itself rather than - * a pre-summary. Requires a REAL `runDir` (that is where the traces live). - * Ignored when `analyzeGeneration` is set explicitly (that wins) or is `null` - * (disabled). Equivalent to `analyzeGeneration: rawTraceDistiller()`; this flag - * is the one-line enable. Default `false` (the distiller stays the default). */ + /** Raw-trace context override. Unset (default): raw-trace findings whenever the + * run is durable (a real `runDir` — that is where the traces live), the ~400-char + * failure digest otherwise. `true` forces `rawTraceDistiller()` even for an + * in-memory run (it emits a loud warning finding instead of paths); `false` + * forces the digest distiller even with a real `runDir`. Ignored when + * `analyzeGeneration` is set explicitly (that wins) or is `null` (disabled). */ rawTraceContext?: boolean /** CODE-surface wiring with prompt-parity DX: name `surface: 'code'`, point at a * repo, and the facade assembles the whole candidate pipeline — git worktrees @@ -253,12 +252,14 @@ function baselineSurfaceFor( } } -/** The default `analyzeGeneration`: distill each generation's failing cells into - * findings for the next proposal round. Deliberately dependency-free — judge notes - * and errors are already the domain's own diagnosis (executable gates put their - * reasons there); a trace-analyst can replace this wholesale via - * `opts.analyzeGeneration`. Falls back to the static seed findings when the - * generation had no failures, so a clean round never wipes the seed context. */ +/** The in-memory-run `analyzeGeneration`: distill each generation's failing cells + * into TYPED `AnalystFinding`s for the next proposal round — the same envelope + * `rawTraceDistiller` and the analyst registry emit, so consumers never branch on + * a second ad-hoc digest shape. Judge notes and errors are already the domain's + * own diagnosis (executable gates put their reasons there); a trace-analyst can + * replace this wholesale via `opts.analyzeGeneration`. Falls back to the static + * seed findings when the generation had no failures, so a clean round never wipes + * the seed context. */ function generationFailureDistiller( staticFindings: unknown[], ): NonNullable['analyzeGeneration']> { @@ -297,10 +298,42 @@ function generationFailureDistiller( } if (failures.length === 0) return staticFindings failures.sort((a, b) => a.composite - b.composite) - return failures.slice(0, CAP) + return failures.slice(0, CAP).map((f) => + makeFinding({ + analyst_id: 'generation-failure-distiller', + severity: f.error !== undefined || f.composite < 0.5 ? 'high' : 'medium', + area: 'generation-failure', + confidence: 1, + subject: f.scenario, + claim: `Scenario ${f.scenario} scored composite ${f.composite}${ + f.notes ? `: ${f.notes}` : '' + }${f.error ? ` (error: ${f.error})` : ''}`, + evidence_refs: [], + metadata: { + scenario: f.scenario, + composite: f.composite, + ...(f.error !== undefined ? { error: f.error } : {}), + }, + }), + ) } } +/** The default `analyzeGeneration` when the caller passed none: raw-trace context + * whenever the run is durable (a real `runDir` is where the traces live — see + * `rawTraceDistiller`), the failure digest for in-memory runs, with + * `rawTraceContext` as the explicit override in either direction. */ +function defaultDistillerFor( + opts: ImproveOptions, + findings: unknown[], +): NonNullable['analyzeGeneration']> { + const durableRun = opts.runDir !== undefined && !opts.runDir.startsWith('mem://') + const useRawTraces = opts.rawTraceContext ?? durableRun + return useRawTraces + ? rawTraceDistiller({ fallbackFindings: findings }) + : generationFailureDistiller(findings) +} + /** Assemble the code-surface proposer from `opts.code`: git worktrees + the * improvement driver + (by default) the full agentic generator. Returns * `undefined` when the surface is not `code` or no code options were given — @@ -438,10 +471,7 @@ export async function improve( ? {} : { analyzeGeneration: - opts.analyzeGeneration ?? - (opts.rawTraceContext - ? rawTraceDistiller({ fallbackFindings: findings }) - : generationFailureDistiller(findings)), + opts.analyzeGeneration ?? defaultDistillerFor(opts, findings), }), }) diff --git a/src/improvement/improvement-driver.ts b/src/improvement/improvement-driver.ts index 2f9699a3..41dc76de 100644 --- a/src/improvement/improvement-driver.ts +++ b/src/improvement/improvement-driver.ts @@ -28,6 +28,7 @@ import type { SurfaceProposer, WorktreeAdapter, } from '@tangle-network/agent-eval/campaign' +import { toAnalystFindings } from './findings' /** The byte-producing seam — the ONE thing that differs between the cheap * reflective path and the full agentic path. A generator makes (uncommitted) @@ -106,12 +107,17 @@ export function improvementDriver(opts: ImprovementDriverOptions): SurfacePropos /** Phase-2 report carries `findings` when present; else fall back to the * loop's `ctx.findings`. The report is opaque to the substrate, so probe it - * structurally. */ + * structurally. Both paths run through `toAnalystFindings` — the wire is + * `unknown[]` at runtime regardless of the generic (static seeds, legacy + * digests), and a bare cast here fed `undefined` claims into build prompts. */ function resolveFindings(ctx: ProposeContext): AnalystFinding[] { const report = ctx.report if (report && typeof report === 'object' && 'findings' in report) { const f = (report as { findings: unknown }).findings - if (Array.isArray(f) && f.length > 0) return f as AnalystFinding[] + if (Array.isArray(f) && f.length > 0) { + const lifted = toAnalystFindings(f, { analystId: 'report-findings', area: 'report' }) + if (lifted.length > 0) return lifted + } } - return ctx.findings + return toAnalystFindings(ctx.findings ?? [], { analystId: 'loop-context', area: 'seed' }) } diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 770e57e0..02a434e8 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -24,10 +24,23 @@ export { type VerifyResult, } from './agentic-generator' export { findingLines, mcpBuildPrompt, toolBuildPrompt } from './build-prompts' +export { + type CampaignOtlpOptions, + type CampaignTraceResolverOptions, + campaignCellSpansToOtlp, + campaignTraceResolver, + convertCampaignDirToOtlp, +} from './campaign-otlp' export { type DriverLoopGeneratorOptions, driverLoopGenerator, } from './driver-loop-generator' +export { + isAnalystFinding, + LIFTED_FINDING_ANALYST_ID, + type ToAnalystFindingsOptions, + toAnalystFindings, +} from './findings' export { type ImproveOptions, type ImproveResult, From 13cb80245a8556f6068a5fb94877849a7c18bbb0 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 13 Jul 2026 00:26:40 -0600 Subject: [PATCH 42/44] feat(lifecycle): memory as a first-class profile surface, served live via stdio MCP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the 'memory' ArtifactKind + AgentMemorySpec payload. applyArtifact mounts it twice: the typed spec on profile.metadata.memory (local extension — the published AgentProfile has no memory field yet) and a live stdio server entry on profile.mcp, so the same-host client (materializeLocalMcp) boots the memory during a scored run and sweEvalRunner({ materializeMcp: true }) measures memory-as-treatment lift with zero scorer changes. The memory MCP server (memory_search / memory_get, deterministic lexical retrieval) serves on the extracted createStdioToolServer core (committed here; the memory server is its first consumer), with a bin entry (agent-runtime-memory-mcp) resolved per install: dist/mcp/ memory-bin.js under node when built, the TS source through tsx in dev/test. Fail-closed: an empty memory never serves; a broken store fails the materialization instead of faking the ablation. Lesson ingestion: memoryArtifactFromLessons / memoryArtifactFromCuratedSurface adapt agent-eval's memoryCurationProposer block (markers duplicated by convention — flagged) into memory artifacts; memoryGenerator is the native CandidateGenerator (observed findings only — judge-derived rows are firewalled; carried memory accumulates; no-change generations emit nothing). AgentMemorySpec.logPath writes a JSONL row per memory_search — the retrieval-log seam for agent-knowledge's RetrievalHoldout, which stays cross-package (not a dependency of this repo). Also fixes a stale improvement-driver test: its partial report-finding fixture now conforms to the P4 toAnalystFindings pass-through contract instead of asserting a pre-lift id. --- docs/api/primitive-catalog.md | 35 ++- package.json | 1 + src/lifecycle/apply.ts | 15 ++ src/lifecycle/index.ts | 20 ++ src/lifecycle/memory.test.ts | 238 ++++++++++++++++++ src/lifecycle/memory.ts | 419 +++++++++++++++++++++++++++++++ src/lifecycle/swe-eval-runner.ts | 5 +- src/lifecycle/types.ts | 17 +- src/mcp/index.ts | 23 ++ src/mcp/memory-bin.ts | 43 ++++ src/mcp/memory-server.test.ts | 172 +++++++++++++ src/mcp/memory-server.ts | 353 ++++++++++++++++++++++++++ src/mcp/tool-server.ts | 195 ++++++++++++++ tests/improvement-driver.test.ts | 15 +- tsup.config.ts | 1 + 15 files changed, 1540 insertions(+), 12 deletions(-) create mode 100644 src/lifecycle/memory.test.ts create mode 100644 src/lifecycle/memory.ts create mode 100644 src/mcp/memory-bin.ts create mode 100644 src/mcp/memory-server.test.ts create mode 100644 src/mcp/memory-server.ts create mode 100644 src/mcp/tool-server.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 39bd5d4c..00f741be 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -692,7 +692,7 @@ Import from `@tangle-network/agent-runtime/analyst-loop` — 15 exports. ### Artifact lifecycle — generate → measure → promote → compose -Import from `@tangle-network/agent-runtime/lifecycle` — 63 exports. +Import from `@tangle-network/agent-runtime/lifecycle` — 81 exports. | Symbol | Kind | Summary | |---|---|---| @@ -705,18 +705,31 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 63 exports. | `driftWatch` | function | Re-measure every `active` artifact and demote those whose held-back lift | | `gepaRefine` | function | Wrap `gepaProposer` as a `RefinePrompt`. The proposer reflects on the | | `heldOutPromotionGate` | function | The paper-grade promotion gate: delegate to agent-eval's `HeldOutGate`, which | +| `lessonsFromCuratedSurface` | function | Extract the lesson lines from a surface carrying a curated-memory block. | | `measureMarginalLift` | function | Run the with/without ablation for `candidate` over `baseline` and return its | +| `memoryArtifactFromCuratedSurface` | function | Parse a proposer surface and build the artifact in one step. Returns | +| `memoryArtifactFromLessons` | function | The curated-lessons → memory-artifact adapter (the seam that gives | +| `memoryGenerator` | function | The `memory` surface's `CandidateGenerator` — the native lifecycle path for | +| `memoryMcpServer` | function | The `AgentProfileMcpServer` entry that SERVES a memory spec — what the | +| `memoryOfProfile` | function | Read (and validate) the memory spec a profile carries, if any. Malformed | | `productionPromptGenerator` | function | Production `promptGenerator`: refine via `gepaProposer`, seed via a | | `promptGenerator` | function | Build a `CandidateGenerator` for the prompt surface. Each generation it pools | +| `readMemoryItemsFile` | function | Read a memory store file: a JSON array, or JSONL (one `MemoryItem` per line). | | `routerSeedAuthor` | function | A router-backed `AuthorDiverseSeeds`: one structured LLM call that authors | | `runLifecycle` | function | Run ONE generation of the artifact lifecycle. | | `skillGenerator` | function | Build a `CandidateGenerator` for the skill surface that distills new skills | | `sweEvalRunner` | function | Build the `EvalRunner` closed over one fixed SWE exam. `runLifecycle` calls | | `thresholdPromotionGate` | function | The simplest honest gate: promote iff the candidate's marginal lift on the | +| `validateMemorySpec` | function | Assert an `AgentMemorySpec` is well-formed and servable. | | `worktreeBuildCandidate` | function | Build the production per-candidate seam for `buildableGenerator`. Each call to | +| `writeMemoryStore` | function | Write a durable JSONL memory store (one `MemoryItem` per line). | +| `CURATED_MEMORY_BLOCK_END` | const | The closing delimiter of the proposer's curated block (see | +| `CURATED_MEMORY_BLOCK_START` | const | The opening delimiter agent-eval's `memoryCurationProposer` writes its | | `lifecycleReasonKey` | const | The metadata key under which the registry records WHY an artifact left the | | `liftMetadataKey` | const | The metadata key under which the registry stores an artifact's measured held- | +| `profileMemoryMetadataKey` | const | The `profile.metadata` key the typed memory spec rides under (the local | | `ArtifactRegistry` | class | A typed, in-memory registry of `ProfileArtifact`s with stable ids. | +| `AgentMemorySpec` | interface | The `memory` artifact payload — HOW a profile's memory is stored and served: | | `ArtifactPayloads` | interface | The payload for each `ArtifactKind`. The shapes are the SAME types the | | `ArtifactQuery` | interface | Filter for `list`. Omit a field to leave that dimension unconstrained. | | `BuiltCandidate` | interface | The result of building ONE candidate in its own worktree. A build either | @@ -729,8 +742,10 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 63 exports. | `EvalResult` | interface | The result of running an eval over ONE profile: a composite score and the cost | | `GenerateContext` | interface | The read-only context a generator sees when proposing candidates. It is the | | `MarginalLift` | interface | The marginal lift of one artifact: the with/without ablation. | +| `MemoryItem` | interface | One row of agent memory: a crisp lesson/fact with provenance. | | `PairStackCheck` | interface | The stacking verdict for one pair of active artifacts. | | `ProfileArtifact` | interface | A discrete, individually-promotable piece of an agent profile. | +| `ProfileWithMemory` | interface | The local profile extension: `AgentProfile` plus the memory spec under | | `PromotionGate` | interface | Decides whether ONE measured candidate is promoted. The lifecycle calls this | | `PromotionVerdict` | interface | The verdict a gate returns for one candidate. | | `PromptDraft` | interface | A proposed prompt instruction line plus the WHY behind it. The `rationale` | @@ -750,7 +765,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 63 exports. | `RefinePrompt` | type | REFINE — incumbent-grounded rewrites. Given the lifecycle context, return | | `RefineSkill` | type | REFINE — improve ONE distilled draft (wording, structure, examples). The | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BuildableGeneratorOptions`, `DedupeResult`, `DriftWatchResult`, `HeldOutPromotionGateOptions`, `MeasureMarginalLiftOptions`, `ProductionPromptGeneratorOptions`, `PromptGeneratorOptions`, `RunLifecycleResult`, `SkillGeneratorOptions`, `SweEvalRunnerOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BuildableGeneratorOptions`, `DedupeResult`, `DriftWatchResult`, `HeldOutPromotionGateOptions`, `MeasureMarginalLiftOptions`, `MemoryArtifactOptions`, `MemoryGeneratorOptions`, `MemoryMcpServerOptions`, `ProductionPromptGeneratorOptions`, `PromptGeneratorOptions`, `RunLifecycleResult`, `SkillGeneratorOptions`, `SweEvalRunnerOptions`. ### Knowledge orchestration — supervised KB updates @@ -828,7 +843,7 @@ Import from `@tangle-network/agent-runtime/platform` — 20 exports. ### MCP servers — delegate / coordination / detached-session -Import from `@tangle-network/agent-runtime/mcp` — 170 exports. +Import from `@tangle-network/agent-runtime/mcp` — 186 exports. | Symbol | Kind | Summary | |---|---|---| @@ -850,8 +865,10 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `createInProcessTransport` | function | In-process pair of `Readable` + `Writable` streams suitable for driving | | `createKbGate` | function | Build a fail-closed KB gate. The returned function runs the built-in floor | | `createMcpServer` | function | Stdio JSON-RPC MCP server exposing the delegation tools (`delegate`, `delegate_feedback`, `delegation_status`, `delegation_history`, optional `delegate_ui_audit`) to sandbox coding-harness agents. | +| `createMemoryToolServer` | function | Build the memory MCP server: `memory_search` (lexical top-k over the rows) | | `createPropagatingTraceEmitter` | function | Create a LoopTraceEmitter that: | | `createSiblingSandboxExecutor` | function | Wrap a raw sandbox SDK client so the kernel emits | +| `createStdioToolServer` | function | Build the generic stdio JSON-RPC tool server. | | `createWorktree` | function | Checkout a fresh git worktree for a delegation run on a new branch under `variantsDir`. | | `detachedSessionDelegate` | function | Build the sandbox-session coder delegate. It drives `runLoop` against the project's | | `detachedTurnEvents` | function | Synthesize the terminal event array a detached turn settles through. Shaped | @@ -864,9 +881,12 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `mcpToolsForRuntimeMcp` | function | Returns the queue-bound delegation tools projected into OpenAI Chat | | `mcpToolsForRuntimeMcpSubset` | function | Subset filter — return only the projected tools whose `function.name` | | `parseDetachedSessionRef` | function | Parse a `detachedSessionRef` string back to parts; throws `ValidationError` on malformed input. | +| `parseMemoryItems` | function | Coerce an untrusted JSON array into validated `MemoryItem` rows. | +| `readMemoryItemsFile` | function | Read a memory store file: a JSON array, or JSONL (one `MemoryItem` per line). | | `readTraceContextFromEnv` | function | Read trace context from the process environment. | | `removeWorktree` | function | Remove a git worktree and delete its branch; tolerates already-removed paths. | | `renderTrace` | function | Render a worker's trace (tool calls + results) into the text an analyst lens reads. Generic over | +| `resolveMemoryFromEnv` | function | Resolve the bin's memory from `AGENT_MEMORY_FILE` (durable store) and/or | | `runCheck` | function | Run ONE lens over a trace → findings. Generic over any kind: prompt = the lens + the agent-eval | | `runDetachedTurn` | function | Dispatch one detached turn and advance it to a terminal state with | | `runLocalHarness` | function | Spawn a local coding harness CLI as a subprocess + collect its output. | @@ -895,12 +915,17 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `DELEGATION_STATUS_TOOL_NAME` | const | MCP tool name for the `delegation_status` synchronous-poll tool. | | `DELEGATION_TRACE_MAX_BYTES` | const | Default cap on the serialized trace payload per record, in bytes. | | `DELEGATION_TRACE_MAX_SPANS` | const | Default cap on spans retained per delegation record. | +| `MEMORY_FILE_ENV` | const | Env var naming the durable row store file the memory bin loads (the | +| `MEMORY_ITEMS_ENV` | const | Env var carrying inline JSON `MemoryItem` rows (win over file rows on id). | +| `MEMORY_LOG_ENV` | const | Env var naming the JSONL retrieval log (one row per `memory_search`). | +| `MEMORY_NAME_ENV` | const | Env var overriding the served display name (default 'agent-memory'). | | `DelegationPersistenceError` | class | A delegation-store read or write failed (filesystem error, store | | `DelegationStateCorruptError` | class | The persisted delegation state exists but cannot be parsed into | | `DelegationTaskQueue` | class | In-process queue for async delegation tasks — submit, cancel, poll status, and read history. | | `FileDelegationStore` | class | JSON-file persistence for the delegation queue. Each write serializes | | `InMemoryDelegationStore` | class | In-memory `DelegationStore` — suitable for single-process use and tests. | | `InMemoryFeedbackStore` | class | In-memory `FeedbackStore` — suitable for single-process use and tests. | +| `AgentMemorySpec` | interface | The `memory` artifact payload — HOW a profile's memory is stored and served: | | `Check` | interface | One lens — a composable analyst kind. Identity fields mirror `TraceAnalystKindSpec` so a kind is | | `CoordinationTools` | interface | The supervisor-side toolbox returned by {@link createCoordinationTools}: the MCP tool | | `DelegateArgs` | interface | Parsed `delegate` tool arguments. | @@ -913,7 +938,9 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `DetachedSessionRefParts` | interface | Decoded `DelegationRecord.detachedSessionRef`. `sandboxId` is absent between | | `DriveTurnCapableBox` | interface | The box surface detached turns need. `SandboxInstance` | | `FleetHandle` | interface | Minimal `SandboxFleet` surface the fleet executor calls. Declared | +| `MemoryItem` | interface | One row of agent memory: a crisp lesson/fact with provenance. | | `ResearchOutputShape` | interface | Loose shape of a research output over the wire — the substrate cannot | +| `ResolvedMemoryEnv` | interface | What the memory bin resolved from its environment. | | `SettledWorker` | interface | A worker the driver has drained via `await_event`. | | `TraceContext` | interface | Trace context propagation for MCP subprocess. | | `UiAuditorDelegationOutput` | interface | Wire-shape of a completed UI-audit delegation. The `findings` array | @@ -927,7 +954,7 @@ Import from `@tangle-network/agent-runtime/mcp` — 170 exports. | `LocalHarness` | type | Local coding harness available inside the sandbox. | | `UiAuditorDelegate` | type | UI-auditor delegate — fully consumer-injected. agent-runtime ships no | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalystRegistry`, `CappedDelegationTrace`, `CheckRunnerOptions`, `CoderReview`, `CoordinationToolsOptions`, `CreateKbGateOptions`, `CreateWorktreeOptions`, `DelegateCodeArgs`, `DelegateCodeResult`, `DelegateFeedbackArgs`, `DelegateFeedbackResult`, `DelegateHandlerOptions`, `DelegateResearchArgs`, `DelegateResearchConfig`, `DelegateResearchResult`, `DelegateRunCtx`, `DelegateUiAuditArgs`, `DelegateUiAuditConfig`, `DelegateUiAuditResult`, `DelegationError`, `DelegationExecutor`, `DelegationFeedbackSnapshot`, `DelegationHistoryArgs`, `DelegationHistoryEntry`, `DelegationHistoryResult`, `DelegationProgress`, `DelegationResumeContext`, `DelegationRunContext`, `DelegationStatusArgs`, `DelegationStatusResult`, `DelegationStore`, `DelegationTaskQueueOptions`, `DelegationTraceCaps`, `DetachedSessionDelegateOptions`, `DetachedTurn`, `DetachedTurnResumeDriverOptions`, `DetectExecutorArgs`, `DiffOptions`, `DiffResult`, `FactCandidate`, `FactJudge`, `FactJudgeVerdict`, `FeedbackEvent`, `FeedbackRating`, `FeedbackRefersTo`, `FeedbackStore`, `FileDelegationStoreOptions`, `FleetWorkspaceExecutorOptions`, `InProcessExecutorDescribePlacement`, `InProcessExecutorOptions`, `JsonRpcMessage`, `JsonRpcResponse`, `KbGateResult`, `LocalHarnessResult`, `McpServer`, `McpServerOptions`, `McpToolDescriptor`, `McpTransport`, `Question`, `QuestionRecord`, `RemoveWorktreeOptions`, `RunDetachedTurnOptions`, `RunLocalHarnessOptions`, `SettleDetachedCoderTurnOptions`, `SiblingSandboxExecutorOptions`, `SubmitInput`, `SubmitOutput`, `WorktreeHandle`, `CoderDelegate`, `DelegationProfile`, `DelegationStatus`, `DetachedWinnerSelection`, `MakeWorkerAgent`, `QuestionDecision`, `QuestionPolicy`, `ResearchSource`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AnalystRegistry`, `CappedDelegationTrace`, `CheckRunnerOptions`, `CoderReview`, `CoordinationToolsOptions`, `CreateKbGateOptions`, `CreateMemoryToolServerOptions`, `CreateWorktreeOptions`, `DelegateCodeArgs`, `DelegateCodeResult`, `DelegateFeedbackArgs`, `DelegateFeedbackResult`, `DelegateHandlerOptions`, `DelegateResearchArgs`, `DelegateResearchConfig`, `DelegateResearchResult`, `DelegateRunCtx`, `DelegateUiAuditArgs`, `DelegateUiAuditConfig`, `DelegateUiAuditResult`, `DelegationError`, `DelegationExecutor`, `DelegationFeedbackSnapshot`, `DelegationHistoryArgs`, `DelegationHistoryEntry`, `DelegationHistoryResult`, `DelegationProgress`, `DelegationResumeContext`, `DelegationRunContext`, `DelegationStatusArgs`, `DelegationStatusResult`, `DelegationStore`, `DelegationTaskQueueOptions`, `DelegationTraceCaps`, `DetachedSessionDelegateOptions`, `DetachedTurn`, `DetachedTurnResumeDriverOptions`, `DetectExecutorArgs`, `DiffOptions`, `DiffResult`, `FactCandidate`, `FactJudge`, `FactJudgeVerdict`, `FeedbackEvent`, `FeedbackRating`, `FeedbackRefersTo`, `FeedbackStore`, `FileDelegationStoreOptions`, `FleetWorkspaceExecutorOptions`, `InProcessExecutorDescribePlacement`, `InProcessExecutorOptions`, `JsonRpcMessage`, `JsonRpcResponse`, `KbGateResult`, `LocalHarnessResult`, `McpServer`, `McpServerOptions`, `McpToolDescriptor`, `McpTransport`, `Question`, `QuestionRecord`, `RemoveWorktreeOptions`, `RunDetachedTurnOptions`, `RunLocalHarnessOptions`, `SettleDetachedCoderTurnOptions`, `SiblingSandboxExecutorOptions`, `StdioToolDescriptor`, `StdioToolServer`, `StdioToolServerOptions`, `SubmitInput`, `SubmitOutput`, `WorktreeHandle`, `CoderDelegate`, `DelegationProfile`, `DelegationStatus`, `DetachedWinnerSelection`, `MakeWorkerAgent`, `QuestionDecision`, `QuestionPolicy`, `ResearchSource`. ## 2. agent-eval — substrate primitives to REUSE diff --git a/package.json b/package.json index bc71b0f9..0ee87109 100644 --- a/package.json +++ b/package.json @@ -72,6 +72,7 @@ }, "bin": { "agent-runtime-mcp": "./dist/mcp/bin.js", + "agent-runtime-memory-mcp": "./dist/mcp/memory-bin.js", "agent-runtime-loop": "./dist/loop-runner-bin.js" }, "files": [ diff --git a/src/lifecycle/apply.ts b/src/lifecycle/apply.ts index 6f5898ca..ddb030e4 100644 --- a/src/lifecycle/apply.ts +++ b/src/lifecycle/apply.ts @@ -10,6 +10,7 @@ */ import type { AgentProfile } from '@tangle-network/agent-interface' +import { memoryMcpServer, profileMemoryMetadataKey } from './memory' import type { ProfileArtifact } from './types' /** @@ -54,6 +55,20 @@ export function applyArtifact(base: AgentProfile, artifact: ProfileArtifact): Ag const key = keyOf(artifact) return { ...base, subagents: { ...base.subagents, [key]: payload.profile } } } + case 'memory': { + // Double mount: the typed spec rides `metadata.memory` (the local + // extension — the published AgentProfile has no memory field yet), and + // the LIVE serving rides `mcp[key]` (a field the profile does have), so + // the same-host materialization boots the memory during a scored run + // with zero scorer changes. See lifecycle/memory.ts for the boundary. + const payload = artifact.payload as ProfileArtifact<'memory'>['payload'] + const key = keyOf(artifact) + return { + ...base, + metadata: { ...base.metadata, [profileMemoryMetadataKey]: payload.spec }, + mcp: { ...base.mcp, [key]: memoryMcpServer(payload.spec) }, + } + } } } diff --git a/src/lifecycle/index.ts b/src/lifecycle/index.ts index f224689a..404334c0 100644 --- a/src/lifecycle/index.ts +++ b/src/lifecycle/index.ts @@ -60,6 +60,26 @@ export { type MeasureMarginalLiftOptions, measureMarginalLift, } from './marginal-lift' +export { + type AgentMemorySpec, + CURATED_MEMORY_BLOCK_END, + CURATED_MEMORY_BLOCK_START, + lessonsFromCuratedSurface, + type MemoryArtifactOptions, + type MemoryGeneratorOptions, + type MemoryItem, + type MemoryMcpServerOptions, + memoryArtifactFromCuratedSurface, + memoryArtifactFromLessons, + memoryGenerator, + memoryMcpServer, + memoryOfProfile, + type ProfileWithMemory, + profileMemoryMetadataKey, + readMemoryItemsFile, + validateMemorySpec, + writeMemoryStore, +} from './memory' export { type AuthorDiverseSeeds, gepaRefine, diff --git a/src/lifecycle/memory.test.ts b/src/lifecycle/memory.test.ts new file mode 100644 index 00000000..4013735c --- /dev/null +++ b/src/lifecycle/memory.test.ts @@ -0,0 +1,238 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { afterAll, describe, expect, it } from 'vitest' +import { ValidationError } from '../errors' +import { materializeLocalMcp } from '../runtime/stdio-mcp-client' +import { applyArtifact } from './apply' +import type { GenerateContext } from './generator' +import { + CURATED_MEMORY_BLOCK_END, + CURATED_MEMORY_BLOCK_START, + lessonsFromCuratedSurface, + memoryArtifactFromCuratedSurface, + memoryArtifactFromLessons, + memoryGenerator, + memoryMcpServer, + memoryOfProfile, + profileMemoryMetadataKey, + readMemoryItemsFile, +} from './memory' +import type { ProfileArtifact } from './types' + +const tmp = mkdtempSync(join(tmpdir(), 'lifecycle-memory-test-')) +afterAll(() => rmSync(tmp, { recursive: true, force: true })) + +const LESSONS = [ + 'Always reproduce the failing test before editing the fix', + 'Read the full traceback before guessing the module', +] + +const asArtifact = (input: ReturnType): ProfileArtifact => ({ + ...input, + id: input.key ?? 'memory', + status: 'candidate', +}) + +describe('memoryArtifactFromLessons', () => { + it('builds an inline-items memory artifact with content-addressed ids', () => { + const a = memoryArtifactFromLessons(LESSONS) + expect(a.kind).toBe('memory') + const items = a.payload.spec.items ?? [] + expect(items).toHaveLength(2) + expect(items[0]?.id).toMatch(/^mem-[0-9a-f]{12}$/) + // Content-addressed: the same lesson always earns the same id. + expect(memoryArtifactFromLessons(LESSONS).payload.spec.items?.[0]?.id).toBe(items[0]?.id) + }) + + it('dedupes near-identical lessons and throws on zero usable ones', () => { + const a = memoryArtifactFromLessons([' Do X. ', 'do x', '']) + expect(a.payload.spec.items).toHaveLength(1) + expect(() => memoryArtifactFromLessons(['', ' '])).toThrow(ValidationError) + }) + + it('writes a durable JSONL store when storePath is given', () => { + const storePath = join(tmp, 'store', 'memory.jsonl') + const a = memoryArtifactFromLessons(LESSONS, { storePath }) + expect(a.payload.spec.path).toBe(storePath) + expect(a.payload.spec.items).toBeUndefined() + expect(readMemoryItemsFile(storePath).map((i) => i.text)).toEqual(LESSONS) + }) +}) + +describe('lessonsFromCuratedSurface (the memoryCurationProposer seam)', () => { + it('parses the exact block shape the proposer emits', () => { + const surface = [ + 'You are a coding agent.', + '', + CURATED_MEMORY_BLOCK_START, + '## Learned from prior runs (curated memory)', + `- ${LESSONS[0]}`, + `- ${LESSONS[1]}`, + CURATED_MEMORY_BLOCK_END, + '', + ].join('\n') + expect(lessonsFromCuratedSurface(surface)).toEqual(LESSONS) + const a = memoryArtifactFromCuratedSurface(surface) + expect(a?.payload.spec.items?.map((i) => i.text)).toEqual(LESSONS) + }) + + it('returns [] / undefined on a surface with no block (gen 0)', () => { + expect(lessonsFromCuratedSurface('plain prompt')).toEqual([]) + expect(memoryArtifactFromCuratedSurface('plain prompt')).toBeUndefined() + }) +}) + +describe('applyArtifact(memory)', () => { + it('mounts the typed spec on metadata.memory AND a live stdio server on mcp', () => { + const base: AgentProfile = { name: 'agent', metadata: { keep: 1 } } + const out = applyArtifact(base, asArtifact(memoryArtifactFromLessons(LESSONS))) + // The local extension (published AgentProfile has no memory field). + const spec = memoryOfProfile(out) + expect(spec?.store).toBe('file') + expect(spec?.items).toHaveLength(2) + expect(out.metadata?.keep).toBe(1) + // The LIVE mount: a stdio entry the same-host client can boot. + const server = out.mcp?.memory + expect(server?.transport).toBe('stdio') + expect(server?.enabled).toBe(true) + expect(server?.env?.AGENT_MEMORY_ITEMS).toContain(LESSONS[0]) + // Base untouched. + expect(base.metadata).toEqual({ keep: 1 }) + expect(base.mcp).toBeUndefined() + }) + + it("store 'mcp' mounts the external server verbatim (enabled defaulted)", () => { + const external = { transport: 'stdio' as const, command: 'my-memory-server' } + const out = applyArtifact( + {}, + { + id: 'memory', + kind: 'memory', + name: 'external', + status: 'candidate', + payload: { spec: { store: 'mcp', server: external } }, + }, + ) + expect(out.mcp?.memory).toEqual({ ...external, enabled: true }) + }) + + it('memoryOfProfile is undefined without memory and throws on a malformed spec', () => { + expect(memoryOfProfile({})).toBeUndefined() + expect(() => + memoryOfProfile({ metadata: { [profileMemoryMetadataKey]: 'not-a-spec' } }), + ).toThrow(ValidationError) + expect(() => + memoryOfProfile({ metadata: { [profileMemoryMetadataKey]: { store: 'file' } } }), + ).toThrow(/path.*items/) + }) + + it('refuses inline items past the env-string cap, naming the file-store fix', () => { + const huge = [`unique prefix ${'x'.repeat(120_000)}`] + expect(() => + memoryMcpServer({ + store: 'file', + items: memoryArtifactFromLessons(huge).payload.spec.items ?? [], + }), + ).toThrow(/writeMemoryStore/) + }) +}) + +describe('memoryGenerator', () => { + const finding = (claim: string, extra: Record = {}) => ({ + schema_version: '1.0.0' as const, + finding_id: `f-${claim.slice(0, 8)}`, + analyst_id: 'failure-mode', + produced_at: new Date().toISOString(), + severity: 'medium' as const, + area: 'agent-reasoning', + claim, + evidence_refs: [], + confidence: 0.8, + ...extra, + }) + const ctx = (over: Partial): GenerateContext => ({ + baseline: {}, + domain: 'swe', + findings: [], + ...over, + }) + + it('curates findings into ONE inline memory candidate, preferring recommended_action', async () => { + const out = await memoryGenerator().generate( + ctx({ + findings: [ + finding('agent guessed the module path', { + recommended_action: 'Read the traceback before editing', + }), + finding('tests were never run'), + ], + }), + ) + expect(out).toHaveLength(1) + expect(out[0]?.payload.spec.items?.map((i) => i.text)).toEqual([ + 'Read the traceback before editing', + 'tests were never run', + ]) + }) + + it('drops judge-derived findings (the firewall) and returns [] with nothing left', async () => { + const out = await memoryGenerator().generate( + ctx({ findings: [finding('judge said so', { derived_from_judge: true })] }), + ) + expect(out).toEqual([]) + }) + + it('accumulates the baseline memory and skips a no-change generation', async () => { + const baseline = applyArtifact({}, asArtifact(memoryArtifactFromLessons(LESSONS))) + // Same lessons re-observed → curated set == carried set → no candidate. + const unchanged = await memoryGenerator().generate( + ctx({ baseline, findings: [finding(LESSONS[0] as string)] }), + ) + expect(unchanged).toEqual([]) + // A genuinely new lesson → one candidate carrying old + new. + const grown = await memoryGenerator().generate( + ctx({ baseline, findings: [finding('Pin the schema version before migrating')] }), + ) + expect(grown).toHaveLength(1) + const texts = grown[0]?.payload.spec.items?.map((i) => i.text) ?? [] + expect(texts).toContain('Pin the schema version before migrating') + for (const lesson of LESSONS) expect(texts).toContain(lesson) + }) + + it('caps at maxLessons by recurrence', async () => { + const out = await memoryGenerator({ maxLessons: 1 }).generate( + ctx({ + findings: [finding('repeated lesson'), finding('repeated lesson'), finding('one-off')], + }), + ) + expect(out[0]?.payload.spec.items?.map((i) => i.text)).toEqual(['repeated lesson']) + }) +}) + +describe('live same-host boot (the Phase-5 smoke)', () => { + it('apply memory artifact → materializeLocalMcp boots the bin → memory_search returns the seeded lesson', { + timeout: 60_000, + }, async () => { + const logPath = join(tmp, 'live', 'retrieval.jsonl') + const profile = applyArtifact({}, asArtifact(memoryArtifactFromLessons(LESSONS, { logPath }))) + // The SAME path sweEvalRunner({ materializeMcp: true }) runs during a + // scored eval: spawn every profile.mcp stdio server next to the worker. + const mat = await materializeLocalMcp(profile) + try { + expect(mat.tools.map((t) => t.function.name)).toEqual([ + 'memory__memory_search', + 'memory__memory_get', + ]) + const hit = await mat.call('memory__memory_search', { query: 'reproduce failing test' }) + expect(hit).toContain(LESSONS[0]) + // The retrieval log captured the live query (the RetrievalHoldout seam). + const rows = readFileSync(logPath, 'utf8').trim().split('\n') + expect(rows).toHaveLength(1) + expect(JSON.parse(rows[0] as string).query).toBe('reproduce failing test') + } finally { + await mat.close() + } + }) +}) diff --git a/src/lifecycle/memory.ts b/src/lifecycle/memory.ts new file mode 100644 index 00000000..36da3b3f --- /dev/null +++ b/src/lifecycle/memory.ts @@ -0,0 +1,419 @@ +/** + * PHASE 5 — memory as a first-class profile surface. + * + * The published `AgentProfile` (`@tangle-network/agent-interface`; source repo + * absent from this workspace) has NO `memory` field, so a trace-learned memory + * had nothing to sit on. This module carries it as a LOCAL profile extension + * and makes it LIVE through a field the profile DOES have: + * + * - `profile.metadata[profileMemoryMetadataKey]` — the typed, auditable + * `AgentMemorySpec` record (`ProfileWithMemory` names the extension); + * - `profile.mcp[]` — a stdio server entry (`memoryMcpServer`) that + * SERVES that spec via the in-repo memory bin (built on + * `createStdioToolServer`), which the same-host client + * (`materializeLocalMcp`) already boots — so + * `sweEvalRunner({ materializeMcp: true })` scores a memory-carrying + * profile with `memory_search`/`memory_get` live, with ZERO scorer changes. + * + * `applyArtifact`'s `memory` case mounts both at once (apply.ts). + * + * >>> CROSS-PACKAGE BOUNDARY (flagged): when agent-interface grows a real + * `AgentProfile.memory` field, the metadata shim moves there; the mcp mount, + * bin, and serving stay unchanged. Do NOT edit node_modules to fake the field. + * + * MEASURING memory-as-treatment: the with/without ablation IS + * + * measureMarginalLift({ + * baseline, + * candidate: registry.register(memoryArtifactFromLessons(lessons)), + * evalRunner: sweEvalRunner({ ...opts, materializeMcp: true }), + * }) + * + * — the candidate arm runs with the memory served live, the baseline without. + * agent-knowledge's `RetrievalHoldout` (the off-policy PER-RETRIEVAL + * estimator) lives in a package this repo does not depend on; the seam it + * needs — a per-query retrieval log — is `AgentMemorySpec.logPath` (JSONL, one + * row per `memory_search`). Flagged as the remaining cross-package hop. + * + * WHERE LESSONS COME FROM: agent-eval's `memoryCurationProposer` writes a + * delimited `curated-memory` block into its candidate surface; + * `memoryArtifactFromCuratedSurface` parses that block into a `memory` + * artifact (the store the proposer's lessons land in). The block markers are + * duplicated here BY CONVENTION — agent-eval does not export them; flagged. + * `memoryGenerator` is the native lifecycle path: a `CandidateGenerator` that + * curates `ctx.findings` (observed analyst findings only — the judge firewall + * holds) plus the baseline's carried memory into one candidate artifact. + */ + +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { AgentProfile, AgentProfileMcpServer } from '@tangle-network/agent-interface' +import { ValidationError } from '../errors' +import { + type AgentMemorySpec, + MEMORY_FILE_ENV, + MEMORY_ITEMS_ENV, + MEMORY_LOG_ENV, + type MemoryItem, + readMemoryItemsFile, +} from '../mcp/memory-server' +import type { CandidateGenerator } from './generator' +import type { ArtifactInput } from './types' + +export type { AgentMemorySpec, MemoryItem } from '../mcp/memory-server' + +/** The `profile.metadata` key the typed memory spec rides under (the local + * extension standing in for the missing `AgentProfile.memory` field). */ +export const profileMemoryMetadataKey = 'memory' + +/** The local profile extension: `AgentProfile` plus the memory spec under + * `metadata.memory`. Purely a naming convenience — any `AgentProfile` a + * `memory` artifact was applied to satisfies it. */ +export interface ProfileWithMemory extends AgentProfile { + metadata?: Record & { [profileMemoryMetadataKey]?: AgentMemorySpec } +} + +/** Read (and validate) the memory spec a profile carries, if any. Malformed + * metadata throws — silently scoring a broken memory as "no memory" would + * fake the ablation. */ +export function memoryOfProfile(profile: AgentProfile): AgentMemorySpec | undefined { + const raw = profile.metadata?.[profileMemoryMetadataKey] + if (raw === undefined || raw === null) return undefined + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new ValidationError( + `memoryOfProfile: profile.metadata.${profileMemoryMetadataKey} is not a memory spec object`, + ) + } + const spec = raw as AgentMemorySpec + validateMemorySpec(spec) + return spec +} + +/** Assert an `AgentMemorySpec` is well-formed and servable. */ +export function validateMemorySpec(spec: AgentMemorySpec): void { + if (spec.store !== 'file' && spec.store !== 'mcp') { + throw new ValidationError( + `memory spec: store must be 'file' or 'mcp' (got '${String((spec as { store?: unknown }).store)}')`, + ) + } + if (spec.store === 'mcp' && !spec.server) { + throw new ValidationError( + "memory spec: store 'mcp' requires `server` (the external memory MCP)", + ) + } + if (spec.store === 'file' && !spec.path && (spec.items?.length ?? 0) === 0) { + throw new ValidationError( + "memory spec: store 'file' requires `path` and/or non-empty `items` — an empty memory is never mounted", + ) + } + const seen = new Set() + for (const item of spec.items ?? []) { + if (!item.id?.trim() || !item.text?.trim()) { + throw new ValidationError('memory spec: every item needs a non-empty id and text') + } + if (seen.has(item.id)) { + throw new ValidationError(`memory spec: duplicate item id '${item.id}'`) + } + seen.add(item.id) + } +} + +/** Linux caps a single env string near 128KiB; refuse before the spawn does. */ +const MAX_INLINE_MEMORY_CHARS = 100_000 + +export interface MemoryMcpServerOptions { + /** Override the serve launch (e.g. the published `agent-runtime-memory-mcp` + * bin on PATH). Default: auto-resolve the in-repo bin — `dist/mcp/memory-bin.js` + * under node when built, the TS source through tsx in dev/test. */ + command?: string + args?: string[] +} + +/** + * The `AgentProfileMcpServer` entry that SERVES a memory spec — what the + * `memory` `applyArtifact` case mounts into `profile.mcp` so the same-host + * client boots the memory during a scored run. `store:'mcp'` mounts the + * external server verbatim; `store:'file'` launches the in-repo memory bin + * with the spec carried on its env (`AGENT_MEMORY_FILE`/`AGENT_MEMORY_ITEMS`). + * + * NB the launch command resolves against THIS install (same-host, like the + * worktree `cwd` a built MCP candidate bakes in) — a memory-mounted profile is + * a same-host profile until the profile schema grows a portable memory field. + */ +export function memoryMcpServer( + spec: AgentMemorySpec, + opts: MemoryMcpServerOptions = {}, +): AgentProfileMcpServer { + validateMemorySpec(spec) + if (spec.store === 'mcp') { + const server = spec.server as AgentProfileMcpServer + return { ...server, enabled: server.enabled ?? true } + } + const launch = opts.command + ? { command: opts.command, args: opts.args ?? [] } + : resolveMemoryBinLaunch() + const inlineJson = spec.items && spec.items.length > 0 ? JSON.stringify(spec.items) : undefined + if (inlineJson && inlineJson.length > MAX_INLINE_MEMORY_CHARS) { + throw new ValidationError( + `memoryMcpServer: inline items serialize to ${inlineJson.length} chars (> ${MAX_INLINE_MEMORY_CHARS}) — write them to a file store (writeMemoryStore) and reference it via spec.path`, + ) + } + return { + transport: 'stdio', + command: launch.command, + ...(launch.args.length > 0 ? { args: launch.args } : {}), + env: { + ...(spec.path ? { [MEMORY_FILE_ENV]: spec.path } : {}), + ...(inlineJson ? { [MEMORY_ITEMS_ENV]: inlineJson } : {}), + ...(spec.logPath ? { [MEMORY_LOG_ENV]: spec.logPath } : {}), + }, + enabled: true, + } +} + +/** Locate the memory bin for THIS install: the built `dist/mcp/memory-bin.js` + * when running from dist, else the TS source through tsx (dev/test). */ +function resolveMemoryBinLaunch(): { command: string; args: string[] } { + const dir = dirname(fileURLToPath(import.meta.url)) + // Built package: this module bundles into dist/*.js; the bin entry sits at + // dist/mcp/memory-bin.js next to it. + const distBin = join(dir, 'mcp', 'memory-bin.js') + if (existsSync(distBin)) { + return { command: process.execPath, args: [distBin] } + } + // Dev/test: this module runs from src/lifecycle; spawn the TS bin via tsx. + const srcBin = join(dir, '..', 'mcp', 'memory-bin.ts') + if (existsSync(srcBin)) { + const tsxCli = createRequire(import.meta.url).resolve('tsx/cli') + return { command: process.execPath, args: [tsxCli, srcBin] } + } + throw new ValidationError( + 'memoryMcpServer: cannot locate the memory bin (no dist/mcp/memory-bin.js, no src/mcp/memory-bin.ts) — pass an explicit `command`', + ) +} + +/** Write a durable JSONL memory store (one `MemoryItem` per line). + * + * CONTAMINATION HAZARD: give every candidate its own path. Overwriting a file + * the BASELINE profile's spec also references changes the without-arm + * mid-ablation and fakes the lift. `memoryGenerator` avoids files entirely + * (inline items are immutable in the artifact) for exactly this reason. */ +export function writeMemoryStore(path: string, items: readonly MemoryItem[]): void { + if (items.length === 0) { + throw new ValidationError('writeMemoryStore: refusing to write an EMPTY memory store') + } + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, `${items.map((i) => JSON.stringify(i)).join('\n')}\n`, 'utf8') +} + +/** Re-exported so lifecycle callers can read a store without reaching into mcp/. */ +export { readMemoryItemsFile } + +/** + * The opening delimiter agent-eval's `memoryCurationProposer` writes its + * curated block after. Duplicated BY CONVENTION — agent-eval does not export + * it (flagged cross-package coupling; a marker drift breaks + * `lessonsFromCuratedSurface`, whose tests pin the published block shape). + */ +export const CURATED_MEMORY_BLOCK_START = + '' +/** The closing delimiter of the proposer's curated block (see + * `CURATED_MEMORY_BLOCK_START` for the convention-coupling flag). */ +export const CURATED_MEMORY_BLOCK_END = '' + +/** Extract the lesson lines from a surface carrying a curated-memory block. + * Returns `[]` when the surface has no block (gen 0 — nothing learned yet). */ +export function lessonsFromCuratedSurface(surfaceText: string): string[] { + const start = surfaceText.indexOf(CURATED_MEMORY_BLOCK_START) + if (start < 0) return [] + const bodyStart = start + CURATED_MEMORY_BLOCK_START.length + const end = surfaceText.indexOf(CURATED_MEMORY_BLOCK_END, bodyStart) + const body = end < 0 ? surfaceText.slice(bodyStart) : surfaceText.slice(bodyStart, end) + return body + .split('\n') + .map((line) => line.replace(/^\s*-\s+/, '').trim()) + .filter((line) => line.length > 0 && !line.startsWith('#')) +} + +export interface MemoryArtifactOptions { + /** The `profile.mcp` key / artifact key the memory mounts under. Default 'memory'. */ + key?: string + /** Artifact display name. Default 'curated-memory'. */ + name?: string + /** Write the items to this JSONL store and reference it via `spec.path` + * (durable, file-backed). Omitted: items ride INLINE in the spec (immutable + * in the artifact — the safe default; see `writeMemoryStore`'s hazard note). */ + storePath?: string + /** Retrieval-log path baked into the spec (`AGENT_MEMORY_LOG`). */ + logPath?: string + /** Provenance stamped on each item. Default 'memory-curation'. */ + source?: string +} + +/** + * The curated-lessons → memory-artifact adapter (the seam that gives + * `memoryCurationProposer` a store to write to). Lessons are normalized, + * deduplicated, and content-addressed (`mem-`), so the same lesson + * gets the same id across generations. Throws on zero usable lessons — a + * memory artifact must never mount an empty memory. + */ +export function memoryArtifactFromLessons( + lessons: readonly string[], + opts: MemoryArtifactOptions = {}, +): ArtifactInput<'memory'> { + const items = itemsFromLessons(lessons, opts.source ?? 'memory-curation') + if (items.length === 0) { + throw new ValidationError( + 'memoryArtifactFromLessons: no non-empty lessons — gen 0 simply has no memory artifact', + ) + } + if (opts.storePath) writeMemoryStore(opts.storePath, items) + const spec: AgentMemorySpec = { + store: 'file', + ...(opts.storePath ? { path: opts.storePath } : { items }), + ...(opts.logPath ? { logPath: opts.logPath } : {}), + } + return { + kind: 'memory', + key: opts.key ?? 'memory', + name: opts.name ?? 'curated-memory', + description: `curated memory of ${items.length} lesson(s), served live as memory_search/memory_get`, + payload: { spec }, + metadata: { + lessonCount: items.length, + ...(opts.storePath ? { storePath: opts.storePath } : {}), + }, + } +} + +/** Parse a proposer surface and build the artifact in one step. Returns + * `undefined` when the surface carries no curated block (nothing learned). */ +export function memoryArtifactFromCuratedSurface( + surfaceText: string, + opts: MemoryArtifactOptions = {}, +): ArtifactInput<'memory'> | undefined { + const lessons = lessonsFromCuratedSurface(surfaceText) + if (lessons.length === 0) return undefined + return memoryArtifactFromLessons(lessons, opts) +} + +export interface MemoryGeneratorOptions { + /** Top-K lessons kept per generation. Default 12 (mirrors `memoryCurationProposer`). */ + maxLessons?: number + /** The `profile.mcp` key / artifact key. Default 'memory'. */ + key?: string + /** Retrieval-log path baked into every candidate spec. */ + logPath?: string +} + +/** + * The `memory` surface's `CandidateGenerator` — the native lifecycle path for + * memory-as-treatment. Mirrors `memoryCurationProposer`'s semantics on the + * artifact plane: + * + * 1. fresh lessons ← `ctx.findings` (`recommended_action` else `claim`), + * OBSERVED findings only — `derived_from_judge` rows are dropped (the + * judge firewall: a held-out verdict must never steer the loop); + * 2. carried lessons ← the baseline profile's existing memory (inline items + * plus its file store, when readable), so memory ACCUMULATES; + * 3. curate: normalize, dedupe, rank by recurrence, keep top-K; + * 4. emit ONE inline-items candidate — or `[]` when nothing new was learned + * (the curated set equals what the baseline already carries). + * + * `runLifecycle` then measures it with the SAME `sweEvalRunner` as every other + * surface (`materializeMcp: true` makes the memory live in the candidate arm). + */ +export function memoryGenerator(opts: MemoryGeneratorOptions = {}): CandidateGenerator<'memory'> { + const maxLessons = opts.maxLessons ?? 12 + if (maxLessons < 1) { + throw new ValidationError(`memoryGenerator: maxLessons must be >= 1 (got ${maxLessons})`) + } + return { + kind: 'memory', + async generate(ctx): Promise[]> { + const fresh: string[] = [] + for (const finding of ctx.findings) { + if (finding.derived_from_judge) continue + const lesson = (finding.recommended_action ?? finding.claim ?? '').trim() + if (lesson) fresh.push(lesson) + } + const carried = carriedLessons(ctx.baseline) + if (fresh.length === 0 && carried.length === 0) return [] + + // Rank by recurrence: carried lessons seed at 1; every fresh repeat + // increments. A lesson seen across many findings outranks a one-off. + const byKey = new Map() + for (const text of carried) { + const k = normalizeLesson(text) + if (k && !byKey.has(k)) byKey.set(k, { text, count: 1 }) + } + for (const text of fresh) { + const k = normalizeLesson(text) + if (!k) continue + const entry = byKey.get(k) + if (entry) entry.count += 1 + else byKey.set(k, { text, count: 1 }) + } + const curated = [...byKey.values()] + .sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)) + .slice(0, maxLessons) + .map((e) => e.text) + if (curated.length === 0) return [] + + // Nothing learned: the curated set is exactly the carried set — the + // candidate would mount an identical memory, a guaranteed zero-delta. + const carriedKeys = new Set(carried.map(normalizeLesson).filter((k) => k.length > 0)) + const curatedKeys = new Set(curated.map(normalizeLesson)) + if ( + curatedKeys.size === carriedKeys.size && + [...curatedKeys].every((k) => carriedKeys.has(k)) + ) { + return [] + } + + return [ + memoryArtifactFromLessons(curated, { + ...(opts.key ? { key: opts.key } : {}), + ...(opts.logPath ? { logPath: opts.logPath } : {}), + source: 'memory-generator', + }), + ] + }, + } +} + +/** The baseline's existing memory as lesson texts (inline items + file store). */ +function carriedLessons(baseline: AgentProfile): string[] { + const spec = memoryOfProfile(baseline) + if (!spec) return [] + const items: MemoryItem[] = [...(spec.items ?? [])] + if (spec.store === 'file' && spec.path && existsSync(spec.path)) { + items.push(...readMemoryItemsFile(spec.path)) + } + return items.map((i) => i.text) +} + +/** Normalize a lesson for dedupe: lowercase, collapse whitespace, strip + * trailing punctuation — the same key `memoryCurationProposer` dedupes on. */ +function normalizeLesson(text: string): string { + return text + .toLowerCase() + .replace(/\s+/g, ' ') + .replace(/[.;:!?\s]+$/, '') + .trim() +} + +/** Deterministic content-addressed items: `mem-`. */ +function itemsFromLessons(lessons: readonly string[], source: string): MemoryItem[] { + const byId = new Map() + for (const lesson of lessons) { + const text = lesson.replace(/\s+/g, ' ').trim() + if (!text) continue + const id = `mem-${createHash('sha256').update(normalizeLesson(text)).digest('hex').slice(0, 12)}` + if (!byId.has(id)) byId.set(id, { id, text, source }) + } + return [...byId.values()] +} diff --git a/src/lifecycle/swe-eval-runner.ts b/src/lifecycle/swe-eval-runner.ts index 8b15a7f8..1bcad3ef 100644 --- a/src/lifecycle/swe-eval-runner.ts +++ b/src/lifecycle/swe-eval-runner.ts @@ -28,7 +28,10 @@ * (`materializeLocalMcp`) for the duration of the eval and its tools are * overlaid on the domain surface — so a worktree-BUILT MCP candidate is LIVE * while the profile is scored, and its marginal lift is real. Default off: - * the prompt-only path spawns nothing. + * the prompt-only path spawns nothing. Phase-5 memory rides this same path: + * a `memory` artifact mounts a stdio memory server into `profile.mcp` + * (lifecycle/memory.ts), so `materializeMcp: true` also makes a learned + * memory (`memory_search`/`memory_get`) live while the profile is scored. * * Phase-1 scope otherwise: returns the scalar `composite` (mean resolved) + * `costUsd` only — no per-task `RunRecord`s — so it pairs with diff --git a/src/lifecycle/types.ts b/src/lifecycle/types.ts index 9a38b47a..d79b75f6 100644 --- a/src/lifecycle/types.ts +++ b/src/lifecycle/types.ts @@ -22,14 +22,17 @@ import type { AgentProfileResourceRef, AgentSubagentProfile, } from '@tangle-network/agent-interface' +import type { AgentMemorySpec } from '../mcp/memory-server' /** * The profile levers an artifact can target. One-to-one with the §1.5 profile - * surface (`prompt + skills + tools + mcp + hooks + subagents`). Each kind maps to - * exactly one field of `AgentProfile`, so an artifact can be applied onto a - * baseline profile deterministically (see `applyArtifact`). + * surface (`prompt + skills + tools + mcp + hooks + subagents`), plus `memory` + * — the Phase-5 surface the published `AgentProfile` does not yet name (it + * mounts as a local extension; see lifecycle/memory.ts). Each kind maps to + * exactly one profile lever, so an artifact can be applied onto a baseline + * profile deterministically (see `applyArtifact`). */ -export type ArtifactKind = 'skill' | 'tool' | 'mcp' | 'hook' | 'subagent' | 'prompt' +export type ArtifactKind = 'skill' | 'tool' | 'mcp' | 'hook' | 'subagent' | 'prompt' | 'memory' /** * The payload for each `ArtifactKind`. The shapes are the SAME types the @@ -42,6 +45,11 @@ export type ArtifactKind = 'skill' | 'tool' | 'mcp' | 'hook' | 'subagent' | 'pro * - `mcp` — one MCP server added under `profile.mcp[name]`. * - `hook` — one or more hook commands added under `profile.hooks[event]`. * - `subagent` — one subagent profile added under `profile.subagents[name]`. + * - `memory` — a memory spec: the typed record lands on + * `profile.metadata.memory` (local extension — the published + * profile has no memory field) AND a stdio server entry that + * SERVES it lands under `profile.mcp[name]`, so the same-host + * materialization boots the memory live during a scored run. */ export interface ArtifactPayloads { prompt: { instruction: string } @@ -50,6 +58,7 @@ export interface ArtifactPayloads { mcp: { server: AgentProfileMcpServer } hook: { event: string; commands: AgentProfileHookCommand[] } subagent: { profile: AgentSubagentProfile } + memory: { spec: AgentMemorySpec } } /** diff --git a/src/mcp/index.ts b/src/mcp/index.ts index 90356a25..c0ee5698 100644 --- a/src/mcp/index.ts +++ b/src/mcp/index.ts @@ -91,6 +91,20 @@ export { } from './kb-gate' export type { LocalHarness, LocalHarnessResult, RunLocalHarnessOptions } from './local-harness' export { runLocalHarness } from './local-harness' +export { + type AgentMemorySpec, + type CreateMemoryToolServerOptions, + createMemoryToolServer, + MEMORY_FILE_ENV, + MEMORY_ITEMS_ENV, + MEMORY_LOG_ENV, + MEMORY_NAME_ENV, + type MemoryItem, + parseMemoryItems, + type ResolvedMemoryEnv, + readMemoryItemsFile, + resolveMemoryFromEnv, +} from './memory-server' export { mcpToolsForRuntimeMcp, mcpToolsForRuntimeMcpSubset } from './openai-tools' export type { JsonRpcMessage, @@ -112,6 +126,15 @@ export type { SubmitOutput, } from './task-queue' export { DelegationTaskQueue, hashIdempotencyInput } from './task-queue' +// The generic stdio JSON-RPC core every in-repo MCP server serves on (the +// memory server uses it; the delegation server predates the extraction). +// Descriptor type aliased: `./server` exports its own McpToolDescriptor above. +export { + createStdioToolServer, + type McpToolDescriptor as StdioToolDescriptor, + type StdioToolServer, + type StdioToolServerOptions, +} from './tool-server' export { type Check, type CheckRunnerOptions, diff --git a/src/mcp/memory-bin.ts b/src/mcp/memory-bin.ts new file mode 100644 index 00000000..d8bfa467 --- /dev/null +++ b/src/mcp/memory-bin.ts @@ -0,0 +1,43 @@ +#!/usr/bin/env node + +/** + * `agent-runtime-memory-mcp` — stdio memory MCP server entry point. + * + * Serves `memory_search` / `memory_get` over an agent's curated memory rows, + * on the generic in-repo JSON-RPC core (`createStdioToolServer`). This is the + * process `memoryMcpServer` (lifecycle/memory.ts) mounts into `profile.mcp`, + * which the same-host client (`materializeLocalMcp`) spawns next to the + * driven worker during a scored run. + * + * Environment variables (the `memoryMcpServer` contract): + * AGENT_MEMORY_FILE optional — path to the durable row store (JSON array + * or JSONL of MemoryItem). + * AGENT_MEMORY_ITEMS optional — inline JSON array of MemoryItem rows; + * wins over file rows on id collision. + * AGENT_MEMORY_LOG optional — JSONL retrieval log, one row per + * memory_search (the retrieval-holdout seam). + * AGENT_MEMORY_NAME optional — server display name (default 'agent-memory'). + * + * At least one of AGENT_MEMORY_FILE / AGENT_MEMORY_ITEMS must resolve rows: + * an EMPTY memory refuses to boot (fail-closed, so a memory-carrying profile + * whose store is missing fails the materialization, never a silent no-tool run). + * + * @experimental + */ + +import { createMemoryToolServer, resolveMemoryFromEnv } from './memory-server' + +async function main(): Promise { + const { items, serverName, logPath } = resolveMemoryFromEnv(process.env) + const server = createMemoryToolServer({ + items, + ...(serverName ? { serverName } : {}), + ...(logPath ? { logPath } : {}), + }) + await server.serve() +} + +main().catch((err: unknown) => { + console.error(`agent-runtime-memory-mcp: ${err instanceof Error ? err.message : String(err)}`) + process.exit(1) +}) diff --git a/src/mcp/memory-server.test.ts b/src/mcp/memory-server.test.ts new file mode 100644 index 00000000..04049d52 --- /dev/null +++ b/src/mcp/memory-server.test.ts @@ -0,0 +1,172 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { ValidationError } from '../errors' +import { + createMemoryToolServer, + MEMORY_FILE_ENV, + MEMORY_ITEMS_ENV, + type MemoryItem, + parseMemoryItems, + readMemoryItemsFile, + resolveMemoryFromEnv, +} from './memory-server' + +const tmp = mkdtempSync(join(tmpdir(), 'memory-server-test-')) +afterAll(() => rmSync(tmp, { recursive: true, force: true })) + +const ITEMS: MemoryItem[] = [ + { id: 'mem-1', text: 'Always run the failing test before editing the fix', tags: ['testing'] }, + { id: 'mem-2', text: 'Prefer reading the traceback over guessing the module', source: 'run-7' }, + { id: 'mem-3', text: 'Pin the schema version before migrating rows', tags: ['db'] }, +] + +describe('createMemoryToolServer', () => { + const server = createMemoryToolServer({ items: ITEMS }) + + it('answers the MCP handshake with server info', async () => { + const res = await server.handle({ jsonrpc: '2.0', id: 1, method: 'initialize' }) + expect(res?.result).toMatchObject({ + protocolVersion: '2024-11-05', + serverInfo: { name: 'agent-memory' }, + }) + }) + + it('lists memory_search and memory_get with schemas', async () => { + const res = await server.handle({ jsonrpc: '2.0', id: 2, method: 'tools/list' }) + const tools = (res?.result as { tools: Array<{ name: string; inputSchema: unknown }> }).tools + expect(tools.map((t) => t.name)).toEqual(['memory_search', 'memory_get']) + expect(tools[0]?.inputSchema).toMatchObject({ required: ['query'] }) + }) + + it('memory_search returns the seeded item ranked by lexical overlap', async () => { + const res = await server.handle({ + jsonrpc: '2.0', + id: 3, + method: 'tools/call', + params: { name: 'memory_search', arguments: { query: 'failing test run' } }, + }) + const out = (res?.result as { structuredContent: { results: Array<{ id: string }> } }) + .structuredContent + expect(out.results[0]?.id).toBe('mem-1') + }) + + it('memory_search respects k and the tags filter', async () => { + const res = await server.handle({ + jsonrpc: '2.0', + id: 4, + method: 'tools/call', + params: { + name: 'memory_search', + arguments: { query: 'schema version test', k: 1, tags: ['db'] }, + }, + }) + const out = (res?.result as { structuredContent: { results: Array<{ id: string }> } }) + .structuredContent + expect(out.results.map((r) => r.id)).toEqual(['mem-3']) + }) + + it('rejects an empty query as invalid params (-32602)', async () => { + const res = await server.handle({ + jsonrpc: '2.0', + id: 5, + method: 'tools/call', + params: { name: 'memory_search', arguments: { query: ' ' } }, + }) + expect(res?.error?.code).toBe(-32602) + }) + + it('memory_get returns the row verbatim and errors on an unknown id', async () => { + const hit = await server.handle({ + jsonrpc: '2.0', + id: 6, + method: 'tools/call', + params: { name: 'memory_get', arguments: { id: 'mem-2' } }, + }) + expect((hit?.result as { structuredContent: MemoryItem }).structuredContent).toEqual(ITEMS[1]) + const miss = await server.handle({ + jsonrpc: '2.0', + id: 7, + method: 'tools/call', + params: { name: 'memory_get', arguments: { id: 'mem-999' } }, + }) + expect(miss?.error?.message).toMatch(/no memory item/) + }) + + it('refuses to serve an EMPTY memory (fail-closed ablation discipline)', () => { + expect(() => createMemoryToolServer({ items: [] })).toThrow(ValidationError) + }) + + it('refuses duplicate item ids', () => { + expect(() => + createMemoryToolServer({ items: [ITEMS[0] as MemoryItem, ITEMS[0] as MemoryItem] }), + ).toThrow(/duplicate/) + }) + + it('appends one JSONL retrieval-log row per memory_search (the holdout seam)', async () => { + const logPath = join(tmp, 'logs', 'retrieval.jsonl') + const logged = createMemoryToolServer({ items: ITEMS, logPath }) + await logged.handle({ + jsonrpc: '2.0', + id: 8, + method: 'tools/call', + params: { name: 'memory_search', arguments: { query: 'failing test' } }, + }) + const rows = readFileSync(logPath, 'utf8') + .trim() + .split('\n') + .map((l) => JSON.parse(l)) + expect(rows).toHaveLength(1) + expect(rows[0]).toMatchObject({ query: 'failing test', k: 5 }) + expect(rows[0].returned[0]).toMatchObject({ id: 'mem-1' }) + }) +}) + +describe('readMemoryItemsFile', () => { + it('reads a JSON array store', () => { + const p = join(tmp, 'items.json') + writeFileSync(p, JSON.stringify(ITEMS)) + expect(readMemoryItemsFile(p)).toEqual(ITEMS) + }) + + it('reads a JSONL store', () => { + const p = join(tmp, 'items.jsonl') + writeFileSync(p, `${ITEMS.map((i) => JSON.stringify(i)).join('\n')}\n`) + expect(readMemoryItemsFile(p)).toEqual(ITEMS) + }) + + it('names the offending row on a malformed store', () => { + const p = join(tmp, 'bad.jsonl') + writeFileSync(p, `${JSON.stringify(ITEMS[0])}\n{"id":"x"}\n`) + expect(() => readMemoryItemsFile(p)).toThrow(/bad\.jsonl:2.*'text'/) + }) + + it('fails loud on a missing file', () => { + expect(() => readMemoryItemsFile(join(tmp, 'nope.jsonl'))).toThrow(/cannot read/) + }) +}) + +describe('parseMemoryItems / resolveMemoryFromEnv', () => { + it('rejects non-array payloads and rows missing id/text', () => { + expect(() => parseMemoryItems({}, 'env')).toThrow(/array/) + expect(() => parseMemoryItems([{ id: 'a' }], 'env')).toThrow(/'text'/) + expect(() => parseMemoryItems([{ id: '', text: 'x' }], 'env')).toThrow(/'id'/) + }) + + it('merges file + inline rows, inline winning on id collision', () => { + const p = join(tmp, 'merge.jsonl') + writeFileSync(p, `${JSON.stringify(ITEMS[0])}\n${JSON.stringify(ITEMS[1])}\n`) + const inline: MemoryItem = { id: 'mem-1', text: 'inline override wins' } + const resolved = resolveMemoryFromEnv({ + [MEMORY_FILE_ENV]: p, + [MEMORY_ITEMS_ENV]: JSON.stringify([inline]), + }) + expect(resolved.items).toHaveLength(2) + expect(resolved.items.find((i) => i.id === 'mem-1')?.text).toBe('inline override wins') + }) + + it('refuses to boot with zero rows resolved', () => { + expect(() => resolveMemoryFromEnv({})).toThrow(/no memory items/) + }) +}) diff --git a/src/mcp/memory-server.ts b/src/mcp/memory-server.ts new file mode 100644 index 00000000..60b3f2e5 --- /dev/null +++ b/src/mcp/memory-server.ts @@ -0,0 +1,353 @@ +/** + * Memory MCP server — the LIVE serving half of the `memory` profile surface + * (Phase 5). A curated memory (lessons distilled from prior runs) is only + * real if the DRIVEN agent can query it mid-task; this module serves a flat + * store of `MemoryItem` rows as `memory_search` / `memory_get` tools over the + * ONE in-repo stdio JSON-RPC core (`createStdioToolServer`), so the exact + * wire protocol the same-host client (`connectStdioMcp` / + * `materializeLocalMcp`, runtime/stdio-mcp-client.ts) speaks is guaranteed by + * construction — serve and consume cannot drift. + * + * Retrieval is DETERMINISTIC lexical overlap (no vectors, no LLM): a lift + * measured with this memory mounted is attributable to the lessons + * themselves, never to retrieval-model noise — the same discipline as + * agent-eval's `memoryCurationProposer` (deterministic curation). Every + * `memory_search` can append one JSONL row to a retrieval log (`logPath`) — + * the per-query record an off-policy retrieval estimator (agent-knowledge's + * `RetrievalHoldout`) consumes. agent-knowledge is NOT a dependency of this + * repo, so the log file is the flagged cross-package seam, not an import. + * + * Fail-loud discipline (mirrors `materializeLocalMcp`): an EMPTY memory is + * never served — a profile without memory simply omits the artifact, and + * silently serving zero rows would fake the with/without ablation. + * + * @experimental + */ + +import { appendFileSync, mkdirSync, readFileSync } from 'node:fs' +import { dirname } from 'node:path' +import type { AgentProfileMcpServer } from '@tangle-network/agent-interface' +import { ValidationError } from '../errors' +import { createStdioToolServer, type McpToolDescriptor, type StdioToolServer } from './tool-server' + +/** One row of agent memory: a crisp lesson/fact with provenance. */ +export interface MemoryItem { + /** Stable id (content-hash by convention; see `memoryArtifactFromLessons`). */ + id: string + /** The lesson itself — one imperative or observation the agent should recall. */ + text: string + /** Optional retrieval tags, matched by `memory_search` alongside the text. */ + tags?: string[] + /** Provenance: the finding / trace / curation pass this row came from. */ + source?: string +} + +/** + * The `memory` artifact payload — HOW a profile's memory is stored and served: + * + * - `store: 'file'` — served by the in-repo memory bin + * (`agent-runtime-memory-mcp`, src/mcp/memory-bin.ts): rows load from + * `path` (a JSON array or JSONL file of `MemoryItem`) and/or the inline + * `items` seed (inline wins on id collision). At least one of + * `path`/`items` is required. + * - `store: 'mcp'` — an EXTERNAL, already-runnable MCP server that exposes + * the memory tools itself; `server` is required and mounts verbatim. + * + * `logPath` makes the served memory append one JSONL row per `memory_search` + * — the retrieval log a holdout estimator reads (see module doc). + */ +export interface AgentMemorySpec { + store: 'file' | 'mcp' + /** `store:'file'` — host path to the durable row store (JSON array or JSONL). */ + path?: string + /** Inline seed rows, served alongside (and winning over) `path` rows. */ + items?: MemoryItem[] + /** `store:'mcp'` — the external server that already serves memory tools. */ + server?: AgentProfileMcpServer + /** JSONL retrieval log: one row per `memory_search` (ts, query, k, returned). */ + logPath?: string +} + +/** Env var naming the durable row store file the memory bin loads (the + * `memoryMcpServer` ↔ memory-bin contract). */ +export const MEMORY_FILE_ENV = 'AGENT_MEMORY_FILE' +/** Env var carrying inline JSON `MemoryItem` rows (win over file rows on id). */ +export const MEMORY_ITEMS_ENV = 'AGENT_MEMORY_ITEMS' +/** Env var naming the JSONL retrieval log (one row per `memory_search`). */ +export const MEMORY_LOG_ENV = 'AGENT_MEMORY_LOG' +/** Env var overriding the served display name (default 'agent-memory'). */ +export const MEMORY_NAME_ENV = 'AGENT_MEMORY_NAME' + +export interface CreateMemoryToolServerOptions { + /** The rows to serve. MUST be non-empty (an empty memory is never served). */ + items: readonly MemoryItem[] + /** Server display name surfaced via `initialize`. Default 'agent-memory'. */ + serverName?: string + /** Server version surfaced via `initialize`. Default '0'. */ + serverVersion?: string + /** Default result count for `memory_search`. Default 5. */ + defaultK?: number + /** Append one JSONL row per `memory_search` (the retrieval-holdout seam). */ + logPath?: string +} + +/** + * Build the memory MCP server: `memory_search` (lexical top-k over the rows) + * and `memory_get` (one row by id) on the generic stdio JSON-RPC core. + */ +export function createMemoryToolServer(opts: CreateMemoryToolServerOptions): StdioToolServer { + if (opts.items.length === 0) { + throw new ValidationError( + 'createMemoryToolServer: refusing to serve an EMPTY memory — a profile without memory omits the artifact; serving zero rows would fake the with/without ablation', + ) + } + const byId = new Map() + for (const item of opts.items) { + if (byId.has(item.id)) { + throw new ValidationError(`createMemoryToolServer: duplicate memory item id '${item.id}'`) + } + byId.set(item.id, item) + } + const items = [...byId.values()] + const defaultK = opts.defaultK ?? 5 + const logPath = opts.logPath + if (logPath) mkdirSync(dirname(logPath), { recursive: true }) + + const search: McpToolDescriptor = { + name: 'memory_search', + description: + 'Search the agent memory of lessons learned from prior runs. Give what you are about to do or the problem you face; returns the top-k lessons ranked by relevance. Consult it BEFORE repeating work a prior run already learned from.', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'The task/problem at hand — matched against lesson text and tags.', + }, + k: { type: 'number', description: `Max results (default ${defaultK}).` }, + tags: { + type: 'array', + items: { type: 'string' }, + description: 'Only return items carrying at least one of these tags.', + }, + }, + required: ['query'], + }, + handler: async (raw) => { + const args = (raw ?? {}) as { query?: unknown; k?: unknown; tags?: unknown } + if (typeof args.query !== 'string' || args.query.trim().length === 0) { + throw new TypeError('memory_search: query must be a non-empty string') + } + const k = args.k === undefined ? defaultK : Math.floor(Number(args.k)) + if (!Number.isFinite(k) || k < 1) { + throw new TypeError('memory_search: k must be a positive integer') + } + let tagFilter: string[] | undefined + if (args.tags !== undefined) { + if (!Array.isArray(args.tags) || args.tags.some((t) => typeof t !== 'string')) { + throw new TypeError('memory_search: tags must be an array of strings') + } + tagFilter = args.tags as string[] + } + const queryTokens = tokenize(args.query) + const pool = tagFilter + ? items.filter((i) => (i.tags ?? []).some((t) => tagFilter.includes(t))) + : items + const results = pool + .map((item) => ({ item, score: scoreItem(queryTokens, item) })) + .filter((r) => r.score > 0) + // Highest score first; stable on ties (curation order is the tiebreak). + .sort((a, b) => b.score - a.score) + .slice(0, k) + .map(({ item, score }) => ({ + id: item.id, + text: item.text, + score: Number(score.toFixed(4)), + ...(item.tags ? { tags: item.tags } : {}), + ...(item.source ? { source: item.source } : {}), + })) + if (logPath) { + // The retrieval log: what was asked, what came back, at what rank — + // exactly the per-query record an off-policy estimator needs. + appendFileSync( + logPath, + `${JSON.stringify({ + ts: new Date().toISOString(), + query: args.query, + k, + returned: results.map((r) => ({ id: r.id, score: r.score })), + })}\n`, + ) + } + return { query: args.query, results } + }, + } + + const get: McpToolDescriptor = { + name: 'memory_get', + description: 'Fetch one memory item verbatim by its id (ids come from memory_search results).', + inputSchema: { + type: 'object', + properties: { id: { type: 'string', description: 'The memory item id.' } }, + required: ['id'], + }, + handler: async (raw) => { + const args = (raw ?? {}) as { id?: unknown } + if (typeof args.id !== 'string' || args.id.trim().length === 0) { + throw new TypeError('memory_get: id must be a non-empty string') + } + const item = byId.get(args.id) + if (!item) throw new Error(`memory_get: no memory item with id '${args.id}'`) + return item + }, + } + + return createStdioToolServer({ + serverName: opts.serverName ?? 'agent-memory', + serverVersion: opts.serverVersion ?? '0', + tools: [search, get], + }) +} + +/** Coerce an untrusted JSON array into validated `MemoryItem` rows. */ +export function parseMemoryItems(value: unknown, source: string): MemoryItem[] { + if (!Array.isArray(value)) { + throw new ValidationError(`${source}: expected a JSON array of memory items`) + } + return value.map((row, i) => coerceMemoryItem(row, `${source}[${i}]`)) +} + +/** Read a memory store file: a JSON array, or JSONL (one `MemoryItem` per line). */ +export function readMemoryItemsFile(path: string): MemoryItem[] { + let raw: string + try { + raw = readFileSync(path, 'utf8') + } catch (err) { + throw new ValidationError( + `readMemoryItemsFile: cannot read '${path}': ${err instanceof Error ? err.message : String(err)}`, + ) + } + const trimmed = raw.trim() + if (trimmed.length === 0) return [] + if (trimmed.startsWith('[')) { + let parsed: unknown + try { + parsed = JSON.parse(trimmed) + } catch (err) { + throw new ValidationError( + `readMemoryItemsFile: '${path}' is not valid JSON: ${(err as Error).message}`, + ) + } + return parseMemoryItems(parsed, path) + } + return trimmed + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line, i) => { + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch (err) { + throw new ValidationError( + `readMemoryItemsFile: '${path}' line ${i + 1} is not valid JSON: ${(err as Error).message}`, + ) + } + return coerceMemoryItem(parsed, `${path}:${i + 1}`) + }) +} + +/** What the memory bin resolved from its environment. */ +export interface ResolvedMemoryEnv { + items: MemoryItem[] + serverName?: string + logPath?: string +} + +/** + * Resolve the bin's memory from `AGENT_MEMORY_FILE` (durable store) and/or + * `AGENT_MEMORY_ITEMS` (inline JSON rows; wins on id collision). Zero rows is + * a boot FAILURE, matching the fail-closed materialization discipline. + */ +export function resolveMemoryFromEnv(env: Record): ResolvedMemoryEnv { + const filePath = env[MEMORY_FILE_ENV] + const inlineRaw = env[MEMORY_ITEMS_ENV] + const fromFile = filePath ? readMemoryItemsFile(filePath) : [] + let inline: MemoryItem[] = [] + if (inlineRaw) { + let parsed: unknown + try { + parsed = JSON.parse(inlineRaw) + } catch (err) { + throw new ValidationError( + `${MEMORY_ITEMS_ENV} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, + ) + } + inline = parseMemoryItems(parsed, MEMORY_ITEMS_ENV) + } + const byId = new Map() + for (const item of fromFile) byId.set(item.id, item) + for (const item of inline) byId.set(item.id, item) + if (byId.size === 0) { + throw new ValidationError( + `memory bin: no memory items — set ${MEMORY_FILE_ENV} and/or ${MEMORY_ITEMS_ENV}; an EMPTY memory must never be served (a profile without memory omits the artifact)`, + ) + } + const serverName = env[MEMORY_NAME_ENV] + const logPath = env[MEMORY_LOG_ENV] + return { + items: [...byId.values()], + ...(serverName ? { serverName } : {}), + ...(logPath ? { logPath } : {}), + } +} + +function coerceMemoryItem(row: unknown, at: string): MemoryItem { + if (!row || typeof row !== 'object' || Array.isArray(row)) { + throw new ValidationError(`${at}: memory item must be an object`) + } + const r = row as Record + if (typeof r.id !== 'string' || r.id.trim().length === 0) { + throw new ValidationError(`${at}: 'id' must be a non-empty string`) + } + if (typeof r.text !== 'string' || r.text.trim().length === 0) { + throw new ValidationError(`${at}: 'text' must be a non-empty string`) + } + let tags: string[] | undefined + if (r.tags !== undefined) { + if (!Array.isArray(r.tags) || r.tags.some((t) => typeof t !== 'string')) { + throw new ValidationError(`${at}: 'tags' must be an array of strings`) + } + tags = r.tags as string[] + } + if (r.source !== undefined && typeof r.source !== 'string') { + throw new ValidationError(`${at}: 'source' must be a string`) + } + return { + id: r.id, + text: r.text, + ...(tags ? { tags } : {}), + ...(typeof r.source === 'string' ? { source: r.source } : {}), + } +} + +/** Unique lowercase alphanumeric tokens of length >= 2. */ +function tokenize(text: string): string[] { + return [ + ...new Set( + text + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t.length >= 2), + ), + ] +} + +/** Fraction of query tokens present in the item's text + tags (0..1). */ +function scoreItem(queryTokens: readonly string[], item: MemoryItem): number { + if (queryTokens.length === 0) return 0 + const hay = new Set(tokenize(`${item.text} ${(item.tags ?? []).join(' ')}`)) + let hit = 0 + for (const t of queryTokens) if (hay.has(t)) hit += 1 + return hit / queryTokens.length +} diff --git a/src/mcp/tool-server.ts b/src/mcp/tool-server.ts new file mode 100644 index 00000000..4ebd8b68 --- /dev/null +++ b/src/mcp/tool-server.ts @@ -0,0 +1,195 @@ +/** + * `createStdioToolServer` — the generic newline-delimited JSON-RPC 2.0 MCP + * server core: `initialize` / `notifications/initialized` / `tools/list` / + * `tools/call` over a stdio-shaped transport, protocol 2024-11-05. + * + * Extracted from `createMcpServer` (server.ts) so every in-repo MCP server — + * the delegation server, the memory server — serves the ONE wire protocol the + * same-host client (`connectStdioMcp`, runtime/stdio-mcp-client.ts) speaks. + * A second hand-rolled serve loop could drift from what the client expects; + * this core makes that impossible by construction. + * + * @experimental + */ + +import { createInterface, type Interface as ReadlineInterface } from 'node:readline' +import { ValidationError } from '../errors' + +const PROTOCOL_VERSION = '2024-11-05' + +/** @experimental */ +export interface McpToolDescriptor { + name: string + description: string + inputSchema: Record + handler: (raw: unknown) => Promise +} + +/** @experimental */ +export interface McpTransport { + input: NodeJS.ReadableStream + output: NodeJS.WritableStream +} + +/** @experimental */ +export interface JsonRpcMessage { + jsonrpc: '2.0' + id?: number | string | null + method: string + params?: unknown +} + +/** @experimental */ +export interface JsonRpcResponse { + jsonrpc: '2.0' + id: number | string | null + result?: unknown + error?: { code: number; message: string; data?: unknown } +} + +/** @experimental */ +export interface StdioToolServerOptions { + /** Server display name surfaced via `initialize`. */ + serverName: string + /** Server version surfaced via `initialize`. */ + serverVersion: string + /** The tools to serve. Duplicate names throw — a silent shadow would hide a tool. */ + tools: readonly McpToolDescriptor[] +} + +/** @experimental */ +export interface StdioToolServer { + /** Tools currently registered, keyed by name. */ + readonly tools: ReadonlyMap + /** Handle a single parsed JSON-RPC message. Returns the response object (or `null` for notifications). */ + handle(message: JsonRpcMessage): Promise + /** Drive the server on a stdio-shaped transport until `stop()` is called. */ + serve(transport?: McpTransport): Promise + /** Stop a `serve` call. Subsequent requests are rejected. */ + stop(): void +} + +/** Build the generic stdio JSON-RPC tool server. */ +export function createStdioToolServer(options: StdioToolServerOptions): StdioToolServer { + const tools = new Map() + for (const tool of options.tools) { + if (tools.has(tool.name)) { + throw new ValidationError(`createStdioToolServer: duplicate tool name "${tool.name}"`) + } + tools.set(tool.name, tool) + } + + let stopped = false + let activeReadline: ReadlineInterface | undefined + + async function handle(message: JsonRpcMessage): Promise { + if (stopped) { + return rpcError(message.id ?? null, -32099, 'server stopped') + } + if (message.method === 'initialize') { + return rpcResult(message.id ?? null, { + protocolVersion: PROTOCOL_VERSION, + capabilities: { tools: {} }, + serverInfo: { name: options.serverName, version: options.serverVersion }, + }) + } + if (message.method === 'notifications/initialized') { + // MCP clients send this after the handshake; it has no id and expects + // no response. + return null + } + if (message.method === 'tools/list') { + return rpcResult(message.id ?? null, { + tools: [...tools.values()].map((tool) => ({ + name: tool.name, + description: tool.description, + inputSchema: tool.inputSchema, + })), + }) + } + if (message.method === 'tools/call') { + const params = (message.params ?? {}) as { name?: unknown; arguments?: unknown } + const name = typeof params.name === 'string' ? params.name : '' + const tool = tools.get(name) + if (!tool) { + return rpcError(message.id ?? null, -32601, `unknown tool: ${name}`) + } + try { + const output = await tool.handler(params.arguments ?? {}) + return rpcResult(message.id ?? null, { + content: [{ type: 'text', text: JSON.stringify(output) }], + structuredContent: output, + isError: false, + }) + } catch (err) { + const reason = err instanceof Error ? err.message : String(err) + const code = err instanceof TypeError || err instanceof RangeError ? -32602 : -32000 + return rpcError(message.id ?? null, code, reason) + } + } + if (message.id === undefined || message.id === null) return null + return rpcError(message.id, -32601, `unknown method: ${message.method}`) + } + + async function serve(transport?: McpTransport): Promise { + const input = transport?.input ?? process.stdin + const output = transport?.output ?? process.stdout + const rl = createInterface({ input, crlfDelay: Number.POSITIVE_INFINITY }) + activeReadline = rl + return new Promise((resolve, reject) => { + rl.on('line', (line) => { + const trimmed = line.trim() + if (!trimmed) return + let parsed: JsonRpcMessage | undefined + try { + parsed = JSON.parse(trimmed) as JsonRpcMessage + } catch (err) { + writeResponse(output, rpcError(null, -32700, `parse error: ${(err as Error).message}`)) + return + } + if (!parsed || parsed.jsonrpc !== '2.0' || typeof parsed.method !== 'string') { + writeResponse(output, rpcError(parsed?.id ?? null, -32600, 'invalid request')) + return + } + void handle(parsed).then((response) => { + if (response) writeResponse(output, response) + }) + }) + rl.on('close', () => resolve()) + rl.on('error', (err) => reject(err)) + if (stopped) { + rl.close() + resolve() + } + }) + } + + function stop(): void { + stopped = true + activeReadline?.close() + activeReadline = undefined + } + + return { tools, handle, serve, stop } +} + +function rpcResult(id: number | string | null, result: unknown): JsonRpcResponse { + return { jsonrpc: '2.0', id, result } +} + +function rpcError( + id: number | string | null, + code: number, + message: string, + data?: unknown, +): JsonRpcResponse { + return { + jsonrpc: '2.0', + id, + error: data === undefined ? { code, message } : { code, message, data }, + } +} + +function writeResponse(output: NodeJS.WritableStream, response: JsonRpcResponse): void { + output.write(`${JSON.stringify(response)}\n`) +} diff --git a/tests/improvement-driver.test.ts b/tests/improvement-driver.test.ts index 263b29be..8e84b788 100644 --- a/tests/improvement-driver.test.ts +++ b/tests/improvement-driver.test.ts @@ -105,14 +105,23 @@ describe('improvementDriver — reflective generator', () => { } const driver = reflectiveDriver(adapter) - const reportFindings = [ + // A CONFORMING envelope passes through `toAnalystFindings` by reference + // (a partial look-alike would be lifted and re-id'd — the P4 accessor's + // fail-closed contract), so the stable id proves the report arm won. + const reportFindings: AnalystFinding[] = [ { + schema_version: '1.0.0', finding_id: 'from-report', - subject: 'system-prompt:rubric', + analyst_id: 'phase2-report', + produced_at: new Date().toISOString(), severity: 'high', + area: 'report', + claim: 'the rubric is too lax', + subject: 'system-prompt:rubric', confidence: 0.9, + evidence_refs: [], }, - ] as unknown as AnalystFinding[] + ] await driver.propose(ctxWith(FINDINGS, { findings: reportFindings, diff: {} })) expect(seen).toHaveLength(1) diff --git a/tsup.config.ts b/tsup.config.ts index 5d4f4f49..fdb2d3d5 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ platform: 'src/platform/index.ts', 'mcp/index': 'src/mcp/index.ts', 'mcp/bin': 'src/mcp/bin.ts', + 'mcp/memory-bin': 'src/mcp/memory-bin.ts', 'loop-runner-bin': 'src/loop-runner-bin.ts', }, format: ['esm'], From 32a3116d5692e45f35495e5695a36e8f4434c1b6 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 13 Jul 2026 00:41:33 -0600 Subject: [PATCH 43/44] feat(lifecycle): external-MCP connection surface + key provisioning for the research optimizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 'connection' ArtifactKind: an ExternalMcpGrant (http url / stdio launch, literal headers/env, secrets by provider KEY NAME, optional hub grant) promotes onto profile.mcp[key] via connectionMcpServer and onto profile.connections (artifact wins per connectionId) - buildableGenerator: BuiltCandidate.remote branches the mcp emit off the hardcoded stdio transport — the adopt-not-build path emits a { transport:'http', url, headers, env } server with adopt provenance - KeyProvider (runtime/key-provider.ts): declarative secretEnv (mcp[key].metadata.secretEnv, names only) resolved at materialize time; envKeyProvider reads the dotenvx-loaded env; materializeLocalMcp injects values into the spawned server child env only — fail-closed on a missing provider/key, values never on the profile or in logs; sweEvalRunner forwards keys so an adopted server scores with its credential live - research seam: driverLoopGenerator gains an optional research{query} tool + researchDriverNote (adopt-before-build doctrine), offered only when wired; mcpBuildPrompt instructs adopt-over-build; no live web backend ships yet (the seam is worktreeBuildCandidate driver.research) --- docs/api/primitive-catalog.md | 18 +- src/improvement/build-prompts.ts | 8 + src/improvement/driver-loop-generator.test.ts | 51 ++++ src/improvement/driver-loop-generator.ts | 42 +++- src/improvement/index.ts | 7 +- src/improvement/optimizer-prompt.ts | 19 ++ src/lifecycle/apply.ts | 20 ++ src/lifecycle/connection.test.ts | 217 ++++++++++++++++++ src/lifecycle/connection.ts | 201 ++++++++++++++++ src/lifecycle/index.ts | 7 + src/lifecycle/swe-eval-runner.ts | 10 +- src/lifecycle/tool-build.ts | 9 +- src/lifecycle/tool-generator.test.ts | 71 ++++++ src/lifecycle/tool-generator.ts | 51 +++- src/lifecycle/types.ts | 22 +- src/runtime/index.ts | 9 + src/runtime/key-provider.ts | 110 +++++++++ src/runtime/stdio-mcp-client.ts | 22 +- 18 files changed, 873 insertions(+), 21 deletions(-) create mode 100644 src/lifecycle/connection.test.ts create mode 100644 src/lifecycle/connection.ts create mode 100644 src/runtime/key-provider.ts diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 00f741be..cd5ccb0a 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 249 exports. +Import from `@tangle-network/agent-runtime` — 250 exports. | Symbol | Kind | Summary | |---|---|---| @@ -115,6 +115,7 @@ Import from `@tangle-network/agent-runtime` — 249 exports. | `LIFTED_FINDING_ANALYST_ID` | const | Analyst id stamped on findings lifted from untyped seed values. | | `optimizerMethod` | const | The shared method block every build/author prompt embeds. Domain framing | | `RESEARCH_SUPERVISOR_SYSTEM_PROMPT` | const | Standing prompt for a supervisor that grows a shared knowledge base through spawned researchers. | +| `researchDriverNote` | const | The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when | | `ROLLOUT_POLICY_BOUNDS` | const | Proposal bounds per dial. These are the SEARCH bounds (what the proposer may | | `ROLLOUT_POLICY_EXTENSION` | const | The profile extensions namespace the policy persists under. | | `strategyAuthorMethod` | const | The senior authoring process for `authorStrategy` — the same method, shaped | @@ -289,7 +290,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 63 exports. ### Recursive atom + loop kernel (alias of ./runtime) -Import from `@tangle-network/agent-runtime/loops` — 467 exports. +Import from `@tangle-network/agent-runtime/loops` — 472 exports. | Symbol | Kind | Summary | |---|---|---| @@ -344,6 +345,7 @@ Import from `@tangle-network/agent-runtime/loops` — 467 exports. | `discriminatingMeans` | function | Strategy means recomputed over the DISCRIMINATING tasks only — tasks where the field | | `driverAgent` | function | Build the intelligent recursive driver. Its `act` is the LLM tool-loop; spawn it as a | | `dumbDriver` | function | `dumbDriver` — the pass/fail-only steering control. | +| `envKeyProvider` | function | The env-backed provider: reads the (dotenvx-loaded) process env. Empty / | | `equalKOnCost` | function | Assert the arms are comparable at EQUAL conserved COST (tokens + usd), NOT raw iteration | | `extractLlmCallEvent` | function | Extract a `RuntimeStreamEvent`-shaped `llm_call` from a sandbox event when | | `failuresAnalyst` | function | The default self-improvement LENS — authored content, not a code path. On each settled worker it hands | @@ -396,6 +398,7 @@ Import from `@tangle-network/agent-runtime/loops` — 467 exports. | `resolveAgentEnvironmentProvider` | function | Resolve a provider instance or registry name, failing loudly when a name is unknown. | | `resolveEntrySymbol` | function | The symbol authored checks are pinned to: `task.meta.entryPoint` when the surface | | `resolveSandboxClient` | function | Resolve a `SandboxClient` for the chosen backend. The generic, dep-light core | +| `resolveSecretEnv` | function | Resolve a declared secret-env map into the real env entries for a server | | `routerBrain` | function | The router as a supervisor BRAIN: the canonical `ToolLoopChat` seam backed by the router's | | `routerChatWithTools` | function | A router completion WITH tool-calling — the operator driver's LLM seam. Passes OpenAI-shape | | `routerChatWithUsage` | function | One OpenAI-compatible chat completion through the Tangle router, returning text + REAL token usage (`undefined` when the provider omits it — never a fabricated 0). | @@ -410,6 +413,7 @@ Import from `@tangle-network/agent-runtime/loops` — 467 exports. | `sandboxClientAsProvider` | function | Adapt a `SandboxClient` into the shared `AgentEnvironmentProvider` contract. | | `sandboxSessionTraceSource` | function | The SANDBOX / fleet trace source: read a box session's message parts and decode the harness's tool | | `sanitizeMcpToolSchema` | function | Coerce an MCP inputSchema to an OpenAI-tool-valid top-level object schema. | +| `secretEnvOfMcpServer` | function | Read (and validate) a server entry's declared secret-env map, if any. | | `selectBestIndex` | function | Argmax by `compareCheckOutcomes`, FIRST index wins ties (deterministic; with zero | | `selectChampion` | function | Search-side champion selection over a tournament report. | | `selectValidWinner` | function | The single content-free valid-only winner selector. Among the gated-VALID children only | @@ -441,6 +445,7 @@ Import from `@tangle-network/agent-runtime/loops` — 467 exports. | `defaultDelegateBudget` | const | The conserved pool a `delegate()` call applies when the caller does not pass its own `budget`. | | `defaultProfileRichnessThresholds` | const | Default thresholds for `ProfileRichnessThresholds` — 600 chars / 6 lines minimum system prompt. | | `defaultStructuralRolloutPolicy` | const | The measured default recipe: 5 samples, 2 guarded repair rounds, 6 authored checks. | +| `mcpSecretEnvMetadataKey` | const | The `AgentProfileMcpServer.metadata` key the declarative secret-env map | | `refine` | const | Built-in `Strategy`: attempt → `observe()` reads the trace → steer the next attempt → repeat (deepen one lineage). | | `sample` | const | Built-in `Strategy`: K independent attempts, keep the best-verifying (best-of-N / resample). | | `sampleThenRefine` | const | The explore-then-exploit MIX: spend ⌈budget/2⌉ on independent samples (kept open), | @@ -511,6 +516,7 @@ Import from `@tangle-network/agent-runtime/loops` — 467 exports. | `InMemoryRunContextOptions` | interface | Options for the in-memory run context. | | `InProcessPromptCtx` | interface | Context handed to each `onPrompt` / `onTask` call. | | `Interval` | interface | A 95%-by-default confidence interval. | +| `KeyProvider` | interface | Resolve named secrets. The ONE seam every secret store adapts to. | | `LeaderboardBenchmarkAdapter` | interface | Structurally `BenchmarkAdapter` (bench registry shape): `name`, | | `LeaderboardBenchScore` | interface | Structurally `BenchScore` (bench registry shape). | | `LeaderboardBenchTask` | interface | Structurally `BenchTask` (bench registry shape) — declared locally so this | @@ -692,7 +698,7 @@ Import from `@tangle-network/agent-runtime/analyst-loop` — 15 exports. ### Artifact lifecycle — generate → measure → promote → compose -Import from `@tangle-network/agent-runtime/lifecycle` — 81 exports. +Import from `@tangle-network/agent-runtime/lifecycle` — 86 exports. | Symbol | Kind | Summary | |---|---|---| @@ -700,6 +706,8 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 81 exports. | `applyArtifacts` | function | Apply many artifacts left-to-right; later artifacts win on key conflicts. | | `buildableGenerator` | function | Build a `CandidateGenerator` for a buildable surface (`tool` / `mcp`). Each | | `composeProfile` | function | Return a new profile with the top-`k` active artifacts (highest measured lift | +| `connectionArtifact` | function | Build a `connection` `ArtifactInput` from a grant — the one-step adapter for | +| `connectionMcpServer` | function | The `AgentProfileMcpServer` entry a grant promotes into `profile.mcp` — the | | `createArtifactRegistry` | function | Construct an empty `ArtifactRegistry`. | | `dedupeArtifacts` | function | Pairwise stack-test the `active` artifacts and retire the redundant half of | | `driftWatch` | function | Re-measure every `active` artifact and demote those whose held-back lift | @@ -720,6 +728,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 81 exports. | `skillGenerator` | function | Build a `CandidateGenerator` for the skill surface that distills new skills | | `sweEvalRunner` | function | Build the `EvalRunner` closed over one fixed SWE exam. `runLifecycle` calls | | `thresholdPromotionGate` | function | The simplest honest gate: promote iff the candidate's marginal lift on the | +| `validateExternalMcpGrant` | function | Assert an `ExternalMcpGrant` is well-formed. Fail-closed: a malformed grant | | `validateMemorySpec` | function | Assert an `AgentMemorySpec` is well-formed and servable. | | `worktreeBuildCandidate` | function | Build the production per-candidate seam for `buildableGenerator`. Each call to | | `writeMemoryStore` | function | Write a durable JSONL memory store (one `MemoryItem` per line). | @@ -740,6 +749,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 81 exports. | `DriftCheck` | interface | Per-artifact record of what the re-measure found and decided. | | `DriftWatchOptions` | interface | `driftWatch` — the scheduled re-measure that DEMOTES decayed artifacts. | | `EvalResult` | interface | The result of running an eval over ONE profile: a composite score and the cost | +| `ExternalMcpGrant` | interface | An external MCP grant — the ADOPT-not-build candidate payload. `http` names | | `GenerateContext` | interface | The read-only context a generator sees when proposing candidates. It is the | | `MarginalLift` | interface | The marginal lift of one artifact: the with/without ablation. | | `MemoryItem` | interface | One row of agent memory: a crisp lesson/fact with provenance. | @@ -765,7 +775,7 @@ Import from `@tangle-network/agent-runtime/lifecycle` — 81 exports. | `RefinePrompt` | type | REFINE — incumbent-grounded rewrites. Given the lifecycle context, return | | `RefineSkill` | type | REFINE — improve ONE distilled draft (wording, structure, examples). The | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BuildableGeneratorOptions`, `DedupeResult`, `DriftWatchResult`, `HeldOutPromotionGateOptions`, `MeasureMarginalLiftOptions`, `MemoryArtifactOptions`, `MemoryGeneratorOptions`, `MemoryMcpServerOptions`, `ProductionPromptGeneratorOptions`, `PromptGeneratorOptions`, `RunLifecycleResult`, `SkillGeneratorOptions`, `SweEvalRunnerOptions`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BuildableGeneratorOptions`, `ConnectionArtifactOptions`, `DedupeResult`, `DriftWatchResult`, `HeldOutPromotionGateOptions`, `MeasureMarginalLiftOptions`, `MemoryArtifactOptions`, `MemoryGeneratorOptions`, `MemoryMcpServerOptions`, `ProductionPromptGeneratorOptions`, `PromptGeneratorOptions`, `RunLifecycleResult`, `SkillGeneratorOptions`, `SweEvalRunnerOptions`. ### Knowledge orchestration — supervised KB updates diff --git a/src/improvement/build-prompts.ts b/src/improvement/build-prompts.ts index 9220744b..2ef606b0 100644 --- a/src/improvement/build-prompts.ts +++ b/src/improvement/build-prompts.ts @@ -65,6 +65,14 @@ export function mcpBuildPrompt(args: FindingsArg): string { '', optimizerMethod, '', + 'RESEARCH FIRST — ADOPT BEFORE BUILD: you may discover and ADOPT an existing external MCP', + 'server if it fits the gaps better than building one. Registries and vendor docs list', + 'maintained servers for most common capabilities (web search, fetch, GitHub, filesystems,', + 'databases). To adopt, deliver a short adoption note instead of an implementation: the', + "server's launch command or HTTP endpoint, and the API key it needs BY NAME (e.g.", + 'EXA_API_KEY) — never a key value; provisioning injects the value at materialize time. If', + 'your environment has no web access, decide from what you already know and say so.', + '', 'THE SURFACE — what a deliverable MCP server looks like here (checked by BOOTING it):', '- it starts over stdio and answers the MCP `initialize` handshake,', '- `tools/list` returns at least one tool with a valid input schema,', diff --git a/src/improvement/driver-loop-generator.test.ts b/src/improvement/driver-loop-generator.test.ts index 08edac20..2677e461 100644 --- a/src/improvement/driver-loop-generator.test.ts +++ b/src/improvement/driver-loop-generator.test.ts @@ -193,6 +193,57 @@ describe('driverLoopGenerator — the driver→worker build atom', () => { expect(result.summary).toBe('') }) + it('offers the research tool + adopt doctrine ONLY when the seam is wired', async () => { + // Wired: the driver researches, the stub's findings flow back as a tool + // message, and the system prompt carries the adopt-before-build doctrine. + const queries: string[] = [] + const research = async (query: string): Promise => { + queries.push(query) + return 'found: mcp.exa.ai — maintained web-search MCP; needs EXA_API_KEY' + } + const wired = scriptedBrain([ + { calls: [{ name: 'research', args: { query: 'web search MCP server' } }] }, + { say: 'Adopt mcp.exa.ai with EXA_API_KEY; no build needed.' }, + ]) + await driverLoopGenerator({ + brain: wired.chat, + runHarness: harnessStub().run, + research, + changedPaths: () => [], + readDiff: () => '', + }).generate(generateArgs([finding('the agent cannot search the web')], 1)) + expect(queries).toEqual(['web search MCP server']) + const flat = wired.seen.flat() + expect(flat.some((m) => m.role === 'tool' && String(m.content).includes('mcp.exa.ai'))).toBe( + true, + ) + expect( + flat.some((m) => m.role === 'system' && String(m.content).includes('ADOPT BEFORE BUILD')), + ).toBe(true) + + // Not wired: no research tool in the specs, no doctrine, and a stray call + // gets the not-provisioned error (live web access is not provisioned). + const unwired = scriptedBrain([ + { calls: [{ name: 'research', args: { query: 'anything' } }] }, + { say: 'ok' }, + ]) + await driverLoopGenerator({ + brain: unwired.chat, + runHarness: harnessStub().run, + changedPaths: () => [], + readDiff: () => '', + }).generate(generateArgs([finding('gap')], 1)) + const unwiredFlat = unwired.seen.flat() + expect( + unwiredFlat.some((m) => m.role === 'tool' && String(m.content).includes('not provisioned')), + ).toBe(true) + expect( + unwiredFlat.some( + (m) => m.role === 'system' && String(m.content).includes('ADOPT BEFORE BUILD'), + ), + ).toBe(false) + }) + it('caps worker sessions at maxShots and tells the driver, without spawning', async () => { const worker = harnessStub() const { chat, seen } = scriptedBrain([ diff --git a/src/improvement/driver-loop-generator.ts b/src/improvement/driver-loop-generator.ts index eae58010..6e75e6ac 100644 --- a/src/improvement/driver-loop-generator.ts +++ b/src/improvement/driver-loop-generator.ts @@ -40,7 +40,7 @@ import { worktreeChangedPaths, } from './agentic-generator' import type { CandidateGenerator } from './improvement-driver' -import { buildDriverSystem } from './optimizer-prompt' +import { buildDriverSystem, researchDriverNote } from './optimizer-prompt' export interface DriverLoopGeneratorOptions { /** The driver-LLM seam — ONE inference turn over the conversation + tool specs (the canonical @@ -62,6 +62,12 @@ export interface DriverLoopGeneratorOptions { /** Max driver inference turns. Default `max(8, 2 + maxShots * 3)` — room for one * observe/rate/decide cycle per worker session plus orientation. */ maxTurns?: number + /** The research seam (adopt-not-build): when set, the driver gets a + * `research{query}` tool + the `researchDriverNote` doctrine, so it can + * discover an EXISTING external MCP instead of building one. Wire a real + * web/search backend here — none is provisioned by default (the build + * harness has no live web access yet; flagged). */ + research?: (query: string) => Promise /** Test seam — inject the harness runner (defaults to `runLocalHarness`). */ runHarness?: typeof runLocalHarness /** Test seam — inject the worktree diff reader (defaults to `git diff` in the worktree). */ @@ -73,6 +79,7 @@ export interface DriverLoopGeneratorOptions { const workerOutputTailChars = 2_000 const diffMaxChars = 6_000 const readFileDefaultBytes = 8_192 +const researchResultMaxChars = 8_000 /** Driver→worker `CandidateGenerator`: an LLM driver on the canonical tool-loop authors, observes, rates, and steers coding-harness sessions in the worktree until the verifier passes or the session budget is spent. */ export function driverLoopGenerator(opts: DriverLoopGeneratorOptions): CandidateGenerator { @@ -150,6 +157,12 @@ export function driverLoopGenerator(opts: DriverLoopGeneratorOptions): Candidate } case 'read_file': return readWorktreeFile(worktreePath, args) + case 'research': { + if (!opts.research) return 'error: research tool is not provisioned in this run' + const query = typeof args.query === 'string' ? args.query.trim() : '' + if (query.length === 0) return 'error: research requires a non-empty `query`' + return truncate(await opts.research(query), researchResultMaxChars) + } case 'run_verifier': { const result = await groundVerify() return JSON.stringify({ @@ -164,10 +177,15 @@ export function driverLoopGenerator(opts: DriverLoopGeneratorOptions): Candidate await runBrainLoop({ chat: opts.brain, - tools: driverToolSpecs, + tools: opts.research ? [...driverToolSpecs, researchToolSpec] : driverToolSpecs, execute, initialMessages: [ - { role: 'system', content: buildDriverSystem }, + { + role: 'system', + content: opts.research + ? `${buildDriverSystem}\n\n${researchDriverNote}` + : buildDriverSystem, + }, { role: 'user', content: [ @@ -245,6 +263,24 @@ const driverToolSpecs = [ }, ] +/** Only offered when `opts.research` is wired — a tool the driver cannot call + * must never appear in its tool list. */ +const researchToolSpec = { + type: 'function' as const, + function: { + name: 'research', + description: + 'Search external sources (MCP registries, vendor docs) for an EXISTING server that closes the capability gap — the adopt-not-build check. Returns text findings.', + parameters: { + type: 'object', + properties: { + query: { type: 'string', description: 'What capability / server to search for.' }, + }, + required: ['query'], + }, + }, +} + /** `git diff` over the worktree (tracked files). Fails loud like `worktreeChangedPaths` — a git * fault on a fresh checkout is a broken setup, not an empty diff. */ function worktreeDiff(worktreePath: string): string { diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 02a434e8..81e391b5 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -53,7 +53,12 @@ export { improvementDriver, } from './improvement-driver' export { type McpServeSpec, mcpServeVerifier } from './mcp-serve-verifier' -export { buildDriverSystem, optimizerMethod, strategyAuthorMethod } from './optimizer-prompt' +export { + buildDriverSystem, + optimizerMethod, + researchDriverNote, + strategyAuthorMethod, +} from './optimizer-prompt' export { type RawTraceDistillerOptions, rawTraceDistiller, diff --git a/src/improvement/optimizer-prompt.ts b/src/improvement/optimizer-prompt.ts index 33e04fb7..78b3c937 100644 --- a/src/improvement/optimizer-prompt.ts +++ b/src/improvement/optimizer-prompt.ts @@ -99,6 +99,25 @@ export const buildDriverSystem = [ 'by code from the verifier exit and the tree state, never from your words.', ].join('\n') +/** + * The driver's ADOPT-not-build doctrine, appended to `buildDriverSystem` when + * a `research` tool is wired into the loop (`DriverLoopGeneratorOptions. + * research`). Kept separate so a driver WITHOUT the tool is never told to + * call a tool it does not have. + */ +export const researchDriverNote = [ + 'RESEARCH — ADOPT BEFORE BUILD. A research{query} tool is provisioned for this run. Before', + 'authoring a from-scratch build, spend one turn researching whether an EXISTING external MCP', + 'server already provides the missing capability — registries and vendor docs list maintained', + 'servers for most common gaps (web search, fetch, GitHub, filesystems, databases).', + '- If a maintained server fits, ADOPT it: report in your final reflection its endpoint or', + ' launch command and the API key it needs BY NAME (e.g. EXA_API_KEY) — never a key value —', + ' so the dispatch can emit a connection candidate with provisioned secrets. Adopting a fit', + ' server beats rebuilding it: less code to verify, maintained upstream, same measured gate.', + '- If nothing fits (unmaintained, wrong tool surface, heavier than the gap), build — and state', + ' in one line why adoption lost.', +].join('\n') + /** * The senior authoring process for `authorStrategy` — the same method, shaped * to the strategy contract (author-blind, conserved budget, one module out). diff --git a/src/lifecycle/apply.ts b/src/lifecycle/apply.ts index ddb030e4..5052c282 100644 --- a/src/lifecycle/apply.ts +++ b/src/lifecycle/apply.ts @@ -10,6 +10,7 @@ */ import type { AgentProfile } from '@tangle-network/agent-interface' +import { connectionMcpServer } from './connection' import { memoryMcpServer, profileMemoryMetadataKey } from './memory' import type { ProfileArtifact } from './types' @@ -69,6 +70,25 @@ export function applyArtifact(base: AgentProfile, artifact: ProfileArtifact): Ag mcp: { ...base.mcp, [key]: memoryMcpServer(payload.spec) }, } } + case 'connection': { + // Promote an EXTERNAL MCP grant (adopt, not build): the server entry — + // secrets referenced by NAME only — lands under `mcp[key]` so the + // existing materialization paths mount it, and an optional hub grant + // lands on `connections` (replacing any prior grant for the same + // connectionId — the artifact wins on conflict, like keyed kinds). + const payload = artifact.payload as ProfileArtifact<'connection'>['payload'] + const key = keyOf(artifact) + const next: AgentProfile = { + ...base, + mcp: { ...base.mcp, [key]: connectionMcpServer(payload.grant) }, + } + const hub = payload.grant.hub + if (hub) { + const others = (base.connections ?? []).filter((c) => c.connectionId !== hub.connectionId) + next.connections = [...others, hub] + } + return next + } } } diff --git a/src/lifecycle/connection.test.ts b/src/lifecycle/connection.test.ts new file mode 100644 index 00000000..1e81155f --- /dev/null +++ b/src/lifecycle/connection.test.ts @@ -0,0 +1,217 @@ +/** + * PHASE-6 proof — the `connection` artifact (adopt an EXTERNAL MCP) + key + * provisioning: + * + * 1. a grant validates fail-closed and becomes a profile `mcp` entry whose + * secrets ride BY NAME only (`metadata.secretEnv`) — a credential value + * never appears anywhere on the artifact or the profile, + * 2. `applyArtifact('connection')` promotes the server entry AND the + * optional hub grant (`profile.connections`, artifact wins on conflict), + * 3. `envKeyProvider`/`resolveSecretEnv` resolve declared names and throw — + * naming the KEY, never a value — on a missing provider or key, + * 4. the LIVE smoke: adopt a stdio "external" server (the in-repo memory + * bin) whose entire dataset arrives ONLY via a provisioned secret env — + * `materializeLocalMcp({ keys })` boots it and a search returns the + * seeded row, proving the key reached the child process env, + * 5. an http grant materializes NOWHERE same-host: fail loud, never a + * silent skip that would fake the with/without ablation. + */ + +import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import type { AgentProfile } from '@tangle-network/agent-interface' +import { describe, expect, it } from 'vitest' +import { ValidationError } from '../errors' +import { envKeyProvider, resolveSecretEnv } from '../runtime/key-provider' +import { materializeLocalMcp } from '../runtime/stdio-mcp-client' +import { applyArtifact } from './apply' +import { + connectionArtifact, + connectionMcpServer, + type ExternalMcpGrant, + validateExternalMcpGrant, +} from './connection' +import type { ProfileArtifact } from './types' + +const FAKE_KEY_VALUE = 'sk-test-9f8e7d6c5b4a' + +const asArtifact = (input: ReturnType): ProfileArtifact => ({ + ...input, + id: input.key ?? 'connection', + status: 'candidate', +}) + +const httpGrant = (over: Partial = {}): ExternalMcpGrant => ({ + transport: 'http', + url: 'https://mcp.exa.ai/mcp', + headers: { 'x-client': 'agent-runtime' }, + secretEnv: { EXA_API_KEY: 'EXA_API_KEY' }, + ...over, +}) + +describe('validateExternalMcpGrant / connectionMcpServer', () => { + it('builds an http server entry with secrets by NAME in metadata', () => { + const server = connectionMcpServer(httpGrant()) + expect(server.transport).toBe('http') + expect(server.url).toBe('https://mcp.exa.ai/mcp') + expect(server.headers).toEqual({ 'x-client': 'agent-runtime' }) + expect(server.enabled).toBe(true) + expect(server.metadata?.secretEnv).toEqual({ EXA_API_KEY: 'EXA_API_KEY' }) + expect(server.command).toBeUndefined() + }) + + it('builds a stdio server entry for an installed external launch', () => { + const server = connectionMcpServer({ + transport: 'stdio', + command: 'npx', + args: ['-y', '@vendor/mcp-server'], + env: { LOG_LEVEL: 'warn' }, + secretEnv: { VENDOR_API_KEY: 'VENDOR_API_KEY' }, + }) + expect(server.transport).toBe('stdio') + expect(server.command).toBe('npx') + expect(server.args).toEqual(['-y', '@vendor/mcp-server']) + expect(server.env).toEqual({ LOG_LEVEL: 'warn' }) + expect(server.metadata?.secretEnv).toEqual({ VENDOR_API_KEY: 'VENDOR_API_KEY' }) + }) + + it('fails closed on malformed grants', () => { + expect(() => validateExternalMcpGrant(httpGrant({ url: 'ftp://x' }))).toThrow(ValidationError) + expect(() => validateExternalMcpGrant(httpGrant({ url: undefined }))).toThrow(/requires/) + expect(() => validateExternalMcpGrant(httpGrant({ command: 'also-a-command' }))).toThrow( + /ambiguous/, + ) + expect(() => validateExternalMcpGrant({ transport: 'stdio' })).toThrow(/requires a `command`/) + expect(() => + validateExternalMcpGrant({ transport: 'stdio', command: 'x', headers: { a: 'b' } }), + ).toThrow(/http-only/) + expect(() => validateExternalMcpGrant(httpGrant({ secretEnv: { '': 'KEY' } }))).toThrow( + /secretEnv/, + ) + expect(() => + validateExternalMcpGrant(httpGrant({ hub: { connectionId: 'c1', capabilities: [] } })), + ).toThrow(/capability/) + }) +}) + +describe('applyArtifact(connection)', () => { + it('promotes the grant onto profile.mcp and never carries a secret VALUE', () => { + const base: AgentProfile = { name: 'agent' } + const out = applyArtifact(base, asArtifact(connectionArtifact(httpGrant(), { key: 'exa' }))) + expect(out.mcp?.exa?.transport).toBe('http') + expect(out.mcp?.exa?.url).toBe('https://mcp.exa.ai/mcp') + expect(out.mcp?.exa?.metadata?.secretEnv).toEqual({ EXA_API_KEY: 'EXA_API_KEY' }) + // The invariant that makes a promoted grant safe to store/log: only the + // key NAME rides the profile — a value like FAKE_KEY_VALUE never can, + // because the grant has no field that holds one. + expect(JSON.stringify(out)).not.toContain(FAKE_KEY_VALUE) + expect(base.mcp).toBeUndefined() + }) + + it('promotes a hub grant onto profile.connections, replacing the same connectionId', () => { + const hub = { connectionId: 'conn-1', capabilities: ['search.web'], alias: 'exa' } + const base: AgentProfile = { + connections: [ + { connectionId: 'conn-1', capabilities: ['old.cap'] }, + { connectionId: 'conn-2', capabilities: ['keep.cap'] }, + ], + } + const out = applyArtifact(base, asArtifact(connectionArtifact(httpGrant({ hub })))) + expect(out.connections).toEqual([{ connectionId: 'conn-2', capabilities: ['keep.cap'] }, hub]) + // Default key derived from the endpoint host. + expect(Object.keys(out.mcp ?? {})).toEqual(['mcp_exa_ai']) + }) + + it('records transport + secret key NAMES in artifact metadata for the audit trail', () => { + const artifact = connectionArtifact(httpGrant()) + expect(artifact.metadata?.transport).toBe('http') + expect(artifact.metadata?.secretKeys).toEqual(['EXA_API_KEY']) + expect(JSON.stringify(artifact)).not.toContain(FAKE_KEY_VALUE) + }) +}) + +describe('envKeyProvider / resolveSecretEnv', () => { + it('resolves named keys from the (dotenvx-loaded) env', async () => { + const keys = envKeyProvider({ EXA_API_KEY: FAKE_KEY_VALUE, EMPTY: ' ' }) + expect(await keys.get('EXA_API_KEY')).toBe(FAKE_KEY_VALUE) + expect(await keys.get('EMPTY')).toBeUndefined() + expect(await keys.get('ABSENT')).toBeUndefined() + const resolved = await resolveSecretEnv({ EXA_API_KEY: 'EXA_API_KEY' }, keys, 'test') + expect(resolved).toEqual({ EXA_API_KEY: FAKE_KEY_VALUE }) + }) + + it('throws on a missing provider or key, naming the key and NEVER a value', async () => { + await expect( + resolveSecretEnv({ X_KEY: 'X_KEY' }, undefined, "profile.mcp['x']"), + ).rejects.toThrow(/no KeyProvider/) + const keys = envKeyProvider({ OTHER: FAKE_KEY_VALUE }) + const failure = await resolveSecretEnv({ X_KEY: 'MISSING_KEY' }, keys, "profile.mcp['x']") + .then(() => undefined) + .catch((e: Error) => e) + expect(failure).toBeInstanceOf(ValidationError) + expect(failure?.message).toContain('MISSING_KEY') + expect(failure?.message).not.toContain(FAKE_KEY_VALUE) + }) +}) + +describe('same-host materialization of a promoted grant', () => { + // The in-repo memory bin stands in for an "external" stdio MCP server whose + // API needs a provisioned credential: its ENTIRE dataset arrives via the + // secret env (AGENT_MEMORY_ITEMS ← key ADOPTED_MEMORY_ITEMS_JSON), and an + // empty memory refuses to boot — so a successful boot + search hit is proof + // the KeyProvider value reached the child process env. + const here = dirname(fileURLToPath(import.meta.url)) + const memoryBin = join(here, '..', 'mcp', 'memory-bin.ts') + const tsxCli = createRequire(import.meta.url).resolve('tsx/cli') + const seededRows = JSON.stringify([ + { id: 'm1', text: 'adopted external memory row: rotate keys quarterly' }, + ]) + + it('boots an adopted stdio grant with the provisioned key in its env (the Phase-6 smoke)', { + timeout: 60_000, + }, async () => { + const grant: ExternalMcpGrant = { + transport: 'stdio', + command: process.execPath, + args: [tsxCli, memoryBin], + secretEnv: { AGENT_MEMORY_ITEMS: 'ADOPTED_MEMORY_ITEMS_JSON' }, + } + const profile = applyArtifact({}, asArtifact(connectionArtifact(grant, { key: 'adopted' }))) + expect(JSON.stringify(profile)).not.toContain('rotate keys quarterly') + + const keys = envKeyProvider({ ADOPTED_MEMORY_ITEMS_JSON: seededRows }) + const mat = await materializeLocalMcp(profile, { keys }) + try { + expect(mat.tools.map((t) => t.function.name)).toEqual([ + 'adopted__memory_search', + 'adopted__memory_get', + ]) + const hit = await mat.call('adopted__memory_search', { query: 'rotate keys' }) + expect(hit).toContain('rotate keys quarterly') + } finally { + await mat.close() + } + }) + + it('fails closed when the grant declares a secret and no provider / key exists', async () => { + const grant: ExternalMcpGrant = { + transport: 'stdio', + command: process.execPath, + args: [tsxCli, memoryBin], + secretEnv: { AGENT_MEMORY_ITEMS: 'ADOPTED_MEMORY_ITEMS_JSON' }, + } + const profile = applyArtifact({}, asArtifact(connectionArtifact(grant, { key: 'adopted' }))) + await expect(materializeLocalMcp(profile)).rejects.toThrow(/no KeyProvider/) + await expect(materializeLocalMcp(profile, { keys: envKeyProvider({}) })).rejects.toThrow( + /ADOPTED_MEMORY_ITEMS_JSON/, + ) + }) + + it('fails loud on a promoted http grant — never a silent skip', async () => { + const profile = applyArtifact({}, asArtifact(connectionArtifact(httpGrant(), { key: 'exa' }))) + await expect( + materializeLocalMcp(profile, { keys: envKeyProvider({ EXA_API_KEY: FAKE_KEY_VALUE }) }), + ).rejects.toThrow(/transport 'http'/) + }) +}) diff --git a/src/lifecycle/connection.ts b/src/lifecycle/connection.ts new file mode 100644 index 00000000..7701183f --- /dev/null +++ b/src/lifecycle/connection.ts @@ -0,0 +1,201 @@ +/** + * PHASE 6 — the `connection` artifact: ADOPT an existing external MCP server + * (instead of building one) as a promotable, measurable profile artifact. + * + * A `connection` candidate carries an `ExternalMcpGrant` — the address of a + * server that already exists (a remote `http` endpoint, or an installed + * `stdio` launch) plus the credential it needs BY NAME. `applyArtifact` + * promotes the grant onto the profile through fields the published + * `AgentProfile` already has: + * + * - `profile.mcp[key]` — the server entry (`connectionMcpServer`), so the + * EXISTING materialization paths mount it: `materializeLocalMcp` spawns a + * stdio grant same-host during a scored run; an http grant is representable + * today and materializes in the sandbox backend (the same-host client has + * no http MCP client yet — it fails loud, never silently skips). + * - `profile.connections` — the optional Tangle Hub grant + * (`AgentProfileConnection`: connectionId + capabilities), when the adopted + * capability is hub-managed rather than a raw server. + * + * SECRETS: `grant.secretEnv` maps env var name → provider KEY NAME. Only the + * names ride the artifact/profile (as `mcp[key].metadata.secretEnv`); the + * VALUE is resolved by a `KeyProvider` at materialize time and injected into + * the spawned server's env only (see runtime/key-provider.ts — fail-closed, + * never logged). So a promoted grant is auditable and replayable without ever + * storing a credential. + * + * Measuring an adopted server is the SAME ablation as every other surface: + * + * measureMarginalLift({ + * baseline, + * candidate: registry.register(connectionArtifact(grant)), + * evalRunner: sweEvalRunner({ ...opts, materializeMcp: true, keys }), + * }) + * + * — the candidate arm runs with the external server live, the baseline + * without, and only a real held-out lift promotes the adoption. + */ + +import type { AgentProfileConnection, AgentProfileMcpServer } from '@tangle-network/agent-interface' +import { ValidationError } from '../errors' +import { mcpSecretEnvMetadataKey } from '../runtime/key-provider' +import type { ArtifactInput } from './types' + +/** + * An external MCP grant — the ADOPT-not-build candidate payload. `http` names + * a remote endpoint (`url`, literal `headers`); `stdio` names an installed + * launch (`command`/`args`). `env` is literal (non-secret) server env; + * `secretEnv` declares credentials by provider key name, resolved at + * materialize time — a secret VALUE must never appear in a grant. + */ +export interface ExternalMcpGrant { + transport: 'http' | 'stdio' + /** `http`: the remote MCP endpoint. */ + url?: string + /** `stdio`: the launch command of an installed external server. */ + command?: string + args?: string[] + /** Literal request headers (`http` only). Secret headers do not exist here — + * route credentials through `secretEnv` so values never ride the profile. */ + headers?: Record + /** Literal (non-secret) env for the server. */ + env?: Record + /** Declarative secrets: env var name → `KeyProvider` key name. Names only. */ + secretEnv?: Record + /** Optional Tangle Hub grant promoted alongside onto `profile.connections`. */ + hub?: AgentProfileConnection +} + +/** Assert an `ExternalMcpGrant` is well-formed. Fail-closed: a malformed grant + * must never become a promotable artifact. */ +export function validateExternalMcpGrant(grant: ExternalMcpGrant): void { + if (grant.transport !== 'http' && grant.transport !== 'stdio') { + throw new ValidationError( + `external MCP grant: transport must be 'http' or 'stdio' (got '${String( + (grant as { transport?: unknown }).transport, + )}')`, + ) + } + if (grant.transport === 'http') { + if (!grant.url || !/^https?:\/\//.test(grant.url)) { + throw new ValidationError("external MCP grant: transport 'http' requires an http(s) `url`") + } + if (grant.command) { + throw new ValidationError( + "external MCP grant: transport 'http' must not carry a `command` (ambiguous grant)", + ) + } + } else { + if (!grant.command || grant.command.trim().length === 0) { + throw new ValidationError("external MCP grant: transport 'stdio' requires a `command`") + } + if (grant.url) { + throw new ValidationError( + "external MCP grant: transport 'stdio' must not carry a `url` (ambiguous grant)", + ) + } + if (grant.headers && Object.keys(grant.headers).length > 0) { + throw new ValidationError( + 'external MCP grant: `headers` are http-only — a stdio server takes env, not headers', + ) + } + } + for (const [envName, keyName] of Object.entries(grant.secretEnv ?? {})) { + if (!envName.trim() || !keyName.trim()) { + throw new ValidationError( + 'external MCP grant: every secretEnv entry needs a non-empty env var name and provider key name', + ) + } + } + if (grant.hub) { + if (!grant.hub.connectionId?.trim()) { + throw new ValidationError('external MCP grant: a hub grant requires a connectionId') + } + if (!Array.isArray(grant.hub.capabilities) || grant.hub.capabilities.length === 0) { + throw new ValidationError( + 'external MCP grant: a hub grant requires at least one capability path', + ) + } + } +} + +/** + * The `AgentProfileMcpServer` entry a grant promotes into `profile.mcp` — the + * profile shape both materialization paths already read. Secret references + * ride `metadata[mcpSecretEnvMetadataKey]` (names only); the value is injected + * at materialize time by the `KeyProvider`. + */ +export function connectionMcpServer(grant: ExternalMcpGrant): AgentProfileMcpServer { + validateExternalMcpGrant(grant) + const secretMeta = + grant.secretEnv && Object.keys(grant.secretEnv).length > 0 + ? { metadata: { [mcpSecretEnvMetadataKey]: { ...grant.secretEnv } } } + : {} + if (grant.transport === 'http') { + return { + transport: 'http', + url: grant.url as string, + ...(grant.headers ? { headers: grant.headers } : {}), + ...(grant.env ? { env: grant.env } : {}), + ...secretMeta, + enabled: true, + } + } + return { + transport: 'stdio', + command: grant.command as string, + ...(grant.args ? { args: grant.args } : {}), + ...(grant.env ? { env: grant.env } : {}), + ...secretMeta, + enabled: true, + } +} + +export interface ConnectionArtifactOptions { + /** The `profile.mcp` key / artifact key the grant mounts under. + * Default: derived from the endpoint host (http) or command basename (stdio). */ + key?: string + /** Artifact display name. Default = key. */ + name?: string + description?: string +} + +/** + * Build a `connection` `ArtifactInput` from a grant — the one-step adapter for + * the adopt path (a research driver's verdict, a hub catalog row, an operator + * decision). Metadata records the transport and the secret KEY NAMES for the + * audit trail; values never appear. + */ +export function connectionArtifact( + grant: ExternalMcpGrant, + opts: ConnectionArtifactOptions = {}, +): ArtifactInput<'connection'> { + validateExternalMcpGrant(grant) + const key = opts.key ?? defaultGrantKey(grant) + return { + kind: 'connection', + key, + name: opts.name ?? key, + description: + opts.description ?? + `adopted external MCP (${grant.transport}: ${grant.transport === 'http' ? grant.url : grant.command})`, + payload: { grant }, + metadata: { + transport: grant.transport, + ...(grant.secretEnv ? { secretKeys: Object.values(grant.secretEnv) } : {}), + ...(grant.hub ? { hubConnectionId: grant.hub.connectionId } : {}), + }, + } +} + +/** A stable default mount key: the endpoint host (http) or the command + * basename (stdio), sanitized to the profile-key alphabet. */ +function defaultGrantKey(grant: ExternalMcpGrant): string { + const raw = + grant.transport === 'http' + ? new URL(grant.url as string).hostname + : ((grant.command as string).split('/').pop() ?? (grant.command as string)) + const key = raw.replace(/[^a-zA-Z0-9_-]/g, '_') + if (!key) throw new ValidationError('connectionArtifact: cannot derive a key — pass opts.key') + return key +} diff --git a/src/lifecycle/index.ts b/src/lifecycle/index.ts index 404334c0..f3b90c3b 100644 --- a/src/lifecycle/index.ts +++ b/src/lifecycle/index.ts @@ -33,6 +33,13 @@ export { applyArtifact, applyArtifacts } from './apply' export { type ComposeProfileOptions, composeProfile } from './compose' +export { + type ConnectionArtifactOptions, + connectionArtifact, + connectionMcpServer, + type ExternalMcpGrant, + validateExternalMcpGrant, +} from './connection' export { type DedupeOptions, type DedupeResult, diff --git a/src/lifecycle/swe-eval-runner.ts b/src/lifecycle/swe-eval-runner.ts index 1bcad3ef..5b6782a7 100644 --- a/src/lifecycle/swe-eval-runner.ts +++ b/src/lifecycle/swe-eval-runner.ts @@ -46,6 +46,7 @@ import { ValidationError } from '../errors' import { type AgenticSurface, type ArtifactHandle, + type KeyProvider, type LocalMcpMaterialization, materializeLocalMcp, refine, @@ -107,6 +108,11 @@ export interface SweEvalRunnerOptions { * server that cannot boot THROWS — scoring it silently without its tools * would fake the with/without ablation. */ materializeMcp?: boolean + /** Provisions declared secrets (`mcp[key].metadata.secretEnv`) into a + * spawned server's env at materialize time — how an ADOPTED external MCP + * (a `connection` artifact) gets its API key during a scored run. Values + * reach only the child env; see runtime/key-provider.ts. */ + keys?: KeyProvider } /** @@ -125,7 +131,9 @@ export function sweEvalRunner( const systemPrompt = renderSystemPrompt(profile, opts.seedPrompt) // Same-host materialization of the profile's MCP surface (opt-in): the // spawned servers live across all tasks of THIS eval and die in finally. - const mcp = opts.materializeMcp ? await materializeLocalMcp(profile) : undefined + const mcp = opts.materializeMcp + ? await materializeLocalMcp(profile, opts.keys ? { keys: opts.keys } : {}) + : undefined const environment = mcp && mcp.tools.length > 0 ? withLocalMcpTools(opts.environment, mcp) : opts.environment try { diff --git a/src/lifecycle/tool-build.ts b/src/lifecycle/tool-build.ts index e15e5af8..1fdb72c1 100644 --- a/src/lifecycle/tool-build.ts +++ b/src/lifecycle/tool-build.ts @@ -70,8 +70,14 @@ export interface WorktreeBuildOptions { * instruction, observes the session's diff + verifier output, rates it, and * decides refine / re-scope / decompose. Unset ⇒ the plain multi-shot * `agenticGenerator` (canned resume notes, no LLM driver) — the offline path. + * `research` wires the adopt-not-build seam (a `research{query}` tool + + * doctrine for the driver) — no live web/search backend ships here yet. */ - driver?: { brain: ToolLoopChat; maxTurns?: number } + driver?: { + brain: ToolLoopChat + maxTurns?: number + research?: (query: string) => Promise + } } /** @@ -104,6 +110,7 @@ export function worktreeBuildCandidate(opts: WorktreeBuildOptions): BuildCandida ...shared, brain: opts.driver.brain, ...(opts.driver.maxTurns !== undefined ? { maxTurns: opts.driver.maxTurns } : {}), + ...(opts.driver.research ? { research: opts.driver.research } : {}), }) : agenticGenerator(shared) diff --git a/src/lifecycle/tool-generator.test.ts b/src/lifecycle/tool-generator.test.ts index c07706d6..a8a9477e 100644 --- a/src/lifecycle/tool-generator.test.ts +++ b/src/lifecycle/tool-generator.test.ts @@ -119,6 +119,77 @@ describe('buildableGenerator — tool/MCP supervisor dispatch', () => { expect(base.mcp).toBeUndefined() }) + it('emits an ADOPTED remote http MCP when the candidate spec reports `remote`', async () => { + const base = baseline() + const exam = examRewardsMcp('exa_search') + + // The adopt-not-build path: the research driver verified an EXISTING + // external server fits, so the candidate spec carries `remote` (endpoint + + // secret KEY NAME) instead of a local serve command. + const adoptOne: BuildCandidate = async (_ctx, index) => ({ + label: 'exa_search', + verified: true, + worktreeRef: `/wt/${index}`, + remote: { + url: 'https://mcp.exa.ai/mcp', + headers: { 'x-client': 'swe' }, + secretEnv: { EXA_API_KEY: 'EXA_API_KEY' }, + }, + }) + + const result = await runLifecycle({ + baseline: base, + domain: 'buildable-domain', + generators: [ + buildableGenerator({ kind: 'mcp', buildCandidate: adoptOne, evalRunner: exam, fanout: 1 }), + ], + evalRunner: exam, + gate: thresholdPromotionGate(), + }) + + expect(result.outcomes).toHaveLength(1) + const winner = result.outcomes[0]! + expect(winner.promoted).toBe(true) + const server = ( + winner.artifact.payload as { + server: { + transport?: string + url?: string + command?: string + headers?: Record + metadata?: Record + } + } + ).server + expect(server.transport).toBe('http') + expect(server.url).toBe('https://mcp.exa.ai/mcp') + expect(server.headers).toEqual({ 'x-client': 'swe' }) + expect(server.command).toBeUndefined() + // Secrets ride by NAME only; adopt provenance is stamped for the audit trail. + expect(server.metadata?.secretEnv).toEqual({ EXA_API_KEY: 'EXA_API_KEY' }) + expect(winner.artifact.metadata?.adopted).toBe(true) + expect(winner.artifact.metadata?.remoteUrl).toBe('https://mcp.exa.ai/mcp') + }) + + it('rejects a build reporting BOTH serve and remote (ambiguous candidate spec)', async () => { + const ambiguous: BuildCandidate = async (_ctx, index) => ({ + label: 'both', + verified: true, + worktreeRef: `/wt/${index}`, + serve: { command: 'node', args: ['server.mjs'] }, + remote: { url: 'https://mcp.example.com/mcp' }, + }) + const gen = buildableGenerator({ + kind: 'mcp', + buildCandidate: ambiguous, + evalRunner: async () => ({ composite: 0, costUsd: 0 }), + fanout: 1, + }) + await expect(gen.generate({ baseline: baseline(), domain: 'd', findings: [] })).rejects.toThrow( + /serve OR remote/, + ) + }) + it('returns no candidate when every build fails verification', async () => { const allFail: BuildCandidate = async (_ctx, index) => ({ label: `c${index}`, diff --git a/src/lifecycle/tool-generator.ts b/src/lifecycle/tool-generator.ts index 1f6ac535..30e11ef3 100644 --- a/src/lifecycle/tool-generator.ts +++ b/src/lifecycle/tool-generator.ts @@ -41,6 +41,7 @@ import type { AgentProfileMcpServer } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' +import { connectionMcpServer } from './connection' import type { CandidateGenerator, GenerateContext } from './generator' import { type EvalRunner, measureMarginalLift } from './marginal-lift' import { ArtifactRegistry } from './registry' @@ -64,11 +65,25 @@ export interface BuiltCandidate { /** The worktree path / git ref holding the built change (provenance). */ worktreeRef: string /** - * For an `mcp` candidate: how to START the built server (stdio transport). - * Becomes the `AgentProfileMcpServer` the artifact carries. REQUIRED for an - * `mcp` build; ignored for a `tool` build. + * For an `mcp` candidate that was BUILT: how to START the built server + * (stdio transport). Becomes the `AgentProfileMcpServer` the artifact + * carries. An `mcp` build must report exactly one of `serve` / `remote`; + * both are ignored for a `tool` build. */ serve?: { command: string; args?: string[]; cwd?: string; env?: Record } + /** + * For an `mcp` candidate that was ADOPTED (the research path found an + * EXISTING external server instead of building one): the remote http + * endpoint. Emits a `{ transport:'http', url, headers, env }` server entry; + * `secretEnv` declares its credential by provider KEY NAME (resolved at + * materialize time by a `KeyProvider` — a value must never appear here). + */ + remote?: { + url: string + headers?: Record + env?: Record + secretEnv?: Record + } /** * For a `tool` candidate: the tool name the grant lands under (the profile * `tools` key). REQUIRED for a `tool` build; ignored for an `mcp` build. @@ -208,10 +223,33 @@ function bareArtifact(kind: BuildableKind, built: BuiltCandidate): ArtifactInput payload: { enabled: true }, } as ArtifactInput } - const serve = built.serve + // The build-vs-adopt branch, selected by the candidate spec: `serve` is a + // locally-BUILT stdio server (launched at its worktree), `remote` is an + // ADOPTED external http server (secrets by name, via connectionMcpServer). + const { serve, remote } = built + if (serve && remote) { + throw new ValidationError( + `buildableGenerator: an 'mcp' build must report serve OR remote, not both (label=${built.label})`, + ) + } + if (remote) { + const server = connectionMcpServer({ + transport: 'http', + url: remote.url, + ...(remote.headers ? { headers: remote.headers } : {}), + ...(remote.env ? { env: remote.env } : {}), + ...(remote.secretEnv ? { secretEnv: remote.secretEnv } : {}), + }) + return { + kind: 'mcp', + key: built.label, + name: built.label, + payload: { server }, + } as ArtifactInput + } if (!serve?.command || serve.command.trim().length === 0) { throw new ValidationError( - `buildableGenerator: a verified 'mcp' build must report a serve command (label=${built.label})`, + `buildableGenerator: a verified 'mcp' build must report a serve command or a remote endpoint (label=${built.label})`, ) } const server: AgentProfileMcpServer = { @@ -249,6 +287,9 @@ function toArtifact( worktreeRef: built.worktreeRef, buildLabel: built.label, fanout, + // Adopt-vs-build provenance: an adopted candidate wired an EXISTING + // external server instead of building one. + ...(built.remote ? { adopted: true, remoteUrl: built.remote.url } : {}), // The INTERNAL rank lift (best-of-N selection), distinct from the // orchestrator's promotion-gate measurement. siblingScoreDelta: scoreDelta, diff --git a/src/lifecycle/types.ts b/src/lifecycle/types.ts index d79b75f6..f6fab100 100644 --- a/src/lifecycle/types.ts +++ b/src/lifecycle/types.ts @@ -23,16 +23,26 @@ import type { AgentSubagentProfile, } from '@tangle-network/agent-interface' import type { AgentMemorySpec } from '../mcp/memory-server' +import type { ExternalMcpGrant } from './connection' /** * The profile levers an artifact can target. One-to-one with the §1.5 profile * surface (`prompt + skills + tools + mcp + hooks + subagents`), plus `memory` * — the Phase-5 surface the published `AgentProfile` does not yet name (it - * mounts as a local extension; see lifecycle/memory.ts). Each kind maps to - * exactly one profile lever, so an artifact can be applied onto a baseline - * profile deterministically (see `applyArtifact`). + * mounts as a local extension; see lifecycle/memory.ts) — and `connection` — + * the Phase-6 adopt-not-build surface: a grant for an EXTERNAL MCP server + * (see lifecycle/connection.ts). Each kind maps deterministically onto the + * profile (see `applyArtifact`). */ -export type ArtifactKind = 'skill' | 'tool' | 'mcp' | 'hook' | 'subagent' | 'prompt' | 'memory' +export type ArtifactKind = + | 'skill' + | 'tool' + | 'mcp' + | 'hook' + | 'subagent' + | 'prompt' + | 'memory' + | 'connection' /** * The payload for each `ArtifactKind`. The shapes are the SAME types the @@ -50,6 +60,9 @@ export type ArtifactKind = 'skill' | 'tool' | 'mcp' | 'hook' | 'subagent' | 'pro * profile has no memory field) AND a stdio server entry that * SERVES it lands under `profile.mcp[name]`, so the same-host * materialization boots the memory live during a scored run. + * - `connection` — an EXTERNAL MCP grant (adopt, not build): the server entry + * lands under `profile.mcp[name]` with secrets by NAME only, + * and an optional hub grant lands on `profile.connections`. */ export interface ArtifactPayloads { prompt: { instruction: string } @@ -59,6 +72,7 @@ export interface ArtifactPayloads { hook: { event: string; commands: AgentProfileHookCommand[] } subagent: { profile: AgentSubagentProfile } memory: { spec: AgentMemorySpec } + connection: { grant: ExternalMcpGrant } } /** diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 6c2bbd64..2eb6ce57 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -153,6 +153,15 @@ export { } from './in-process-sandbox-client' // The one pseudo-box adapter: any non-box Executor → a SandboxClient for runLoop. export { inlineSandboxClient } from './inline-sandbox-client' +// API-key provisioning for adopted external MCP servers: secrets ride the +// profile by NAME only; a KeyProvider resolves values at materialize time. +export { + envKeyProvider, + type KeyProvider, + mcpSecretEnvMetadataKey, + resolveSecretEnv, + secretEnvOfMcpServer, +} from './key-provider' // The same-host pseudo-box: a router-brain tool loop with the profile's stdio // MCP servers spawned as LOCAL children — the one client that can reach an MCP // server built into a host worktree. diff --git a/src/runtime/key-provider.ts b/src/runtime/key-provider.ts new file mode 100644 index 00000000..48a4d4e5 --- /dev/null +++ b/src/runtime/key-provider.ts @@ -0,0 +1,110 @@ +/** + * `KeyProvider` — API-key provisioning for adopted external MCP servers. + * + * An adopted server (a `connection` artifact, or a `buildableGenerator` remote + * emit) usually needs a credential. The credential must never ride the profile + * or the artifact — both are logged, diffed, and stored as audit records — so + * the profile carries only a DECLARATIVE reference: + * + * `profile.mcp[key].metadata[mcpSecretEnvMetadataKey]` + * = { ENV_VAR_NAME: 'PROVIDER_KEY_NAME', … } + * + * and the VALUE is resolved at materialize time (`materializeLocalMcp`'s + * `keys` option) and injected straight into the spawned server child's env. + * Values exist only in the child process env — never in the profile, the + * artifact registry, an error message, or a log line (errors name the KEY + * NAME only). + * + * Fail-closed at every hop: a declared secret with no provider, or a provider + * that does not hold the named key, THROWS — booting an external server + * keyless would fail opaquely mid-eval or silently score a broken candidate. + * + * `envKeyProvider` is the same-host default: it reads the process env, which + * the operator loads via dotenvx (the secrets files never touch the repo). + * + * >>> REAL SECRET STORES PLUG IN HERE (flagged): the sandbox SDK's + * `client.secrets` (`SecretsManager.get(name)`) and the platform hub's + * short-lived capability tokens (`PlatformHubClient.mintToken` → + * `{ ENV: token }`) both satisfy `KeyProvider` with a one-line adapter; both + * need a live service, so neither is constructed here. + */ + +import type { AgentProfileMcpServer } from '@tangle-network/agent-interface' +import { ValidationError } from '../errors' + +/** Resolve named secrets. The ONE seam every secret store adapts to. */ +export interface KeyProvider { + /** The value for `name`, or `undefined` when this provider does not hold it. */ + get(name: string): Promise +} + +/** The env-backed provider: reads the (dotenvx-loaded) process env. Empty / + * whitespace-only values count as absent — fail loud, not with a blank key. */ +export function envKeyProvider(env: Record = process.env): KeyProvider { + return { + async get(name: string): Promise { + const value = env[name] + return value !== undefined && value.trim().length > 0 ? value : undefined + }, + } +} + +/** The `AgentProfileMcpServer.metadata` key the declarative secret-env map + * rides under: `{ ENV_VAR_NAME: 'PROVIDER_KEY_NAME' }`. Names only — values + * are resolved at materialize time and never stored. */ +export const mcpSecretEnvMetadataKey = 'secretEnv' + +/** Read (and validate) a server entry's declared secret-env map, if any. + * Malformed metadata throws — a half-declared secret must never half-boot. */ +export function secretEnvOfMcpServer( + server: AgentProfileMcpServer, +): Record | undefined { + const raw = server.metadata?.[mcpSecretEnvMetadataKey] + if (raw === undefined || raw === null) return undefined + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new ValidationError( + `secretEnvOfMcpServer: metadata.${mcpSecretEnvMetadataKey} must be an object mapping env var name -> key name`, + ) + } + const entries = Object.entries(raw as Record) + if (entries.length === 0) return undefined + for (const [envName, keyName] of entries) { + if (!envName.trim() || typeof keyName !== 'string' || !keyName.trim()) { + throw new ValidationError( + `secretEnvOfMcpServer: metadata.${mcpSecretEnvMetadataKey}['${envName}'] must name a non-empty provider key`, + ) + } + } + return raw as Record +} + +/** + * Resolve a declared secret-env map into the real env entries for a server + * spawn. Fail-closed: no provider or a missing key throws, naming the KEY + * NAME only (the value never appears in any message). `label` names the + * server for the error (e.g. `profile.mcp['exa']`). + */ +export async function resolveSecretEnv( + secretEnv: Record, + keys: KeyProvider | undefined, + label: string, +): Promise> { + const entries = Object.entries(secretEnv) + if (entries.length === 0) return {} + if (!keys) { + throw new ValidationError( + `${label} declares secret env (${entries.map(([e]) => e).join(', ')}) but no KeyProvider was supplied — refusing to boot an external server keyless`, + ) + } + const resolved: Record = {} + for (const [envName, keyName] of entries) { + const value = await keys.get(keyName) + if (value === undefined) { + throw new ValidationError( + `${label}: the KeyProvider holds no value for '${keyName}' (wanted for env ${envName}) — provision the key or drop the grant`, + ) + } + resolved[envName] = value + } + return resolved +} diff --git a/src/runtime/stdio-mcp-client.ts b/src/runtime/stdio-mcp-client.ts index d2f9815f..8e30e648 100644 --- a/src/runtime/stdio-mcp-client.ts +++ b/src/runtime/stdio-mcp-client.ts @@ -34,6 +34,7 @@ import { spawn } from 'node:child_process' import { createInterface } from 'node:readline' import type { AgentProfile } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' +import { type KeyProvider, resolveSecretEnv, secretEnvOfMcpServer } from './key-provider' import { sanitizeMcpToolSchema } from './mcp-environment' import type { AgenticTool } from './strategy' @@ -235,6 +236,12 @@ export interface MaterializeLocalMcpOptions { timeoutMs?: number /** Cap on a tool result's text fed back to the worker. Default 2000 chars. */ maxResultChars?: number + /** Resolves a server's DECLARED secrets (`metadata.secretEnv`: env var name → + * provider key name) at spawn time. The resolved values reach ONLY the child + * process env — never the profile, the logs, or an error message. Fail-closed: + * a server declaring secrets without a provider (or with a missing key) + * throws instead of booting keyless. */ + keys?: KeyProvider } /** The live same-host materialization of a profile's `mcp` surface. */ @@ -273,7 +280,7 @@ export async function materializeLocalMcp( const transport = server.transport ?? 'stdio' if (transport !== 'stdio') { throw new ValidationError( - `materializeLocalMcp: profile.mcp['${key}'] has transport '${transport}' — the same-host client only spawns stdio servers (remote servers materialize in the sandbox backend)`, + `materializeLocalMcp: profile.mcp['${key}'] has transport '${transport}' — the same-host client only spawns stdio servers; a remote (http/sse) server needs an http MCP client (not yet built here) or the sandbox backend. Failing loud: scoring this profile without its declared server would fake the with/without ablation`, ) } if (!server.command || server.command.trim().length === 0) { @@ -281,11 +288,22 @@ export async function materializeLocalMcp( `materializeLocalMcp: profile.mcp['${key}'] declares a stdio server with no command`, ) } + // Provision declared secrets NOW, into the spawn env only. The resolved + // values live in this local and the child env — nowhere else. + const secretRefs = secretEnvOfMcpServer(server) + const provisioned = secretRefs + ? await resolveSecretEnv( + secretRefs, + opts.keys, + `materializeLocalMcp: profile.mcp['${key}']`, + ) + : undefined + const env = server.env || provisioned ? { ...server.env, ...provisioned } : undefined const conn = await connectStdioMcp({ command: server.command, ...(server.args ? { args: server.args } : {}), ...(server.cwd ? { cwd: server.cwd } : {}), - ...(server.env ? { env: server.env } : {}), + ...(env ? { env } : {}), ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}), }) connections.push(conn) From fb636d710c96f513a25cd5af63e76c9f45d96979 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 13 Jul 2026 00:54:48 -0600 Subject: [PATCH 44/44] fix(bench): canonical self-improve invocation resolves local src The e2e driver imports sweEvalRunner, which lives in ../src and resolves only through bench/tsconfig path aliases (tsx applies them at cwd=bench). Running tsx bench/src/... from the repo root resolved the published agent-runtime in bench/node_modules and crashed on the missing export. Add a bench package script (self-improve) so the invocation always runs at cwd=bench, and document the requirement in the driver header. --- bench/package.json | 3 ++- bench/src/self-improve-swe.mts | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bench/package.json b/bench/package.json index a511ee9a..ad41ba9b 100644 --- a/bench/package.json +++ b/bench/package.json @@ -15,7 +15,8 @@ "gate-cli": "tsx src/gate-cli.mts", "run-benchmarks": "tsx src/run-benchmarks-cli.mts", "gate-report": "tsx src/corpus-report.mts corpus/finsearch.jsonl", - "terminal-compare": "tsx src/terminal-compare.ts" + "terminal-compare": "tsx src/terminal-compare.ts", + "self-improve": "tsx src/self-improve-swe.mts" }, "dependencies": { "@tangle-network/agent-eval": "^0.106.3", diff --git a/bench/src/self-improve-swe.mts b/bench/src/self-improve-swe.mts index d4820ba2..7fb2af37 100644 --- a/bench/src/self-improve-swe.mts +++ b/bench/src/self-improve-swe.mts @@ -33,6 +33,14 @@ * swe-bench-env + lifecycle incl. sweEvalRunner + the same-host MCP client) * loaded; print READY on stderr and exit 0 WITHOUT a clone / model call / * dataset read. + * + * RUN FROM bench/ (cwd=bench): this driver imports `sweEvalRunner`, which lives + * in THIS worktree's `../src` and is reachable only through bench/tsconfig's + * path aliases — tsx applies them when cwd is bench. `bench/node_modules` pins + * the PUBLISHED agent-runtime (no `sweEvalRunner`), so running from the repo + * root resolves the published dist and crashes on the missing export. Use the + * package script `pnpm --filter ./bench self-improve` (or `cd bench && tsx + * src/self-improve-swe.mts`), never `tsx bench/src/...` from the root. */ import type { AgentProfile } from '@tangle-network/agent-interface' import {