diff --git a/docs/api/index.md b/docs/api/index.md index 1b1713b5..dd62d123 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -11290,7 +11290,7 @@ Per-generation findings producer passthrough (see selfImprove.analyzeGeneration) > `optional` **rawTraceContext?**: `boolean` -META-HARNESS mode: instead of the ~400-char distilled findings, feed the +META-HARNESS mode: instead of the ~1500-char distilled findings, feed the proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's real run traces under `runDir` (per-cell `spans.jsonl` event logs + `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose @@ -13295,7 +13295,7 @@ Build the starting instruction for a coder agent tasked with implementing a new > **applyImprovementWinnerToProfile**(`profile`, `surface`, `winner`): `AgentProfile` -Defined in: [improvement/improve.ts:466](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L466) +Defined in: [improvement/improve.ts:478](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L478) Apply a promoted winner surface back into the profile field for `surface`. Returns a shallow copy; never mutates the input profile. @@ -13324,7 +13324,7 @@ Apply a promoted winner surface back into the profile field for `surface`. > **improve**\<`TScenario`, `TArtifact`\>(`profile`, `findings`, `opts`): `Promise`\<[`ImproveResult`](#improveresult)\<`TScenario`, `TArtifact`\>\> -Defined in: [improvement/improve.ts:529](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L529) +Defined in: [improvement/improve.ts:541](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/improve.ts#L541) Run the held-out-gated self-improvement loop on ONE profile surface. diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 7f983170..1f074138 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -585,6 +585,63 @@ describe('improve() — default proposer resolution (substrate export drift guar } }) + it('the distiller keeps traceback-sized notes intact and clips at the 1500/500 caps', async () => { + // A realistic executable-judge note (~1000 chars) must survive whole; a + // runaway note is clipped to exactly 1500; a cell error is clipped to 500. + const intactNote = `Traceback (most recent call last):\n${' assert tour == expected\n'.repeat(38)}` + expect(intactNote.length).toBeGreaterThan(900) + expect(intactNote.length).toBeLessThan(1500) + const runawayNote = 'n'.repeat(1600) + const longError = 'e'.repeat(800) + const cappingJudge: JudgeConfig<{ text: string }, Scenario> = { + name: 'capping-judge', + dimensions: [{ key: 'q', description: 'fixture quality' }], + score: ({ scenario }) => { + if (scenario.id === 'a') return { dimensions: { q: 0 }, composite: 0, notes: intactNote } + if (scenario.id === 'b') return { dimensions: { q: 0 }, composite: 0, notes: runawayNote } + return { dimensions: { q: 1 }, composite: 1, notes: 'ok' } + }, + } + const findingsSeen: unknown[][] = [] + const stubProposer = { + kind: 'stub-recorder', + async propose(ctx: { findings: unknown[]; populationSize: number }) { + findingsSeen.push(ctx.findings) + return [{ surface: `candidate-${findingsSeen.length}`, label: 'stub', rationale: 'stub' }] + }, + } + const failingAgent = async (surface: unknown, scenario: Scenario, ctx: DispatchContext) => { + // The baseline must stay complete (an incomplete incumbent is refused), + // so the error lands on a generation-1 candidate cell instead. + if (scenario.id === 'c' && typeof surface === 'string' && surface.startsWith('candidate-')) { + throw new Error(longError) + } + return stubAgent(surface, scenario, ctx) + } + await improve(promptProfile(), [], { + surface: 'prompt', + scenarios, + judge: cappingJudge, + agent: failingAgent, + generator: stubProposer as never, + budget: { generations: 2, populationSize: 1, holdoutFraction: 0.25 }, + }) + expect(findingsSeen.length).toBeGreaterThanOrEqual(1) + const rows = findingsSeen.flat() as Array<{ + scenario: string + notes?: string + error?: string + }> + const intact = rows.find((row) => row.scenario === 'a') + expect(intact?.notes).toBe(intactNote) // below the cap ⇒ untouched + const clipped = rows.find((row) => row.scenario === 'b') + expect(clipped?.notes).toBe('n'.repeat(1500)) // at the cap ⇒ exactly 1500 + const errored = rows.find((row) => row.scenario === 'c') + expect(errored?.error).toBeDefined() + expect(errored?.error).toContain('e'.repeat(400)) + expect(errored?.error?.length).toBeLessThanOrEqual(500) + }) + it("surface 'code' + opts.code assembles the worktree pipeline and measures a candidate", async () => { const { execSync } = await import('node:child_process') const { mkdtempSync, readFileSync, rmSync, writeFileSync } = await import('node:fs') diff --git a/src/improvement/improve.ts b/src/improvement/improve.ts index 72a83616..f259905f 100644 --- a/src/improvement/improve.ts +++ b/src/improvement/improve.ts @@ -104,7 +104,7 @@ export type ImproveOptions = Omit< * trace-analyst over the runDir's traces) to replace it; pass `null` to disable * and keep the static `findings` all the way through. */ analyzeGeneration?: SelfImproveOptions['analyzeGeneration'] | null - /** META-HARNESS mode: instead of the ~400-char distilled findings, feed the + /** META-HARNESS mode: instead of the ~1500-char distilled findings, feed the * proposer RAW-TRACE FILESYSTEM CONTEXT — the PATHS into the prior generation's * real run traces under `runDir` (per-cell `spans.jsonl` event logs + * `cached-result.json` scores + artifacts) plus a `grep`/`cat`-to-diagnose @@ -262,6 +262,16 @@ function baselineSurfaceFor( } } +/** Slice bound for distilled judge notes: wide enough that a real traceback / + * failing-assertion note survives intact — clipping mid-traceback would leave + * the proposer trace-blind. Referenced by the `rawTraceContext` docs. */ +const DISTILLED_NOTES_MAX_CHARS = 1500 + +/** Slice bound for a cell's error string — tighter than notes (errors are + * usually one line; the full text stays on the raw cell). Also bounds the + * error-derived `claim` fallback. */ +const DISTILLED_ERROR_MAX_CHARS = 500 + /** The default `analyzeGeneration`: distill each generation's failing cells into * findings for the next proposal round. Deliberately dependency-free — judge notes * and errors are already the domain's own diagnosis (executable gates put their @@ -300,14 +310,16 @@ function generationFailureDistiller( .map((j) => j.notes) .filter((n): n is string => typeof n === 'string' && n.length > 0) .join('; ') - .slice(0, 400) - const claim = notes || (error ? `Scenario ${scenario} failed: ${error.slice(0, 200)}` : '') + .slice(0, DISTILLED_NOTES_MAX_CHARS) + const claim = + notes || + (error ? `Scenario ${scenario} failed: ${error.slice(0, DISTILLED_ERROR_MAX_CHARS)}` : '') failures.push({ scenario, composite: Number(composite.toFixed(3)), notes, ...(claim ? { claim } : {}), - ...(error ? { error: error.slice(0, 200) } : {}), + ...(error ? { error: error.slice(0, DISTILLED_ERROR_MAX_CHARS) } : {}), }) } } diff --git a/src/improvement/raw-trace-distiller.ts b/src/improvement/raw-trace-distiller.ts index 22f625ae..5d7db35b 100644 --- a/src/improvement/raw-trace-distiller.ts +++ b/src/improvement/raw-trace-distiller.ts @@ -3,7 +3,7 @@ * `rawTraceDistiller` — the meta-harness `analyzeGeneration` producer. * * The default `generationFailureDistiller` (in `improve.ts`) COMPRESSES each - * generation's failing cells into ~400-char structured findings before the next + * generation's failing cells into ~1500-char structured findings before the next * proposal round. That is the ACE-style recipe: a small summary is the proposer's * whole view of what went wrong. This producer does the opposite — the * meta-harness recipe (yoonholee.com/meta-harness): it does NOT summarize. It @@ -11,7 +11,7 @@ * disk under `runDir` — the durable per-cell `spans.jsonl` event logs, * `cached-result.json` scores, and any artifacts the substrate persisted — and * instructs the agent to `grep`/`cat`/`ls` them to diagnose the failures itself - * (up to the harness's full context, ~millions of tokens, vs a ~400-char digest). + * (up to the harness's full context, ~millions of tokens, vs a ~1500-char digest). * * It emits `AnalystFinding[]` so it drops into the SAME `opts.analyzeGeneration` * slot the default distiller uses, and renders through the same