diff --git a/docs/api/index.md b/docs/api/index.md index 8ac41cfc..6129081b 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -5896,7 +5896,7 @@ Defined in: [improvement/agentic-generator.ts:116](https://github.com/tangle-net ### AgenticGeneratorOptions -Defined in: [improvement/agentic-generator.ts:119](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L119) +Defined in: [improvement/agentic-generator.ts:159](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L159) `@tangle-network/agent-runtime` improvement — the CODE-surface proposer for agent-eval's improvement loop. @@ -5914,7 +5914,7 @@ producer, which mutates an isolated git worktree via a pluggable > `optional` **harness?**: [`LocalHarness`](mcp.md#localharness) -Defined in: [improvement/agentic-generator.ts:121](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L121) +Defined in: [improvement/agentic-generator.ts:161](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L161) Local coding harness to run in the worktree. Default `claude`. @@ -5922,7 +5922,7 @@ Local coding harness to run in the worktree. Default `claude`. > `optional` **profile?**: `AgentProfile` -Defined in: [improvement/agentic-generator.ts:124](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L124) +Defined in: [improvement/agentic-generator.ts:164](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L164) Author profile rendered through the canonical harness mapper. Required for reproducible Codex so model and reasoning settings are explicit. @@ -5931,7 +5931,7 @@ Author profile rendered through the canonical harness mapper. Required > `optional` **codexReproducible?**: `boolean` -Defined in: [improvement/agentic-generator.ts:127](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L127) +Defined in: [improvement/agentic-generator.ts:167](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L167) Run Codex with isolated configuration, exact prompt evidence, and required terminal token usage. Requires `harness: 'codex'` and `profile`. @@ -5940,19 +5940,21 @@ Run Codex with isolated configuration, exact prompt evidence, and required > `optional` **codexReadDeniedPaths?**: readonly `string`[] \| ((`worktreePath`) => readonly `string`[]) -Defined in: [improvement/agentic-generator.ts:130](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L130) +Defined in: [improvement/agentic-generator.ts:170](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L170) Absolute paths reproducible Codex must not read. A function can derive candidate-specific paths after the driver creates its worktree. ##### onShotCompleted? -> `optional` **onShotCompleted?**: (`receipt`) => `void` \| `Promise`\<`void`\> +> `optional` **onShotCompleted?**: (`receipt`, `execution`) => `void` \| `Promise`\<`void`\> -Defined in: [improvement/agentic-generator.ts:133](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L133) +Defined in: [improvement/agentic-generator.ts:175](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L175) Awaited once for every attempted author shot, including process failures. - Throwing aborts the candidate so receipt persistence can fail closed. + The second argument preserves the exact harness result, including stdout + and stderr, before worktree inspection or verification can reject the + shot. Throwing aborts the candidate so evidence persistence fails closed. ###### Parameters @@ -5960,6 +5962,33 @@ Awaited once for every attempted author shot, including process failures. [`AgenticGeneratorShotReceipt`](#agenticgeneratorshotreceipt) +###### execution + +`Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> \| `null` + +###### Returns + +`void` \| `Promise`\<`void`\> + +##### onShotDisposition? + +> `optional` **onShotDisposition?**: (`receipt`, `disposition`) => `void` \| `Promise`\<`void`\> + +Defined in: [improvement/agentic-generator.ts:181](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L181) + +Awaited after worktree inspection and before the shot is accepted, + retried, or discarded. Throwing aborts the candidate. + +###### Parameters + +###### receipt + +[`AgenticGeneratorShotReceipt`](#agenticgeneratorshotreceipt) + +###### disposition + +[`AgenticGeneratorShotDisposition`](#agenticgeneratorshotdisposition) + ###### Returns `void` \| `Promise`\<`void`\> @@ -5968,7 +5997,7 @@ Awaited once for every attempted author shot, including process failures. > `optional` **maximumCharge?**: `MaximumCharge` -Defined in: [improvement/agentic-generator.ts:138](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L138) +Defined in: [improvement/agentic-generator.ts:189](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L189) Optional hard upper bound passed to the run-wide CostLedger before each author shot. This MUST be enforced by the provider or executor; a planning @@ -5979,7 +6008,7 @@ Optional hard upper bound passed to the run-wide CostLedger before each > `optional` **timeoutMs?**: `number` -Defined in: [improvement/agentic-generator.ts:140](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L140) +Defined in: [improvement/agentic-generator.ts:191](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L191) Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). @@ -5987,7 +6016,7 @@ Per-shot wall-clock timeout (ms). Default = `runLocalHarness` default (5m). > `optional` **buildPrompt?**: (`args`) => `string` -Defined in: [improvement/agentic-generator.ts:143](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L143) +Defined in: [improvement/agentic-generator.ts:194](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L194) Build the harness task prompt from the report + findings. Override for domain phrasing; the default turns findings into a concrete coder task. @@ -6012,7 +6041,7 @@ Build the harness task prompt from the report + findings. Override for > `optional` **verify?**: [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:149](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L149) +Defined in: [improvement/agentic-generator.ts:200](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L200) Verify the worktree after each dirtying shot. When set, a candidate that fails verification is NOT returned — the failure feeds the next shot @@ -6024,7 +6053,7 @@ Verify the worktree after each dirtying shot. When set, a candidate that > `optional` **runHarness?**: (`options`) => `Promise`\<[`LocalHarnessResult`](mcp.md#localharnessresult)\> -Defined in: [improvement/agentic-generator.ts:151](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L151) +Defined in: [improvement/agentic-generator.ts:202](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L202) Test seam — inject the harness runner (defaults to `runLocalHarness`). @@ -6060,7 +6089,7 @@ Does NOT throw when: > `optional` **isDirty?**: (`worktreePath`) => `boolean` -Defined in: [improvement/agentic-generator.ts:153](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L153) +Defined in: [improvement/agentic-generator.ts:204](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L204) Test seam — inject the worktree-dirty check (defaults to `git status`). @@ -11159,6 +11188,31 @@ Verifies the edited worktree. Sync or async; throws only on a setup fault *** +### AgenticGeneratorShotExecution + +> **AgenticGeneratorShotExecution** = `Readonly`\<`Omit`\<[`LocalHarnessResult`](mcp.md#localharnessresult), `"usage"` \| `"evidence"`\> & `object`\> + +Defined in: [improvement/agentic-generator.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L123) + +Frozen exact harness result for an author shot: full streams, process state, + token usage, and execution-policy evidence. + The `onShotCompleted` callback receives `null` when execution failed before + the harness returned. + +*** + +### AgenticGeneratorShotDisposition + +> **AgenticGeneratorShotDisposition** = \{ `kind`: `"clean"`; `worktreePath`: `string`; \} \| \{ `kind`: `"rejected"`; `worktreePath`: `string`; `stage`: `"raw-trace-evidence"` \| `"verification"`; `feedback`: `string` \| `null`; \} \| \{ `kind`: `"accepted"`; `worktreePath`: `string`; `verified`: `boolean`; \} \| \{ `kind`: `"setup-error"`; `worktreePath`: `string`; `stage`: `"worktree-inspection"` \| `"raw-trace-evidence"` \| `"verification"`; `error`: \{ `name`: `string`; `message`: `string`; \}; \} + +Defined in: [improvement/agentic-generator.ts:136](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L136) + +Worktree decision emitted before a completed shot is retried, accepted, or + discarded. The callback runs while `worktreePath` is still available, so + callers can persist the exact diff. + +*** + ### ImproveSurface > **ImproveSurface** = `"prompt"` \| `"skills"` \| `"tools"` \| `"mcp"` \| `"hooks"` \| `"subagents"` \| `"agent-profile"` \| `"memory"` \| `"code"` @@ -13138,7 +13192,7 @@ Wire integration: > **agenticGenerator**(`opts?`): [`CandidateGenerator`](#candidategenerator) -Defined in: [improvement/agentic-generator.ts:157](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L157) +Defined in: [improvement/agentic-generator.ts:208](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L208) Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a real coding harness inside the candidate worktree so the agent makes the change in place. @@ -13158,7 +13212,7 @@ Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a rea > **commandVerifier**(`command`, `args?`, `timeoutMs?`): [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:693](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L693) +Defined in: [improvement/agentic-generator.ts:860](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L860) A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other exit ⇒ failed with stdout+stderr as feedback. The common case — verify by diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index beb67a33..d7fbf389 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.94.10` and `@tangle-network/agent-eval@0.117.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.94.11` and `@tangle-network/agent-eval@0.117.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -15,7 +15,7 @@ Every subpath this package declares in `package.json` `exports`. Reach for these ### Root — task lifecycle, conversation, RSI verbs, observability -Import from `@tangle-network/agent-runtime` — 339 exports. +Import from `@tangle-network/agent-runtime` — 341 exports. | Symbol | Kind | Summary | |---|---|---| @@ -199,6 +199,8 @@ Import from `@tangle-network/agent-runtime` — 339 exports. | `AgentCandidateModelLimits` | type | Limits mechanically enforced by the evaluator-owned model gateway. | | `AgentCandidateProfileSource` | type | A complete profile that can be frozen without losing behavior. | | `AgentEvalErrorCode` | type | Error taxonomy for `@tangle-network/agent-eval`. | +| `AgenticGeneratorShotDisposition` | type | Worktree decision emitted before a completed shot is retried, accepted, or | +| `AgenticGeneratorShotExecution` | type | Frozen exact harness result for an author shot: full streams, process state, | | `ImproveSurface` | type | The executable agent lever `improve` optimizes. Profile fields remain | | `OpenAIChatResponseFormat` | type | `response_format` parameter for OpenAI-compatible chat endpoints. Use | | `OpenAIChatToolChoice` | type | `tool_choice` parameter for OpenAI-compat chat. Same shape as the OpenAI | diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 67c0d5a8..9e055b63 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.94.10.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.117.1 <0.118.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <0.26.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.94.11.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.117.1 <0.118.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <0.26.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > diff --git a/package.json b/package.json index 4e214c6a..983014e4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.94.10", + "version": "0.94.11", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index f8e4e391..6d63722c 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -116,6 +116,46 @@ export interface AgenticGeneratorShotReceipt { readonly error: { readonly name: string; readonly message: string } | null } +/** Frozen exact harness result for an author shot: full streams, process state, + * token usage, and execution-policy evidence. + * The `onShotCompleted` callback receives `null` when execution failed before + * the harness returned. */ +export type AgenticGeneratorShotExecution = Readonly< + Omit & { + readonly usage?: Readonly + readonly evidence?: Readonly> & { + readonly readDeniedPaths: ReadonlyArray + readonly policy: Readonly + } + } +> + +/** Worktree decision emitted before a completed shot is retried, accepted, or + * discarded. The callback runs while `worktreePath` is still available, so + * callers can persist the exact diff. */ +export type AgenticGeneratorShotDisposition = + | { + readonly kind: 'clean' + readonly worktreePath: string + } + | { + readonly kind: 'rejected' + readonly worktreePath: string + readonly stage: 'raw-trace-evidence' | 'verification' + readonly feedback: string | null + } + | { + readonly kind: 'accepted' + readonly worktreePath: string + readonly verified: boolean + } + | { + readonly kind: 'setup-error' + readonly worktreePath: string + readonly stage: 'worktree-inspection' | 'raw-trace-evidence' | 'verification' + readonly error: { readonly name: string; readonly message: string } + } + export interface AgenticGeneratorOptions { /** Local coding harness to run in the worktree. Default `claude`. */ harness?: LocalHarness @@ -129,8 +169,19 @@ export interface AgenticGeneratorOptions { * candidate-specific paths after the driver creates its worktree. */ codexReadDeniedPaths?: ReadonlyArray | ((worktreePath: string) => ReadonlyArray) /** Awaited once for every attempted author shot, including process failures. - * Throwing aborts the candidate so receipt persistence can fail closed. */ - onShotCompleted?: (receipt: AgenticGeneratorShotReceipt) => void | Promise + * The second argument preserves the exact harness result, including stdout + * and stderr, before worktree inspection or verification can reject the + * shot. Throwing aborts the candidate so evidence persistence fails closed. */ + onShotCompleted?: ( + receipt: AgenticGeneratorShotReceipt, + execution: AgenticGeneratorShotExecution | null, + ) => void | Promise + /** Awaited after worktree inspection and before the shot is accepted, + * retried, or discarded. Throwing aborts the candidate. */ + onShotDisposition?: ( + receipt: AgenticGeneratorShotReceipt, + disposition: AgenticGeneratorShotDisposition, + ) => void | Promise /** Optional hard upper bound passed to the run-wide CostLedger before each * author shot. This MUST be enforced by the provider or executor; a planning * estimate is not an admissible bound. Omit for an uncapped ledger. A capped @@ -296,40 +347,72 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG } catch (cause) { shotError = cause instanceof Error ? cause : new Error(String(cause)) } - await emitShotReceipt( - opts.onShotCompleted, - shotReceipt({ - generation, - candidateIndex, - shot, - maxShots: shots, - harness, - profile: opts.profile, - prompt: exactPrompt, - startedAt, - completedAt: new Date(), - result: harnessResult, - profileWorkspaceReceipt, - costCallId, - costReceipt, - error: shotError, - }), - shotError, - ) - - if (!harnessResult) { + const execution = shotExecutionSnapshot(harnessResult) + const receipt = shotReceipt({ + generation, + candidateIndex, + shot, + maxShots: shots, + harness, + profile: opts.profile, + prompt: exactPrompt, + startedAt, + completedAt: new Date(), + result: execution, + profileWorkspaceReceipt, + costCallId, + costReceipt, + error: shotError, + }) + await emitShotReceipt(opts.onShotCompleted, receipt, execution, shotError) + + if (!execution) { throw new Error('agenticGenerator: author shot completed without a harness result') } + let worktreeChanged: boolean + try { + worktreeChanged = dirty(worktreePath) + } catch (cause) { + return rethrowShotSetupError( + opts.onShotDisposition, + receipt, + worktreePath, + 'worktree-inspection', + cause, + ) + } + // The worktree IS the signal: no edits ⇒ tell the next shot to act. - if (!dirty(worktreePath)) { + if (!worktreeChanged) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'clean', + worktreePath, + }) attemptNote = EMPTY_TREE_NOTE continue } if (needsRawTraceEvidence) { - const problem = rawTraceEvidenceProblem(worktreePath, findings) + let problem: string | null + try { + problem = rawTraceEvidenceProblem(worktreePath, findings) + } catch (cause) { + return rethrowShotSetupError( + opts.onShotDisposition, + receipt, + worktreePath, + 'raw-trace-evidence', + cause, + ) + } if (problem) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'rejected', + worktreePath, + stage: 'raw-trace-evidence', + feedback: problem, + }) attemptNote = problem continue } @@ -338,12 +421,39 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG // Dirty: with no verifier the diff IS the candidate (we trust the diff, // not the harness's stdout). With a verifier the candidate must pass it. if (!verify) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'accepted', + worktreePath, + verified: false, + }) return { applied: true, summary: summarize(findings) } } - const result = await verify(worktreePath) + let result: VerifyResult + try { + result = await verify(worktreePath) + } catch (cause) { + return rethrowShotSetupError( + opts.onShotDisposition, + receipt, + worktreePath, + 'verification', + cause, + ) + } if (result.ok) { + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'accepted', + worktreePath, + verified: true, + }) return { applied: true, summary: summarize(findings) } } + await emitShotDisposition(opts.onShotDisposition, receipt, { + kind: 'rejected', + worktreePath, + stage: 'verification', + feedback: result.feedback ?? null, + }) // Dirty but failing — resume next shot atop these edits with the error. attemptNote = failureNote(result.feedback) } @@ -463,10 +573,11 @@ async function withAuthorProfileResources( async function emitShotReceipt( callback: AgenticGeneratorOptions['onShotCompleted'], receipt: AgenticGeneratorShotReceipt, + execution: AgenticGeneratorShotExecution | null, primaryError: Error | null, ): Promise { try { - await callback?.(receipt) + await callback?.(receipt, execution) } catch (callbackError) { if (primaryError !== null) { throw new AggregateError( @@ -479,6 +590,62 @@ async function emitShotReceipt( if (primaryError !== null) throw primaryError } +async function emitShotDisposition( + callback: AgenticGeneratorOptions['onShotDisposition'], + receipt: AgenticGeneratorShotReceipt, + disposition: AgenticGeneratorShotDisposition, +): Promise { + await callback?.(receipt, disposition) +} + +async function rethrowShotSetupError( + callback: AgenticGeneratorOptions['onShotDisposition'], + receipt: AgenticGeneratorShotReceipt, + worktreePath: string, + stage: Extract['stage'], + cause: unknown, +): Promise { + const error = cause instanceof Error ? cause : new Error(String(cause)) + try { + await emitShotDisposition(callback, receipt, { + kind: 'setup-error', + worktreePath, + stage, + error: { name: error.name, message: error.message }, + }) + } catch (callbackError) { + throw new AggregateError( + [cause, callbackError], + 'agenticGenerator: shot processing failed and its worktree disposition could not be persisted', + ) + } + throw cause +} + +function shotExecutionSnapshot( + result: LocalHarnessResult | null, +): AgenticGeneratorShotExecution | null { + if (result === null) return null + const usage = result.usage ? Object.freeze({ ...result.usage }) : undefined + const evidence = result.evidence + ? Object.freeze({ + ...result.evidence, + readDeniedPaths: Object.freeze([...result.evidence.readDeniedPaths]), + policy: Object.freeze({ ...result.evidence.policy }), + }) + : undefined + return Object.freeze({ + exitCode: result.exitCode, + stdout: result.stdout, + stderr: result.stderr, + killedBySignal: result.killedBySignal, + durationMs: result.durationMs, + timedOut: result.timedOut, + ...(usage ? { usage } : {}), + ...(evidence ? { evidence } : {}), + }) +} + function shotReceipt(input: { readonly generation: number | undefined readonly candidateIndex: number | undefined @@ -489,7 +656,7 @@ function shotReceipt(input: { readonly prompt: string readonly startedAt: Date readonly completedAt: Date - readonly result: LocalHarnessResult | null + readonly result: AgenticGeneratorShotExecution | null readonly profileWorkspaceReceipt: WorkspacePlanReceipt | null readonly costCallId: string | null readonly costReceipt: CostReceipt | null diff --git a/src/improvement/index.ts b/src/improvement/index.ts index 62f5570e..0b916825 100644 --- a/src/improvement/index.ts +++ b/src/improvement/index.ts @@ -13,6 +13,8 @@ export { AGENTIC_PROFILE_RESOURCE_ROOT, type AgenticGeneratorOptions, + type AgenticGeneratorShotDisposition, + type AgenticGeneratorShotExecution, type AgenticGeneratorShotReceipt, agenticGenerator, commandVerifier, diff --git a/tests/agentic-generator.test.ts b/tests/agentic-generator.test.ts index b6253244..6c85d9e3 100644 --- a/tests/agentic-generator.test.ts +++ b/tests/agentic-generator.test.ts @@ -8,6 +8,8 @@ import { gitWorktreeAdapter, type ProposeContext } from '@tangle-network/agent-e import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AGENTIC_PROFILE_RESOURCE_ROOT, + type AgenticGeneratorShotDisposition, + type AgenticGeneratorShotExecution, type AgenticGeneratorShotReceipt, agenticGenerator, commandVerifier, @@ -236,6 +238,211 @@ describe('agenticGenerator — runs a harness in the worktree', () => { ]) }) + it('exposes exact execution before a clean-tree shot is rejected', async () => { + const events: string[] = [] + const executions: AgenticGeneratorShotExecution[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] + const exactExecution: LocalHarnessResult = { + exitCode: 0, + stdout: 'exact stdout\nwith a second line\n', + stderr: 'exact stderr\n', + killedBySignal: null, + durationMs: 4321, + timedOut: false, + usage: CODEX_USAGE, + evidence: CODEX_EVIDENCE, + } + const isDirty = vi.fn(() => { + events.push('dirty-check') + expect(events).toEqual(['callback-complete', 'dirty-check']) + return false + }) + const gen = agenticGenerator({ + runHarness: (async () => exactExecution) as never, + isDirty, + onShotCompleted: async (receipt, execution) => { + await Promise.resolve() + expect(execution).not.toBeNull() + if (!execution) throw new Error('expected a completed execution') + executions.push(execution) + expect(execution).toEqual(exactExecution) + expect(Object.isFrozen(execution)).toBe(true) + expect(Object.isFrozen(execution.usage)).toBe(true) + expect(Object.isFrozen(execution.evidence)).toBe(true) + expect(Object.isFrozen(execution.evidence?.readDeniedPaths)).toBe(true) + expect(Object.isFrozen(execution.evidence?.policy)).toBe(true) + expect(() => Object.assign(execution, { stdout: 'mutated' })).toThrow() + expect(receipt.stdoutBytes).toBe(Buffer.byteLength(execution.stdout)) + expect(receipt.stderrBytes).toBe(Buffer.byteLength(execution.stderr)) + expect(receipt.stdoutSha256).toBe( + `sha256:${createHash('sha256').update(execution.stdout).digest('hex')}`, + ) + expect(receipt.stderrSha256).toBe( + `sha256:${createHash('sha256').update(execution.stderr).digest('hex')}`, + ) + events.push('callback-complete') + }, + onShotDisposition: async (_receipt, disposition) => { + await Promise.resolve() + expect(disposition.kind).toBe('clean') + expect(disposition.worktreePath).toContain('exact-clean-shot') + dispositions.push(disposition) + events.push('disposition-complete') + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'exact-clean-shot', + }) + + const out = await gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }) + + expect(out.applied).toBe(false) + expect(executions).toEqual([exactExecution]) + expect(dispositions).toEqual([ + expect.objectContaining({ kind: 'clean', worktreePath: wt.path }), + ]) + expect(events).toEqual(['callback-complete', 'dirty-check', 'disposition-complete']) + expect(isDirty).toHaveBeenCalledTimes(1) + }) + + it('awaits shot evidence before verification can return a candidate', async () => { + let evidencePersisted = false + let dispositionPersisted = false + const runHarness = vi.fn(async ({ cwd }: { cwd: string }) => { + writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') + return HARNESS_OK + }) + const verify = vi.fn(() => { + expect(evidencePersisted).toBe(true) + return { ok: true } + }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotCompleted: async () => { + await Promise.resolve() + evidencePersisted = true + }, + onShotDisposition: async (_receipt, disposition) => { + await Promise.resolve() + expect(disposition).toEqual({ + kind: 'accepted', + worktreePath: expect.stringContaining('evidence-before-verify'), + verified: true, + }) + dispositionPersisted = true + }, + verify, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'evidence-before-verify', + }) + + const out = await gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }) + + expect(out.applied).toBe(true) + expect(dispositionPersisted).toBe(true) + expect(verify).toHaveBeenCalledTimes(1) + }) + + it('fails closed when shot evidence persistence throws', async () => { + const isDirty = vi.fn(() => false) + const gen = agenticGenerator({ + runHarness: (async () => HARNESS_OK) as never, + isDirty, + onShotCompleted: () => { + throw new Error('shot evidence persistence failed') + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'evidence-failure', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('shot evidence persistence failed') + expect(isDirty).not.toHaveBeenCalled() + }) + + it('fails closed when worktree disposition persistence throws', async () => { + const gen = agenticGenerator({ + runHarness: (async () => HARNESS_OK) as never, + isDirty: () => false, + onShotDisposition: () => { + throw new Error('shot disposition persistence failed') + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'disposition-failure', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('shot disposition persistence failed') + }) + + it('persists a setup-error disposition before rethrowing inspection failure', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] + const gen = agenticGenerator({ + runHarness: (async () => HARNESS_OK) as never, + isDirty: () => { + throw new Error('git status unavailable') + }, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) + const wt = await gitWorktreeAdapter({ repoRoot }).create({ + baseRef: 'main', + label: 'inspection-error', + }) + + await expect( + gen.generate({ + worktreePath: wt.path, + report: undefined, + findings: FINDINGS, + maxShots: 1, + signal: new AbortController().signal, + }), + ).rejects.toThrow('git status unavailable') + expect(dispositions).toEqual([ + { + kind: 'setup-error', + worktreePath: wt.path, + stage: 'worktree-inspection', + error: { name: 'Error', message: 'git status unavailable' }, + }, + ]) + }) + it('rejects profile files outside the dedicated ephemeral root before dispatch', () => { expect(() => agenticGenerator({ @@ -259,11 +466,15 @@ describe('agenticGenerator — runs a harness in the worktree', () => { it('emits a failed shot receipt before rethrowing a harness failure', async () => { const receipts: unknown[] = [] + const executions: Array = [] const gen = agenticGenerator({ runHarness: (async () => { throw new Error('author process failed') }) as never, - onShotCompleted: (receipt) => receipts.push(receipt), + onShotCompleted: (receipt, execution) => { + receipts.push(receipt) + executions.push(execution) + }, }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'failed' }) @@ -277,6 +488,7 @@ describe('agenticGenerator — runs a harness in the worktree', () => { }), ).rejects.toThrow('author process failed') expect(receipts).toHaveLength(1) + expect(executions).toEqual([null]) expect(receipts[0]).toMatchObject({ usage: null, evidence: null, @@ -486,6 +698,7 @@ describe('agenticGenerator — runs a harness in the worktree', () => { }) it('returns applied when the harness changes the worktree', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] // The harness "edits" by writing into its cwd (the worktree). We stub the // subprocess (the only process boundary) but use a REAL git dirty check. const runHarness = vi.fn( @@ -505,7 +718,12 @@ describe('agenticGenerator — runs a harness in the worktree', () => { return HARNESS_OK }, ) - const gen = agenticGenerator({ runHarness: runHarness as never }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'cand' }) const out = await gen.generate({ @@ -519,11 +737,18 @@ describe('agenticGenerator — runs a harness in the worktree', () => { expect(runHarness).toHaveBeenCalledTimes(1) expect(out.applied).toBe(true) expect(out.summary).toContain('x should be 2') + expect(dispositions).toEqual([{ kind: 'accepted', worktreePath: wt.path, verified: false }]) }) it('retries up to maxShots when the harness produces no change, then gives up', async () => { + const dispositions: AgenticGeneratorShotDisposition[] = [] const runHarness = vi.fn(async () => HARNESS_OK) // never edits the worktree - const gen = agenticGenerator({ runHarness: runHarness as never }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'noop' }) const out = await gen.generate({ @@ -536,6 +761,11 @@ describe('agenticGenerator — runs a harness in the worktree', () => { expect(runHarness).toHaveBeenCalledTimes(3) expect(out.applied).toBe(false) + expect(dispositions).toEqual([ + { kind: 'clean', worktreePath: wt.path }, + { kind: 'clean', worktreePath: wt.path }, + { kind: 'clean', worktreePath: wt.path }, + ]) }) it('stops retrying as soon as a shot produces a change', async () => { @@ -625,6 +855,7 @@ describe('agenticGenerator — verify-in-session loop', () => { it('feeds the verifier failure into the next shot, then ships when it passes', async () => { let shot = 0 const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { prompts.push(taskPrompt) shot++ @@ -635,7 +866,13 @@ describe('agenticGenerator — verify-in-session loop', () => { const verify = vi.fn(() => shot === 1 ? { ok: false, feedback: 'TS2322: x must be 2' } : { ok: true }, ) - const gen = agenticGenerator({ runHarness: runHarness as never, verify }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + verify, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'vresume' }) const out = await gen.generate({ @@ -653,6 +890,15 @@ describe('agenticGenerator — verify-in-session loop', () => { expect(prompts[1]).toContain('TS2322: x must be 2') // The first shot's prompt is the clean base — no failure note yet. expect(prompts[0]).not.toContain('verification FAILED') + expect(dispositions).toEqual([ + { + kind: 'rejected', + worktreePath: wt.path, + stage: 'verification', + feedback: 'TS2322: x must be 2', + }, + { kind: 'accepted', worktreePath: wt.path, verified: true }, + ]) }) it('discards (applied:false) a candidate that never verifies within maxShots', async () => { @@ -703,12 +949,18 @@ describe('agenticGenerator — raw-trace evidence discipline', () => { it('retries and discards a raw-trace candidate that edits code without citing inspected traces', async () => { const prompts: string[] = [] + const dispositions: AgenticGeneratorShotDisposition[] = [] const runHarness = vi.fn(async ({ cwd, taskPrompt }: { cwd: string; taskPrompt: string }) => { prompts.push(taskPrompt) writeFileSync(join(cwd, 'app.ts'), 'export const x = 2\n') return HARNESS_OK }) - const gen = agenticGenerator({ runHarness: runHarness as never }) + const gen = agenticGenerator({ + runHarness: runHarness as never, + onShotDisposition: (_receipt, disposition) => { + dispositions.push(disposition) + }, + }) const wt = await gitWorktreeAdapter({ repoRoot }).create({ baseRef: 'main', label: 'rt-miss' }) const out = await gen.generate({ @@ -724,6 +976,21 @@ describe('agenticGenerator — raw-trace evidence discipline', () => { expect(prompts[0]).toContain('Raw trace evidence requirement') expect(prompts[0]).toContain('.improve/raw-trace-diagnosis.md') expect(prompts[1]).toContain('raw-trace mode requires .improve/raw-trace-diagnosis.md') + expect(dispositions).toHaveLength(2) + expect(dispositions).toEqual([ + expect.objectContaining({ + kind: 'rejected', + worktreePath: wt.path, + stage: 'raw-trace-evidence', + feedback: expect.stringContaining('requires .improve/raw-trace-diagnosis.md'), + }), + expect.objectContaining({ + kind: 'rejected', + worktreePath: wt.path, + stage: 'raw-trace-evidence', + feedback: expect.stringContaining('requires .improve/raw-trace-diagnosis.md'), + }), + ]) }) it('rejects a raw-trace candidate that only writes the diagnosis artifact', async () => {