From 9b57af6bffcab5dc04550386c7f11a91b3c3f547 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 15 Jul 2026 10:21:18 -0600 Subject: [PATCH] fix(improvement): honor configured dispatch deadlines --- docs/api/index.md | 4 +- docs/api/primitive-catalog.md | 24 ++++++++---- docs/canonical-api.md | 2 +- package.json | 6 +-- pnpm-lock.yaml | 54 +++++++++++++++++++++++---- src/improvement/agentic-generator.ts | 19 ++++++---- src/improvement/improve.test.ts | 1 - src/improvement/improvement-driver.ts | 4 +- tests/improve.test.ts | 35 +++++++++++++++++ 9 files changed, 118 insertions(+), 31 deletions(-) diff --git a/docs/api/index.md b/docs/api/index.md index 6129081b..2b9eeee3 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -6419,7 +6419,7 @@ Improvement-loop coordinates. Present when called through improvementDriver. ###### costLedger? -`CostLedger` +`CostLedgerHandle` Shared run-wide paid-call account supplied by agent-eval 0.117+. @@ -13212,7 +13212,7 @@ Full-agentic `CandidateGenerator` (the `shots=N, sandbox=on` setting): run a rea > **commandVerifier**(`command`, `args?`, `timeoutMs?`): [`Verifier`](#verifier) -Defined in: [improvement/agentic-generator.ts:860](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L860) +Defined in: [improvement/agentic-generator.ts:863](https://github.com/tangle-network/agent-runtime/blob/main/src/improvement/agentic-generator.ts#L863) A `Verifier` that runs a command in the worktree: exit 0 ⇒ ok, any other exit ⇒ failed with stdout+stderr as feedback. The common case — verify by diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index ddeac247..c6cde362 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.94.11` and `@tangle-network/agent-eval@0.117.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.94.12` and `@tangle-network/agent-eval@0.120.1` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -1228,7 +1228,7 @@ Import from `@tangle-network/agent-eval` — 51 exports. ### CAMPAIGN — profile matrix, gates, improvement loop -Import from `@tangle-network/agent-eval/campaign` — 379 exports. +Import from `@tangle-network/agent-eval/campaign` — 391 exports. | Symbol | Kind | Summary | |---|---|---| @@ -1236,7 +1236,9 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `acquireSingleRunLock` | function | Acquire the lock or throw naming the live holder. A stale lock (holder pid | | `analyzeCrossSurfaceInteractions` | function | Build the complete cross-surface evidence matrix and derive all three frozen | | `applySkillPatch` | function | Apply a SkillOpt patch to a text surface. Ops apply in array order against | -| `assertCodeSurfaceIdentity` | function | Fail when a code surface does not carry a complete immutable identity. | +| `assertCampaignDesign` | function | Reject campaign designs whose denominator cannot be identified exactly. | +| `assertCampaignSplitIdentity` | function | Refuse a campaign whose retained task identities contradict its split digest. | +| `assertCodeSurfaceIdentity` | function | Validate the immutable identity shape; the owning executor verifies the Git objects and patch. | | `assertPolicyEditAuthorContextBudget` | function | Serialize once and fail before dispatch when author context exceeds its budget. | | `buildAnalystSurfaceDispatch` | function | Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop | | `buildEvidenceVector` | function | The Evidence Bus. For each objective, pair candidate vs baseline by full | @@ -1244,6 +1246,10 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `callbackGovernor` | function | The LLM-supervisor slot: a governor whose `decide` defers to a caller-supplied | | `campaignBreakdown` | function | Per-candidate evidence a reflective/patch proposer grounds its next proposal | | `campaignMeanComposite` | function | Mean composite across a campaign: per cell, the mean of its finite, | +| `campaignMeasurementDigest` | function | Digest the exact campaign fields that can affect a measured comparison. | +| `campaignScenarioIdentity` | function | Redacted but independently verifiable identity of one complete scenario. | +| `campaignSplitDigest` | function | Canonical identity of the exact scenario payloads and replicate count. | +| `campaignSplitDigestFromIdentities` | function | Canonical split identity reconstructed from redacted scenario identities. | | `classifyUngroundedLiterals` | function | Scan revised artifact text for single-quoted single-word literals (the | | `codeSurfaceIdentityMaterial` | function | Canonical, location-independent identity of a finalized code candidate. | | `compareProposers` | function | Run a head-to-head lift benchmark across surface proposers on a shared holdout, returning per-proposer lift CIs and pairwise "who wins" verdicts. | @@ -1253,7 +1259,6 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `createReferenceEquivalenceJudge` | function | Build the campaign-native expected-answer judge. | | `createRunCostLedger` | function | Open the durable spend account stored beside a logical run. | | `defaultProductionGate` | function | Opinionated production gate composing held-out significance, red-team, reward-hacking, and canary checks into a single `Gate.decide` decision. | -| `defaultRenderDiff` | function | Default surface diff renderer: produces a unified baseline/winner text diff for prompt surfaces or a worktree-ref summary for code surfaces. | | `detectScale` | function | Detect the native scale of a set of scores: 0-100 when any magnitude clears | | `dimensionRegressions` | function | Per-critical-dimension regression guard. For each dimension, pair the | | `discoverEvalFixtures` | function | Walk `evalsDir` and return the relative name of every fixture directory (one containing an exact-case `PROMPT.md`). | @@ -1283,6 +1288,7 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `llmPolicyEditProposer` | function | LLM-backed PolicyEdit author. It reads only the current JSON surface, | | `loadEvalFixture` | function | Load ONE fixture by name: reads `PROMPT.md` (plus `EVAL.ts`/`EVAL.tsx` and `package.json` under | | `loadEvalFixtureScenarios` | function | Load fixtures (all discovered, or just `names`) as campaign `Scenario`s tagged `eval-fixture`. | +| `loopProvenanceArgsFromResult` | function | One translation from a completed improvement loop into durable evidence. | | `loopProvenanceSpans` | function | Build the loop's OTLP-ingestable spans from a provenance record. One root | | `makePlaybackDispatch` | function | Adapt a `PlaybackDriver` into a `runProfileMatrix` dispatch. The artifact the | | `memLineageStore` | function | In-memory store (default; for tests and ephemeral runs). | @@ -1304,6 +1310,7 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `provenanceRecordPath` | function | Canonical durable paths under the run dir. | | `provenanceSpansPath` | function | Canonical path for the durable OTLP spans JSONL file under a loop run directory. | | `renderScoreboardMarkdown` | function | Render the scoreboard as a launch-readiness Markdown document — the literal | +| `renderSurfaceDiff` | function | Canonical customer-visible description of the exact before/after surfaces. | | `resolveRunDir` | function | Resolve a campaign `runDir`. An absolute path is honored as-is (the caller | | `resolveWorktreePath` | function | Resolve a code candidate for evaluation only after verifying its immutable | | `rolloutArgumentDiff` | function | Deterministic per-field diff of call arguments between passing and failing | @@ -1332,6 +1339,7 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `validatePolicyEditCandidateRecord` | function | _(no summary — add a TSDoc line at the declaration)_ | | `validateSearchLedgerEvent` | function | Validate and return a canonical copy. Arrays whose order is not semantic are | | `verifyCodeSurface` | function | Verify a finalized code surface against its current checkout. This rejects | +| `verifyLoopProvenanceRecord` | function | Recompute and validate the self-addressed durable record. | | `DEFAULT_POLICY_EDIT_HISTORY_LIMITS` | const | _(no summary — add a TSDoc line at the declaration)_ | | `paretoPolicy` | const | The default strategy: symmetric multi-objective Pareto significance. Ship iff | | `POLICY_EDIT_CANDIDATE_RECORD_SCHEMA` | const | _(no summary — add a TSDoc line at the declaration)_ | @@ -1351,6 +1359,7 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `AnalystScenario` | interface | A labeled trace scenario: a FIXED trace corpus plus the failure modes a | | `CampaignArtifactWriter` | interface | Scoped artifact writer — `write(path, content)` lands under | | `CampaignCostMeter` | interface | Cell-scoped paid-call entry point. The dispatch places every paid operation | +| `CampaignScenarioIdentity` | interface | Redacted identity of a complete scenario payload retained in campaign results. | | `CampaignStorage` | interface | `CampaignStorage` — the filesystem seam `runCampaign` writes through | | `CampaignTraceWriter` | interface | Scoped trace writer handed to each dispatch — every span | | `CodeSurface` | interface | A tier-4 code surface — a finalized candidate change to the agent's | @@ -1420,6 +1429,7 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `UserStoryVerdict` | interface | A scored user story — the completion verdict plus its human title. | | `AxisVerdict` | type | Per-axis verdict from the good-direction paired bootstrap. | | `CampaignTokenUsage` | type | Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage` | +| `CostLedgerHandle` | type | Public callback surface for a shared cost ledger. | | `CrossSurfaceAttemptCompleteness` | type | Whether one candidate attempt produced a usable executable outcome. | | `DispatchFn` | type | One function: scenario + ctx → artifact. Dispatcher chooses | | `FapoOptimizationLevel` | type | FAPO (Fully Autonomous Prompt Optimization) is an orchestration policy, not | @@ -1437,7 +1447,7 @@ Import from `@tangle-network/agent-eval/campaign` — 379 exports. | `SequentialDecision` | type | Anytime-valid sequential promotion gate — an e-process (betting | | `SkillPatchOp` | type | A single bounded edit against a skill surface. | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `AnalyzeCrossSurfaceInteractionsInput`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareProposersOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `Governor`, `GovernorContext`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `HeuristicGovernorOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LineageEdge`, `LineageGraph`, `LineageStore`, `LlmJudgeOptions`, `LlmPolicyEditProposerOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceBackend`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PolicyEditCandidateSummary`, `PolicyEditHistoryCandidateContext`, `PolicyEditHistoryGenerationContext`, `PolicyEditHistoryProjectionOptions`, `PolicyEditObjective`, `PolicyEditOutcomeContext`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunImprovementLoopResult`, `RunLineageLoopOptions`, `RunLineageLoopResult`, `RunLineageOptions`, `RunLineageResult`, `RunLineageSeed`, `RunLineageStepResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SelectPolicyEditAuthorRowsOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SerializedJsonBudget`, `SingleRunLock`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `GovernorOp`, `JsonPolicyEditTargetSurface`, `JsonPrimitive`, `JsonValue`, `PolicyEditFindingSource`, `RedactionStatus`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcceptedEdit`, `AnalyzeCrossSurfaceInteractionsInput`, `ApplySkillPatchResult`, `AxisEvidence`, `BuildAnalystSurfaceDispatchOptions`, `BuildEvidenceVectorOptions`, `BuildLoopProvenanceArgs`, `CampaignAggregates`, `CampaignBreakdown`, `CampaignCellResult`, `CampaignResult`, `CampaignRunPlan`, `CampaignRunPlanCell`, `CodeSurfaceVerification`, `CompareProposersOptions`, `CrossSurfaceAdditionDecision`, `CrossSurfaceBestSingleSelection`, `CrossSurfaceBootstrapPolicy`, `CrossSurfaceCandidateComparison`, `CrossSurfaceCandidateEvidence`, `CrossSurfaceCandidateOutcome`, `CrossSurfaceCandidateSummary`, `CrossSurfaceCompositionStep`, `CrossSurfaceDistribution`, `CrossSurfaceEligibility`, `CrossSurfaceEvidenceBreakdown`, `CrossSurfaceInteractionAwareSelection`, `CrossSurfaceInteractionEffect`, `CrossSurfaceInteractionReport`, `CrossSurfaceInteractionTask`, `CrossSurfaceNaiveStackSelection`, `CrossSurfacePairCompatibility`, `CrossSurfacePairEvidence`, `CrossSurfacePairwiseEntry`, `CrossSurfaceRankedSingle`, `CrossSurfaceRelativeCost`, `CrossSurfaceSelections`, `DimensionRegression`, `DiscriminationScore`, `EmitLoopProvenanceArgs`, `EmitLoopProvenanceResult`, `EvalFixture`, `EvalFixtureFile`, `EvalFixtureLoadOptions`, `EvalFixtureScenario`, `EvidenceVector`, `FailureModeRecallJudgeOptions`, `FapoAttributionSignals`, `FapoFailureCluster`, `FapoProposerOptions`, `FapoReviewInput`, `FapoReviewIssue`, `FapoReviewResult`, `FapoScopeContract`, `GateContext`, `GateResult`, `GenerationRecord`, `GepaProposerOptions`, `GitWorktreeAdapterOptions`, `Governor`, `GovernorContext`, `HeldOutGateOptions`, `HeldoutSignificance`, `HeldoutSignificanceOptions`, `HeuristicGovernorOptions`, `JudgeAggregate`, `JudgeDimension`, `LabeledScenarioRecord`, `LabeledScenarioSampleArgs`, `LabeledScenarioStore`, `LineageEdge`, `LineageGraph`, `LineageStore`, `LlmJudgeOptions`, `LlmPolicyEditProposerOptions`, `LoadEvalFixtureScenariosOptions`, `LoopProvenanceArgsFromResult`, `LoopProvenanceBackend`, `LoopProvenanceEvidence`, `NeutralizationGateOptions`, `OpenAutoPrResult`, `OpenSearchLedgerOptions`, `OptimizerConfig`, `ParameterCandidate`, `ParameterChange`, `ParameterSweepProposerOptions`, `ParetoSignificanceGateOptions`, `PlanCampaignRunOptions`, `PlanEvalFixtureRunOptions`, `PolicyEditCandidateSummary`, `PolicyEditHistoryCandidateContext`, `PolicyEditHistoryGenerationContext`, `PolicyEditHistoryProjectionOptions`, `PolicyEditObjective`, `PolicyEditOutcomeContext`, `PowerPreflight`, `ProfileSummary`, `PromotionObjective`, `ProposePatchesArgs`, `ProposerComparison`, `ProposerPairwise`, `ProposerScore`, `ReferenceEquivalenceJudgeOptions`, `ReferenceEquivalenceScenario`, `RolloutArgumentDiff`, `RolloutArgumentDiffOptions`, `RunImprovementLoopResult`, `RunLineageLoopOptions`, `RunLineageLoopResult`, `RunLineageOptions`, `RunLineageResult`, `RunLineageSeed`, `RunLineageStepResult`, `RunOptimizationResult`, `RunProfileMatrixOptions`, `RunProfileMatrixResult`, `RunSkillOptResult`, `ScenarioAggregate`, `ScenarioRollup`, `ScoreboardRenderOptions`, `SearchAttemptAccounting`, `SearchCandidateDecidedEvent`, `SearchCandidateLineage`, `SearchCandidateRegisteredEvent`, `SearchCandidateSlot`, `SearchCandidateSlotClosedEvent`, `SearchCandidateSurface`, `SearchCompletedEvent`, `SearchFailureReason`, `SearchLedger`, `SearchLedgerAppendResult`, `SearchLedgerEntry`, `SearchLedgerReplay`, `SearchModelIdentity`, `SearchOperationRecordedEvent`, `SearchPlan`, `SearchPlannedEvent`, `SearchPlannedOperation`, `SearchPlannedTask`, `SearchTaskAttemptedEvent`, `SelectPolicyEditAuthorRowsOptions`, `SequentialDecideFn`, `SequentialDecideOptions`, `SequentialObservation`, `SequentialPairedGate`, `SequentialPairedGateOptions`, `SerializedJsonBudget`, `SingleRunLock`, `SkillOptEpochRecord`, `SkillOptProposer`, `SkillOptProposerOptions`, `SkillPatchRejection`, `TraceSpan`, `UngroundedLiteralReport`, `Worktree`, `WorktreeAdapter`, `CrossSurfaceAdditionRejectionReason`, `CrossSurfaceIneligibilityReason`, `CrossSurfacePairIncompatibilityReason`, `EvalFixtureRunPlan`, `EvalFixtureValidationMode`, `GovernorOp`, `JsonPolicyEditTargetSurface`, `JsonPrimitive`, `JsonValue`, `PolicyEditFindingSource`, `RedactionStatus`, `RunOptimizationOptions`, `SearchAccountingAudit`, `SearchCostAccounting`, `SearchLedgerEvent`, `SearchLedgerHash`, `SearchOperationKind`, `SearchSurfaceEffect`, `SearchSurfaceKind`, `SearchTaskOutcome`, `SearchTokenAccounting`. ### TOKEN / USAGE — usage extraction + run-record usage types @@ -1445,8 +1455,8 @@ Import from `@tangle-network/agent-eval` — 5 exports. | Symbol | Kind | Summary | |---|---|---| -| `extractUsage` | function | Pull `{ input, output, cached? }` from a parsed chat-completions response | +| `extractUsage` | function | Pull `{ input, output, cached?, cacheWrite? }` from a parsed response | | `extractUsageFromResponse` | function | Extract usage from an HTTP `Response` without consuming the caller's body: | -| `extractUsageFromSse` | function | Sum token usage across an SSE response body. Each `data:` line is parsed and | +| `extractUsageFromSse` | function | Extract token usage from a complete SSE response body using the shared SSE | **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `LlmUsage`, `RunTokenUsage`. diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 9e055b63..d930db64 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -2,7 +2,7 @@ -> **Version 0.94.11.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.117.1 <0.118.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <0.26.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. +> **Version 0.94.12.** The export inventory + per-symbol signatures live in the generated `docs/api/` reference: **`docs/api/primitive-catalog.md`** is the never-stale, grouped list of every primitive to reuse (own surface + the agent-eval judge / authenticity / verification / statistics / campaign / token-usage surfaces), with each one's import path and one-line summary read live from source; the per-module pages hold the full signatures. The pinned substrate is agent-eval `>=0.120.1 <0.121.0`; the sandbox substrate that materializes profiles into harness shapes is `@tangle-network/sandbox` (peer `>=0.8.0 <1.0.0`). The neutral contract types (`AgentProfile`, `AgentProfileMcpServer`, `HarnessType`, `ReasoningEffort`, `Part`/`ToolPart`/`ToolState`, plus environment-provider types) are owned by **`@tangle-network/agent-interface`** (peer `>=0.25.0 <0.26.0`) — the single source of truth. Substrate primitives are re-exported through `@tangle-network/agent-eval/contract` (or `/campaign`), not local to this package — the catalog's §2 shows exactly which subpath each lives under. > > **`./loops` is the runtime barrel** — `package.json` maps it to `src/runtime/index.ts`. Everything below labelled `/loops` is the recursive-atom + loop-kernel surface. > diff --git a/package.json b/package.json index 1c49ccde..a1223031 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.94.11", + "version": "0.94.12", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -116,7 +116,7 @@ }, "devDependencies": { "@biomejs/biome": "^2.4.15", - "@tangle-network/agent-eval": "0.117.1", + "@tangle-network/agent-eval": "0.120.1", "@tangle-network/agent-interface": "^0.25.0", "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.9.3", @@ -147,7 +147,7 @@ "license": "MIT", "packageManager": "pnpm@10.28.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.117.1 <0.118.0", + "@tangle-network/agent-eval": ">=0.120.1 <0.121.0", "@tangle-network/agent-interface": ">=0.25.0 <0.26.0", "@tangle-network/sandbox": ">=0.8.0 <1.0.0", "playwright": "^1.40.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18e1045c..2e91f453 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,8 +22,8 @@ importers: specifier: ^2.4.15 version: 2.4.15 '@tangle-network/agent-eval': - specifier: 0.117.1 - version: 0.117.1(typescript@5.9.3) + specifier: 0.120.1 + version: 0.120.1(typescript@5.9.3) '@tangle-network/agent-interface': specifier: ^0.25.0 version: 0.25.0 @@ -77,6 +77,15 @@ packages: zod: optional: true + '@ax-llm/ax@23.0.1': + resolution: {integrity: sha512-9glLBupY1XthY/WQGaUa3fK5BYYKTqc07Dxf4ZQmt0PDvX+s0XGyiua1jNnJGUlUeROyKN8k5IuVM84VXgS6Hw==} + hasBin: true + peerDependencies: + zod: ^3.24.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + '@biomejs/biome@2.4.15': resolution: {integrity: sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==} engines: {node: '>=14.21.3'} @@ -649,13 +658,16 @@ packages: '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} + '@tangle-network/agent-core@0.4.15': + resolution: {integrity: sha512-sSYFh1N9jqRlDX/s2Dudv70ZK/Pg6naHLTAgzujPOgc7QvWLTwaKUxTcJqlYPA5/1JRH8vSKL+m2gQcy3J751g==} + '@tangle-network/agent-eval@0.115.1': resolution: {integrity: sha512-Yd487u8v0c8/+On9kaYGASS88oLhOJ5p0zwRMGdbWRSWC1UODbIgk1Fqdyq6EI60v+lZ+yDg/7LHkIrNi4kjPA==} engines: {node: '>=20'} hasBin: true - '@tangle-network/agent-eval@0.117.1': - resolution: {integrity: sha512-mtBdGhGIC+nrPZbseLQo0Iako1LR66ZeIr2UjVBJcNj/lHYfib2jbhdWy3RmbQcHR0Cxdqqydn7pLJ5dSXVYtQ==} + '@tangle-network/agent-eval@0.120.1': + resolution: {integrity: sha512-XprtRhV6ncw5TPPJSJhSzfE2CoKALjUmKgSYiGtrl9bwvNN+uAPCraKv0tG9s77Ze3qKSePZaOu47gWFC4Pv3g==} engines: {node: '>=20'} hasBin: true @@ -671,6 +683,12 @@ packages: '@tangle-network/agent-interface@0.25.0': resolution: {integrity: sha512-bMKjyjk8bk2651HleiTEUOTPgNE2bWxCEl9VBVUyub/UkP87Y09byXOxeDEsNGc4TIT19Y5Uo0b+k1kRWXkMVQ==} + '@tangle-network/agent-interface@0.26.1': + resolution: {integrity: sha512-NnQIWtudBHt1P6UXeLMAXUUUxjbYlMLuO3x7yGUGuGWkGcJ0EK0Gpv6aI81g+uqpsV1aHQ4gYfDTCVpdNICUeA==} + + '@tangle-network/agent-interface@0.27.2': + resolution: {integrity: sha512-GzJJ3AO6S22Sf4hd5lGoQK7ft2hb6IvbcIiZ91vUfwlPNWO4oWK7eFVhKYE2YqHi/Tad2X+uE+Mvyhdfx/jH1w==} + '@tangle-network/agent-knowledge@1.12.1': resolution: {integrity: sha512-j5riyIvz5C+ZBELTtNVtmUbcKAMpgTHMNmDQ12MhOIPJv5Y7AeX2oCWuF6c8aPqg1hOrsLqgqXSr1zkV0ePsrA==} engines: {node: '>=20'} @@ -1340,6 +1358,12 @@ snapshots: optionalDependencies: zod: 4.4.3 + '@ax-llm/ax@23.0.1(zod@4.4.3)': + dependencies: + '@opentelemetry/api': 1.9.1 + optionalDependencies: + zod: 4.4.3 + '@biomejs/biome@2.4.15': optionalDependencies: '@biomejs/cli-darwin-arm64': 2.4.15 @@ -1699,6 +1723,11 @@ snapshots: '@tangle-network/agent-interface': 0.17.1 zod: 4.4.3 + '@tangle-network/agent-core@0.4.15': + dependencies: + '@tangle-network/agent-interface': 0.27.2 + zod: 4.4.3 + '@tangle-network/agent-eval@0.115.1(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) @@ -1717,12 +1746,13 @@ snapshots: - typescript - utf-8-validate - '@tangle-network/agent-eval@0.117.1(typescript@5.9.3)': + '@tangle-network/agent-eval@0.120.1(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) - '@ax-llm/ax': 19.0.45(zod@4.4.3) + '@ax-llm/ax': 23.0.1(zod@4.4.3) '@hono/node-server': 2.0.8(hono@4.12.30) - '@tangle-network/agent-interface': 0.22.0 + '@tangle-network/agent-core': 0.4.15 + '@tangle-network/agent-interface': 0.26.1 '@tangle-network/tcloud': 0.4.14(typescript@5.9.3)(zod@4.4.3) hono: 4.12.30 zod: 4.4.3 @@ -1751,6 +1781,16 @@ snapshots: dependencies: zod: 4.4.3 + '@tangle-network/agent-interface@0.26.1': + dependencies: + '@noble/hashes': 1.8.0 + zod: 4.4.3 + + '@tangle-network/agent-interface@0.27.2': + dependencies: + '@noble/hashes': 1.8.0 + zod: 4.4.3 + '@tangle-network/agent-knowledge@1.12.1(typescript@5.9.3)': dependencies: '@tangle-network/agent-eval': 0.115.1(typescript@5.9.3) diff --git a/src/improvement/agentic-generator.ts b/src/improvement/agentic-generator.ts index 6d63722c..6e89c63b 100644 --- a/src/improvement/agentic-generator.ts +++ b/src/improvement/agentic-generator.ts @@ -38,7 +38,7 @@ import { existsSync, readFileSync, rmSync } from 'node:fs' import { join, resolve, sep } from 'node:path' import type { AnalystFinding, - CostLedger, + CostLedgerHandle, CostReceipt, CostReceiptInput, MaximumCharge, @@ -246,10 +246,14 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG costLedger, costPhase, }) { - if (opts.codexReproducible && !costLedger) { - throw new Error( - 'agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval 0.117+', - ) + let reproducibleCostLedger: CostLedgerHandle | undefined + if (opts.codexReproducible) { + if (!costLedger) { + throw new Error( + 'agenticGenerator: reproducible Codex requires the run-wide CostLedger supplied by agent-eval 0.117+', + ) + } + reproducibleCostLedger = costLedger } const basePrompt = appendProfileResourcePaths( buildPrompt({ report, findings }), @@ -314,13 +318,12 @@ export function agenticGenerator(opts: AgenticGeneratorOptions = {}): CandidateG return harnessResult } - if (opts.codexReproducible) { - const ledger = costLedger as CostLedger + if (reproducibleCostLedger) { const model = opts.profile?.model?.default if (!model) { throw new Error('agenticGenerator: reproducible Codex requires profile.model.default') } - const paid = await ledger.runPaidCall({ + const paid = await reproducibleCostLedger.runPaidCall({ channel: 'driver', phase: costPhase ?? 'search.proposal', actor: `agentic-generator:${harness}`, diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 898030cd..7f983170 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -474,7 +474,6 @@ describe('improve() — default proposer resolution (substrate export drift guar onProvenance: () => { provenanceCalls += 1 }, - collectWorkerRecords: () => [], expectUsage: 'assert', captureSource: 'eval-run', autoOnPromote: 'none', diff --git a/src/improvement/improvement-driver.ts b/src/improvement/improvement-driver.ts index b3ea5c8a..6881c5e4 100644 --- a/src/improvement/improvement-driver.ts +++ b/src/improvement/improvement-driver.ts @@ -21,7 +21,7 @@ */ import { spawnSync } from 'node:child_process' -import type { AnalystFinding, CostLedger } from '@tangle-network/agent-eval' +import type { AnalystFinding, CostLedgerHandle } from '@tangle-network/agent-eval' import { type CodeSurface, type LabeledScenarioStore, @@ -66,7 +66,7 @@ export interface CandidateGenerator { generation?: number candidateIndex?: number /** Shared run-wide paid-call account supplied by agent-eval 0.117+. */ - costLedger?: CostLedger + costLedger?: CostLedgerHandle /** Receipt attribution phase supplied alongside `costLedger`. */ costPhase?: string }): Promise<{ applied: boolean; summary: string }> diff --git a/tests/improve.test.ts b/tests/improve.test.ts index 18fe736c..897cebaf 100644 --- a/tests/improve.test.ts +++ b/tests/improve.test.ts @@ -88,6 +88,41 @@ describe('improve() — facade over selfImprove', () => { expect(out.profile.prompt?.systemPrompt).toBe('BASELINE') }) + it('forwards dispatchTimeoutMs through improve() to abort a stalled cell', async () => { + let observedAbort = false + const stalledAgent = ( + _surface: MutableSurface, + _scenario: DemoScenario, + ctx: DispatchContext, + ): Promise => + new Promise(() => { + ctx.signal.addEventListener( + 'abort', + () => { + observedAbort = true + }, + { once: true }, + ) + }) + + await expect( + improve(profileWith('BASELINE'), [], { + surface: 'prompt', + gate: 'none', + generator: scriptedWinner, + scenarios: scenarios.slice(0, 2), + judge, + agent: stalledAgent, + budget: { holdoutFraction: 0.5 }, + dispatchTimeoutMs: 25, + expectUsage: 'off', + maxConcurrency: 1, + }), + ).rejects.toThrow(/dispatch exceeded 25ms/) + + expect(observedAbort).toBe(true) + }) + it('applies the promoted prompt back into the profile on a ship verdict', async () => { const profile = profileWith('BASELINE') const out = await improve(profile, [], {