From 1a89652a1d8b003ed98d9c4332b719e58a8b79a2 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 14 Jul 2026 23:14:25 -0600 Subject: [PATCH 1/2] feat(primeintellect): run runtime programs on Verifiers v1 --- README.md | 66 +- docs/api/README.md | 1 + docs/api/primeintellect.md | 830 ++++++++++++++++++++++ docs/api/primitive-catalog.md | 20 + package.json | 6 + scripts/gen-primitive-catalog.mjs | 1 + scripts/verify-package-exports.mjs | 24 + scripts/verify-primeintellect-v1.mjs | 192 +++++ src/primeintellect/index.ts | 35 + src/primeintellect/package.ts | 554 +++++++++++++++ src/primeintellect/primeintellect.test.ts | 369 ++++++++++ src/primeintellect/runner.ts | 297 ++++++++ src/primeintellect/traces.ts | 377 ++++++++++ src/primeintellect/types.ts | 124 ++++ tsup.config.ts | 1 + typedoc.json | 1 + 16 files changed, 2897 insertions(+), 1 deletion(-) create mode 100644 docs/api/primeintellect.md create mode 100644 scripts/verify-primeintellect-v1.mjs create mode 100644 src/primeintellect/index.ts create mode 100644 src/primeintellect/package.ts create mode 100644 src/primeintellect/primeintellect.test.ts create mode 100644 src/primeintellect/runner.ts create mode 100644 src/primeintellect/traces.ts create mode 100644 src/primeintellect/types.ts diff --git a/README.md b/README.md index a3ac111c..709ab998 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ pnpm add @tangle-network/agent-runtime @tangle-network/agent-eval @tangle-networ - [Supervise a team of agents](#supervise-a-team-of-agents) - [Improve an agent](#improve-an-agent) - [Improve a knowledge base](#improve-a-knowledge-base) +- [Run on PrimeIntellect](#run-on-primeintellect) - [How it works](#how-it-works-the-short-version) - [Examples](#examples) - [Where to go next](#where-to-go-next) @@ -33,6 +34,7 @@ pnpm tsx examples/driver-loop/driver-loop.ts | Have one agent **supervise a team of agents** toward a goal | `supervise(profile, task, opts)` | | **Improve** an agent and prove the gain on fresh tasks | `improve(profile, findings, opts)` | | **Improve** a knowledge base with agents, checks, and safe promotion | `runKnowledgeImprovementJob(...)` | +| Evaluate or train the same agent on **PrimeIntellect** | `createPrimeIntellectPackage(...)` | ### Run a chat turn @@ -113,6 +115,67 @@ console.log(result.promoted, result.measurement.supervisedSpent) Use it when the product needs one knob for "make this knowledge base better" instead of wiring `improveKnowledgeBase`, a runtime supervisor, candidate workspaces, readiness checks, and promotion tracking by hand. +### Run on PrimeIntellect + +`@tangle-network/agent-runtime/primeintellect` packages typed train and eval tasks as a Verifiers v1 environment. +Prime launches your actual runtime program against an intercepted model endpoint, so `runPersonified`, `runAgentic`, product agents, tool calls, and multiple rounds stay intact. +Reference answers remain in Prime's task process and never enter the agent workspace. +The runner file must be one executable bundle containing the app and its runtime dependencies. + +```ts +import { readFile } from 'node:fs/promises' +import { + createPrimeIntellectPackage, + writePrimeIntellectPackage, +} from '@tangle-network/agent-runtime/primeintellect' + +const bundledRunner = await readFile('./dist/prime-runner.mjs', 'utf8') +const bundle = createPrimeIntellectPackage({ + name: 'support-agent-v1', + version: '1.0.0', + tasks: [ + { + id: 'train-refund-policy', + split: 'train', + prompt: 'Can a subscription renewal be refunded?', + answer: 'No', + }, + { + id: 'eval-final-sale', + split: 'eval', + prompt: 'Can a final-sale order be refunded?', + answer: 'No', + }, + ], + scoring: { kind: 'exact', normalization: 'trim-casefold' }, + runner: { + image: 'node:22-bookworm-slim', + files: { 'runner.mjs': bundledRunner }, + command: ['node', 'runner.mjs'], + }, +}) + +await writePrimeIntellectPackage(bundle, './prime/support-agent-v1') +``` + +The runner reads the episode and uses the normal runtime APIs: +Here, `runProductAgent` is the application's existing entry point, not another loop supplied by this adapter. + +```ts +import { + createPrimeIntellectBackend, + runPrimeIntellectProgram, +} from '@tangle-network/agent-runtime/primeintellect' + +await runPrimeIntellectProgram(async (episode) => { + const backend = createPrimeIntellectBackend(episode) + return runProductAgent({ task: episode.task, backend }) +}) +``` + +Prime writes complete `traces.jsonl` rows. +Use `importPrimeIntellectTraces(...)` to convert them to agent-eval `RunRecord`s for the existing reports and release checks. + ## How it works (the short version) - **One agent, run two ways.** The same agent runs at "do the task" speed and at "get better at the task" speed. "Driver", "worker", and "coordinator" are roles one agent plays, not separate types. @@ -134,6 +197,7 @@ Runnable, grouped by what they show. Copy the one nearest your task: | Trace + bill + effort-gate the WebCode benchmark (the Intelligence SDK) | [`intelligence-webcode`](./examples/intelligence-webcode) | | Self-improve an agent, gated on a held-out set | [`improve`](./examples/improve) · [`self-improving-coder`](./examples/self-improving-coder) | | Improve a KB, wiki, or RAG corpus with runtime agents | [`docs/canonical-api.md`](./docs/canonical-api.md) | +| Evaluate or train a runtime program on PrimeIntellect | `@tangle-network/agent-runtime/primeintellect` | | Study coordination vs raw compute | [`ablation-suite`](./examples/ablation-suite) | All 29 live in [`examples/`](./examples). @@ -143,7 +207,7 @@ All 29 live in [`examples/`](./examples). - New here? [`docs/concepts.md`](./docs/concepts.md), the mental model in plain terms. - [`docs/canonical-api.md`](./docs/canonical-api.md), find the primitive: "I want to ___ → use ___". - [`docs/api/primitive-catalog.md`](./docs/api/primitive-catalog.md), every export in one generated, never-stale list with its import path. Check it before building anything new. -- Import subpaths: the root export is the product surface (`handleChatTurn`, `improve`); deeper capabilities ship as subpaths: `/loops` (multi-agent + the loop kernel), `/knowledge` (KB improvement), `/mcp` (tool servers), `/intelligence` (observability drop-in), `/lifecycle`, `/agent`, `/profiles`, `/platform`, `/analyst-loop`, `/environment-provider`. +- Import subpaths: the root export is the product surface (`handleChatTurn`, `improve`); deeper capabilities ship as subpaths: `/loops` (multi-agent + the loop kernel), `/knowledge` (KB improvement), `/primeintellect` (Prime task, runtime, and trace adapter), `/mcp` (tool servers), `/intelligence` (observability drop-in), `/lifecycle`, `/agent`, `/profiles`, `/platform`, `/analyst-loop`, `/environment-provider`. - [`docs/architecture.md`](./docs/architecture.md), the design, end to end. - [`bench/HARNESS.md`](./bench/HARNESS.md), the experiment harness and how to run a benchmark. diff --git a/docs/api/README.md b/docs/api/README.md index 4d679de6..a5ebd062 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -15,6 +15,7 @@ - [lifecycle](lifecycle.md) - [mcp](mcp.md) - [platform](platform.md) +- [primeintellect](primeintellect.md) - [profiles](profiles.md) - [runtime/environment-provider](runtime/environment-provider.md) - [runtime](runtime.md) diff --git a/docs/api/primeintellect.md b/docs/api/primeintellect.md new file mode 100644 index 00000000..c556ed81 --- /dev/null +++ b/docs/api/primeintellect.md @@ -0,0 +1,830 @@ +[**@tangle-network/agent-runtime**](README.md) + +*** + +[@tangle-network/agent-runtime](README.md) / primeintellect + +# primeintellect + +## Interfaces + +### WritePrimeIntellectPackageOptions + +Defined in: [primeintellect/package.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L22) + +#### Properties + +##### replace? + +> `optional` **replace?**: `boolean` + +Defined in: [primeintellect/package.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L24) + +Replace an existing generated package and restore it if the final swap fails. + +*** + +### RunPrimeIntellectProgramOptions + +Defined in: [primeintellect/runner.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L18) + +#### Properties + +##### env? + +> `optional` **env?**: `ProcessEnv` + +Defined in: [primeintellect/runner.ts:19](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L19) + +*** + +### PrimeIntellectTrace + +Defined in: [primeintellect/traces.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L23) + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [primeintellect/traces.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L24) + +##### task + +> **task**: `object` + +Defined in: [primeintellect/traces.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L25) + +###### type + +> **type**: `string` + +###### data + +> **data**: `object` + +###### Index Signature + +\[`key`: `string`\]: `unknown` + +###### data.idx + +> **idx**: `number` + +###### data.name? + +> `optional` **name?**: `string` \| `null` + +###### data.split? + +> `optional` **split?**: `"train"` \| `"eval"` + +###### data.prompt? + +> `optional` **prompt?**: `unknown` + +###### data.system\_prompt? + +> `optional` **system\_prompt?**: `string` \| `null` + +###### data.metadata? + +> `optional` **metadata?**: `Record`\<`string`, `unknown`\> + +##### runtime? + +> `optional` **runtime?**: `unknown` + +Defined in: [primeintellect/traces.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L37) + +##### nodes + +> **nodes**: `PrimeTraceNode`[] + +Defined in: [primeintellect/traces.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L38) + +##### rewards + +> **rewards**: `Record`\<`string`, `number`\> + +Defined in: [primeintellect/traces.ts:39](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L39) + +##### metrics + +> **metrics**: `Record`\<`string`, `number`\> + +Defined in: [primeintellect/traces.ts:40](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L40) + +##### info? + +> `optional` **info?**: `Record`\<`string`, `unknown`\> + +Defined in: [primeintellect/traces.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L41) + +##### extra\_usage? + +> `optional` **extra\_usage?**: `PrimeUsage`[] + +Defined in: [primeintellect/traces.ts:42](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L42) + +##### is\_completed? + +> `optional` **is\_completed?**: `boolean` + +Defined in: [primeintellect/traces.ts:43](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L43) + +##### stop\_condition? + +> `optional` **stop\_condition?**: `string` \| `null` + +Defined in: [primeintellect/traces.ts:44](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L44) + +##### errors? + +> `optional` **errors?**: `object`[] + +Defined in: [primeintellect/traces.ts:45](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L45) + +###### type + +> **type**: `string` + +###### message + +> **message**: `string` + +###### traceback? + +> `optional` **traceback?**: `string` \| `null` + +##### timing? + +> `optional` **timing?**: `object` + +Defined in: [primeintellect/traces.ts:46](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L46) + +###### start? + +> `optional` **start?**: `number` + +###### setup? + +> `optional` **setup?**: `PrimeTimeSpan` + +###### generation? + +> `optional` **generation?**: `PrimeTimeSpan` + +###### finalize? + +> `optional` **finalize?**: `PrimeTimeSpan` + +###### scoring? + +> `optional` **scoring?**: `PrimeTimeSpan` + +*** + +### PrimeIntellectTraceImportOptions + +Defined in: [primeintellect/traces.ts:55](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L55) + +#### Properties + +##### experimentId + +> **experimentId**: `string` + +Defined in: [primeintellect/traces.ts:56](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L56) + +##### candidateId + +> **candidateId**: `string` + +Defined in: [primeintellect/traces.ts:57](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L57) + +##### seed + +> **seed**: `number` + +Defined in: [primeintellect/traces.ts:58](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L58) + +##### model + +> **model**: `string` + +Defined in: [primeintellect/traces.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L60) + +Snapshot-pinned model id required by RunRecord validation. + +##### promptHash + +> **promptHash**: `string` + +Defined in: [primeintellect/traces.ts:61](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L61) + +##### configHash + +> **configHash**: `string` + +Defined in: [primeintellect/traces.ts:62](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L62) + +##### commitSha + +> **commitSha**: `string` + +Defined in: [primeintellect/traces.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L63) + +*** + +### PrimeIntellectTask + +Defined in: [primeintellect/types.ts:32](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L32) + +One immutable problem. References stay inside Prime's task process. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [primeintellect/types.ts:33](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L33) + +##### split + +> **split**: [`PrimeIntellectSplit`](#primeintellectsplit) + +Defined in: [primeintellect/types.ts:34](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L34) + +##### prompt + +> **prompt**: `string` \| [`PrimeIntellectMessage`](#primeintellectmessage)[] + +Defined in: [primeintellect/types.ts:35](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L35) + +##### systemPrompt? + +> `optional` **systemPrompt?**: `string` + +Defined in: [primeintellect/types.ts:36](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L36) + +##### answer? + +> `optional` **answer?**: `string` \| `string`[] + +Defined in: [primeintellect/types.ts:37](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L37) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, [`PrimeIntellectJson`](#primeintellectjson)\> + +Defined in: [primeintellect/types.ts:38](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L38) + +*** + +### PrimeIntellectRunner + +Defined in: [primeintellect/types.ts:63](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L63) + +Files and commands that make the caller's real agent program runnable. + +#### Properties + +##### command + +> **command**: readonly \[`string`, `string`\] + +Defined in: [primeintellect/types.ts:64](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L64) + +##### files? + +> `optional` **files?**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [primeintellect/types.ts:65](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L65) + +##### setup? + +> `optional` **setup?**: readonly [`PrimeIntellectSetupCommand`](#primeintellectsetupcommand)[] + +Defined in: [primeintellect/types.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L66) + +##### forwardEnv? + +> `optional` **forwardEnv?**: readonly `string`[] + +Defined in: [primeintellect/types.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L67) + +##### image + +> **image**: `string` + +Defined in: [primeintellect/types.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L69) + +Container image used by the generated eval config. + +*** + +### PrimeIntellectPackageOptions + +Defined in: [primeintellect/types.ts:72](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L72) + +#### Properties + +##### name + +> **name**: `string` + +Defined in: [primeintellect/types.ts:73](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L73) + +##### version + +> **version**: `string` + +Defined in: [primeintellect/types.ts:74](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L74) + +##### description? + +> `optional` **description?**: `string` + +Defined in: [primeintellect/types.ts:75](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L75) + +##### tasks + +> **tasks**: readonly [`PrimeIntellectTask`](#primeintellecttask)[] + +Defined in: [primeintellect/types.ts:76](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L76) + +##### scoring + +> **scoring**: [`PrimeIntellectScoring`](#primeintellectscoring) + +Defined in: [primeintellect/types.ts:77](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L77) + +##### runner + +> **runner**: [`PrimeIntellectRunner`](#primeintellectrunner) + +Defined in: [primeintellect/types.ts:78](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L78) + +##### maxTurns? + +> `optional` **maxTurns?**: `number` + +Defined in: [primeintellect/types.ts:80](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L80) + +Prime-enforced model turn cap. Default 16. + +##### maxInputTokens? + +> `optional` **maxInputTokens?**: `number` + +Defined in: [primeintellect/types.ts:81](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L81) + +##### maxOutputTokens? + +> `optional` **maxOutputTokens?**: `number` + +Defined in: [primeintellect/types.ts:82](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L82) + +##### maxTotalTokens? + +> `optional` **maxTotalTokens?**: `number` + +Defined in: [primeintellect/types.ts:83](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L83) + +##### rolloutTimeoutSeconds? + +> `optional` **rolloutTimeoutSeconds?**: `number` + +Defined in: [primeintellect/types.ts:84](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L84) + +##### scoringTimeoutSeconds? + +> `optional` **scoringTimeoutSeconds?**: `number` + +Defined in: [primeintellect/types.ts:85](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L85) + +*** + +### PrimeIntellectPackageManifest + +Defined in: [primeintellect/types.ts:88](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L88) + +#### Properties + +##### schema + +> **schema**: `"tangle.primeintellect.package/v1"` + +Defined in: [primeintellect/types.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L89) + +##### name + +> **name**: `string` + +Defined in: [primeintellect/types.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L90) + +##### moduleName + +> **moduleName**: `string` + +Defined in: [primeintellect/types.ts:91](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L91) + +##### version + +> **version**: `string` + +Defined in: [primeintellect/types.ts:92](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L92) + +##### verifiers + +> **verifiers**: `">=0.2.0,<0.3.0"` + +Defined in: [primeintellect/types.ts:93](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L93) + +##### taskCount + +> **taskCount**: `number` + +Defined in: [primeintellect/types.ts:94](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L94) + +##### splits + +> **splits**: `Record`\<[`PrimeIntellectSplit`](#primeintellectsplit), `number`\> + +Defined in: [primeintellect/types.ts:95](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L95) + +##### taskIdsSha256 + +> **taskIdsSha256**: `string` + +Defined in: [primeintellect/types.ts:96](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L96) + +##### filesSha256 + +> **filesSha256**: `Record`\<`string`, `string`\> + +Defined in: [primeintellect/types.ts:97](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L97) + +*** + +### PrimeIntellectPackageBundle + +Defined in: [primeintellect/types.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L100) + +#### Properties + +##### manifest + +> **manifest**: [`PrimeIntellectPackageManifest`](#primeintellectpackagemanifest) + +Defined in: [primeintellect/types.ts:101](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L101) + +##### files + +> **files**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [primeintellect/types.ts:103](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L103) + +Relative package path to UTF-8 contents. + +*** + +### PrimeIntellectPublicTask + +Defined in: [primeintellect/types.ts:107](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L107) + +The answer-free task exposed to the caller's runtime program. + +#### Properties + +##### id + +> **id**: `string` + +Defined in: [primeintellect/types.ts:108](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L108) + +##### split + +> **split**: [`PrimeIntellectSplit`](#primeintellectsplit) + +Defined in: [primeintellect/types.ts:109](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L109) + +##### prompt + +> **prompt**: `string` \| [`PrimeIntellectMessage`](#primeintellectmessage)[] + +Defined in: [primeintellect/types.ts:110](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L110) + +##### systemPrompt? + +> `optional` **systemPrompt?**: `string` + +Defined in: [primeintellect/types.ts:111](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L111) + +##### metadata? + +> `optional` **metadata?**: `Record`\<`string`, [`PrimeIntellectJson`](#primeintellectjson)\> + +Defined in: [primeintellect/types.ts:112](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L112) + +*** + +### PrimeIntellectEpisodeContext + +Defined in: [primeintellect/types.ts:115](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L115) + +#### Properties + +##### protocol + +> **protocol**: `"tangle.primeintellect.episode/v1"` + +Defined in: [primeintellect/types.ts:116](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L116) + +##### task + +> **task**: [`PrimeIntellectPublicTask`](#primeintellectpublictask) + +Defined in: [primeintellect/types.ts:117](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L117) + +##### model + +> **model**: `object` + +Defined in: [primeintellect/types.ts:118](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L118) + +###### name + +> **name**: `string` + +###### baseUrl + +> **baseUrl**: `string` + +###### apiKey + +> **apiKey**: `string` + +##### mcpServers + +> **mcpServers**: `Readonly`\<`Record`\<`string`, `string`\>\> + +Defined in: [primeintellect/types.ts:123](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L123) + +## Type Aliases + +### PrimeIntellectBackendOptions + +> **PrimeIntellectBackendOptions** = `Omit`\<`Parameters`\<*typeof* [`createOpenAICompatibleBackend`](index.md#createopenaicompatiblebackend)\>\[`0`\], `"apiKey"` \| `"baseUrl"` \| `"model"`\> + +Defined in: [primeintellect/runner.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L22) + +*** + +### PrimeIntellectImportDefaults + +> **PrimeIntellectImportDefaults** = [`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) + +Defined in: [primeintellect/traces.ts:66](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L66) + +*** + +### PrimeIntellectSplit + +> **PrimeIntellectSplit** = `"train"` \| `"eval"` + +Defined in: [primeintellect/types.ts:1](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L1) + +*** + +### PrimeIntellectJson + +> **PrimeIntellectJson** = `null` \| `boolean` \| `number` \| `string` \| [`PrimeIntellectJson`](#primeintellectjson)[] \| \{\[`key`: `string`\]: [`PrimeIntellectJson`](#primeintellectjson); \} + +Defined in: [primeintellect/types.ts:3](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L3) + +*** + +### PrimeIntellectContent + +> **PrimeIntellectContent** = `string` \| (\{ `type`: `"text"`; `text`: `string`; \} \| \{ `type`: `"image_url"`; `image_url`: \{ `url`: `string`; \}; \})[] + +Defined in: [primeintellect/types.ts:11](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L11) + +*** + +### PrimeIntellectMessage + +> **PrimeIntellectMessage** = \{ `role`: `"system"` \| `"user"`; `content`: [`PrimeIntellectContent`](#primeintellectcontent); \} \| \{ `role`: `"assistant"`; `content?`: `string` \| `null`; `reasoning_content?`: `string` \| `null`; `tool_calls?`: `object`[]; `provider_state?`: `Record`\<`string`, [`PrimeIntellectJson`](#primeintellectjson)\>[]; \} \| \{ `role`: `"tool"`; `tool_call_id`: `string`; `content`: [`PrimeIntellectContent`](#primeintellectcontent); `name?`: `string`; \} + +Defined in: [primeintellect/types.ts:15](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L15) + +*** + +### PrimeIntellectScoring + +> **PrimeIntellectScoring** = \{ `kind`: `"exact"`; `normalization?`: `"none"` \| `"trim"` \| `"trim-casefold"`; \} \| \{ `kind`: `"reference-judge"`; `model`: `string`; `prompt?`: `string`; `view?`: `"last_reply"` \| `"full_trace"`; \} \| \{ `kind`: `"command"`; `command`: readonly \[`string`, `...string[]`\]; `files?`: `Readonly`\<`Record`\<`string`, `string`\>\>; `forwardEnv?`: readonly `string`[]; `timeoutSeconds?`: `number`; \} + +Defined in: [primeintellect/types.ts:41](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L41) + +*** + +### PrimeIntellectSetupCommand + +> **PrimeIntellectSetupCommand** = readonly \[`string`, `...string[]`\] + +Defined in: [primeintellect/types.ts:60](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/types.ts#L60) + +## Functions + +### createPrimeIntellectPackage() + +> **createPrimeIntellectPackage**(`options`): [`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) + +Defined in: [primeintellect/package.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L28) + +Build a complete PrimeIntellect Verifiers v1 package without writing to disk. + +#### Parameters + +##### options + +[`PrimeIntellectPackageOptions`](#primeintellectpackageoptions) + +#### Returns + +[`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) + +*** + +### writePrimeIntellectPackage() + +> **writePrimeIntellectPackage**(`bundle`, `outputDirectory`, `options?`): `Promise`\<`string`\> + +Defined in: [primeintellect/package.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L89) + +Write a bundle through a sibling temporary directory, then rename it into place. + +#### Parameters + +##### bundle + +[`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) + +##### outputDirectory + +`string` + +##### options? + +[`WritePrimeIntellectPackageOptions`](#writeprimeintellectpackageoptions) = `{}` + +#### Returns + +`Promise`\<`string`\> + +*** + +### readPrimeIntellectEpisodeContext() + +> **readPrimeIntellectEpisodeContext**(`env?`): [`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) + +Defined in: [primeintellect/runner.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L28) + +Read and validate the private process contract installed by the generated Prime harness. + +#### Parameters + +##### env? + +`ProcessEnv` = `process.env` + +#### Returns + +[`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) + +*** + +### createPrimeIntellectBackend() + +> **createPrimeIntellectBackend**(`context`, `options?`): [`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> + +Defined in: [primeintellect/runner.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L51) + +Build the existing runtime backend against Prime's intercepted model endpoint. + +#### Parameters + +##### context + +[`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) + +##### options? + +[`PrimeIntellectBackendOptions`](#primeintellectbackendoptions) = `{}` + +#### Returns + +[`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> + +*** + +### runPrimeIntellectProgram() + +> **runPrimeIntellectProgram**\<`Result`\>(`run`, `options?`): `Promise`\<`Result`\> + +Defined in: [primeintellect/runner.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L68) + +Execute the caller's canonical runtime program inside a Prime rollout. +The callback may call runPersonified, runAgentic, runLoop, or any product wrapper. + +#### Type Parameters + +##### Result + +`Result` + +#### Parameters + +##### run + +(`context`) => `Promise`\<`Result`\> + +##### options? + +[`RunPrimeIntellectProgramOptions`](#runprimeintellectprogramoptions) = `{}` + +#### Returns + +`Promise`\<`Result`\> + +*** + +### parsePrimeIntellectTraces() + +> **parsePrimeIntellectTraces**(`jsonl`): [`PrimeIntellectTrace`](#primeintellecttrace)[] + +Defined in: [primeintellect/traces.ts:69](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L69) + +Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. + +#### Parameters + +##### jsonl + +`string` + +#### Returns + +[`PrimeIntellectTrace`](#primeintellecttrace)[] + +*** + +### importPrimeIntellectTraces() + +> **importPrimeIntellectTraces**(`jsonl`, `defaults`): `RunRecord`[] + +Defined in: [primeintellect/traces.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L90) + +Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. + +#### Parameters + +##### jsonl + +`string` + +##### defaults + +[`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) + +#### Returns + +`RunRecord`[] + +*** + +### primeIntellectTraceToRunRecord() + +> **primeIntellectTraceToRunRecord**(`trace`, `options`): `RunRecord` + +Defined in: [primeintellect/traces.ts:100](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/traces.ts#L100) + +Project one complete Prime trace into the common agent-eval analysis row. + +#### Parameters + +##### trace + +[`PrimeIntellectTrace`](#primeintellecttrace) + +##### options + +[`PrimeIntellectTraceImportOptions`](#primeintellecttraceimportoptions) + +#### Returns + +`RunRecord` diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index d7fbf389..b26bd8f2 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -872,6 +872,26 @@ Import from `@tangle-network/agent-runtime/platform` — 20 exports. **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AuthorizeUrlOptions`, `CatalogResult`, `ConnectionHealth`, `ConnectionHealthResult`, `ExchangeCodeResult`, `ExecInput`, `MintTokenInput`, `MintTokenResult`, `PlatformHubStatus`, `StartAuthInput`, `StartAuthResult`. +### PrimeIntellect: Verifiers v1 package and trace adapter + +Import from `@tangle-network/agent-runtime/primeintellect` — 27 exports. + +| Symbol | Kind | Summary | +|---|---|---| +| `createPrimeIntellectBackend` | function | Build the existing runtime backend against Prime's intercepted model endpoint. | +| `createPrimeIntellectPackage` | function | Build a complete PrimeIntellect Verifiers v1 package without writing to disk. | +| `importPrimeIntellectTraces` | function | Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. | +| `parsePrimeIntellectTraces` | function | Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. | +| `primeIntellectTraceToRunRecord` | function | Project one complete Prime trace into the common agent-eval analysis row. | +| `readPrimeIntellectEpisodeContext` | function | Read and validate the private process contract installed by the generated Prime harness. | +| `runPrimeIntellectProgram` | function | Execute the caller's canonical runtime program inside a Prime rollout. | +| `writePrimeIntellectPackage` | function | Write a bundle through a sibling temporary directory, then rename it into place. | +| `PrimeIntellectPublicTask` | interface | The answer-free task exposed to the caller's runtime program. | +| `PrimeIntellectRunner` | interface | Files and commands that make the caller's real agent program runnable. | +| `PrimeIntellectTask` | interface | One immutable problem. References stay inside Prime's task process. | + +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `PrimeIntellectEpisodeContext`, `PrimeIntellectPackageBundle`, `PrimeIntellectPackageManifest`, `PrimeIntellectPackageOptions`, `PrimeIntellectTrace`, `PrimeIntellectTraceImportOptions`, `RunPrimeIntellectProgramOptions`, `WritePrimeIntellectPackageOptions`, `PrimeIntellectBackendOptions`, `PrimeIntellectContent`, `PrimeIntellectImportDefaults`, `PrimeIntellectJson`, `PrimeIntellectMessage`, `PrimeIntellectScoring`, `PrimeIntellectSetupCommand`, `PrimeIntellectSplit`. + ### Candidate execution — immutable prepare, run, grade, and receipt Import from `@tangle-network/agent-runtime/candidate-execution` — 95 exports. diff --git a/package.json b/package.json index 983014e4..262d4407 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,11 @@ "import": "./dist/platform.js", "default": "./dist/platform.js" }, + "./primeintellect": { + "types": "./dist/primeintellect/index.d.ts", + "import": "./dist/primeintellect/index.js", + "default": "./dist/primeintellect/index.js" + }, "./candidate-execution": { "types": "./dist/candidate-execution/index.d.ts", "import": "./dist/candidate-execution/index.js", @@ -98,6 +103,7 @@ "typecheck": "tsc --noEmit && pnpm run typecheck:examples", "typecheck:examples": "tsc --noEmit -p tsconfig.examples.json", "verify:package": "node scripts/verify-package-exports.mjs", + "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect-v1.mjs", "docs:api": "typedoc && node scripts/gen-primitive-catalog.mjs", "docs:freshness": "node scripts/check-docs-freshness.mjs", "docs:check": "pnpm run docs:api && git diff --exit-code -- docs/api && pnpm run docs:freshness" diff --git a/scripts/gen-primitive-catalog.mjs b/scripts/gen-primitive-catalog.mjs index bd2cd68f..98dc85b3 100644 --- a/scripts/gen-primitive-catalog.mjs +++ b/scripts/gen-primitive-catalog.mjs @@ -69,6 +69,7 @@ const ownSurfaceLabels = { './analyst-loop': 'Analyst loop — trace findings on a running loop', './lifecycle': 'Artifact lifecycle — generate → measure → promote → compose', './knowledge': 'Knowledge orchestration — supervised KB updates', + './primeintellect': 'PrimeIntellect: Verifiers v1 package and trace adapter', './profiles': 'Built-in agent profiles', './platform': 'Platform glue', './candidate-execution': 'Candidate execution — immutable prepare, run, grade, and receipt', diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index c45d42b0..fff84292 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -49,6 +49,7 @@ try { './intelligence': ['import', 'types'], './loops': ['import', 'types'], './environment-provider': ['import', 'types'], + './primeintellect': ['import', 'types'], './profiles': ['import', 'types'], './mcp': ['import', 'types'], } @@ -97,6 +98,29 @@ try { ], appDir, ) + run( + process.execPath, + [ + '--input-type=module', + '--eval', + ` + const prime = await import('@tangle-network/agent-runtime/primeintellect') + for (const name of [ + 'createPrimeIntellectPackage', + 'writePrimeIntellectPackage', + 'readPrimeIntellectEpisodeContext', + 'createPrimeIntellectBackend', + 'runPrimeIntellectProgram', + 'parsePrimeIntellectTraces', + 'primeIntellectTraceToRunRecord', + 'importPrimeIntellectTraces', + ]) { + if (typeof prime[name] !== 'function') throw new Error('missing PrimeIntellect export ' + name) + } + `, + ], + appDir, + ) run( process.execPath, [ diff --git a/scripts/verify-primeintellect-v1.mjs b/scripts/verify-primeintellect-v1.mjs new file mode 100644 index 00000000..9955d191 --- /dev/null +++ b/scripts/verify-primeintellect-v1.mjs @@ -0,0 +1,192 @@ +import { spawnSync } from 'node:child_process' +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + createPrimeIntellectPackage, + writePrimeIntellectPackage, +} from '../dist/primeintellect/index.js' + +const root = mkdtempSync(join(tmpdir(), 'agent-runtime-primeintellect-v1-')) +const output = join(root, 'strict-exact-v1') +const referenceOutput = join(root, 'reference-v1') +const commandOutput = join(root, 'command-v1') + +try { + const bundle = createPrimeIntellectPackage({ + name: 'strict-exact-v1', + version: '1.0.0', + tasks: [ + { id: 'train', split: 'train', prompt: 'Is a subscription refundable?', answer: 'No' }, + { id: 'eval', split: 'eval', prompt: 'Is a final-sale item refundable?', answer: 'No' }, + ], + scoring: { kind: 'exact', normalization: 'trim-casefold' }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': 'process.exit(0)\n' }, + }, + }) + await writePrimeIntellectPackage(bundle, output) + await writePrimeIntellectPackage( + createPrimeIntellectPackage({ + name: 'reference-v1', + version: '1.0.0', + tasks: [ + { id: 'train-ref', split: 'train', prompt: 'Practice reference?', answer: 'No' }, + { id: 'eval-ref', split: 'eval', prompt: 'Held-out reference?', answer: 'No' }, + ], + scoring: { kind: 'reference-judge', model: 'openai/gpt-5.4-nano' }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': 'process.exit(0)\n' }, + }, + }), + referenceOutput, + ) + await writePrimeIntellectPackage( + createPrimeIntellectPackage({ + name: 'command-v1', + version: '1.0.0', + tasks: [ + { id: 'train-command', split: 'train', prompt: 'Practice command?' }, + { id: 'eval-command', split: 'eval', prompt: 'Held-out command?' }, + ], + scoring: { + kind: 'command', + command: ['python', 'scoring/score.py'], + files: { + 'score.py': + 'import json, sys\nrequest = json.load(sys.stdin)\nassert request["protocol"] == "tangle.primeintellect.score/v1"\nprint(json.dumps({"reward": 0.25, "metrics": {"custom": 0.75}}))\n', + }, + }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': 'process.exit(0)\n' }, + }, + }), + commandOutput, + ) + const check = ` +import asyncio +import json +import sys +from pathlib import Path +from types import SimpleNamespace + +import verifiers.v1 as vf +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "reference-v1")) +sys.path.insert(0, str(ROOT / "command-v1")) +from command_v1 import TangleTaskset as CommandTaskset +from command_v1.taskset import TangleTasksetConfig as CommandTasksetConfig +from reference_v1 import TangleTaskset as ReferenceTaskset +from reference_v1.taskset import TangleTasksetConfig as ReferenceTasksetConfig +from strict_exact_v1 import TangleRuntimeHarness, TangleTaskset +from strict_exact_v1.harness import TangleRuntimeHarnessConfig +from strict_exact_v1.taskset import TangleTasksetConfig + +train = TangleTaskset(TangleTasksetConfig(id="strict-exact-v1", split="train")).load() +heldout = TangleTaskset(TangleTasksetConfig(id="strict-exact-v1", split="eval")).load() +assert [task.data.name for task in train] == ["train"] +assert [task.data.name for task in heldout] == ["eval"] + +class ScoreTrace: + def __init__(self, reply): + self.last_reply = reply + +task = heldout[0] +assert asyncio.run(task.task_reward(ScoreTrace("No"))) == 1.0 +assert asyncio.run(task.task_reward(ScoreTrace("No, but the policy allows it"))) == 0.0 +assert asyncio.run(task.task_reward(ScoreTrace("Yes, it is allowed"))) == 0.0 + +reference_task = ReferenceTaskset( + ReferenceTasksetConfig(id="reference-v1", split="eval") +).load()[0] +reference_trace = SimpleNamespace(last_reply="", transcript="") +assert asyncio.run(reference_task.task_reward(reference_trace)) == 0.0 + +class CommandTrace: + def __init__(self): + self.recorded = {} + def model_dump(self, **kwargs): + return {"id": "command-trace", "nodes": []} + def record_metrics(self, metrics): + self.recorded.update(metrics) + +command_task = CommandTaskset( + CommandTasksetConfig(id="command-v1", split="eval") +).load()[0] +command_trace = CommandTrace() +assert asyncio.run(command_task.task_reward(command_trace)) == 0.25 +assert command_trace.recorded == {"custom": 0.75} + +class Runtime: + def __init__(self): + self.env = None + self.files = {} + async def write(self, path, contents): + self.files[path] = contents + async def run(self, command, env): + return vf.ProgramResult(exit_code=0, stdout="", stderr="") + async def run_program(self, command, env): + self.env = env + return vf.ProgramResult(exit_code=0, stdout="", stderr="") + +runtime = Runtime() +harness = TangleRuntimeHarness(TangleRuntimeHarnessConfig(id="strict-exact-v1")) +asyncio.run(harness.setup(runtime)) +asyncio.run(harness.launch( + SimpleNamespace(model="openai/gpt-5.4-20260601"), + SimpleNamespace(task=SimpleNamespace(data=task.data)), + runtime, + "http://127.0.0.1:9000/v1", + "secret", + {}, +)) +public = json.loads(runtime.env["TANGLE_PRIME_TASK_JSON"]) +assert "answer" not in public +assert "reference" not in public +assert public["id"] == "eval" +assert runtime.files["runner.mjs"] == b"process.exit(0)\\n" +print(json.dumps({ + "train": len(train), + "eval": len(heldout), + "strict_scores": [1, 0, 0], + "reference_empty": 0, + "command_score": 0.25, + "command_metric": 0.75, +})) +` + const checkPath = join(output, 'verify.py') + writeFileSync(checkPath, check) + const result = spawnSync( + 'uv', + [ + 'run', + '--project', + output, + '--with', + 'verifiers @ git+https://github.com/PrimeIntellect-ai/verifiers.git@v0.2.0', + 'python', + checkPath, + ], + { cwd: output, encoding: 'utf8' }, + ) + if (result.status !== 0) { + throw new Error( + [ + 'PrimeIntellect Verifiers v0.2.0 integration check failed', + result.stdout.trim(), + result.stderr.trim(), + ] + .filter(Boolean) + .join('\n'), + ) + } + process.stdout.write(result.stdout) +} finally { + rmSync(root, { recursive: true, force: true }) +} diff --git a/src/primeintellect/index.ts b/src/primeintellect/index.ts new file mode 100644 index 00000000..5f7e7ee3 --- /dev/null +++ b/src/primeintellect/index.ts @@ -0,0 +1,35 @@ +export { + createPrimeIntellectPackage, + type WritePrimeIntellectPackageOptions, + writePrimeIntellectPackage, +} from './package' +export { + createPrimeIntellectBackend, + type PrimeIntellectBackendOptions, + type RunPrimeIntellectProgramOptions, + readPrimeIntellectEpisodeContext, + runPrimeIntellectProgram, +} from './runner' +export { + importPrimeIntellectTraces, + type PrimeIntellectImportDefaults, + type PrimeIntellectTrace, + type PrimeIntellectTraceImportOptions, + parsePrimeIntellectTraces, + primeIntellectTraceToRunRecord, +} from './traces' +export type { + PrimeIntellectContent, + PrimeIntellectEpisodeContext, + PrimeIntellectJson, + PrimeIntellectMessage, + PrimeIntellectPackageBundle, + PrimeIntellectPackageManifest, + PrimeIntellectPackageOptions, + PrimeIntellectPublicTask, + PrimeIntellectRunner, + PrimeIntellectScoring, + PrimeIntellectSetupCommand, + PrimeIntellectSplit, + PrimeIntellectTask, +} from './types' diff --git a/src/primeintellect/package.ts b/src/primeintellect/package.ts new file mode 100644 index 00000000..98e52428 --- /dev/null +++ b/src/primeintellect/package.ts @@ -0,0 +1,554 @@ +import { createHash, randomUUID } from 'node:crypto' +import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' +import { basename, dirname, isAbsolute, join, normalize, relative, resolve, sep } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import type { + PrimeIntellectJson, + PrimeIntellectPackageBundle, + PrimeIntellectPackageManifest, + PrimeIntellectPackageOptions, + PrimeIntellectScoring, + PrimeIntellectTask, +} from './types' + +const VERIFIERS_RANGE = '>=0.2.0,<0.3.0' as const +const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/ +const PACKAGE_NAME = /^[a-z][a-z0-9-]{0,62}$/ +const VERSION = /^\d+\.\d+\.\d+(?:[-+][a-zA-Z0-9.-]+)?$/ +const DEFAULT_MAX_TURNS = 16 +const DEFAULT_ROLLOUT_TIMEOUT = 3_600 +const DEFAULT_SCORING_TIMEOUT = 300 + +export interface WritePrimeIntellectPackageOptions { + /** Replace an existing generated package and restore it if the final swap fails. */ + replace?: boolean +} + +/** Build a complete PrimeIntellect Verifiers v1 package without writing to disk. */ +export function createPrimeIntellectPackage( + options: PrimeIntellectPackageOptions, +): PrimeIntellectPackageBundle { + const validated = validateOptions(options) + const moduleName = validated.name.replaceAll('-', '_') + const rows = validated.tasks.map((task, idx) => taskRow(task, idx)) + const runnerFiles = validated.runner.files ?? {} + const scoringFiles = validated.scoring.kind === 'command' ? (validated.scoring.files ?? {}) : {} + const files: Record = { + 'pyproject.toml': renderPyproject(validated, moduleName), + 'prime.eval.toml': renderPrimeConfig(validated, 'eval'), + 'prime.train.toml': renderPrimeConfig(validated, 'train'), + 'README.md': renderReadme(validated), + [`${moduleName}/__init__.py`]: renderInit(moduleName), + [`${moduleName}/taskset.py`]: renderTaskset(moduleName, validated.scoring), + [`${moduleName}/harness.py`]: renderHarness(moduleName), + [`${moduleName}/tasks.jsonl`]: `${rows.map((row) => JSON.stringify(row)).join('\n')}\n`, + [`${moduleName}/runner.json`]: `${JSON.stringify( + { + command: validated.runner.command, + files: runnerFiles, + setup: validated.runner.setup ?? [], + forwardEnv: validated.runner.forwardEnv ?? [], + }, + null, + 2, + )}\n`, + } + for (const [path, contents] of Object.entries(scoringFiles)) { + files[`${moduleName}/scoring/${path}`] = contents + } + + const filesSha256 = Object.fromEntries( + Object.entries(files) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([path, contents]) => [path, sha256(contents)]), + ) + const manifest: PrimeIntellectPackageManifest = { + schema: 'tangle.primeintellect.package/v1', + name: validated.name, + moduleName, + version: validated.version, + verifiers: VERIFIERS_RANGE, + taskCount: validated.tasks.length, + splits: { + train: validated.tasks.filter((task) => task.split === 'train').length, + eval: validated.tasks.filter((task) => task.split === 'eval').length, + }, + taskIdsSha256: sha256( + validated.tasks + .map((task) => `${task.split}:${task.id}`) + .sort() + .join('\n'), + ), + filesSha256, + } + files['manifest.json'] = `${JSON.stringify(manifest, null, 2)}\n` + return { manifest, files: Object.freeze(files) } +} + +/** Write a bundle through a sibling temporary directory, then rename it into place. */ +export async function writePrimeIntellectPackage( + bundle: PrimeIntellectPackageBundle, + outputDirectory: string, + options: WritePrimeIntellectPackageOptions = {}, +): Promise { + const output = resolve(outputDirectory) + const parent = dirname(output) + await mkdir(parent, { recursive: true }) + const replacing = await pathExists(output) + if (replacing) { + if (!options.replace) throw new Error(`PrimeIntellect output already exists: ${output}`) + await assertGeneratedPackage(output) + } + + const temporary = join(parent, `.${basename(output)}.${randomUUID()}.tmp`) + const backup = replacing ? join(parent, `.${basename(output)}.${randomUUID()}.backup`) : undefined + try { + await mkdir(temporary) + for (const [path, contents] of Object.entries(bundle.files)) { + assertRelativePath(path, 'bundle file') + const target = resolve(temporary, path) + if (target !== temporary && !target.startsWith(`${temporary}${sep}`)) { + throw new Error(`bundle file escapes output directory: ${path}`) + } + await mkdir(dirname(target), { recursive: true }) + await writeFile(target, contents, 'utf8') + } + if (backup) await rename(output, backup) + try { + await rename(temporary, output) + } catch (error) { + if (backup) { + try { + await rename(backup, output) + } catch (restoreError) { + throw new AggregateError( + [error, restoreError], + `failed to install PrimeIntellect package and restore ${output}`, + ) + } + } + throw error + } + if (backup) await rm(backup, { recursive: true }) + } catch (error) { + await rm(temporary, { recursive: true, force: true }) + throw error + } + return output +} + +function validateOptions(options: PrimeIntellectPackageOptions): PrimeIntellectPackageOptions { + if (!PACKAGE_NAME.test(options.name)) { + throw new Error('PrimeIntellect package name must match /^[a-z][a-z0-9-]{0,62}$/') + } + if (!VERSION.test(options.version)) { + throw new Error('PrimeIntellect package version must be a numeric semantic version') + } + if (!Array.isArray(options.tasks) || options.tasks.length === 0) { + throw new Error('PrimeIntellect package requires tasks') + } + const tasks: readonly PrimeIntellectTask[] = options.tasks + const seen = new Set() + const seenInputs = new Map() + const splitCounts = { train: 0, eval: 0 } + for (const [index, task] of tasks.entries()) { + validateTask(task, index, options.scoring) + if (seen.has(task.id)) throw new Error(`duplicate PrimeIntellect task id: ${task.id}`) + seen.add(task.id) + const input = canonicalJson({ + prompt: task.prompt, + systemPrompt: task.systemPrompt ?? null, + metadata: task.metadata ?? {}, + }) + const duplicate = seenInputs.get(input) + if (duplicate) { + throw new Error( + `PrimeIntellect tasks ${duplicate.id} (${duplicate.split}) and ${task.id} (${task.split}) expose the same public input`, + ) + } + seenInputs.set(input, { id: task.id, split: task.split }) + splitCounts[task.split] += 1 + } + if (splitCounts.train === 0 || splitCounts.eval === 0) { + throw new Error('PrimeIntellect package requires non-empty, disjoint train and eval splits') + } + validateScoring(options.scoring) + validateCommand(options.runner.command, 'runner.command') + validateFiles(options.runner.files ?? {}, 'runner.files') + for (const [index, command] of (options.runner.setup ?? []).entries()) { + validateCommand(command, `runner.setup[${index}]`) + } + validateEnvNames(options.runner.forwardEnv ?? [], 'runner.forwardEnv') + if (typeof options.runner.image !== 'string' || options.runner.image.trim().length === 0) { + throw new Error('runner.image must be a non-empty container image') + } + if (/(^|:)latest$/i.test(options.runner.image)) { + throw new Error('runner.image must not use the mutable latest tag') + } + positiveInteger(options.maxTurns ?? DEFAULT_MAX_TURNS, 'maxTurns') + for (const [name, value] of [ + ['maxInputTokens', options.maxInputTokens], + ['maxOutputTokens', options.maxOutputTokens], + ['maxTotalTokens', options.maxTotalTokens], + ] as const) { + if (value !== undefined) positiveInteger(value, name) + } + positiveNumber(options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT, 'rolloutTimeoutSeconds') + positiveNumber(options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, 'scoringTimeoutSeconds') + return options +} + +function validateTask( + task: PrimeIntellectTask, + index: number, + scoring: PrimeIntellectScoring, +): void { + const path = `tasks[${index}]` + if (typeof task.id !== 'string' || task.id.trim().length === 0) { + throw new Error(`${path}.id must be a non-empty string`) + } + if (task.split !== 'train' && task.split !== 'eval') { + throw new Error(`${path}.split must be train or eval`) + } + if (typeof task.prompt === 'string') { + if (task.prompt.length === 0) throw new Error(`${path}.prompt must not be empty`) + } else if (!Array.isArray(task.prompt) || task.prompt.length === 0) { + throw new Error(`${path}.prompt must be a non-empty string or message array`) + } else { + task.prompt.forEach((message, messageIndex) => { + validateMessage(message, `${path}.prompt[${messageIndex}]`) + }) + if ( + task.systemPrompt !== undefined && + task.prompt.some((message) => message.role === 'system') + ) { + throw new Error(`${path} must not set systemPrompt and include a system message`) + } + } + validateJson(task.prompt, `${path}.prompt`) + if (task.systemPrompt !== undefined && typeof task.systemPrompt !== 'string') { + throw new Error(`${path}.systemPrompt must be a string`) + } + if (task.metadata !== undefined) validateJson(task.metadata, `${path}.metadata`) + if (scoring.kind !== 'command') { + const answers = Array.isArray(task.answer) ? task.answer : [task.answer] + if ( + answers.length === 0 || + answers.some((answer) => typeof answer !== 'string' || answer.length === 0) + ) { + throw new Error(`${path}.answer is required for ${scoring.kind} scoring`) + } + } +} + +function validateScoring(scoring: PrimeIntellectScoring): void { + if (scoring.kind === 'exact') return + if (scoring.kind === 'reference-judge') { + if (typeof scoring.model !== 'string' || scoring.model.length === 0) { + throw new Error('reference-judge scoring requires a model') + } + return + } + validateCommand(scoring.command, 'scoring.command') + validateFiles(scoring.files ?? {}, 'scoring.files') + validateEnvNames(scoring.forwardEnv ?? [], 'scoring.forwardEnv') + positiveNumber(scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, 'scoring.timeoutSeconds') +} + +function validateCommand(command: readonly string[], path: string): void { + if (!Array.isArray(command) || command.length === 0) { + throw new Error(`${path} must be a non-empty argv array`) + } + for (const [index, argument] of command.entries()) { + if (typeof argument !== 'string' || argument.length === 0 || argument.includes('\0')) { + throw new Error(`${path}[${index}] must be a non-empty string without NUL bytes`) + } + } +} + +function validateFiles(files: Readonly>, path: string): void { + for (const [file, contents] of Object.entries(files)) { + assertRelativePath(file, path) + if (typeof contents !== 'string') throw new Error(`${path}.${file} must be a string`) + } +} + +function validateEnvNames(names: readonly string[], path: string): void { + const seen = new Set() + for (const [index, name] of names.entries()) { + if (!ENV_NAME.test(name)) throw new Error(`${path}[${index}] is not a valid environment name`) + if (seen.has(name)) throw new Error(`${path} contains duplicate name ${name}`) + seen.add(name) + } +} + +function assertRelativePath(path: string, label: string): void { + const normalized = normalize(path) + if ( + path.length === 0 || + path.includes('\0') || + isAbsolute(path) || + normalized === '..' || + normalized.startsWith(`..${sep}`) || + relative('.', normalized).startsWith('..') + ) { + throw new Error(`${label} contains unsafe path: ${path}`) + } +} + +function validateJson(value: unknown, path: string): void { + if (value === null || ['string', 'boolean'].includes(typeof value)) return + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`) + return + } + if (Array.isArray(value)) { + value.forEach((entry, index) => { + validateJson(entry, `${path}[${index}]`) + }) + return + } + if (typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) validateJson(entry, `${path}.${key}`) + return + } + throw new Error(`${path} is not JSON serializable`) +} + +function validateMessage(value: unknown, path: string): void { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be a message object`) + } + const message = value as Record + const role = message.role + if (!['system', 'user', 'assistant', 'tool'].includes(String(role))) { + throw new Error(`${path}.role is invalid`) + } + if (role === 'system' || role === 'user') { + assertOnlyKeys(message, ['role', 'content'], path) + validateContent(message.content, `${path}.content`) + } else if (role === 'assistant') { + assertOnlyKeys( + message, + ['role', 'content', 'reasoning_content', 'tool_calls', 'provider_state'], + path, + ) + if ( + message.content !== undefined && + message.content !== null && + typeof message.content !== 'string' + ) { + throw new Error(`${path}.content must be a string or null`) + } + if ( + message.reasoning_content !== undefined && + message.reasoning_content !== null && + typeof message.reasoning_content !== 'string' + ) { + throw new Error(`${path}.reasoning_content must be a string or null`) + } + if (message.tool_calls !== undefined) { + if (!Array.isArray(message.tool_calls)) throw new Error(`${path}.tool_calls must be an array`) + for (const [index, rawCall] of message.tool_calls.entries()) { + if (rawCall === null || typeof rawCall !== 'object' || Array.isArray(rawCall)) { + throw new Error(`${path}.tool_calls[${index}] must be an object`) + } + const call = rawCall as Record + assertOnlyKeys(call, ['id', 'name', 'arguments'], `${path}.tool_calls[${index}]`) + for (const field of ['id', 'name', 'arguments']) { + if (typeof call[field] !== 'string' || call[field].length === 0) { + throw new Error(`${path}.tool_calls[${index}].${field} must be a non-empty string`) + } + } + } + } + if (message.provider_state !== undefined) { + validateProviderState(message.provider_state, `${path}.provider_state`) + } + } else { + assertOnlyKeys(message, ['role', 'tool_call_id', 'content', 'name'], path) + if (typeof message.tool_call_id !== 'string' || message.tool_call_id.length === 0) { + throw new Error(`${path}.tool_call_id must be a non-empty string`) + } + if (message.name !== undefined && typeof message.name !== 'string') { + throw new Error(`${path}.name must be a string`) + } + validateContent(message.content, `${path}.content`) + } +} + +function validateProviderState(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, entry] of value.entries()) { + if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) { + throw new Error(`${path}[${index}] must be an object`) + } + validateJson(entry, `${path}[${index}]`) + } +} + +function validateContent(value: unknown, path: string): void { + if (typeof value === 'string') return + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a string or non-empty content array`) + } + for (const [index, rawPart] of value.entries()) { + if (rawPart === null || typeof rawPart !== 'object' || Array.isArray(rawPart)) { + throw new Error(`${path}[${index}] must be a content object`) + } + const part = rawPart as Record + if (part.type === 'text' && typeof part.text === 'string') { + assertOnlyKeys(part, ['type', 'text'], `${path}[${index}]`) + continue + } + if ( + part.type === 'image_url' && + part.image_url !== null && + typeof part.image_url === 'object' && + !Array.isArray(part.image_url) && + typeof (part.image_url as Record).url === 'string' + ) { + assertOnlyKeys(part, ['type', 'image_url'], `${path}[${index}]`) + assertOnlyKeys( + part.image_url as Record, + ['url'], + `${path}[${index}].image_url`, + ) + continue + } + throw new Error(`${path}[${index}] is not a supported text or image_url content part`) + } +} + +function assertOnlyKeys( + value: Record, + allowed: readonly string[], + path: string, +): void { + for (const key of Object.keys(value)) { + if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`) + } +} + +function taskRow(task: PrimeIntellectTask, idx: number): Record { + return { + idx, + name: task.id, + prompt: task.prompt as PrimeIntellectJson, + system_prompt: task.systemPrompt ?? null, + split: task.split, + answer: task.answer ?? null, + metadata: task.metadata ?? {}, + } +} + +function renderPyproject(options: PrimeIntellectPackageOptions, moduleName: string): string { + const description = options.description ?? `PrimeIntellect tasks for ${options.name}` + return `[project]\nname = ${toml(options.name)}\nversion = ${toml(options.version)}\ndescription = ${toml(description)}\nrequires-python = ">=3.11,<3.14"\ndependencies = ["verifiers${VERIFIERS_RANGE}"]\n\n[build-system]\nrequires = ["hatchling"]\nbuild-backend = "hatchling.build"\n\n[tool.hatch.build.targets.wheel]\npackages = [${toml(moduleName)}]\n\n[tool.uv]\nprerelease = "allow"\n` +} + +function renderPrimeConfig(options: PrimeIntellectPackageOptions, split: 'train' | 'eval'): string { + const limits = [ + `max_turns = ${options.maxTurns ?? DEFAULT_MAX_TURNS}`, + options.maxInputTokens === undefined + ? undefined + : `max_input_tokens = ${options.maxInputTokens}`, + options.maxOutputTokens === undefined + ? undefined + : `max_output_tokens = ${options.maxOutputTokens}`, + options.maxTotalTokens === undefined + ? undefined + : `max_total_tokens = ${options.maxTotalTokens}`, + ].filter((line): line is string => line !== undefined) + return `${limits.join('\n')}\npush = false\n\n[timeout]\nrollout = ${options.rolloutTimeoutSeconds ?? DEFAULT_ROLLOUT_TIMEOUT}\nscoring = ${options.scoringTimeoutSeconds ?? DEFAULT_SCORING_TIMEOUT}\n\n[taskset]\nid = ${toml(options.name)}\nsplit = ${toml(split)}\n\n[harness]\nid = ${toml(options.name)}\nprogram = ${tomlArray(options.runner.command)}\nforward_env = ${tomlArray(options.runner.forwardEnv ?? [])}\n\n[harness.runtime]\ntype = "docker"\nimage = ${toml(options.runner.image)}\n` +} + +function renderInit(moduleName: string): string { + return `from ${moduleName}.harness import TangleRuntimeHarness\nfrom ${moduleName}.taskset import TangleTaskset\n\n__all__ = ["TangleRuntimeHarness", "TangleTaskset"]\n` +} + +function renderTaskset(moduleName: string, scoring: PrimeIntellectScoring): string { + const config = scoringConfig(scoring) + return `import asyncio\nimport json\nimport math\nimport os\nfrom importlib.resources import files\nfrom pathlib import Path\nfrom typing import Any, Literal\n\nimport verifiers.v1 as vf\n\nSCORING = json.loads(${pythonString(JSON.stringify(config))})\nPACKAGE_ROOT = Path(__file__).resolve().parent\n\n\nclass TangleTaskData(vf.TaskData):\n split: Literal["train", "eval"]\n answer: str | list[str] | None = None\n metadata: dict[str, Any] = {}\n\n\nclass TangleTaskConfig(vf.TaskConfig):\n scoring: Literal["exact", "reference-judge", "command"] = SCORING["kind"]\n normalization: Literal["none", "trim", "trim-casefold"] = SCORING.get("normalization", "trim")\n judge_model: str = SCORING.get("model", "openai/gpt-5.4-nano")\n judge_prompt: str | None = SCORING.get("prompt")\n judge_view: Literal["last_reply", "full_trace"] = SCORING.get("view", "last_reply")\n score_program: list[str] = SCORING.get("command", [])\n score_forward_env: list[str] = SCORING.get("forwardEnv", [])\n score_timeout_seconds: float = SCORING.get("timeoutSeconds", 300)\n\n\ndef _normalize(value: str, mode: str) -> str:\n if mode == "none":\n return value\n value = value.strip()\n return value.casefold() if mode == "trim-casefold" else value\n\n\nasync def _run_score_command(config: TangleTaskConfig, data: TangleTaskData, trace: vf.Trace) -> float:\n if not config.score_program:\n raise ValueError("command scoring requires score_program")\n safe_env = {\n key: os.environ[key]\n for key in ("PATH", "HOME", "TMPDIR", "LANG")\n if key in os.environ\n }\n safe_env.update(\n {key: os.environ[key] for key in config.score_forward_env if key in os.environ}\n )\n request = {\n "protocol": "tangle.primeintellect.score/v1",\n "task": data.model_dump(mode="json", exclude_none=True),\n "trace": trace.model_dump(mode="json", exclude_none=True),\n }\n process = await asyncio.create_subprocess_exec(\n *config.score_program,\n cwd=PACKAGE_ROOT,\n env=safe_env,\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n payload = json.dumps(request, separators=(",", ":")).encode()\n try:\n stdout, stderr = await asyncio.wait_for(\n process.communicate(payload), timeout=config.score_timeout_seconds\n )\n except TimeoutError:\n process.kill()\n await process.communicate()\n raise RuntimeError(\n f"score command timed out after {config.score_timeout_seconds}s"\n )\n if process.returncode != 0:\n detail = (stderr or stdout).decode(errors="replace").strip()[-2000:]\n raise RuntimeError(f"score command exited {process.returncode}: {detail}")\n try:\n result = json.loads(stdout)\n except json.JSONDecodeError as error:\n raise ValueError(f"score command returned invalid JSON: {error}") from error\n if not isinstance(result, dict):\n raise ValueError("score command must return an object")\n reward = result.get("reward")\n if isinstance(reward, bool) or not isinstance(reward, (int, float)) or not math.isfinite(reward):\n raise ValueError("score command reward must be a finite number")\n metrics = result.get("metrics", {})\n if not isinstance(metrics, dict):\n raise ValueError("score command metrics must be an object")\n for name, value in metrics.items():\n if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):\n raise ValueError(f"score command metric {name!r} must be a finite number")\n trace.record_metrics(metrics)\n return float(reward)\n\n\nclass TangleTask(vf.Task[TangleTaskData, vf.State, TangleTaskConfig]):\n @vf.reward(weight=1.0)\n async def task_reward(self, trace: vf.Trace) -> float:\n if self.config.scoring == "command":\n return await _run_score_command(self.config, self.data, trace)\n if self.data.answer is None:\n raise ValueError(f"task {self.data.name!r} has no reference answer")\n if self.config.scoring == "reference-judge":\n judge = vf.ReferenceJudge(\n vf.ReferenceJudgeConfig(\n model=self.config.judge_model,\n prompt=self.config.judge_prompt,\n view=self.config.judge_view,\n )\n )\n return await judge.score(self.data, trace)\n expected = self.data.answer if isinstance(self.data.answer, list) else [self.data.answer]\n actual = _normalize(trace.last_reply, self.config.normalization)\n return float(any(actual == _normalize(answer, self.config.normalization) for answer in expected))\n\n\nclass TangleTasksetConfig(vf.TasksetConfig):\n split: Literal["train", "eval"] = "eval"\n task: TangleTaskConfig = TangleTaskConfig()\n\n\nclass TangleTaskset(vf.Taskset[TangleTask, TangleTasksetConfig]):\n def load(self) -> list[TangleTask]:\n resource = files(${pythonString(moduleName)}).joinpath("tasks.jsonl")\n tasks: list[TangleTask] = []\n with resource.open("r", encoding="utf-8") as handle:\n for line in handle:\n if not line.strip():\n continue\n row = json.loads(line)\n if row["split"] != self.config.split:\n continue\n tasks.append(TangleTask(TangleTaskData.model_validate(row), self.config.task))\n if not tasks:\n raise ValueError(f"task split {self.config.split!r} is empty")\n return tasks\n\n\n__all__ = ["TangleTaskset"]\n` +} + +function renderHarness(moduleName: string): string { + return `import json\nfrom importlib.resources import files\n\nimport verifiers.v1 as vf\n\nRUNNER = json.loads(files(${pythonString(moduleName)}).joinpath("runner.json").read_text(encoding="utf-8"))\n\n\nclass TangleRuntimeHarnessConfig(vf.HarnessConfig):\n program: list[str] = RUNNER["command"]\n setup_commands: list[list[str]] = RUNNER["setup"]\n forward_env: list[str] = RUNNER["forwardEnv"]\n\n\nclass TangleRuntimeHarness(vf.Harness[TangleRuntimeHarnessConfig]):\n APPENDS_SYSTEM_PROMPT = True\n SUPPORTS_MCP = True\n SUPPORTS_MESSAGE_PROMPT = True\n\n async def setup(self, runtime: vf.Runtime) -> None:\n for path, contents in RUNNER["files"].items():\n await runtime.write(path, contents.encode())\n for command in self.config.setup_commands:\n result = await runtime.run(command, self.config.resolved_env)\n if result.exit_code != 0:\n detail = (result.stderr or result.stdout).strip()[-2000:]\n raise RuntimeError(\n f"runner setup command {command[0]!r} exited {result.exit_code}: {detail}"\n )\n\n async def launch(\n self,\n ctx: vf.ModelContext,\n trace: vf.Trace,\n runtime: vf.Runtime,\n endpoint: str,\n secret: str,\n mcp_urls: dict[str, str],\n ) -> vf.ProgramResult:\n data = trace.task.data\n public_task = {\n "id": data.name or str(data.idx),\n "split": data.split,\n "prompt": data.prompt,\n "metadata": data.metadata,\n }\n if data.system_prompt is not None:\n public_task["systemPrompt"] = data.system_prompt\n env = {\n **self.config.resolved_env,\n "OPENAI_BASE_URL": endpoint,\n "OPENAI_API_KEY": secret,\n "OPENAI_MODEL": ctx.model,\n "TANGLE_PRIME_TASK_JSON": json.dumps(public_task, separators=(",", ":")),\n "TANGLE_PRIME_MCP_SERVERS_JSON": json.dumps(mcp_urls, separators=(",", ":")),\n }\n if not self.config.program:\n raise ValueError("Tangle runtime harness requires a program argv")\n return await runtime.run_program(self.config.program, env)\n\n\n__all__ = ["TangleRuntimeHarness"]\n` +} + +function scoringConfig(scoring: PrimeIntellectScoring): Record { + if (scoring.kind === 'exact') { + return { kind: scoring.kind, normalization: scoring.normalization ?? 'trim' } + } + if (scoring.kind === 'reference-judge') { + return { + kind: scoring.kind, + model: scoring.model, + prompt: scoring.prompt ?? null, + view: scoring.view ?? 'last_reply', + } + } + return { + kind: scoring.kind, + command: [...scoring.command], + forwardEnv: [...(scoring.forwardEnv ?? [])], + timeoutSeconds: scoring.timeoutSeconds ?? DEFAULT_SCORING_TIMEOUT, + } +} + +function renderReadme(options: PrimeIntellectPackageOptions): string { + return `# ${options.name}\n\nPrimeIntellect Verifiers v1 tasks that run the caller's Tangle agent program.\n\n## Evaluate\n\n\`\`\`bash\nuv run eval @ prime.eval.toml --model \n\`\`\`\n\n## Train\n\nUse \`prime.train.toml\` as the environment config for Prime RL. The train and eval rows are disjoint and selected by \`taskset.split\`.\n\nThe runner receives only the prompt, metadata, intercepted model endpoint, and MCP URLs. Reference answers remain in the task process and are never written into the runner workspace or environment.\n` +} + +function toml(value: string): string { + return JSON.stringify(value) +} + +function tomlArray(values: readonly string[]): string { + return `[${values.map(toml).join(', ')}]` +} + +function pythonString(value: string): string { + return JSON.stringify(value) +} + +function positiveInteger(value: number, path: string): void { + if (!Number.isSafeInteger(value) || value <= 0) + throw new Error(`${path} must be a positive integer`) +} + +function positiveNumber(value: number, path: string): void { + if (!Number.isFinite(value) || value <= 0) throw new Error(`${path} must be positive`) +} + +async function pathExists(path: string): Promise { + try { + await stat(path) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false + throw error + } +} + +async function assertGeneratedPackage(output: string): Promise { + let manifest: unknown + try { + manifest = JSON.parse(await readFile(join(output, 'manifest.json'), 'utf8')) + } catch (error) { + throw new Error( + `refusing to replace ${output}: it is not a generated PrimeIntellect package (${error instanceof Error ? error.message : String(error)})`, + ) + } + if ( + manifest === null || + typeof manifest !== 'object' || + (manifest as Record).schema !== 'tangle.primeintellect.package/v1' + ) { + throw new Error(`refusing to replace ${output}: manifest schema does not match`) + } +} + +function sha256(value: string): string { + return createHash('sha256').update(value, 'utf8').digest('hex') +} diff --git a/src/primeintellect/primeintellect.test.ts b/src/primeintellect/primeintellect.test.ts new file mode 100644 index 00000000..ff893d15 --- /dev/null +++ b/src/primeintellect/primeintellect.test.ts @@ -0,0 +1,369 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { createPrimeIntellectPackage, writePrimeIntellectPackage } from './package' +import { + createPrimeIntellectBackend, + readPrimeIntellectEpisodeContext, + runPrimeIntellectProgram, +} from './runner' +import { + importPrimeIntellectTraces, + type PrimeIntellectTrace, + parsePrimeIntellectTraces, + primeIntellectTraceToRunRecord, +} from './traces' +import type { PrimeIntellectPackageOptions } from './types' + +function packageOptions(): PrimeIntellectPackageOptions { + return { + name: 'support-agent-v1', + version: '1.0.0', + tasks: [ + { + id: 'train-refund', + split: 'train', + prompt: 'Can this order be refunded?', + answer: 'No', + metadata: { policy: 'refunds' }, + }, + { + id: 'eval-refund', + split: 'eval', + prompt: 'Is a final-sale order refundable?', + answer: 'No', + }, + ], + scoring: { kind: 'exact', normalization: 'trim-casefold' }, + runner: { + image: 'node:22-bookworm-slim', + files: { 'runner.mjs': 'console.log("runner")\n' }, + command: ['node', 'runner.mjs'], + }, + } +} + +describe('PrimeIntellect v1 package', () => { + it('builds a split-safe package around the caller runtime program', () => { + const bundle = createPrimeIntellectPackage(packageOptions()) + + expect(bundle.manifest.splits).toEqual({ train: 1, eval: 1 }) + expect(bundle.manifest.verifiers).toBe('>=0.2.0,<0.3.0') + expect(bundle.files['prime.eval.toml']).toContain('max_turns = 16') + expect(bundle.files['prime.eval.toml']).toContain('type = "docker"') + expect(bundle.files['support_agent_v1/taskset.py']).toContain('import verifiers.v1 as vf') + expect(bundle.files['support_agent_v1/harness.py']).toContain('runtime.run_program') + expect(bundle.files['support_agent_v1/harness.py']).not.toContain('data.answer') + expect(bundle.files['support_agent_v1/harness.py']).not.toContain('reference') + }) + + it('requires both splits and rejects duplicate ids and unsafe runner files', () => { + const options = packageOptions() + expect(() => + createPrimeIntellectPackage({ ...options, tasks: options.tasks.slice(0, 1) }), + ).toThrow(/non-empty, disjoint train and eval splits/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [options.tasks[0]!, { ...options.tasks[1]!, id: options.tasks[0]!.id }], + }), + ).toThrow(/duplicate PrimeIntellect task id/) + expect(() => + createPrimeIntellectPackage({ + ...options, + runner: { ...options.runner, files: { '../answer.txt': 'secret' } }, + }), + ).toThrow(/unsafe path/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [options.tasks[0]!, { ...options.tasks[0]!, id: 'eval-copy', split: 'eval' }], + }), + ).toThrow(/expose the same public input/) + }) + + it('rejects malformed messages and duplicate system prompts', () => { + const options = packageOptions() + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [{ ...options.tasks[0]!, prompt: [{ role: 'user' }] as never }, options.tasks[1]!], + }), + ).toThrow(/content/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [ + { + ...options.tasks[0]!, + systemPrompt: 'First system prompt', + prompt: [{ role: 'system', content: 'Second system prompt' }], + }, + options.tasks[1]!, + ], + }), + ).toThrow(/must not set systemPrompt and include a system message/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [ + { + ...options.tasks[0]!, + prompt: [{ role: 'assistant', content: null, unsupported: true }] as never, + }, + options.tasks[1]!, + ], + }), + ).toThrow(/unsupported is not supported/) + expect(() => + createPrimeIntellectPackage({ + ...options, + tasks: [ + { + ...options.tasks[0]!, + prompt: [ + { + role: 'assistant', + content: null, + provider_state: [{ signature: 'opaque-provider-state' }], + }, + ], + }, + options.tasks[1]!, + ], + }), + ).not.toThrow() + }) + + it('writes atomically and refuses an existing destination by default', async () => { + const root = await mkdtemp(join(tmpdir(), 'prime-package-test-')) + const output = join(root, 'generated') + try { + const bundle = createPrimeIntellectPackage(packageOptions()) + await writePrimeIntellectPackage(bundle, output) + expect(JSON.parse(await readFile(join(output, 'manifest.json'), 'utf8'))).toEqual( + bundle.manifest, + ) + await expect(writePrimeIntellectPackage(bundle, output)).rejects.toThrow(/already exists/) + await expect(writePrimeIntellectPackage(bundle, output, { replace: true })).resolves.toBe( + output, + ) + + const unrelated = join(root, 'user-owned') + await mkdir(unrelated) + await writeFile(join(unrelated, 'important.txt'), 'keep me', 'utf8') + await expect( + writePrimeIntellectPackage(bundle, unrelated, { replace: true }), + ).rejects.toThrow(/not a generated PrimeIntellect package/) + await expect(readFile(join(unrelated, 'important.txt'), 'utf8')).resolves.toBe('keep me') + } finally { + await rm(root, { recursive: true, force: true }) + } + }) +}) + +describe('PrimeIntellect runtime program contract', () => { + const env = { + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'eval-refund', + split: 'eval', + prompt: 'Is this refundable?', + systemPrompt: 'Use the policy tools.', + metadata: { tenant: 'test' }, + }), + TANGLE_PRIME_MCP_SERVERS_JSON: JSON.stringify({ policy: 'http://127.0.0.1:3210/mcp' }), + OPENAI_MODEL: 'openai/gpt-5.4-20260601', + OPENAI_BASE_URL: 'http://127.0.0.1:9000/v1', + OPENAI_API_KEY: 'interception-secret', + } + + it('reads an answer-free episode and creates the existing backend', async () => { + const context = readPrimeIntellectEpisodeContext(env) + expect(context.task.id).toBe('eval-refund') + expect(context.mcpServers.policy).toBe('http://127.0.0.1:3210/mcp') + expect(createPrimeIntellectBackend(context).kind).toBe('primeintellect') + expect(createPrimeIntellectBackend(context, { kind: 'product-runtime' }).kind).toBe( + 'product-runtime', + ) + await expect( + runPrimeIntellectProgram(async (episode) => episode.task.id, { env }), + ).resolves.toBe('eval-refund') + }) + + it('fails if the task process exposes a private scoring field', () => { + expect(() => + readPrimeIntellectEpisodeContext({ + ...env, + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'leaked', + split: 'eval', + prompt: 'Question', + answer: 'private', + }), + }), + ).toThrow(/exposed private field answer/) + }) + + it('rejects conflicting system prompts and malformed tool calls', () => { + expect(() => + readPrimeIntellectEpisodeContext({ + ...env, + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'conflicting-system', + split: 'eval', + prompt: [{ role: 'system', content: 'In prompt' }], + systemPrompt: 'Alongside prompt', + }), + }), + ).toThrow(/must not set systemPrompt and include a system message/) + expect(() => + readPrimeIntellectEpisodeContext({ + ...env, + TANGLE_PRIME_TASK_JSON: JSON.stringify({ + id: 'bad-tool-call', + split: 'eval', + prompt: [{ role: 'assistant', tool_calls: [{ id: 'call-1', name: 'search' }] }], + }), + }), + ).toThrow(/arguments/) + }) +}) + +function primeTrace(): PrimeIntellectTrace { + return { + id: 'prime-trace-1', + task: { + type: 'TangleTask', + data: { idx: 1, name: 'eval-refund', split: 'eval', prompt: 'Question' }, + }, + nodes: [ + { + parent: null, + sampled: false, + usage: null, + }, + { + parent: 0, + sampled: true, + usage: { + prompt_tokens: 100, + completion_tokens: 20, + cached_input_tokens: 40, + reasoning_tokens: 5, + cost: 0.03, + }, + }, + { + parent: 1, + sampled: true, + usage: { + prompt_tokens: 50, + completion_tokens: 10, + cost: 0.02, + }, + }, + ], + rewards: { task_reward: 0.8 }, + metrics: { citation_precision: 0.75 }, + extra_usage: [{ prompt_tokens: 10, completion_tokens: 2, reasoning_tokens: 1, cost: 0.01 }], + is_completed: true, + stop_condition: 'agent_completed', + errors: [], + timing: { + start: 100, + generation: { start: 101, end: 104 }, + scoring: { start: 104, end: 105.5 }, + }, + } +} + +const importOptions = { + experimentId: 'prime-eval-1', + candidateId: 'support-agent-a', + seed: 42, + model: 'openai/gpt-5.4-20260601', + promptHash: 'a'.repeat(64), + configHash: 'b'.repeat(64), + commitSha: 'c'.repeat(40), +} + +describe('PrimeIntellect trace import', () => { + it('preserves score, metrics, turns, tokens, cost, duration, scenario, and split', () => { + const record = primeIntellectTraceToRunRecord(primeTrace(), importOptions) + + expect(record.splitTag).toBe('holdout') + expect(record.scenarioId).toBe('eval-refund') + expect(record.outcome.holdoutScore).toBe(0.8) + expect(record.outcome.raw).toMatchObject({ + reward: 0.8, + 'prime.turns': 2, + 'prime.branches': 1, + 'metric.citation_precision': 0.75, + }) + expect(record.tokenUsage).toEqual({ input: 200, output: 32, reasoning: 6, cached: 40 }) + expect(record.costUsd).toBeCloseTo(0.06) + expect(record.costProvenance?.kind).toBe('observed') + expect(record.costProvenance?.usd).toBeCloseTo(0.06) + expect(record.wallMs).toBe(5_500) + }) + + it('parses durable JSONL and refuses empty or malformed inputs', () => { + const jsonl = `${JSON.stringify(primeTrace())}\n` + expect(parsePrimeIntellectTraces(jsonl)).toHaveLength(1) + expect(importPrimeIntellectTraces(jsonl, importOptions)).toHaveLength(1) + expect(() => parsePrimeIntellectTraces('\n')).toThrow(/contains no traces/) + expect(() => parsePrimeIntellectTraces('{oops}\n')).toThrow(/line 1 is invalid JSON/) + }) + + it('marks partial cost as uncaptured while retaining the reported subtotal', () => { + const trace = primeTrace() + delete trace.nodes[2]!.usage!.cost + const record = primeIntellectTraceToRunRecord(trace, importOptions) + + expect(record.costUsd).toBe(0) + expect(record.costProvenance).toEqual({ kind: 'uncaptured', usd: null }) + expect(record.outcome.raw).toMatchObject({ + 'prime.cost_complete': 0, + 'prime.reported_cost_usd': 0.04, + }) + }) + + it('rejects malformed graph nodes and usage instead of undercounting them', () => { + expect(() => + parsePrimeIntellectTraces(`${JSON.stringify({ ...primeTrace(), nodes: [null] })}\n`), + ).toThrow(/nodes\[0\] must be an object/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ + ...primeTrace(), + nodes: [{ parent: 1, sampled: true, usage: null }], + })}\n`, + ), + ).toThrow(/parent must reference an earlier node/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ + ...primeTrace(), + nodes: [ + { + parent: null, + sampled: true, + usage: { prompt_tokens: -1, completion_tokens: 0 }, + }, + ], + })}\n`, + ), + ).toThrow(/prompt_tokens must be non-negative/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ ...primeTrace(), errors: [{ type: 'timeout' }] })}\n`, + ), + ).toThrow(/message must be a non-empty string/) + expect(() => + parsePrimeIntellectTraces( + `${JSON.stringify({ ...primeTrace(), timing: { start: 'now' } })}\n`, + ), + ).toThrow(/timing.start must be a finite number/) + }) +}) diff --git a/src/primeintellect/runner.ts b/src/primeintellect/runner.ts new file mode 100644 index 00000000..285abe91 --- /dev/null +++ b/src/primeintellect/runner.ts @@ -0,0 +1,297 @@ +import { createOpenAICompatibleBackend } from '../backends' +import type { + PrimeIntellectEpisodeContext, + PrimeIntellectJson, + PrimeIntellectMessage, + PrimeIntellectPublicTask, + PrimeIntellectSplit, +} from './types' + +const ENV = { + task: 'TANGLE_PRIME_TASK_JSON', + model: 'OPENAI_MODEL', + baseUrl: 'OPENAI_BASE_URL', + apiKey: 'OPENAI_API_KEY', + mcpServers: 'TANGLE_PRIME_MCP_SERVERS_JSON', +} as const + +export interface RunPrimeIntellectProgramOptions { + env?: NodeJS.ProcessEnv +} + +export type PrimeIntellectBackendOptions = Omit< + Parameters[0], + 'apiKey' | 'baseUrl' | 'model' +> + +/** Read and validate the private process contract installed by the generated Prime harness. */ +export function readPrimeIntellectEpisodeContext( + env: NodeJS.ProcessEnv = process.env, +): PrimeIntellectEpisodeContext { + const rawTask = requiredEnv(env, ENV.task) + const rawMcp = env[ENV.mcpServers] ?? '{}' + const parsedTask = parseJson(rawTask, ENV.task) + const parsedMcp = parseJson(rawMcp, ENV.mcpServers) + const task = validatePublicTask(parsedTask) + const mcpServers = validateStringMap(parsedMcp, ENV.mcpServers) + + return { + protocol: 'tangle.primeintellect.episode/v1', + task, + model: { + name: requiredEnv(env, ENV.model), + baseUrl: requiredEnv(env, ENV.baseUrl), + apiKey: requiredEnv(env, ENV.apiKey), + }, + mcpServers, + } +} + +/** Build the existing runtime backend against Prime's intercepted model endpoint. */ +export function createPrimeIntellectBackend( + context: PrimeIntellectEpisodeContext, + options: PrimeIntellectBackendOptions = {}, +) { + return createOpenAICompatibleBackend({ + ...options, + apiKey: context.model.apiKey, + baseUrl: context.model.baseUrl, + model: context.model.name, + kind: options.kind ?? 'primeintellect', + }) +} + +/** + * Execute the caller's canonical runtime program inside a Prime rollout. + * The callback may call runPersonified, runAgentic, runLoop, or any product wrapper. + */ +export async function runPrimeIntellectProgram( + run: (context: PrimeIntellectEpisodeContext) => Promise, + options: RunPrimeIntellectProgramOptions = {}, +): Promise { + return run(readPrimeIntellectEpisodeContext(options.env)) +} + +function requiredEnv(env: NodeJS.ProcessEnv, name: string): string { + const value = env[name] + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`PrimeIntellect runner requires ${name}`) + } + return value +} + +function parseJson(value: string, name: string): unknown { + try { + return JSON.parse(value) + } catch (error) { + throw new Error( + `${name} must contain valid JSON: ${error instanceof Error ? error.message : String(error)}`, + ) + } +} + +function validatePublicTask(value: unknown): PrimeIntellectPublicTask { + const task = record(value, ENV.task) + for (const privateField of ['answer', 'reference', 'scoring', 'score']) { + if (privateField in task) { + throw new Error(`${ENV.task} exposed private field ${privateField}`) + } + } + const id = nonEmptyString(task.id, `${ENV.task}.id`) + const split = validateSplit(task.split, `${ENV.task}.split`) + const prompt = validatePrompt(task.prompt, `${ENV.task}.prompt`) + const systemPrompt = optionalString(task.systemPrompt, `${ENV.task}.systemPrompt`) + if ( + systemPrompt !== undefined && + Array.isArray(prompt) && + prompt.some((message) => message.role === 'system') + ) { + throw new Error(`${ENV.task} must not set systemPrompt and include a system message`) + } + const metadata = + task.metadata === undefined + ? undefined + : (validateJsonObject(task.metadata, `${ENV.task}.metadata`) as Record< + string, + PrimeIntellectJson + >) + return { + id, + split, + prompt, + ...(systemPrompt !== undefined ? { systemPrompt } : {}), + ...(metadata !== undefined ? { metadata } : {}), + } +} + +function validatePrompt(value: unknown, path: string): string | PrimeIntellectMessage[] { + if (typeof value === 'string' && value.length > 0) return value + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a non-empty string or message array`) + } + for (const [index, message] of value.entries()) { + const item = record(message, `${path}[${index}]`) + const role = item.role + if (!['system', 'user', 'assistant', 'tool'].includes(String(role))) { + throw new Error(`${path}[${index}].role is invalid`) + } + if (role === 'system' || role === 'user') { + assertOnlyKeys(item, ['role', 'content'], `${path}[${index}]`) + validateContent(item.content, `${path}[${index}].content`) + } else if (role === 'assistant') { + assertOnlyKeys( + item, + ['role', 'content', 'reasoning_content', 'tool_calls', 'provider_state'], + `${path}[${index}]`, + ) + if (item.content !== undefined && item.content !== null && typeof item.content !== 'string') { + throw new Error(`${path}[${index}].content must be a string or null`) + } + if ( + item.reasoning_content !== undefined && + item.reasoning_content !== null && + typeof item.reasoning_content !== 'string' + ) { + throw new Error(`${path}[${index}].reasoning_content must be a string or null`) + } + if (item.tool_calls !== undefined) { + validateToolCalls(item.tool_calls, `${path}[${index}].tool_calls`) + } + if (item.provider_state !== undefined) { + validateProviderState(item.provider_state, `${path}[${index}].provider_state`) + } + } else { + assertOnlyKeys(item, ['role', 'tool_call_id', 'content', 'name'], `${path}[${index}]`) + nonEmptyString(item.tool_call_id, `${path}[${index}].tool_call_id`) + if (item.name !== undefined && typeof item.name !== 'string') { + throw new Error(`${path}[${index}].name must be a string`) + } + validateContent(item.content, `${path}[${index}].content`) + } + validateJson(item, `${path}[${index}]`) + } + return value as PrimeIntellectMessage[] +} + +function validateToolCalls(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, rawCall] of value.entries()) { + const call = record(rawCall, `${path}[${index}]`) + assertOnlyKeys(call, ['id', 'name', 'arguments'], `${path}[${index}]`) + for (const field of ['id', 'name', 'arguments']) { + nonEmptyString(call[field], `${path}[${index}].${field}`) + } + } +} + +function validateProviderState(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, rawState] of value.entries()) { + const state = record(rawState, `${path}[${index}]`) + validateJson(state, `${path}[${index}]`) + } +} + +function validateContent(value: unknown, path: string): void { + if (typeof value === 'string') return + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a string or non-empty content array`) + } + for (const [index, rawPart] of value.entries()) { + const part = record(rawPart, `${path}[${index}]`) + if (part.type === 'text' && typeof part.text === 'string') { + assertOnlyKeys(part, ['type', 'text'], `${path}[${index}]`) + continue + } + if ( + part.type === 'image_url' && + part.image_url !== null && + typeof part.image_url === 'object' && + !Array.isArray(part.image_url) && + typeof (part.image_url as Record).url === 'string' + ) { + assertOnlyKeys(part, ['type', 'image_url'], `${path}[${index}]`) + assertOnlyKeys( + part.image_url as Record, + ['url'], + `${path}[${index}].image_url`, + ) + continue + } + throw new Error(`${path}[${index}] is not a supported text or image_url content part`) + } +} + +function assertOnlyKeys( + value: Record, + allowed: readonly string[], + path: string, +): void { + for (const key of Object.keys(value)) { + if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`) + } +} + +function validateSplit(value: unknown, path: string): PrimeIntellectSplit { + if (value !== 'train' && value !== 'eval') { + throw new Error(`${path} must be train or eval`) + } + return value +} + +function validateStringMap(value: unknown, path: string): Readonly> { + const input = record(value, path) + const output: Record = {} + for (const [key, entry] of Object.entries(input)) { + if (typeof entry !== 'string' || entry.length === 0) { + throw new Error(`${path}.${key} must be a non-empty string`) + } + output[key] = entry + } + return output +} + +function validateJsonObject(value: unknown, path: string): Record { + const output = record(value, path) + validateJson(output, path) + return output as Record +} + +function validateJson(value: unknown, path: string): void { + if (value === null || ['string', 'boolean'].includes(typeof value)) return + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`) + return + } + if (Array.isArray(value)) { + value.forEach((entry, index) => { + validateJson(entry, `${path}[${index}]`) + }) + return + } + if (typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) validateJson(entry, `${path}.${key}`) + return + } + throw new Error(`${path} is not JSON serializable`) +} + +function record(value: unknown, path: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be an object`) + } + return value as Record +} + +function nonEmptyString(value: unknown, path: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${path} must be a non-empty string`) + } + return value +} + +function optionalString(value: unknown, path: string): string | undefined { + if (value === undefined) return undefined + if (typeof value !== 'string') throw new Error(`${path} must be a string`) + return value +} diff --git a/src/primeintellect/traces.ts b/src/primeintellect/traces.ts new file mode 100644 index 00000000..003b9477 --- /dev/null +++ b/src/primeintellect/traces.ts @@ -0,0 +1,377 @@ +import { type RunRecord, validateRunRecord } from '@tangle-network/agent-eval' + +interface PrimeUsage { + prompt_tokens: number + completion_tokens: number + cached_input_tokens?: number | null + reasoning_tokens?: number | null + cost?: number | null +} + +interface PrimeTraceNode { + parent?: number | null + sampled?: boolean + usage?: PrimeUsage | null + message?: unknown +} + +interface PrimeTimeSpan { + start?: number + end?: number +} + +export interface PrimeIntellectTrace { + id: string + task: { + type: string + data: { + idx: number + name?: string | null + split?: 'train' | 'eval' + prompt?: unknown + system_prompt?: string | null + metadata?: Record + [key: string]: unknown + } + } + runtime?: unknown + nodes: PrimeTraceNode[] + rewards: Record + metrics: Record + info?: Record + extra_usage?: PrimeUsage[] + is_completed?: boolean + stop_condition?: string | null + errors?: Array<{ type: string; message: string; traceback?: string | null }> + timing?: { + start?: number + setup?: PrimeTimeSpan + generation?: PrimeTimeSpan + finalize?: PrimeTimeSpan + scoring?: PrimeTimeSpan + } +} + +export interface PrimeIntellectTraceImportOptions { + experimentId: string + candidateId: string + seed: number + /** Snapshot-pinned model id required by RunRecord validation. */ + model: string + promptHash: string + configHash: string + commitSha: string +} + +export type PrimeIntellectImportDefaults = PrimeIntellectTraceImportOptions + +/** Parse Prime's durable `traces.jsonl` and reject malformed rows with a line number. */ +export function parsePrimeIntellectTraces(jsonl: string): PrimeIntellectTrace[] { + const traces: PrimeIntellectTrace[] = [] + for (const [lineIndex, line] of jsonl.split('\n').entries()) { + if (!line.trim()) continue + let parsed: unknown + try { + parsed = JSON.parse(line) + } catch (error) { + throw new Error( + `PrimeIntellect trace line ${lineIndex + 1} is invalid JSON: ${ + error instanceof Error ? error.message : String(error) + }`, + ) + } + traces.push(validateTrace(parsed, lineIndex + 1)) + } + if (traces.length === 0) throw new Error('PrimeIntellect traces.jsonl contains no traces') + return traces +} + +/** Convert all Prime traces to agent-eval RunRecords while retaining one shared run config. */ +export function importPrimeIntellectTraces( + jsonl: string, + defaults: PrimeIntellectImportDefaults, +): RunRecord[] { + return parsePrimeIntellectTraces(jsonl).map((trace) => + primeIntellectTraceToRunRecord(trace, defaults), + ) +} + +/** Project one complete Prime trace into the common agent-eval analysis row. */ +export function primeIntellectTraceToRunRecord( + trace: PrimeIntellectTrace, + options: PrimeIntellectTraceImportOptions, +): RunRecord { + const split = trace.task.data.split + if (split !== 'train' && split !== 'eval') { + throw new Error(`PrimeIntellect trace ${trace.id} has no train/eval split`) + } + const reward = sumFinite(Object.values(trace.rewards), `trace ${trace.id} rewards`) + const usage = aggregateUsage([ + ...trace.nodes.map((node) => node.usage).filter((value): value is PrimeUsage => value != null), + ...(trace.extra_usage ?? []), + ]) + const errors = trace.errors ?? [] + const raw: Record = { + reward, + 'prime.turns': trace.nodes.filter((node) => node.sampled === true).length, + 'prime.branches': countBranches(trace.nodes), + 'prime.errors': errors.length, + 'prime.completed': trace.is_completed ? 1 : 0, + 'prime.cost_complete': usage.costComplete ? 1 : 0, + 'prime.reported_cost_usd': usage.reportedCostUsd, + } + for (const [name, value] of Object.entries(trace.rewards)) { + raw[`reward.${name}`] = finite(value, `trace ${trace.id} reward ${name}`) + } + for (const [name, value] of Object.entries(trace.metrics)) { + raw[`metric.${name}`] = finite(value, `trace ${trace.id} metric ${name}`) + } + + const record: RunRecord = { + runId: trace.id, + experimentId: options.experimentId, + candidateId: options.candidateId, + seed: options.seed, + model: options.model, + promptHash: options.promptHash, + configHash: options.configHash, + commitSha: options.commitSha, + wallMs: traceWallMs(trace), + costUsd: usage.costComplete ? usage.reportedCostUsd : 0, + costProvenance: usage.costComplete + ? { kind: 'observed', usd: usage.reportedCostUsd } + : { kind: 'uncaptured', usd: null }, + tokenUsage: { + input: usage.input, + output: usage.output, + ...(usage.reasoning !== undefined ? { reasoning: usage.reasoning } : {}), + ...(usage.cached !== undefined ? { cached: usage.cached } : {}), + }, + outcome: split === 'eval' ? { holdoutScore: reward, raw } : { searchScore: reward, raw }, + splitTag: split === 'eval' ? 'holdout' : 'search', + scenarioId: trace.task.data.name ?? String(trace.task.data.idx), + ...(errors[0] ? { failureMode: `primeintellect:${errors[0].type}:${errors[0].message}` } : {}), + } + return validateRunRecord(record) +} + +function validateTrace(value: unknown, line: number): PrimeIntellectTrace { + const trace = object(value, `PrimeIntellect trace line ${line}`) + nonEmptyString(trace.id, `PrimeIntellect trace line ${line}.id`) + const task = object(trace.task, `PrimeIntellect trace line ${line}.task`) + nonEmptyString(task.type, `PrimeIntellect trace line ${line}.task.type`) + const data = object(task.data, `PrimeIntellect trace line ${line}.task.data`) + if (!Number.isSafeInteger(data.idx) || (data.idx as number) < 0) { + throw new Error( + `PrimeIntellect trace line ${line}.task.data.idx must be a non-negative integer`, + ) + } + if (!Array.isArray(trace.nodes)) { + throw new Error(`PrimeIntellect trace line ${line}.nodes must be an array`) + } + for (const [index, rawNode] of trace.nodes.entries()) { + const node = object(rawNode, `PrimeIntellect trace line ${line}.nodes[${index}]`) + const parent = node.parent + if ( + parent !== undefined && + parent !== null && + (!Number.isSafeInteger(parent) || (parent as number) < 0 || (parent as number) >= index) + ) { + throw new Error( + `PrimeIntellect trace line ${line}.nodes[${index}].parent must reference an earlier node`, + ) + } + if (node.sampled !== undefined && typeof node.sampled !== 'boolean') { + throw new Error(`PrimeIntellect trace line ${line}.nodes[${index}].sampled must be boolean`) + } + if (node.usage !== undefined && node.usage !== null) { + validateUsage(node.usage, `PrimeIntellect trace line ${line}.nodes[${index}].usage`) + } + } + validateNumberMap(trace.rewards, `PrimeIntellect trace line ${line}.rewards`) + validateNumberMap(trace.metrics, `PrimeIntellect trace line ${line}.metrics`) + if (trace.extra_usage !== undefined && !Array.isArray(trace.extra_usage)) { + throw new Error(`PrimeIntellect trace line ${line}.extra_usage must be an array`) + } + for (const [index, usage] of (trace.extra_usage ?? []).entries()) { + validateUsage(usage, `PrimeIntellect trace line ${line}.extra_usage[${index}]`) + } + if (trace.errors !== undefined && !Array.isArray(trace.errors)) { + throw new Error(`PrimeIntellect trace line ${line}.errors must be an array`) + } + for (const [index, rawError] of (trace.errors ?? []).entries()) { + const error = object(rawError, `PrimeIntellect trace line ${line}.errors[${index}]`) + nonEmptyString(error.type, `PrimeIntellect trace line ${line}.errors[${index}].type`) + nonEmptyString(error.message, `PrimeIntellect trace line ${line}.errors[${index}].message`) + if ( + error.traceback !== undefined && + error.traceback !== null && + typeof error.traceback !== 'string' + ) { + throw new Error( + `PrimeIntellect trace line ${line}.errors[${index}].traceback must be a string or null`, + ) + } + } + if (trace.is_completed !== undefined && typeof trace.is_completed !== 'boolean') { + throw new Error(`PrimeIntellect trace line ${line}.is_completed must be boolean`) + } + if ( + trace.stop_condition !== undefined && + trace.stop_condition !== null && + typeof trace.stop_condition !== 'string' + ) { + throw new Error(`PrimeIntellect trace line ${line}.stop_condition must be a string or null`) + } + if (trace.timing !== undefined) validateTiming(trace.timing, line) + return value as PrimeIntellectTrace +} + +function validateTiming(value: unknown, line: number): void { + const timing = object(value, `PrimeIntellect trace line ${line}.timing`) + optionalFinite(timing.start, `PrimeIntellect trace line ${line}.timing.start`) + for (const phase of ['setup', 'generation', 'finalize', 'scoring']) { + if (timing[phase] === undefined) continue + const span = object(timing[phase], `PrimeIntellect trace line ${line}.timing.${phase}`) + optionalFinite(span.start, `PrimeIntellect trace line ${line}.timing.${phase}.start`) + optionalFinite(span.end, `PrimeIntellect trace line ${line}.timing.${phase}.end`) + } +} + +function aggregateUsage(usages: PrimeUsage[]): { + input: number + output: number + reasoning?: number + cached?: number + reportedCostUsd: number + costComplete: boolean +} { + let input = 0 + let output = 0 + let reasoning = 0 + let cached = 0 + let costUsd = 0 + let sawReasoning = false + let sawCached = false + let costsReported = 0 + for (const [index, usage] of usages.entries()) { + const prompt = nonNegative(usage.prompt_tokens, `usage[${index}].prompt_tokens`) + const completion = nonNegative(usage.completion_tokens, `usage[${index}].completion_tokens`) + const cachedInput = optionalNonNegative( + usage.cached_input_tokens, + `usage[${index}].cached_input_tokens`, + ) + const reasoningTokens = optionalNonNegative( + usage.reasoning_tokens, + `usage[${index}].reasoning_tokens`, + ) + const cost = optionalNonNegative(usage.cost, `usage[${index}].cost`) + input += prompt + (cachedInput ?? 0) + output += completion + if (cachedInput !== undefined) { + cached += cachedInput + sawCached = true + } + if (reasoningTokens !== undefined) { + reasoning += reasoningTokens + sawReasoning = true + } + if (cost !== undefined) { + costUsd += cost + costsReported += 1 + } + } + if (reasoning > output) { + throw new Error( + `PrimeIntellect reasoning token total ${reasoning} exceeds output total ${output}`, + ) + } + return { + input, + output, + ...(sawReasoning ? { reasoning } : {}), + ...(sawCached ? { cached } : {}), + reportedCostUsd: costUsd, + costComplete: usages.length > 0 && costsReported === usages.length, + } +} + +function validateUsage(value: unknown, path: string): void { + const usage = object(value, path) + nonNegative(usage.prompt_tokens, `${path}.prompt_tokens`) + nonNegative(usage.completion_tokens, `${path}.completion_tokens`) + optionalNonNegative(usage.cached_input_tokens, `${path}.cached_input_tokens`) + optionalNonNegative(usage.reasoning_tokens, `${path}.reasoning_tokens`) + optionalNonNegative(usage.cost, `${path}.cost`) +} + +function traceWallMs(trace: PrimeIntellectTrace): number { + const timing = trace.timing + if (!timing || typeof timing.start !== 'number' || !Number.isFinite(timing.start)) return 0 + const ends = [ + timing.setup?.end, + timing.generation?.end, + timing.finalize?.end, + timing.scoring?.end, + ].filter((value): value is number => typeof value === 'number' && Number.isFinite(value)) + if (ends.length === 0) return 0 + return Math.max(0, (Math.max(...ends) - timing.start) * 1000) +} + +function countBranches(nodes: PrimeTraceNode[]): number { + if (nodes.length === 0) return 0 + const parents = new Set( + nodes + .map((node) => node.parent) + .filter( + (parent): parent is number => Number.isSafeInteger(parent) && (parent as number) >= 0, + ), + ) + return nodes.reduce((count, _node, index) => count + (parents.has(index) ? 0 : 1), 0) +} + +function validateNumberMap(value: unknown, path: string): void { + const map = object(value, path) + for (const [key, entry] of Object.entries(map)) finite(entry, `${path}.${key}`) +} + +function sumFinite(values: unknown[], path: string): number { + return values.reduce((sum, value, index) => sum + finite(value, `${path}[${index}]`), 0) +} + +function finite(value: unknown, path: string): number { + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`${path} must be a finite number`) + } + return value +} + +function nonNegative(value: unknown, path: string): number { + const parsed = finite(value, path) + if (parsed < 0) throw new Error(`${path} must be non-negative`) + return parsed +} + +function optionalNonNegative(value: unknown, path: string): number | undefined { + if (value === undefined || value === null) return undefined + return nonNegative(value, path) +} + +function optionalFinite(value: unknown, path: string): number | undefined { + if (value === undefined || value === null) return undefined + return finite(value, path) +} + +function object(value: unknown, path: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be an object`) + } + return value as Record +} + +function nonEmptyString(value: unknown, path: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${path} must be a non-empty string`) + } + return value +} diff --git a/src/primeintellect/types.ts b/src/primeintellect/types.ts new file mode 100644 index 00000000..f3df50c6 --- /dev/null +++ b/src/primeintellect/types.ts @@ -0,0 +1,124 @@ +export type PrimeIntellectSplit = 'train' | 'eval' + +export type PrimeIntellectJson = + | null + | boolean + | number + | string + | PrimeIntellectJson[] + | { [key: string]: PrimeIntellectJson } + +export type PrimeIntellectContent = + | string + | Array<{ type: 'text'; text: string } | { type: 'image_url'; image_url: { url: string } }> + +export type PrimeIntellectMessage = + | { role: 'system' | 'user'; content: PrimeIntellectContent } + | { + role: 'assistant' + content?: string | null + reasoning_content?: string | null + tool_calls?: Array<{ id: string; name: string; arguments: string }> + provider_state?: Array> + } + | { + role: 'tool' + tool_call_id: string + content: PrimeIntellectContent + name?: string + } + +/** One immutable problem. References stay inside Prime's task process. */ +export interface PrimeIntellectTask { + id: string + split: PrimeIntellectSplit + prompt: string | PrimeIntellectMessage[] + systemPrompt?: string + answer?: string | string[] + metadata?: Record +} + +export type PrimeIntellectScoring = + | { + kind: 'exact' + normalization?: 'none' | 'trim' | 'trim-casefold' + } + | { + kind: 'reference-judge' + model: string + prompt?: string + view?: 'last_reply' | 'full_trace' + } + | { + kind: 'command' + command: readonly [string, ...string[]] + files?: Readonly> + forwardEnv?: readonly string[] + timeoutSeconds?: number + } + +export type PrimeIntellectSetupCommand = readonly [string, ...string[]] + +/** Files and commands that make the caller's real agent program runnable. */ +export interface PrimeIntellectRunner { + command: readonly [string, ...string[]] + files?: Readonly> + setup?: readonly PrimeIntellectSetupCommand[] + forwardEnv?: readonly string[] + /** Container image used by the generated eval config. */ + image: string +} + +export interface PrimeIntellectPackageOptions { + name: string + version: string + description?: string + tasks: readonly PrimeIntellectTask[] + scoring: PrimeIntellectScoring + runner: PrimeIntellectRunner + /** Prime-enforced model turn cap. Default 16. */ + maxTurns?: number + maxInputTokens?: number + maxOutputTokens?: number + maxTotalTokens?: number + rolloutTimeoutSeconds?: number + scoringTimeoutSeconds?: number +} + +export interface PrimeIntellectPackageManifest { + schema: 'tangle.primeintellect.package/v1' + name: string + moduleName: string + version: string + verifiers: '>=0.2.0,<0.3.0' + taskCount: number + splits: Record + taskIdsSha256: string + filesSha256: Record +} + +export interface PrimeIntellectPackageBundle { + manifest: PrimeIntellectPackageManifest + /** Relative package path to UTF-8 contents. */ + files: Readonly> +} + +/** The answer-free task exposed to the caller's runtime program. */ +export interface PrimeIntellectPublicTask { + id: string + split: PrimeIntellectSplit + prompt: string | PrimeIntellectMessage[] + systemPrompt?: string + metadata?: Record +} + +export interface PrimeIntellectEpisodeContext { + protocol: 'tangle.primeintellect.episode/v1' + task: PrimeIntellectPublicTask + model: { + name: string + baseUrl: string + apiKey: string + } + mcpServers: Readonly> +} diff --git a/tsup.config.ts b/tsup.config.ts index 100602eb..19747b40 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -12,6 +12,7 @@ export default defineConfig({ knowledge: 'src/knowledge/index.ts', profiles: 'src/profiles/index.ts', platform: 'src/platform/index.ts', + 'primeintellect/index': 'src/primeintellect/index.ts', 'candidate-execution/index': 'src/candidate-execution/index.ts', 'mcp/index': 'src/mcp/index.ts', 'mcp/bin': 'src/mcp/bin.ts', diff --git a/typedoc.json b/typedoc.json index 363a14f7..b4319cf1 100644 --- a/typedoc.json +++ b/typedoc.json @@ -11,6 +11,7 @@ "src/knowledge/index.ts", "src/profiles/index.ts", "src/platform/index.ts", + "src/primeintellect/index.ts", "src/candidate-execution/index.ts", "src/mcp/index.ts" ], From 06ae27da00342d3593403d2575fe1b9a4bb3b97a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 14 Jul 2026 23:24:23 -0600 Subject: [PATCH 2/2] refactor(primeintellect): share message schema validation --- docs/api/primeintellect.md | 20 ++-- src/primeintellect/package.ts | 155 ++---------------------------- src/primeintellect/runner.ts | 148 ++--------------------------- src/primeintellect/validation.ts | 156 +++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 298 deletions(-) create mode 100644 src/primeintellect/validation.ts diff --git a/docs/api/primeintellect.md b/docs/api/primeintellect.md index c556ed81..45dd6c11 100644 --- a/docs/api/primeintellect.md +++ b/docs/api/primeintellect.md @@ -10,7 +10,7 @@ ### WritePrimeIntellectPackageOptions -Defined in: [primeintellect/package.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L22) +Defined in: [primeintellect/package.ts:23](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L23) #### Properties @@ -18,7 +18,7 @@ Defined in: [primeintellect/package.ts:22](https://github.com/tangle-network/age > `optional` **replace?**: `boolean` -Defined in: [primeintellect/package.ts:24](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L24) +Defined in: [primeintellect/package.ts:25](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L25) Replace an existing generated package and restore it if the final swap fails. @@ -26,7 +26,7 @@ Replace an existing generated package and restore it if the final swap fails. ### RunPrimeIntellectProgramOptions -Defined in: [primeintellect/runner.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L18) +Defined in: [primeintellect/runner.ts:17](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L17) #### Properties @@ -34,7 +34,7 @@ Defined in: [primeintellect/runner.ts:18](https://github.com/tangle-network/agen > `optional` **env?**: `ProcessEnv` -Defined in: [primeintellect/runner.ts:19](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L19) +Defined in: [primeintellect/runner.ts:18](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L18) *** @@ -580,7 +580,7 @@ Defined in: [primeintellect/types.ts:123](https://github.com/tangle-network/agen > **PrimeIntellectBackendOptions** = `Omit`\<`Parameters`\<*typeof* [`createOpenAICompatibleBackend`](index.md#createopenaicompatiblebackend)\>\[`0`\], `"apiKey"` \| `"baseUrl"` \| `"model"`\> -Defined in: [primeintellect/runner.ts:22](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L22) +Defined in: [primeintellect/runner.ts:21](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L21) *** @@ -644,7 +644,7 @@ Defined in: [primeintellect/types.ts:60](https://github.com/tangle-network/agent > **createPrimeIntellectPackage**(`options`): [`PrimeIntellectPackageBundle`](#primeintellectpackagebundle) -Defined in: [primeintellect/package.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L28) +Defined in: [primeintellect/package.ts:29](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L29) Build a complete PrimeIntellect Verifiers v1 package without writing to disk. @@ -664,7 +664,7 @@ Build a complete PrimeIntellect Verifiers v1 package without writing to disk. > **writePrimeIntellectPackage**(`bundle`, `outputDirectory`, `options?`): `Promise`\<`string`\> -Defined in: [primeintellect/package.ts:89](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L89) +Defined in: [primeintellect/package.ts:90](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/package.ts#L90) Write a bundle through a sibling temporary directory, then rename it into place. @@ -692,7 +692,7 @@ Write a bundle through a sibling temporary directory, then rename it into place. > **readPrimeIntellectEpisodeContext**(`env?`): [`PrimeIntellectEpisodeContext`](#primeintellectepisodecontext) -Defined in: [primeintellect/runner.ts:28](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L28) +Defined in: [primeintellect/runner.ts:27](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L27) Read and validate the private process contract installed by the generated Prime harness. @@ -712,7 +712,7 @@ Read and validate the private process contract installed by the generated Prime > **createPrimeIntellectBackend**(`context`, `options?`): [`AgentExecutionBackend`](index.md#agentexecutionbackend)\<[`AgentBackendInput`](index.md#agentbackendinput)\> -Defined in: [primeintellect/runner.ts:51](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L51) +Defined in: [primeintellect/runner.ts:50](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L50) Build the existing runtime backend against Prime's intercepted model endpoint. @@ -736,7 +736,7 @@ Build the existing runtime backend against Prime's intercepted model endpoint. > **runPrimeIntellectProgram**\<`Result`\>(`run`, `options?`): `Promise`\<`Result`\> -Defined in: [primeintellect/runner.ts:68](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L68) +Defined in: [primeintellect/runner.ts:67](https://github.com/tangle-network/agent-runtime/blob/main/src/primeintellect/runner.ts#L67) Execute the caller's canonical runtime program inside a Prime rollout. The callback may call runPersonified, runAgentic, runLoop, or any product wrapper. diff --git a/src/primeintellect/package.ts b/src/primeintellect/package.ts index 98e52428..9e5a6122 100644 --- a/src/primeintellect/package.ts +++ b/src/primeintellect/package.ts @@ -10,6 +10,7 @@ import type { PrimeIntellectScoring, PrimeIntellectTask, } from './types' +import { validatePrimeIntellectJson, validatePrimeIntellectPrompt } from './validation' const VERIFIERS_RANGE = '>=0.2.0,<0.3.0' as const const ENV_NAME = /^[A-Z_][A-Z0-9_]*$/ @@ -210,26 +211,18 @@ function validateTask( if (task.split !== 'train' && task.split !== 'eval') { throw new Error(`${path}.split must be train or eval`) } - if (typeof task.prompt === 'string') { - if (task.prompt.length === 0) throw new Error(`${path}.prompt must not be empty`) - } else if (!Array.isArray(task.prompt) || task.prompt.length === 0) { - throw new Error(`${path}.prompt must be a non-empty string or message array`) - } else { - task.prompt.forEach((message, messageIndex) => { - validateMessage(message, `${path}.prompt[${messageIndex}]`) - }) - if ( - task.systemPrompt !== undefined && - task.prompt.some((message) => message.role === 'system') - ) { + const prompt = validatePrimeIntellectPrompt(task.prompt, `${path}.prompt`) + if (Array.isArray(prompt)) { + if (task.systemPrompt !== undefined && prompt.some((message) => message.role === 'system')) { throw new Error(`${path} must not set systemPrompt and include a system message`) } } - validateJson(task.prompt, `${path}.prompt`) if (task.systemPrompt !== undefined && typeof task.systemPrompt !== 'string') { throw new Error(`${path}.systemPrompt must be a string`) } - if (task.metadata !== undefined) validateJson(task.metadata, `${path}.metadata`) + if (task.metadata !== undefined) { + validatePrimeIntellectJson(task.metadata, `${path}.metadata`) + } if (scoring.kind !== 'command') { const answers = Array.isArray(task.answer) ? task.answer : [task.answer] if ( @@ -296,140 +289,6 @@ function assertRelativePath(path: string, label: string): void { } } -function validateJson(value: unknown, path: string): void { - if (value === null || ['string', 'boolean'].includes(typeof value)) return - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`) - return - } - if (Array.isArray(value)) { - value.forEach((entry, index) => { - validateJson(entry, `${path}[${index}]`) - }) - return - } - if (typeof value === 'object') { - for (const [key, entry] of Object.entries(value)) validateJson(entry, `${path}.${key}`) - return - } - throw new Error(`${path} is not JSON serializable`) -} - -function validateMessage(value: unknown, path: string): void { - if (value === null || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`${path} must be a message object`) - } - const message = value as Record - const role = message.role - if (!['system', 'user', 'assistant', 'tool'].includes(String(role))) { - throw new Error(`${path}.role is invalid`) - } - if (role === 'system' || role === 'user') { - assertOnlyKeys(message, ['role', 'content'], path) - validateContent(message.content, `${path}.content`) - } else if (role === 'assistant') { - assertOnlyKeys( - message, - ['role', 'content', 'reasoning_content', 'tool_calls', 'provider_state'], - path, - ) - if ( - message.content !== undefined && - message.content !== null && - typeof message.content !== 'string' - ) { - throw new Error(`${path}.content must be a string or null`) - } - if ( - message.reasoning_content !== undefined && - message.reasoning_content !== null && - typeof message.reasoning_content !== 'string' - ) { - throw new Error(`${path}.reasoning_content must be a string or null`) - } - if (message.tool_calls !== undefined) { - if (!Array.isArray(message.tool_calls)) throw new Error(`${path}.tool_calls must be an array`) - for (const [index, rawCall] of message.tool_calls.entries()) { - if (rawCall === null || typeof rawCall !== 'object' || Array.isArray(rawCall)) { - throw new Error(`${path}.tool_calls[${index}] must be an object`) - } - const call = rawCall as Record - assertOnlyKeys(call, ['id', 'name', 'arguments'], `${path}.tool_calls[${index}]`) - for (const field of ['id', 'name', 'arguments']) { - if (typeof call[field] !== 'string' || call[field].length === 0) { - throw new Error(`${path}.tool_calls[${index}].${field} must be a non-empty string`) - } - } - } - } - if (message.provider_state !== undefined) { - validateProviderState(message.provider_state, `${path}.provider_state`) - } - } else { - assertOnlyKeys(message, ['role', 'tool_call_id', 'content', 'name'], path) - if (typeof message.tool_call_id !== 'string' || message.tool_call_id.length === 0) { - throw new Error(`${path}.tool_call_id must be a non-empty string`) - } - if (message.name !== undefined && typeof message.name !== 'string') { - throw new Error(`${path}.name must be a string`) - } - validateContent(message.content, `${path}.content`) - } -} - -function validateProviderState(value: unknown, path: string): void { - if (!Array.isArray(value)) throw new Error(`${path} must be an array`) - for (const [index, entry] of value.entries()) { - if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) { - throw new Error(`${path}[${index}] must be an object`) - } - validateJson(entry, `${path}[${index}]`) - } -} - -function validateContent(value: unknown, path: string): void { - if (typeof value === 'string') return - if (!Array.isArray(value) || value.length === 0) { - throw new Error(`${path} must be a string or non-empty content array`) - } - for (const [index, rawPart] of value.entries()) { - if (rawPart === null || typeof rawPart !== 'object' || Array.isArray(rawPart)) { - throw new Error(`${path}[${index}] must be a content object`) - } - const part = rawPart as Record - if (part.type === 'text' && typeof part.text === 'string') { - assertOnlyKeys(part, ['type', 'text'], `${path}[${index}]`) - continue - } - if ( - part.type === 'image_url' && - part.image_url !== null && - typeof part.image_url === 'object' && - !Array.isArray(part.image_url) && - typeof (part.image_url as Record).url === 'string' - ) { - assertOnlyKeys(part, ['type', 'image_url'], `${path}[${index}]`) - assertOnlyKeys( - part.image_url as Record, - ['url'], - `${path}[${index}].image_url`, - ) - continue - } - throw new Error(`${path}[${index}] is not a supported text or image_url content part`) - } -} - -function assertOnlyKeys( - value: Record, - allowed: readonly string[], - path: string, -): void { - for (const key of Object.keys(value)) { - if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`) - } -} - function taskRow(task: PrimeIntellectTask, idx: number): Record { return { idx, diff --git a/src/primeintellect/runner.ts b/src/primeintellect/runner.ts index 285abe91..2cc78960 100644 --- a/src/primeintellect/runner.ts +++ b/src/primeintellect/runner.ts @@ -1,11 +1,10 @@ import { createOpenAICompatibleBackend } from '../backends' import type { PrimeIntellectEpisodeContext, - PrimeIntellectJson, - PrimeIntellectMessage, PrimeIntellectPublicTask, PrimeIntellectSplit, } from './types' +import { validatePrimeIntellectJsonObject, validatePrimeIntellectPrompt } from './validation' const ENV = { task: 'TANGLE_PRIME_TASK_JSON', @@ -97,9 +96,14 @@ function validatePublicTask(value: unknown): PrimeIntellectPublicTask { throw new Error(`${ENV.task} exposed private field ${privateField}`) } } + for (const field of Object.keys(task)) { + if (!['id', 'split', 'prompt', 'systemPrompt', 'metadata'].includes(field)) { + throw new Error(`${ENV.task}.${field} is not supported`) + } + } const id = nonEmptyString(task.id, `${ENV.task}.id`) const split = validateSplit(task.split, `${ENV.task}.split`) - const prompt = validatePrompt(task.prompt, `${ENV.task}.prompt`) + const prompt = validatePrimeIntellectPrompt(task.prompt, `${ENV.task}.prompt`) const systemPrompt = optionalString(task.systemPrompt, `${ENV.task}.systemPrompt`) if ( systemPrompt !== undefined && @@ -111,10 +115,7 @@ function validatePublicTask(value: unknown): PrimeIntellectPublicTask { const metadata = task.metadata === undefined ? undefined - : (validateJsonObject(task.metadata, `${ENV.task}.metadata`) as Record< - string, - PrimeIntellectJson - >) + : validatePrimeIntellectJsonObject(task.metadata, `${ENV.task}.metadata`) return { id, split, @@ -124,114 +125,6 @@ function validatePublicTask(value: unknown): PrimeIntellectPublicTask { } } -function validatePrompt(value: unknown, path: string): string | PrimeIntellectMessage[] { - if (typeof value === 'string' && value.length > 0) return value - if (!Array.isArray(value) || value.length === 0) { - throw new Error(`${path} must be a non-empty string or message array`) - } - for (const [index, message] of value.entries()) { - const item = record(message, `${path}[${index}]`) - const role = item.role - if (!['system', 'user', 'assistant', 'tool'].includes(String(role))) { - throw new Error(`${path}[${index}].role is invalid`) - } - if (role === 'system' || role === 'user') { - assertOnlyKeys(item, ['role', 'content'], `${path}[${index}]`) - validateContent(item.content, `${path}[${index}].content`) - } else if (role === 'assistant') { - assertOnlyKeys( - item, - ['role', 'content', 'reasoning_content', 'tool_calls', 'provider_state'], - `${path}[${index}]`, - ) - if (item.content !== undefined && item.content !== null && typeof item.content !== 'string') { - throw new Error(`${path}[${index}].content must be a string or null`) - } - if ( - item.reasoning_content !== undefined && - item.reasoning_content !== null && - typeof item.reasoning_content !== 'string' - ) { - throw new Error(`${path}[${index}].reasoning_content must be a string or null`) - } - if (item.tool_calls !== undefined) { - validateToolCalls(item.tool_calls, `${path}[${index}].tool_calls`) - } - if (item.provider_state !== undefined) { - validateProviderState(item.provider_state, `${path}[${index}].provider_state`) - } - } else { - assertOnlyKeys(item, ['role', 'tool_call_id', 'content', 'name'], `${path}[${index}]`) - nonEmptyString(item.tool_call_id, `${path}[${index}].tool_call_id`) - if (item.name !== undefined && typeof item.name !== 'string') { - throw new Error(`${path}[${index}].name must be a string`) - } - validateContent(item.content, `${path}[${index}].content`) - } - validateJson(item, `${path}[${index}]`) - } - return value as PrimeIntellectMessage[] -} - -function validateToolCalls(value: unknown, path: string): void { - if (!Array.isArray(value)) throw new Error(`${path} must be an array`) - for (const [index, rawCall] of value.entries()) { - const call = record(rawCall, `${path}[${index}]`) - assertOnlyKeys(call, ['id', 'name', 'arguments'], `${path}[${index}]`) - for (const field of ['id', 'name', 'arguments']) { - nonEmptyString(call[field], `${path}[${index}].${field}`) - } - } -} - -function validateProviderState(value: unknown, path: string): void { - if (!Array.isArray(value)) throw new Error(`${path} must be an array`) - for (const [index, rawState] of value.entries()) { - const state = record(rawState, `${path}[${index}]`) - validateJson(state, `${path}[${index}]`) - } -} - -function validateContent(value: unknown, path: string): void { - if (typeof value === 'string') return - if (!Array.isArray(value) || value.length === 0) { - throw new Error(`${path} must be a string or non-empty content array`) - } - for (const [index, rawPart] of value.entries()) { - const part = record(rawPart, `${path}[${index}]`) - if (part.type === 'text' && typeof part.text === 'string') { - assertOnlyKeys(part, ['type', 'text'], `${path}[${index}]`) - continue - } - if ( - part.type === 'image_url' && - part.image_url !== null && - typeof part.image_url === 'object' && - !Array.isArray(part.image_url) && - typeof (part.image_url as Record).url === 'string' - ) { - assertOnlyKeys(part, ['type', 'image_url'], `${path}[${index}]`) - assertOnlyKeys( - part.image_url as Record, - ['url'], - `${path}[${index}].image_url`, - ) - continue - } - throw new Error(`${path}[${index}] is not a supported text or image_url content part`) - } -} - -function assertOnlyKeys( - value: Record, - allowed: readonly string[], - path: string, -): void { - for (const key of Object.keys(value)) { - if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`) - } -} - function validateSplit(value: unknown, path: string): PrimeIntellectSplit { if (value !== 'train' && value !== 'eval') { throw new Error(`${path} must be train or eval`) @@ -251,31 +144,6 @@ function validateStringMap(value: unknown, path: string): Readonly { - const output = record(value, path) - validateJson(output, path) - return output as Record -} - -function validateJson(value: unknown, path: string): void { - if (value === null || ['string', 'boolean'].includes(typeof value)) return - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`) - return - } - if (Array.isArray(value)) { - value.forEach((entry, index) => { - validateJson(entry, `${path}[${index}]`) - }) - return - } - if (typeof value === 'object') { - for (const [key, entry] of Object.entries(value)) validateJson(entry, `${path}.${key}`) - return - } - throw new Error(`${path} is not JSON serializable`) -} - function record(value: unknown, path: string): Record { if (value === null || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`${path} must be an object`) diff --git a/src/primeintellect/validation.ts b/src/primeintellect/validation.ts new file mode 100644 index 00000000..53a60b26 --- /dev/null +++ b/src/primeintellect/validation.ts @@ -0,0 +1,156 @@ +import type { PrimeIntellectJson, PrimeIntellectMessage } from './types' + +/** Validate the Verifiers v1 prompt schema shared by package creation and runner input. */ +export function validatePrimeIntellectPrompt( + value: unknown, + path: string, +): string | PrimeIntellectMessage[] { + if (typeof value === 'string' && value.length > 0) return value + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a non-empty string or message array`) + } + for (const [index, message] of value.entries()) { + validateMessage(message, `${path}[${index}]`) + } + return value as PrimeIntellectMessage[] +} + +export function validatePrimeIntellectJson(value: unknown, path: string): void { + if (value === null || ['string', 'boolean'].includes(typeof value)) return + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new Error(`${path} contains a non-finite number`) + return + } + if (Array.isArray(value)) { + value.forEach((entry, index) => { + validatePrimeIntellectJson(entry, `${path}[${index}]`) + }) + return + } + if (typeof value === 'object') { + for (const [key, entry] of Object.entries(value)) { + validatePrimeIntellectJson(entry, `${path}.${key}`) + } + return + } + throw new Error(`${path} is not JSON serializable`) +} + +export function validatePrimeIntellectJsonObject( + value: unknown, + path: string, +): Record { + const output = record(value, path) + validatePrimeIntellectJson(output, path) + return output as Record +} + +function validateMessage(value: unknown, path: string): void { + const message = record(value, path) + const role = message.role + if (!['system', 'user', 'assistant', 'tool'].includes(String(role))) { + throw new Error(`${path}.role is invalid`) + } + if (role === 'system' || role === 'user') { + assertOnlyKeys(message, ['role', 'content'], path) + validateContent(message.content, `${path}.content`) + } else if (role === 'assistant') { + assertOnlyKeys( + message, + ['role', 'content', 'reasoning_content', 'tool_calls', 'provider_state'], + path, + ) + optionalNullableString(message.content, `${path}.content`) + optionalNullableString(message.reasoning_content, `${path}.reasoning_content`) + if (message.tool_calls !== undefined) { + validateToolCalls(message.tool_calls, `${path}.tool_calls`) + } + if (message.provider_state !== undefined) { + validateProviderState(message.provider_state, `${path}.provider_state`) + } + } else { + assertOnlyKeys(message, ['role', 'tool_call_id', 'content', 'name'], path) + nonEmptyString(message.tool_call_id, `${path}.tool_call_id`) + if (message.name !== undefined && typeof message.name !== 'string') { + throw new Error(`${path}.name must be a string`) + } + validateContent(message.content, `${path}.content`) + } + validatePrimeIntellectJson(message, path) +} + +function validateToolCalls(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, rawCall] of value.entries()) { + const call = record(rawCall, `${path}[${index}]`) + assertOnlyKeys(call, ['id', 'name', 'arguments'], `${path}[${index}]`) + for (const field of ['id', 'name', 'arguments']) { + nonEmptyString(call[field], `${path}[${index}].${field}`) + } + } +} + +function validateProviderState(value: unknown, path: string): void { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`) + for (const [index, rawState] of value.entries()) { + const state = record(rawState, `${path}[${index}]`) + validatePrimeIntellectJson(state, `${path}[${index}]`) + } +} + +function validateContent(value: unknown, path: string): void { + if (typeof value === 'string') return + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${path} must be a string or non-empty content array`) + } + for (const [index, rawPart] of value.entries()) { + const part = record(rawPart, `${path}[${index}]`) + if (part.type === 'text' && typeof part.text === 'string') { + assertOnlyKeys(part, ['type', 'text'], `${path}[${index}]`) + continue + } + const image = part.image_url + if ( + part.type === 'image_url' && + image !== null && + typeof image === 'object' && + !Array.isArray(image) && + typeof (image as Record).url === 'string' + ) { + assertOnlyKeys(part, ['type', 'image_url'], `${path}[${index}]`) + assertOnlyKeys(image as Record, ['url'], `${path}[${index}].image_url`) + continue + } + throw new Error(`${path}[${index}] is not a supported text or image_url content part`) + } +} + +function optionalNullableString(value: unknown, path: string): void { + if (value !== undefined && value !== null && typeof value !== 'string') { + throw new Error(`${path} must be a string or null`) + } +} + +function nonEmptyString(value: unknown, path: string): string { + if (typeof value !== 'string' || value.length === 0) { + throw new Error(`${path} must be a non-empty string`) + } + return value +} + +function record(value: unknown, path: string): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${path} must be an object`) + } + return value as Record +} + +function assertOnlyKeys( + value: Record, + allowed: readonly string[], + path: string, +): void { + for (const key of Object.keys(value)) { + if (!allowed.includes(key)) throw new Error(`${path}.${key} is not supported`) + } +}