diff --git a/CHANGELOG.md b/CHANGELOG.md index 16b0fd81..159367b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## Unreleased + +### Breaking + +- Removed `@tangle-network/agent-eval/primeintellect`, which generated a legacy single-turn Verifiers package with unsafe substring scoring. Use `@tangle-network/agent-runtime/primeintellect` for Verifiers v1 task packaging, real runtime execution, and full trace import. + ## [0.119.1] — 2026-07-14 — portable improvement evidence ### Added diff --git a/README.md b/README.md index 6f5ef38a..300ce7cd 100644 --- a/README.md +++ b/README.md @@ -95,31 +95,6 @@ const traceRuns = fromOtelSpans({ spans: yourOtelSpans }) await analyzeRuns({ runs: [...runs, ...traceRuns], raterScores }) ``` -### 4. Export to PrimeIntellect - -Use this when you want PrimeIntellect Verifiers to run or train against tasks produced by Tangle evals, runtime runs, or knowledge runs. - -```ts -import { - primeIntellectRowsFromRunRecords, - writePrimeIntellectEnvironmentPackage, -} from '@tangle-network/agent-eval/primeintellect' - -const rows = primeIntellectRowsFromRunRecords({ - records, // RunRecord[] with scenarioId - scenarios, // prompts plus answer or requiredSubstrings -}) - -await writePrimeIntellectEnvironmentPackage('prime-envs/refund-policy', { - name: 'refund-policy', - rows, - runRecords: records, -}) -``` - -The generated package contains `load_environment()`, `data/dataset.jsonl`, `data/run_records.jsonl`, and a manifest. -Run it with `uv run vf-eval refund-policy` or upload it with `prime env push`. - --- ## Core concepts @@ -149,7 +124,6 @@ Each example: `README.md` + a single `index.ts` runnable via `pnpm tsx`. Prints |---|---| | `…/contract` | **The headline, frozen surface — new code starts here.** `defineAgentEval`, `selfImprove`, `analyzeRuns`, `runEval`, `runCampaign`, `runImprovementLoop`, `diffRuns`; intake adapters (`fromFeedbackTable`, `fromOtelSpans`); proposers (`gepaProposer`, `evolutionaryProposer`); gates (`defaultProductionGate`, `heldOutGate`, `paretoSignificanceGate`, `neutralizationGate`, `composeGate`); the deployment-outcome store; storage; and the five core types `Scenario` / `Dispatch` / `JudgeConfig` / `SurfaceProposer` / `Gate`. | | `…/hosted` | `createHostedClient` / `hostedClientFromEnv` + the wire types to ship eval-run events + trace spans to a hosted orchestrator (ours or your own implementation of the spec) | -| `…/primeintellect` | Export validated `RunRecord`s plus scenario prompts into a PrimeIntellect Verifiers package: Python `load_environment()`, `data/dataset.jsonl`, `data/run_records.jsonl`, and export manifest | | `…/adapters/otel` | `createOtelBridge` — forwards OpenTelemetry-shape spans into the hosted-tier ingest, no `@opentelemetry/*` dependency | | `…/adapters/langchain` | Wrap any LangChain `Runnable` as a `Dispatch` (or `JudgeConfig`), no `@langchain/core` peer dep | | `…/adapters/http` | `httpDispatch` + `runDispatchServer` — run a campaign's worker on another machine (multi-region, remote worker execution) | diff --git a/package.json b/package.json index de6f2d00..e826c0a5 100644 --- a/package.json +++ b/package.json @@ -119,11 +119,6 @@ "import": "./dist/hosted/index.js", "default": "./dist/hosted/index.js" }, - "./primeintellect": { - "types": "./dist/primeintellect/index.d.ts", - "import": "./dist/primeintellect/index.js", - "default": "./dist/primeintellect/index.js" - }, "./openapi.json": { "default": "./dist/openapi.json" } diff --git a/scripts/build-entries.mjs b/scripts/build-entries.mjs index 0039e32a..8c956da7 100644 --- a/scripts/build-entries.mjs +++ b/scripts/build-entries.mjs @@ -20,6 +20,5 @@ export const buildEntries = { 'belief-state/index': 'src/belief-state/index.ts', 'contract/index': 'src/contract/index.ts', 'hosted/index': 'src/hosted/index.ts', - 'primeintellect/index': 'src/primeintellect/index.ts', cli: 'src/cli.ts', } diff --git a/src/primeintellect/index.ts b/src/primeintellect/index.ts deleted file mode 100644 index 61a63195..00000000 --- a/src/primeintellect/index.ts +++ /dev/null @@ -1,464 +0,0 @@ -import { mkdir, writeFile } from 'node:fs/promises' -import { dirname, join } from 'node:path' -import { type RunRecord, validateRunRecord } from '../run-record' - -export type PrimeIntellectJson = - | null - | boolean - | number - | string - | PrimeIntellectJson[] - | { [key: string]: PrimeIntellectJson } - -export interface PrimeIntellectMessage { - role: 'system' | 'user' | 'assistant' | 'tool' - content: string -} - -export interface PrimeIntellectScenario { - id: string - prompt: string | readonly PrimeIntellectMessage[] - answer?: string - requiredSubstrings?: readonly string[] - split?: string - info?: Record -} - -export interface PrimeIntellectDatasetRow { - id: string - prompt: PrimeIntellectMessage[] - answer?: string - required_substrings?: string[] - split?: string - info: Record -} - -export type PrimeIntellectScenarioMap = - | ReadonlyMap - | Record - | readonly PrimeIntellectScenario[] - -export interface PrimeIntellectRowsFromRunRecordsOptions { - records: readonly RunRecord[] - scenarios: PrimeIntellectScenarioMap - requireScorable?: boolean - includeFailed?: boolean -} - -export interface PrimeIntellectEnvironmentPackageInput { - name: string - version?: string - description?: string - tags?: readonly string[] - moduleName?: string - rows: readonly PrimeIntellectDatasetRow[] - runRecords?: readonly RunRecord[] - systemPrompt?: string - verifiersVersion?: string - dependencies?: readonly string[] - readme?: string -} - -export interface PrimeIntellectPackageFile { - path: string - content: string -} - -export interface PrimeIntellectPackageManifest { - schemaVersion: 1 - name: string - moduleName: string - version: string - rowCount: number - runRecordCount: number - artifactKinds: readonly ['environment', 'dataset', 'run_records'] -} - -export interface PrimeIntellectEnvironmentPackage { - files: PrimeIntellectPackageFile[] - manifest: PrimeIntellectPackageManifest -} - -export class PrimeIntellectBridgeError extends Error { - constructor(message: string) { - super(message) - this.name = 'PrimeIntellectBridgeError' - } -} - -export function primeIntellectRowsFromRunRecords( - options: PrimeIntellectRowsFromRunRecordsOptions, -): PrimeIntellectDatasetRow[] { - const scenarioById = normalizeScenarios(options.scenarios) - const requireScorable = options.requireScorable ?? true - const includeFailed = options.includeFailed ?? true - const rows: PrimeIntellectDatasetRow[] = [] - - for (const record of options.records) { - const validRecord = validateRunRecord(record) - if (!includeFailed && validRecord.failureMode) continue - - const scenarioId = validRecord.scenarioId - if (!scenarioId) { - throw new PrimeIntellectBridgeError( - `RunRecord ${validRecord.runId} is missing scenarioId; PrimeIntellect rows need stable task ids`, - ) - } - - const scenario = scenarioById.get(scenarioId) - if (!scenario) { - throw new PrimeIntellectBridgeError( - `RunRecord ${validRecord.runId} references unknown scenarioId ${JSON.stringify(scenarioId)}`, - ) - } - - const hasAnswer = typeof scenario.answer === 'string' && scenario.answer.trim().length > 0 - const hasRequiredSubstrings = (scenario.requiredSubstrings?.length ?? 0) > 0 - if (requireScorable && !hasAnswer && !hasRequiredSubstrings) { - throw new PrimeIntellectBridgeError( - `Scenario ${scenario.id} has no answer or requiredSubstrings; generated environments would always score 0`, - ) - } - - const score = validRecord.outcome.holdoutScore ?? validRecord.outcome.searchScore ?? null - const split = scenario.split ?? validRecord.splitTag - const tangleInfo = { - run_id: validRecord.runId, - experiment_id: validRecord.experimentId, - candidate_id: validRecord.candidateId, - scenario_id: scenarioId, - split, - score, - cost_usd: validRecord.costUsd, - wall_ms: validRecord.wallMs, - failure_mode: validRecord.failureMode ?? null, - model: validRecord.model, - commit_sha: validRecord.commitSha, - } satisfies Record - - rows.push({ - id: `${scenarioId}:${validRecord.runId}`, - prompt: normalizePrompt(scenario.prompt), - ...(hasAnswer ? { answer: scenario.answer } : {}), - ...(hasRequiredSubstrings - ? { required_substrings: [...(scenario.requiredSubstrings ?? [])] } - : {}), - split, - info: { - ...(scenario.info ?? {}), - tangle: tangleInfo, - }, - }) - } - - return rows -} - -export function buildPrimeIntellectEnvironmentPackage( - input: PrimeIntellectEnvironmentPackageInput, -): PrimeIntellectEnvironmentPackage { - if (input.rows.length === 0) { - throw new PrimeIntellectBridgeError('cannot build a PrimeIntellect environment with zero rows') - } - - for (const record of input.runRecords ?? []) { - validateRunRecord(record) - } - - const moduleName = input.moduleName ?? toPythonModuleName(input.name) - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(moduleName)) { - throw new PrimeIntellectBridgeError( - `moduleName ${JSON.stringify(moduleName)} is not a valid Python module name`, - ) - } - - const version = input.version ?? '0.1.0' - const description = - input.description ?? 'PrimeIntellect Verifiers environment exported from Tangle runs.' - const verifiersVersion = input.verifiersVersion ?? '>=0.2.0,<0.3.0' - const dependencies = unique([ - 'datasets', - `verifiers${verifiersVersion}`, - ...(input.dependencies ?? []), - ]) - const tags = unique(['tangle', 'agent-eval', ...(input.tags ?? [])]) - const manifest: PrimeIntellectPackageManifest = { - schemaVersion: 1, - name: input.name, - moduleName, - version, - rowCount: input.rows.length, - runRecordCount: input.runRecords?.length ?? 0, - artifactKinds: ['environment', 'dataset', 'run_records'], - } - - const files: PrimeIntellectPackageFile[] = [ - { - path: 'pyproject.toml', - content: renderPyproject({ - name: input.name, - version, - description, - dependencies, - tags, - }), - }, - { - path: 'README.md', - content: input.readme ?? renderReadme({ name: input.name, description }), - }, - { - path: `${moduleName}.py`, - content: renderEnvironmentModule({ - systemPrompt: input.systemPrompt, - }), - }, - { - path: 'data/dataset.jsonl', - content: toJsonl(input.rows), - }, - { - path: 'data/run_records.jsonl', - content: toJsonl(input.runRecords ?? []), - }, - { - path: 'tangle-primeintellect-manifest.json', - content: `${stableJson(manifest)}\n`, - }, - ] - - return { files, manifest } -} - -export async function writePrimeIntellectEnvironmentPackage( - root: string, - input: PrimeIntellectEnvironmentPackageInput, -): Promise { - const pkg = buildPrimeIntellectEnvironmentPackage(input) - await Promise.all( - pkg.files.map(async (file) => { - const target = join(root, file.path) - await mkdir(dirname(target), { recursive: true }) - await writeFile(target, file.content, 'utf8') - }), - ) - return pkg -} - -function normalizeScenarios( - scenarios: PrimeIntellectScenarioMap, -): Map { - const map = new Map() - if (scenarios instanceof Map) { - for (const [id, scenario] of scenarios.entries()) { - map.set(id, { ...scenario, id: scenario.id ?? id }) - } - return map - } - - const values = Array.isArray(scenarios) ? scenarios : Object.values(scenarios) - for (const scenario of values) { - if (!scenario.id) { - throw new PrimeIntellectBridgeError('every PrimeIntellect scenario needs an id') - } - if (map.has(scenario.id)) { - throw new PrimeIntellectBridgeError(`duplicate PrimeIntellect scenario id ${scenario.id}`) - } - map.set(scenario.id, scenario) - } - return map -} - -function normalizePrompt( - prompt: string | readonly PrimeIntellectMessage[], -): PrimeIntellectMessage[] { - if (typeof prompt === 'string') { - return [{ role: 'user', content: prompt }] - } - return prompt.map((message) => ({ role: message.role, content: message.content })) -} - -function toPythonModuleName(name: string): string { - return name - .trim() - .toLowerCase() - .replace(/[^a-z0-9_]+/g, '_') - .replace(/^_+|_+$/g, '') - .replace(/^[0-9]/, '_$&') -} - -function unique(values: readonly string[]): string[] { - return [...new Set(values)] -} - -function toJsonl(values: readonly unknown[]): string { - if (values.length === 0) return '' - return `${values.map((value) => stableJson(value)).join('\n')}\n` -} - -function stableJson(value: unknown): string { - return JSON.stringify(sortJson(value)) -} - -function sortJson(value: unknown): unknown { - if (Array.isArray(value)) return value.map(sortJson) - if (value === null || typeof value !== 'object') return value - return Object.fromEntries( - Object.entries(value as Record) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, nested]) => [key, sortJson(nested)]), - ) -} - -function renderPyproject(input: { - name: string - version: string - description: string - dependencies: readonly string[] - tags: readonly string[] -}): string { - return `[project] -name = ${tomlString(input.name)} -version = ${tomlString(input.version)} -description = ${tomlString(input.description)} -readme = "README.md" -requires-python = ">=3.11" -license = "MIT" -keywords = ${tomlArray(input.tags)} -dependencies = ${tomlArray(input.dependencies)} - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build] -include = ["*.py", "README.md", "data/*.jsonl", "tangle-primeintellect-manifest.json"] - -[tool.uv] -prerelease = "allow" - -[tool.verifiers.eval] -num_examples = 5 -rollouts_per_example = 3 -` -} - -function renderReadme(input: { name: string; description: string }): string { - return `# ${input.name} - -${input.description} - -Generated by \`@tangle-network/agent-eval/primeintellect\` from validated Tangle \`RunRecord\` rows. - -## Files - -- \`data/dataset.jsonl\`: prompts and scoring references for PrimeIntellect Verifiers. -- \`data/run_records.jsonl\`: original Tangle run rows for provenance and analysis. -- \`tangle-primeintellect-manifest.json\`: export metadata. - -## Local check - -\`\`\`sh -uv pip install --prerelease=allow -e . -uv run vf-eval ${input.name} -\`\`\` - -## Upload - -\`\`\`sh -prime login -prime env push -\`\`\` -` -} - -function renderEnvironmentModule(input: { systemPrompt?: string }): string { - const systemPrompt = input.systemPrompt === undefined ? 'None' : pyString(input.systemPrompt) - return `import json -import re -from pathlib import Path - -from datasets import Dataset -import verifiers as vf - - -DATA_PATH = Path(__file__).parent / "data" / "dataset.jsonl" -DEFAULT_SYSTEM_PROMPT = ${systemPrompt} - - -def _load_rows(dataset_path=None, dataset_split=None): - path = Path(dataset_path) if dataset_path is not None else DATA_PATH - rows = [] - with path.open("r", encoding="utf-8") as handle: - for line in handle: - row = json.loads(line) - if dataset_split is None or row.get("split") == dataset_split: - rows.append(row) - if not rows: - raise ValueError(f"No dataset rows found in {path} for split={dataset_split!r}") - return rows - - -def _message_text(message): - if isinstance(message, dict): - content = message.get("content", "") - else: - content = getattr(message, "content", "") - if isinstance(content, list): - return " ".join( - str(part.get("text", part)) if isinstance(part, dict) else str(part) - for part in content - ) - return str(content) - - -def _completion_text(completion): - if isinstance(completion, str): - return completion - if isinstance(completion, list): - return "\\n".join(_message_text(message) for message in completion) - return _message_text(completion) - - -def _normalize(text): - return re.sub(r"\\s+", " ", str(text).strip().lower()) - - -async def tangle_reward(completion, answer=None, required_substrings=None, **kwargs): - response = _normalize(_completion_text(completion)) - scores = [] - if answer: - expected = _normalize(answer) - scores.append(1.0 if expected == response or expected in response else 0.0) - if required_substrings: - required = [_normalize(item) for item in required_substrings] - hits = sum(1 for item in required if item and item in response) - scores.append(hits / len(required)) - return max(scores) if scores else 0.0 - - -def build_dataset(dataset_path=None, dataset_split=None): - return Dataset.from_list(_load_rows(dataset_path=dataset_path, dataset_split=dataset_split)) - - -def load_environment(dataset_path=None, dataset_split=None, system_prompt=DEFAULT_SYSTEM_PROMPT): - rubric = vf.Rubric(funcs=[tangle_reward]) - return vf.SingleTurnEnv( - dataset=lambda: build_dataset(dataset_path=dataset_path, dataset_split=dataset_split), - system_prompt=system_prompt, - rubric=rubric, - ) -` -} - -function tomlString(value: string): string { - return JSON.stringify(value) -} - -function tomlArray(values: readonly string[]): string { - return `[${values.map(tomlString).join(', ')}]` -} - -function pyString(value: string): string { - return JSON.stringify(value) -} diff --git a/tests/primeintellect.test.ts b/tests/primeintellect.test.ts deleted file mode 100644 index 748ecd62..00000000 --- a/tests/primeintellect.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { mkdtemp, readFile, rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import { join } from 'node:path' -import { describe, expect, it } from 'vitest' -import { - buildPrimeIntellectEnvironmentPackage, - PrimeIntellectBridgeError, - type PrimeIntellectScenario, - primeIntellectRowsFromRunRecords, - writePrimeIntellectEnvironmentPackage, -} from '../src/primeintellect' -import type { RunRecord } from '../src/run-record' - -function run(overrides: Partial = {}): RunRecord { - const base: RunRecord = { - runId: '11111111-2222-3333-4444-555555555555', - experimentId: 'exp-primeintellect', - candidateId: 'baseline', - seed: 7, - model: 'gpt-4o-2024-11-20', - promptHash: 'p'.repeat(64), - configHash: 'c'.repeat(64), - commitSha: 'cafebabe', - wallMs: 1234, - costUsd: 0.01, - tokenUsage: { input: 10, output: 20 }, - outcome: { searchScore: 0.75, raw: { pass: 1 } }, - splitTag: 'search', - scenarioId: 'scenario-1', - } - return { ...base, ...overrides, outcome: overrides.outcome ?? base.outcome } -} - -const scenario: PrimeIntellectScenario = { - id: 'scenario-1', - prompt: 'What is the refund policy?', - answer: 'Refunds are available within 30 days.', - requiredSubstrings: ['refunds', '30 days'], - split: 'train', - info: { source: 'agent-knowledge' }, -} - -describe('PrimeIntellect bridge', () => { - it('turns validated RunRecords plus scenarios into Verifiers dataset rows', () => { - const rows = primeIntellectRowsFromRunRecords({ - records: [run()], - scenarios: [scenario], - }) - - expect(rows).toEqual([ - { - id: 'scenario-1:11111111-2222-3333-4444-555555555555', - prompt: [{ role: 'user', content: 'What is the refund policy?' }], - answer: 'Refunds are available within 30 days.', - required_substrings: ['refunds', '30 days'], - split: 'train', - info: { - source: 'agent-knowledge', - tangle: { - candidate_id: 'baseline', - commit_sha: 'cafebabe', - cost_usd: 0.01, - experiment_id: 'exp-primeintellect', - failure_mode: null, - model: 'gpt-4o-2024-11-20', - run_id: '11111111-2222-3333-4444-555555555555', - scenario_id: 'scenario-1', - score: 0.75, - split: 'train', - wall_ms: 1234, - }, - }, - }, - ]) - }) - - it('fails loudly when an environment row cannot be scored', () => { - expect(() => - primeIntellectRowsFromRunRecords({ - records: [run()], - scenarios: [{ id: 'scenario-1', prompt: 'Unscored prompt' }], - }), - ).toThrow(PrimeIntellectBridgeError) - }) - - it('builds a PrimeIntellect Verifiers package with dataset and run-record artifacts', () => { - const record = run() - const rows = primeIntellectRowsFromRunRecords({ records: [record], scenarios: [scenario] }) - const pkg = buildPrimeIntellectEnvironmentPackage({ - name: 'tangle-refund-policy', - rows, - runRecords: [record], - systemPrompt: 'Answer using the supplied policy.', - }) - - expect(pkg.manifest).toEqual({ - schemaVersion: 1, - name: 'tangle-refund-policy', - moduleName: 'tangle_refund_policy', - version: '0.1.0', - rowCount: 1, - runRecordCount: 1, - artifactKinds: ['environment', 'dataset', 'run_records'], - }) - expect(pkg.files.map((file) => file.path).sort()).toEqual([ - 'README.md', - 'data/dataset.jsonl', - 'data/run_records.jsonl', - 'pyproject.toml', - 'tangle-primeintellect-manifest.json', - 'tangle_refund_policy.py', - ]) - expect(pkg.files.find((file) => file.path === 'pyproject.toml')?.content).toContain( - '"verifiers>=0.2.0,<0.3.0"', - ) - expect(pkg.files.find((file) => file.path === 'tangle_refund_policy.py')?.content).toContain( - 'def load_environment(', - ) - expect(pkg.files.find((file) => file.path === 'data/dataset.jsonl')?.content).toContain( - '"required_substrings":["refunds","30 days"]', - ) - }) - - it('writes the generated package to disk', async () => { - const root = await mkdtemp(join(tmpdir(), 'agent-eval-primeintellect-')) - try { - const record = run() - const rows = primeIntellectRowsFromRunRecords({ records: [record], scenarios: [scenario] }) - await writePrimeIntellectEnvironmentPackage(root, { - name: 'tangle-refund-policy', - rows, - runRecords: [record], - }) - - const manifest = JSON.parse( - await readFile(join(root, 'tangle-primeintellect-manifest.json'), 'utf8'), - ) - const dataset = await readFile(join(root, 'data/dataset.jsonl'), 'utf8') - - expect(manifest.rowCount).toBe(1) - expect(dataset.split('\n').filter(Boolean)).toHaveLength(1) - } finally { - await rm(root, { recursive: true, force: true }) - } - }) -})