diff --git a/README.md b/README.md index 709ab998..70a60a26 100644 --- a/README.md +++ b/README.md @@ -207,7 +207,7 @@ All 29 live in [`examples/`](./examples). - New here? [`docs/concepts.md`](./docs/concepts.md), the mental model in plain terms. - [`docs/canonical-api.md`](./docs/canonical-api.md), find the primitive: "I want to ___ → use ___". - [`docs/api/primitive-catalog.md`](./docs/api/primitive-catalog.md), every export in one generated, never-stale list with its import path. Check it before building anything new. -- Import subpaths: the root export is the product surface (`handleChatTurn`, `improve`); deeper capabilities ship as subpaths: `/loops` (multi-agent + the loop kernel), `/knowledge` (KB improvement), `/primeintellect` (Prime task, runtime, and trace adapter), `/mcp` (tool servers), `/intelligence` (observability drop-in), `/lifecycle`, `/agent`, `/profiles`, `/platform`, `/analyst-loop`, `/environment-provider`. +- Import subpaths: the root export is the product surface (`handleChatTurn`, `improve`); deeper capabilities ship as subpaths: `/loops` (multi-agent + the loop kernel), `/conversation` (multi-turn conversations), `/knowledge` (KB improvement), `/primeintellect` (Prime task, runtime, and trace adapter), `/mcp` (tool servers), `/intelligence` (observability drop-in), `/lifecycle`, `/agent`, `/profiles`, `/platform`, `/analyst-loop`, `/environment-provider`. - [`docs/architecture.md`](./docs/architecture.md), the design, end to end. - [`bench/HARNESS.md`](./bench/HARNESS.md), the experiment harness and how to run a benchmark. diff --git a/docs/api/README.md b/docs/api/README.md index a5ebd062..ac376904 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -9,6 +9,7 @@ - [agent](agent.md) - [analyst-loop](analyst-loop.md) - [candidate-execution](candidate-execution.md) +- [conversation](conversation.md) - [index](index.md) - [intelligence](intelligence.md) - [knowledge](knowledge.md) diff --git a/docs/api/conversation.md b/docs/api/conversation.md new file mode 100644 index 00000000..e2b79361 --- /dev/null +++ b/docs/api/conversation.md @@ -0,0 +1,325 @@ +[**@tangle-network/agent-runtime**](README.md) + +*** + +[@tangle-network/agent-runtime](README.md) / conversation + +# conversation + +## References + +### BackendCallPolicy + +Re-exports [BackendCallPolicy](index.md#backendcallpolicy) + +*** + +### CircuitBreakerConfig + +Re-exports [CircuitBreakerConfig](index.md#circuitbreakerconfig) + +*** + +### CircuitBreakerState + +Re-exports [CircuitBreakerState](index.md#circuitbreakerstate) + +*** + +### CircuitOpenError + +Re-exports [CircuitOpenError](index.md#circuitopenerror) + +*** + +### computeBackoff + +Re-exports [computeBackoff](index.md#computebackoff) + +*** + +### DeadlineExceededError + +Re-exports [DeadlineExceededError](index.md#deadlineexceedederror) + +*** + +### defaultIsRetryable + +Re-exports [defaultIsRetryable](index.md#defaultisretryable) + +*** + +### makePerAttemptSignal + +Re-exports [makePerAttemptSignal](index.md#makeperattemptsignal) + +*** + +### RetryableErrorPredicate + +Re-exports [RetryableErrorPredicate](index.md#retryableerrorpredicate) + +*** + +### RetryBackoff + +Re-exports [RetryBackoff](index.md#retrybackoff) + +*** + +### sleep + +Re-exports [sleep](index.md#sleep) + +*** + +### createConversationBackend + +Re-exports [createConversationBackend](index.md#createconversationbackend) + +*** + +### defineConversation + +Re-exports [defineConversation](index.md#defineconversation) + +*** + +### buildForwardHeaders + +Re-exports [buildForwardHeaders](index.md#buildforwardheaders) + +*** + +### DEFAULT\_MAX\_DEPTH + +Re-exports [DEFAULT_MAX_DEPTH](index.md#default_max_depth) + +*** + +### FORWARD\_HEADERS + +Re-exports [FORWARD_HEADERS](index.md#forward_headers) + +*** + +### ForwardHeaderName + +Re-exports [ForwardHeaderName](index.md#forwardheadername) + +*** + +### isDepthExceeded + +Re-exports [isDepthExceeded](index.md#isdepthexceeded) + +*** + +### PropagatedHeaders + +Re-exports [PropagatedHeaders](index.md#propagatedheaders) + +*** + +### readDepth + +Re-exports [readDepth](index.md#readdepth) + +*** + +### ConversationJournal + +Re-exports [ConversationJournal](index.md#conversationjournal) + +*** + +### ConversationJournalEntry + +Re-exports [ConversationJournalEntry](index.md#conversationjournalentry) + +*** + +### FileConversationJournal + +Re-exports [FileConversationJournal](index.md#fileconversationjournal) + +*** + +### InMemoryConversationJournal + +Re-exports [InMemoryConversationJournal](index.md#inmemoryconversationjournal) + +*** + +### D1DatabaseLike + +Re-exports [D1DatabaseLike](index.md#d1databaselike) + +*** + +### D1StmtLike + +Re-exports [D1StmtLike](index.md#d1stmtlike) + +*** + +### d1ToSqlAdapter + +Re-exports [d1ToSqlAdapter](index.md#d1tosqladapter) + +*** + +### SqlAdapter + +Re-exports [SqlAdapter](index.md#sqladapter) + +*** + +### SqlConversationJournal + +Re-exports [SqlConversationJournal](index.md#sqlconversationjournal) + +*** + +### runConversation + +Re-exports [runConversation](index.md#runconversation) + +*** + +### runConversationStream + +Re-exports [runConversationStream](index.md#runconversationstream) + +*** + +### PersonaConversationResult + +Re-exports [PersonaConversationResult](index.md#personaconversationresult) + +*** + +### PersonaDriver + +Re-exports [PersonaDriver](index.md#personadriver) + +*** + +### RunPersonaConfig + +Re-exports [RunPersonaConfig](index.md#runpersonaconfig) + +*** + +### RunPersonaConversationOptions + +Re-exports [RunPersonaConversationOptions](index.md#runpersonaconversationoptions) + +*** + +### runPersonaConversation + +Re-exports [runPersonaConversation](index.md#runpersonaconversation) + +*** + +### runPersonaDispatch + +Re-exports [runPersonaDispatch](index.md#runpersonadispatch) + +*** + +### slugifySpeaker + +Re-exports [slugifySpeaker](index.md#slugifyspeaker) + +*** + +### turnId + +Re-exports [turnId](index.md#turnid-1) + +*** + +### AuthSource + +Re-exports [AuthSource](index.md#authsource-1) + +*** + +### Conversation + +Re-exports [Conversation](index.md#conversation) + +*** + +### ConversationDriveState + +Re-exports [ConversationDriveState](index.md#conversationdrivestate) + +*** + +### ConversationParticipant + +Re-exports [ConversationParticipant](index.md#conversationparticipant) + +*** + +### ConversationPolicy + +Re-exports [ConversationPolicy](index.md#conversationpolicy) + +*** + +### ConversationResult + +Re-exports [ConversationResult](index.md#conversationresult) + +*** + +### ConversationStreamEvent + +Re-exports [ConversationStreamEvent](index.md#conversationstreamevent) + +*** + +### ConversationTurn + +Re-exports [ConversationTurn](index.md#conversationturn) + +*** + +### HaltContext + +Re-exports [HaltContext](index.md#haltcontext) + +*** + +### HaltPredicate + +Re-exports [HaltPredicate](index.md#haltpredicate) + +*** + +### HaltReason + +Re-exports [HaltReason](index.md#haltreason) + +*** + +### HaltSignal + +Re-exports [HaltSignal](index.md#haltsignal) + +*** + +### RunConversationOptions + +Re-exports [RunConversationOptions](index.md#runconversationoptions) + +*** + +### TurnOrder + +Re-exports [TurnOrder](index.md#turnorder) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b26bd8f2..ddeac247 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -258,6 +258,47 @@ Import from `@tangle-network/agent-runtime/agent` — 48 exports. **Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AgentRubric`, `AgentRunContext`, `AgentRunInvocation`, `AgentRuntime`, `AnalystConfig`, `AutoApplyPolicy`, `CreateSandboxActOptions`, `CreateSurfaceImprovementAdapterOpts`, `DraftPatchInput`, `DraftPatchOutput`, `JudgeConfig`, `OutcomeMeasurementOpts`, `ResolvedSurface`, `RubricDimension`, `KnownAgentProfileMaterializationAxis`. +### Multi-turn conversations + +Import from `@tangle-network/agent-runtime/conversation` — 53 exports. + +| Symbol | Kind | Summary | +|---|---|---| +| `buildForwardHeaders` | function | Build the headers to emit on an outbound participant call, given the | +| `computeBackoff` | function | Compute the delay before the next attempt. Default: 250ms exponential with jitter. | +| `createConversationBackend` | function | Wrap a `Conversation` so it satisfies `AgentExecutionBackend`. The result is | +| `d1ToSqlAdapter` | function | Adapt a Cloudflare D1 binding to the SqlAdapter shape. Lives here so D1 | +| `defineConversation` | function | Declarative constructor for a multi-agent `Conversation`. Validates inputs | +| `isDepthExceeded` | function | Refuse further forwarding when the inbound depth has reached the limit. | +| `makePerAttemptSignal` | function | Build a per-attempt AbortSignal linked to the parent signal AND fired when | +| `readDepth` | function | Read the depth counter off an inbound request. Missing → 0 (caller is the | +| `runConversation` | function | Conversation orchestrator. Drives N participants in turn through their own | +| `runConversationStream` | function | Streaming conversation orchestrator: drives N participants in turn through their own backends, enforcing `maxTurns` / `maxCreditsCents` / `haltOn`, yielding per-event stream markers. | +| `runPersonaConversation` | function | Run one worker profile against one persona as a multi-round conversation. | +| `runPersonaDispatch` | function | Wrap {@link runPersonaConversation} as a `ProfileDispatchFn` for | +| `sleep` | function | Resolve after `ms` milliseconds — used for retry backoff in conversation call policy. | +| `slugifySpeaker` | function | Reduce a speaker name to ASCII alphanumerics + dashes. Preserves enough | +| `turnId` | function | Deterministic turn identifier. Stable across retries of the same logical | +| `DEFAULT_MAX_DEPTH` | const | Hard cap on chained gateway hops; refused beyond this. Default keeps recursion bounded. | +| `defaultIsRetryable` | const | Default retryable classification — network/timeout class errors. Errors | +| `FORWARD_HEADERS` | const | Standard names — lowercased so Headers maps interop on every runtime. | +| `CircuitBreakerState` | class | Live circuit-breaker state — one instance per (participant, conversation run). | +| `CircuitOpenError` | class | Thrown when the circuit breaker is open for a participant and no retry is allowed yet. | +| `DeadlineExceededError` | class | Thrown when a backend call exceeds its per-attempt deadline. | +| `FileConversationJournal` | class | JSONL on disk. One line per record; first line is the `begin`, subsequent | +| `InMemoryConversationJournal` | class | In-memory `ConversationJournal` — suitable for testing and single-process runs. | +| `SqlConversationJournal` | class | SQL-backed ConversationJournal. Two tables — runs (one row per runId, holds | +| `CircuitBreakerConfig` | interface | Circuit-breaker tuning. `failuresToOpen` consecutive failures opens it; closed only after `cooldownMs`. | +| `ConversationJournalEntry` | interface | Durable conversation transcript — survives a driver process crash mid-run. | +| `D1DatabaseLike` | interface | Structural type matching the surface of `D1Database` we depend on, so the | +| `SqlAdapter` | interface | Minimal SQL driver shape. Implementations forward to whichever client the | +| `PersonaDriver` | type | A persona that drives the conversation: either a full driver `AgentProfile` | +| `PropagatedHeaders` | type | Header bag carried through `AgentBackendContext.propagatedHeaders` so | +| `RetryableErrorPredicate` | type | Pure judgment of whether an error is worth retrying. Defaults: TimeoutError, AbortError, fetch-level network errors. | +| `RetryBackoff` | type | Backoff between attempts. Constant ms, or `(attempt: 1-indexed) => ms`. | + +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `BackendCallPolicy`, `Conversation`, `ConversationDriveState`, `ConversationJournal`, `ConversationParticipant`, `ConversationPolicy`, `ConversationResult`, `ConversationTurn`, `D1StmtLike`, `HaltContext`, `HaltSignal`, `PersonaConversationResult`, `RunConversationOptions`, `RunPersonaConfig`, `RunPersonaConversationOptions`, `AuthSource`, `ConversationStreamEvent`, `ForwardHeaderName`, `HaltPredicate`, `HaltReason`, `TurnOrder`. + ### Intelligence SDK — Observe + provable-OFF billing Import from `@tangle-network/agent-runtime/intelligence` — 85 exports. diff --git a/package.json b/package.json index 262d4407..1c49ccde 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,11 @@ "import": "./dist/agent.js", "default": "./dist/agent.js" }, + "./conversation": { + "types": "./dist/conversation.d.ts", + "import": "./dist/conversation.js", + "default": "./dist/conversation.js" + }, "./intelligence": { "types": "./dist/intelligence.d.ts", "import": "./dist/intelligence.js", @@ -104,6 +109,7 @@ "typecheck:examples": "tsc --noEmit -p tsconfig.examples.json", "verify:package": "node scripts/verify-package-exports.mjs", "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect-v1.mjs", + "verify:primeintellect:live": "pnpm build && node scripts/verify-primeintellect-live.mjs", "docs:api": "typedoc && node scripts/gen-primitive-catalog.mjs", "docs:freshness": "node scripts/check-docs-freshness.mjs", "docs:check": "pnpm run docs:api && git diff --exit-code -- docs/api && pnpm run docs:freshness" diff --git a/scripts/gen-primitive-catalog.mjs b/scripts/gen-primitive-catalog.mjs index 98dc85b3..1b9509b2 100644 --- a/scripts/gen-primitive-catalog.mjs +++ b/scripts/gen-primitive-catalog.mjs @@ -63,6 +63,7 @@ const pkg = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')) const ownSurfaceLabels = { '.': 'Root — task lifecycle, conversation, RSI verbs, observability', './agent': 'Vertical agent — manifest + improvement adapter', + './conversation': 'Multi-turn conversations', './intelligence': 'Intelligence SDK — Observe + provable-OFF billing', './loops': 'Recursive atom + loop kernel (alias of ./runtime)', './environment-provider': 'Environment provider adapters — generic sandbox/compute bridge', diff --git a/scripts/verify-package-exports.mjs b/scripts/verify-package-exports.mjs index fff84292..b761fa21 100644 --- a/scripts/verify-package-exports.mjs +++ b/scripts/verify-package-exports.mjs @@ -45,6 +45,7 @@ try { const requiredExports = { '.': ['import', 'types'], './agent': ['import', 'types'], + './conversation': ['import', 'types'], './candidate-execution': ['import', 'types'], './intelligence': ['import', 'types'], './loops': ['import', 'types'], diff --git a/scripts/verify-primeintellect-live.mjs b/scripts/verify-primeintellect-live.mjs new file mode 100644 index 00000000..3f26a2da --- /dev/null +++ b/scripts/verify-primeintellect-live.mjs @@ -0,0 +1,312 @@ +import { createHash } from 'node:crypto' +import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { spawnSync } from 'node:child_process' +import { build } from 'tsup' +import { + createPrimeIntellectPackage, + importPrimeIntellectTraces, + parsePrimeIntellectTraces, + writePrimeIntellectPackage, +} from '../dist/primeintellect/index.js' + +const VERIFIERS = + 'verifiers @ git+https://github.com/PrimeIntellect-ai/verifiers.git@v0.2.0' +const model = process.env.PRIME_LIVE_MODEL ?? 'deepseek/deepseek-v4-flash' +const providerBaseUrl = + process.env.PRIME_LIVE_BASE_URL ?? 'https://api.pinference.ai/api/v1' +const apiKeyVariable = process.env.PRIME_LIVE_API_KEY_VAR ?? 'PRIME_API_KEY' +const providerKey = process.env[apiKeyVariable] + +if (!providerKey) { + throw new Error(`Prime live verification requires ${apiKeyVariable}`) +} + +const root = process.env.PRIME_LIVE_RUN_DIR + ? resolve(process.env.PRIME_LIVE_RUN_DIR) + : await mkdtemp(join(tmpdir(), 'agent-runtime-prime-live-')) +const bundleDirectory = join(root, 'bundle') +const packageDirectory = join(root, 'environment') +const outputDirectory = join(root, 'output') +const runnerSource = join(root, 'runner.ts') +const runnerBundle = join(bundleDirectory, 'runner.js') + +await mkdir(root, { recursive: true }) +await writeFile(runnerSource, renderRunner(), 'utf8') +await build({ + entry: [runnerSource], + outDir: bundleDirectory, + format: ['esm'], + platform: 'node', + target: 'node22', + bundle: true, + splitting: false, + clean: true, + dts: false, + sourcemap: false, + minify: false, + noExternal: [/.*/], + silent: true, +}) + +const bundledRunner = await readFile(runnerBundle, 'utf8') +if (/from\s+["']@tangle-network\//.test(bundledRunner)) { + throw new Error('Prime live runner retained an external Tangle package import') +} + +const scoringProgram = `import json +import sys + +request = json.load(sys.stdin) +assert request["protocol"] == "tangle.primeintellect.score/v1" +trace = request["trace"] +sampled = [node for node in trace["nodes"] if node.get("sampled") is True] +contents = [str((node.get("message") or {}).get("content") or "").strip() for node in sampled] +solver_handoff = bool(contents and "REVIEW REQUEST:" in contents[0]) +exact_final = bool(contents and contents[-1] == "FINAL: 42") +multi_turn = len(sampled) >= 2 +reward = float(multi_turn and solver_handoff and exact_final and not trace.get("errors")) +print(json.dumps({ + "reward": reward, + "metrics": { + "multi_turn": float(multi_turn), + "sampled_turns": float(len(sampled)), + "solver_handoff": float(solver_handoff), + "exact_final": float(exact_final), + }, +})) +` + +const taskPrompt = `Run this two-stage workflow. + +The solver checks 17 + 25 and sends the reviewer a complete handoff beginning with REVIEW REQUEST:. +The reviewer independently checks that handoff and replies exactly FINAL: 42 with no other text. +Do not skip either stage.` + +const environment = createPrimeIntellectPackage({ + name: 'tangle-runtime-live-v1', + version: '1.0.0', + description: 'Live multi-turn Tangle runtime execution through PrimeIntellect', + tasks: [ + { + id: 'train-two-stage-arithmetic', + split: 'train', + prompt: taskPrompt.replace('17 + 25', '19 + 23'), + }, + { + id: 'eval-two-stage-arithmetic', + split: 'eval', + prompt: taskPrompt, + }, + ], + scoring: { + kind: 'command', + command: ['python', 'scoring/score.py'], + files: { 'score.py': scoringProgram }, + timeoutSeconds: 30, + }, + runner: { + image: 'node:22-bookworm-slim', + command: ['node', 'runner.mjs'], + files: { 'runner.mjs': bundledRunner }, + }, + maxTurns: 4, + maxOutputTokens: 512, + maxTotalTokens: 4_096, + rolloutTimeoutSeconds: 180, + scoringTimeoutSeconds: 30, +}) +await writePrimeIntellectPackage(environment, packageDirectory) + +const dryRun = runPrime([ + '--dry-run', + 'true', + '--output-dir', + outputDirectory, +]) +if (dryRun.status !== 0) fail('Prime live config validation failed', dryRun) + +const liveRun = runPrime([ + '--output-dir', + outputDirectory, + '--server', + 'true', + '--pool.type', + 'static', + '--pool.num-workers', + '2', + '--sampling.temperature', + '0', + '--sampling.max-tokens', + '256', +]) +if (liveRun.status !== 0) fail('Prime live rollout failed', liveRun) + +const tracePath = join(outputDirectory, 'traces.jsonl') +const traceJsonl = await readFile(tracePath, 'utf8') +const traces = parsePrimeIntellectTraces(traceJsonl) +if (traces.length !== 1) throw new Error(`expected one Prime trace, received ${traces.length}`) + +const commitSha = git('rev-parse', 'HEAD') +const observedDate = new Date().toISOString().slice(0, 10) +const records = importPrimeIntellectTraces(traceJsonl, { + experimentId: `prime-live-${observedDate}`, + candidateId: 'tangle-runtime-live-v1', + seed: 0, + model: `${model}@observed-${observedDate}`, + promptHash: sha256(taskPrompt), + configHash: sha256(environment.files['prime.eval.toml']), + commitSha, +}) +const trace = traces[0] +const record = records[0] +const raw = record.outcome.raw ?? {} +const reward = record.outcome.holdoutScore + +if (trace.errors?.length) { + throw new Error(`Prime trace contains errors: ${JSON.stringify(trace.errors)}`) +} +if (trace.runtime === undefined) throw new Error('Prime trace did not record its runtime') +if (reward !== 1) throw new Error(`Prime live reward was ${String(reward)}, expected 1`) +if ((raw['prime.turns'] ?? 0) < 2) { + throw new Error(`Prime trace recorded ${String(raw['prime.turns'])} sampled turns, expected >= 2`) +} +for (const metric of ['multi_turn', 'solver_handoff', 'exact_final']) { + if (raw[`metric.${metric}`] !== 1) { + throw new Error(`Prime live metric ${metric} was ${String(raw[`metric.${metric}`])}`) + } +} + +process.stdout.write( + `${JSON.stringify( + { + runDirectory: root, + tracePath, + model, + workerMode: 'server', + workerCount: 2, + runtime: trace.runtime, + reward, + sampledTurns: raw['prime.turns'], + branches: raw['prime.branches'], + metrics: Object.fromEntries( + Object.entries(raw).filter(([name]) => name.startsWith('metric.')), + ), + tokenUsage: record.tokenUsage, + costUsd: record.costUsd, + costProvenance: record.costProvenance, + completed: raw['prime.completed'], + errors: trace.errors ?? [], + importedRunRecord: record.runId, + }, + null, + 2, + )}\n`, +) + +function runPrime(extraArguments) { + return spawnSync( + 'uv', + [ + 'run', + '--project', + packageDirectory, + '--with', + VERIFIERS, + 'eval', + '@', + 'prime.eval.toml', + '--model', + model, + '--num-tasks', + '1', + '--num-rollouts', + '1', + '--max-concurrent', + '1', + '--rich', + 'false', + '--push', + 'false', + '--client.base-url', + providerBaseUrl, + '--client.api-key-var', + apiKeyVariable, + ...extraArguments, + ], + { + cwd: packageDirectory, + env: { ...process.env, [apiKeyVariable]: providerKey }, + encoding: 'utf8', + maxBuffer: 16 * 1024 * 1024, + }, + ) +} + +function fail(message, result) { + const detail = [result.stdout, result.stderr].filter(Boolean).join('\n').trim() + throw new Error(`${message}${detail ? `\n${detail}` : ''}`) +} + +function git(...args) { + const result = spawnSync('git', args, { encoding: 'utf8' }) + if (result.status !== 0) fail(`git ${args.join(' ')} failed`, result) + return result.stdout.trim() +} + +function sha256(value) { + return createHash('sha256').update(value, 'utf8').digest('hex') +} + +function renderRunner() { + const conversationEntry = resolve('dist/conversation.js') + const primeEntry = resolve('dist/primeintellect/index.js') + return `import { runConversation } from ${JSON.stringify(conversationEntry)} +import { createPrimeIntellectBackend, runPrimeIntellectProgram } from ${JSON.stringify(primeEntry)} + +function roleBackend(base, kind, instruction) { + return { + kind, + async *stream(input, context) { + const existing = input.messages ?? [{ role: 'user', content: input.message ?? context.task.intent }] + yield* base.stream( + { ...input, messages: [{ role: 'system', content: instruction }, ...existing] }, + context, + ) + }, + } +} + +await runPrimeIntellectProgram(async (episode) => { + if (typeof episode.task.prompt !== 'string') throw new Error('live proof expects a string prompt') + const base = createPrimeIntellectBackend(episode, { + temperature: 0, + maxTokens: 192, + retry: { maxAttempts: 2, requestTimeoutMs: 60_000 }, + }) + const solver = roleBackend( + base, + 'prime-solver', + 'You are the solver. Calculate the arithmetic and send one complete handoff beginning with REVIEW REQUEST:. Do not emit FINAL:.', + ) + const reviewer = roleBackend( + base, + 'prime-reviewer', + 'You are the reviewer. Independently check the arithmetic in the solver handoff. Reply exactly FINAL: 42 only when the handoff is correct, with no other text.', + ) + const result = await runConversation( + { + participants: [ + { name: 'solver', backend: solver, authSource: 'agent-owned' }, + { name: 'reviewer', backend: reviewer, authSource: 'agent-owned' }, + ], + policy: { maxTurns: 2, turnOrder: 'alternate' }, + }, + { seed: episode.task.prompt, runId: 'prime:' + episode.task.id }, + ) + if (result.turns !== 2) throw new Error('expected two completed runtime turns, received ' + result.turns) + process.stdout.write(JSON.stringify({ turns: result.turns, halted: result.halted }) + '\\n') +}) +` +} diff --git a/src/conversation/run-conversation.test.ts b/src/conversation/run-conversation.test.ts index 21521dee..e0d88e68 100644 --- a/src/conversation/run-conversation.test.ts +++ b/src/conversation/run-conversation.test.ts @@ -62,6 +62,48 @@ function alwaysThrowsBackend(name: string, message: string): AgentExecutionBacke }) } +function emptyBackend(name: string): AgentExecutionBackend { + return createIterableBackend({ + kind: `empty-${name}`, + async *stream() {}, + }) +} + +function failedFinalBackend(name: string): AgentExecutionBackend { + return createIterableBackend({ + kind: `failed-${name}`, + async *stream(_input, context) { + yield { + type: 'final', + task: context.task, + session: context.session, + status: 'failed', + reason: 'provider failed', + error: { kind: 'transport', message: 'provider failed', status: 503 }, + timestamp: new Date().toISOString(), + } satisfies RuntimeStreamEvent + }, + }) +} + +function backendErrorEventBackend(name: string): AgentExecutionBackend { + return createIterableBackend({ + kind: `error-event-${name}`, + async *stream(_input, context) { + yield { + type: 'backend_error', + task: context.task, + session: context.session, + backend: `error-event-${name}`, + message: 'provider unavailable', + recoverable: true, + error: { kind: 'transport', message: 'provider unavailable', status: 503 }, + timestamp: new Date().toISOString(), + } satisfies RuntimeStreamEvent + }, + }) +} + describe('defineConversation', () => { const okBackend = fakeBackend('x', ['hi']) @@ -228,6 +270,57 @@ describe('runConversation — halting', () => { expect(result.transcript[0]?.speaker).toBe('a') }) + it('halts when a backend completes without text', async () => { + const conv = defineConversation({ + participants: [ + { name: 'a', backend: emptyBackend('a') }, + { name: 'b', backend: fakeBackend('b', ['unused']) }, + ], + policy: { maxTurns: 2 }, + }) + const result = await runConversation(conv, { seed: 'go' }) + expect(result.halted).toEqual({ + kind: 'participant_error', + participant: 'a', + message: "backend 'empty-a' completed without text", + }) + expect(result.transcript).toHaveLength(0) + }) + + it('halts when a backend emits a failed final event', async () => { + const conv = defineConversation({ + participants: [ + { name: 'a', backend: failedFinalBackend('a') }, + { name: 'b', backend: fakeBackend('b', ['unused']) }, + ], + policy: { maxTurns: 2 }, + }) + const result = await runConversation(conv, { seed: 'go' }) + expect(result.halted).toEqual({ + kind: 'participant_error', + participant: 'a', + message: 'provider failed', + }) + expect(result.transcript).toHaveLength(0) + }) + + it('halts when a backend emits a backend error event', async () => { + const conv = defineConversation({ + participants: [ + { name: 'a', backend: backendErrorEventBackend('a') }, + { name: 'b', backend: fakeBackend('b', ['unused']) }, + ], + policy: { maxTurns: 2 }, + }) + const result = await runConversation(conv, { seed: 'go' }) + expect(result.halted).toEqual({ + kind: 'participant_error', + participant: 'a', + message: 'provider unavailable', + }) + expect(result.transcript).toHaveLength(0) + }) + it('halts on abort signal', async () => { const controller = new AbortController() const conv = defineConversation({ diff --git a/src/conversation/run-conversation.ts b/src/conversation/run-conversation.ts index b0bd82d5..54b1e001 100644 --- a/src/conversation/run-conversation.ts +++ b/src/conversation/run-conversation.ts @@ -453,10 +453,29 @@ async function* driveSingleAttempt( yield { text: event.text, timestamp: event.timestamp } } else if (event.type === 'llm_call') { args.aggregator.recordUsage(event) + } else if (event.type === 'backend_error') { + throw new BackendTransportError(args.speaker.backend.kind, event.message, { + status: event.error?.status, + body: event.error?.body, + }) } else if (event.type === 'final') { + if (event.status !== 'completed') { + throw new BackendTransportError( + args.speaker.backend.kind, + event.error?.message ?? event.reason, + { status: event.error?.status, body: event.error?.body }, + ) + } args.aggregator.adoptFinalText(event.text) } } + + if (!args.aggregator.hasText()) { + throw new BackendTransportError( + args.speaker.backend.kind, + `backend '${args.speaker.backend.kind}' completed without text`, + ) + } } class TurnAggregator { @@ -491,6 +510,10 @@ class TurnAggregator { this.adoptedFinal = true } + hasText(): boolean { + return this.text.trim().length > 0 + } + recordUsage(event: { model?: string tokensIn?: number diff --git a/tsup.config.ts b/tsup.config.ts index 19747b40..08b386ec 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -4,6 +4,7 @@ export default defineConfig({ entry: { index: 'src/index.ts', agent: 'src/agent/index.ts', + conversation: 'src/conversation/index.ts', intelligence: 'src/intelligence/index.ts', loops: 'src/runtime/index.ts', // the loop kernel + recursive atom surface (the runtime/ directory) 'environment-provider': 'src/runtime/environment-provider.ts', diff --git a/typedoc.json b/typedoc.json index b4319cf1..5fa697fd 100644 --- a/typedoc.json +++ b/typedoc.json @@ -3,6 +3,7 @@ "entryPoints": [ "src/index.ts", "src/agent/index.ts", + "src/conversation/index.ts", "src/intelligence/index.ts", "src/runtime/index.ts", "src/runtime/environment-provider.ts",