From 7f859bdfe115bc57133ebe5f02dcab19af7d7249 Mon Sep 17 00:00:00 2001 From: ga1ien Date: Thu, 23 Jul 2026 09:01:34 -0700 Subject: [PATCH] fix: make internal research runs resumable --- README.md | 8 +- ...ernal-agent-research-release-2026-07-23.md | 67 ++++ package-lock.json | 4 +- package.json | 2 +- skills/run-braintied-research/SKILL.md | 16 +- .../references/runtime.md | 29 +- .../scripts/run-internal-research.mjs | 290 +++++++++++++++--- test/internal-runner-preflight.test.ts | 229 +++++++++++--- 8 files changed, 556 insertions(+), 89 deletions(-) create mode 100644 docs/internal-agent-research-release-2026-07-23.md diff --git a/README.md b/README.md index 626165a..439e882 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,12 @@ node skills/run-braintied-research/scripts/run-internal-research.mjs \ The internal runner uses Braintied Agent Auth from `BRAINTIED_AGENT_TOKEN` or macOS Keychain and keeps all model/search provider credentials in Cortex Worker. +Live execution requires the authenticated catalog's durable protocol v2: the +client submits one idempotent request, records the server-owned run ID, and +polls the tenant-bound result. A dropped response or rolling deployment is +retried with the same request ID instead of starting a second paid run. Preserve +the printed request ID; `--request-id ` explicitly reattaches after a local +timeout or interruption. `run-research.mjs` remains available as an explicit local-provider fallback; it supports allowlisted interactive-shell environment loading and build-freshness checks. Exact lane preflight requires an as-of date: @@ -198,7 +204,7 @@ npm run pack:release # → releases/braintied-research-.tgz ```jsonc // consumer package.json -"@braintied/research": "file:vendor/braintied-research-0.6.4.tgz" +"@braintied/research": "file:vendor/braintied-research-0.8.1.tgz" ``` ## Development diff --git a/docs/internal-agent-research-release-2026-07-23.md b/docs/internal-agent-research-release-2026-07-23.md new file mode 100644 index 0000000..2b501b6 --- /dev/null +++ b/docs/internal-agent-research-release-2026-07-23.md @@ -0,0 +1,67 @@ +# Internal Agent Research Release — July 23, 2026 + +## Outcome + +`@braintied/research` 0.8.1 replaces the long-lived HTTP execution contract +with resumable, idempotent durable polling. It is the client-side half of Cortex +internal-tools protocol v2 and is required for paid agent research. + +## Root cause + +Version 0.8.0 kept a deep research request attached to one Fly Machine. JSON +whitespace heartbeats prevented idle proxy timeouts, but a rolling Machine +replacement still severed the live request after provider work had begun. The +client could not determine whether work or billing continued and a retry risked +starting a second attempt. + +## Durable contract + +- The authenticated catalog must advertise protocol v2 and exact same-origin + submission/status paths. +- Submission uses one caller request ID as its idempotency key. The client may + retry a dropped submission without creating another durable record. +- The server returns a run UUID; the client polls its authenticated status and + validates the final result before writing any artifact. +- `--request-id` reattaches a later CLI process to the same attempt. +- Temporary transport, `404`, `408`, `425`, `429`, `502`, `503`, and `504` + responses use bounded retry delays within the caller's deadline. +- Metadata and stdout expose the request ID and durable run ID, never the Agent + Auth token, provider credentials, raw trusted recall, or provider error body. +- A protocol-v1/stream-only deployment is rejected before a paid call. + +## Deployment exclusion + +The matching Cortex release persists tenant-bound runs before Inngest dispatch. +Its deployment script acquires an atomic database lease only when no unexpired +run is queued/running. The same locked row pauses new submissions until the +rolling deploy completes, closing the check-then-deploy race. Lease expiry is +bounded so a dead deploy process cannot create an indefinite outage. + +## Operator recovery + +When a client deadline or local process interruption occurs, do not infer zero +cost and do not issue a new request ID. Repeat the identical command with the +reported `--request-id`; the client will retrieve or continue polling the +existing durable record. A fresh ID is a deliberately separate paid run. + +## Acceptance gates + +Package acceptance passed: + +- unit and contract tests: 56/56; +- Instagram/Bright Data boundary tests: 8/8; +- TypeScript and production build; +- CommonJS, ESM, packaged CLI, and exact version import from a clean tarball + consumer; +- clean production dependency audit: zero vulnerabilities; +- exact 14-file tarball audit plus Gitleaks and high-confidence provider, + Agent Auth, Swishh, GitHub, Slack, and private-key signatures: no leaks; +- archive: `releases/braintied-research-0.8.1.tgz`; +- SHA-256: + `901a1734195d060e47fe5f34bb9b5fb540f7afa3a81e374fed9aebff7e0c05ea`; +- npm integrity: + `sha512-w/GrdkCn56vmlQ+gExVPE4F/NHDbttS2xAkgE33N7fKBJcbwZ8eBEacYKXJbfwF0FQaTxBVNUmFuoTw2gd9zPw==`. + +Production acceptance additionally requires a protocol-v2 catalog probe plus a +transport-loss canary that destroys the first submission socket after +persistence and proves the retry returns the same run. diff --git a/package-lock.json b/package-lock.json index b1395df..437cfb8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@braintied/research", - "version": "0.8.0", + "version": "0.8.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@braintied/research", - "version": "0.8.0", + "version": "0.8.1", "license": "UNLICENSED", "dependencies": { "@anthropic-ai/sdk": "^0.104.1", diff --git a/package.json b/package.json index 940c063..31e2868 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@braintied/research", - "version": "0.8.0", + "version": "0.8.1", "description": "Deep-research engine — multi-provider search, quote extraction, synthesis, critique, grounded report assembly, and research-driven document generation (PRDs, briefs, specs)", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/skills/run-braintied-research/SKILL.md b/skills/run-braintied-research/SKILL.md index b8f3de0..541c424 100644 --- a/skills/run-braintied-research/SKILL.md +++ b/skills/run-braintied-research/SKILL.md @@ -53,7 +53,8 @@ model providers; treat every brief as outbound data. ``` A token-presence check alone is not deployment health. Do not continue on a - missing route, failed authentication, or catalog without `research.run`. + missing route, failed authentication, or a catalog that lacks durable + protocol v2 submission/status endpoints for `research.run`. 5. **Run only after the outbound brief, credentials, and remote catalog are authorized.** Pass `--allow-external`, an explicit report path, and a metadata path: @@ -72,6 +73,14 @@ model providers; treat every brief as outbound data. --allow-external ``` + The server persists the authenticated input before dispatch and returns a + durable run ID. Submission and status requests retry bounded transport and + temporary-service failures with the same request ID. If the local deadline + or process interrupts the client, rerun the identical command with the + printed `--request-id ` to reattach; never choose a fresh ID for the same + paid attempt. Cortex deploys atomically exclude queued/running research, so a + normal rolling release cannot cut a paid run in half. + Use `run-research.mjs` only as an explicitly authorized local fallback when the internal service is unavailable. That fallback requires package build freshness plus its own provider credentials; `--load-shell-env` imports only @@ -187,6 +196,11 @@ report **incurred cost `$0.00`** and say that engine `cost_usd` metadata and grounding were **not produced**; do not describe either as `null`. If standalone provider probes were made, report their known quota/credit cost separately. +For an interrupted internal run, distinguish client transport from server +execution. Preserve its request ID and use `--request-id` to retrieve the same +durable record. Do not report `$0.00`, infer provider cost, or start a replacement +run until the durable status proves the original attempt terminal. + Run `node skills/run-braintied-research/scripts/run-internal-research.mjs --help` for the default internal surface and `run-research.mjs --help` for the local fallback. Read [references/runtime.md](references/runtime.md) when diff --git a/skills/run-braintied-research/references/runtime.md b/skills/run-braintied-research/references/runtime.md index 56faaca..2ad2631 100644 --- a/skills/run-braintied-research/references/runtime.md +++ b/skills/run-braintied-research/references/runtime.md @@ -58,13 +58,28 @@ tool request requires an agent identity, `cortex` product access, and an execution-capable scope (`execute`, `write`, `admin`, `master`, or `*`). The initial registry exposes `research.run`; provider keys are never returned. -Deep requests default to a 3,600-second client deadline; other kinds default to -1,200 seconds. The Cortex route commits response headers immediately for long -work and emits JSON-whitespace heartbeats every 15 seconds before one final JSON -envelope. Do not place a buffering proxy in this path. A transport failure -reports only its sanitized error name/cause code and request ID. A streamed -HTTP 200 is not itself success: the client requires `ok: true` and a non-empty -report before it writes any artifact. +Live runs require internal-tools protocol v2. The client POSTs the request once +to the catalog-advertised durable submission path, then polls the advertised +tenant-bound status path until it receives a terminal result. Cortex persists +the validated input before Inngest dispatch, binds idempotency to the complete +organization/user/agent/request identity, and retains results for 24 hours. +Submission retries, temporary `404`/`429`/`5xx` responses, and transport loss +reuse the same request ID. A response lost after persistence therefore +reattaches instead of paying for a second run. + +Deep requests default to a 3,600-second local client deadline; other kinds +default to 1,200 seconds. The deadline stops polling, not the server-owned run. +Preserve the request ID from the diagnostic and repeat the same invocation with +`--request-id ` to resume. Metadata records both `request_id` and +`durable_run_id`. A successful HTTP response is not itself success: the client +requires the strict completed result envelope and a non-empty report before it +writes artifacts. + +Cortex deployments acquire an atomic database exclusion lease. Lease +acquisition fails while any unexpired research run is queued/running, and new +submissions receive retryable `503` responses while the lease is held. The +lease has a bounded expiry so a killed deploy client cannot pause research +indefinitely. ## Local provider fallback: package and build diff --git a/skills/run-braintied-research/scripts/run-internal-research.mjs b/skills/run-braintied-research/scripts/run-internal-research.mjs index ee81975..664d402 100644 --- a/skills/run-braintied-research/scripts/run-internal-research.mjs +++ b/skills/run-braintied-research/scripts/run-internal-research.mjs @@ -37,6 +37,7 @@ Options: --endpoint Internal tool endpoint --timeout-seconds Request timeout, 1-3600 (default: 3600 for deep, 1200 otherwise) + --request-id Optional durable idempotency key for resuming a run --keychain-service macOS Keychain service (default: braintied-agent-auth) --keychain-account macOS Keychain account (default: codex) --output Required Markdown report output @@ -72,6 +73,7 @@ function parseCli() { 'as-of': { type: 'string' }, endpoint: { type: 'string' }, 'timeout-seconds': { type: 'string' }, + 'request-id': { type: 'string' }, 'keychain-service': { type: 'string', default: DEFAULT_KEYCHAIN_SERVICE }, 'keychain-account': { type: 'string', default: DEFAULT_KEYCHAIN_ACCOUNT }, output: { type: 'string' }, @@ -131,6 +133,14 @@ function parseAsOf(raw) { return raw; } +function parseRequestId(raw) { + if (raw === undefined) return undefined; + if (!/^[A-Za-z0-9._:-]{1,128}$/.test(raw)) { + throw new Error('--request-id must contain 1-128 URL-safe identifier characters.'); + } + return raw; +} + function resolveEndpoint(raw) { const endpoint = new URL(raw ?? process.env.BRAINTIED_INTERNAL_TOOLS_URL ?? DEFAULT_ENDPOINT); if (endpoint.username !== '' || endpoint.password !== '' || endpoint.search !== '' || endpoint.hash !== '') { @@ -171,6 +181,7 @@ async function probeInternalCatalog({ endpoint, token, timeoutSeconds }) { http_status: null, protocol_version: null, research_run_available: false, + durable_execution: null, error: 'Internal tool catalog is unavailable.', }; } @@ -185,17 +196,36 @@ async function probeInternalCatalog({ endpoint, token, timeoutSeconds }) { http_status: response.status, protocol_version: null, research_run_available: false, + durable_execution: null, error: `Internal tool catalog returned HTTP ${response.status} with invalid JSON.`, }; } const tools = Array.isArray(payload?.tools) ? payload.tools : []; - const researchRunAvailable = tools.some((tool) => tool?.name === 'research.run'); - const ready = response.ok && payload?.ok === true && researchRunAvailable; + const researchTool = tools.find((tool) => tool?.name === 'research.run'); + const researchRunAvailable = researchTool !== undefined; + const execution = researchTool?.execution; + const durableExecution = execution !== null && typeof execution === 'object' + && execution.mode === 'durable-polling' + && execution.submitPath === '/internal/tools/runs' + && execution.statusPathTemplate === '/internal/tools/runs/{runId}' + && Number.isSafeInteger(execution.pollAfterMs) + && execution.pollAfterMs >= 250 + && execution.pollAfterMs <= 10_000 + && Number.isSafeInteger(execution.retentionHours) + && execution.retentionHours >= 1; + const ready = response.ok + && payload?.ok === true + && payload?.protocolVersion === '2' + && researchRunAvailable + && durableExecution; let error = null; if (!response.ok) error = `Internal tool catalog returned HTTP ${response.status}.`; else if (payload?.ok !== true) error = 'Internal tool catalog returned an invalid success envelope.'; else if (!researchRunAvailable) error = 'Internal tool catalog does not advertise research.run.'; + else if (payload?.protocolVersion !== '2' || !durableExecution) { + error = 'Internal tool catalog does not advertise durable research execution.'; + } return { ready, @@ -203,6 +233,15 @@ async function probeInternalCatalog({ endpoint, token, timeoutSeconds }) { http_status: response.status, protocol_version: typeof payload?.protocolVersion === 'string' ? payload.protocolVersion : null, research_run_available: researchRunAvailable, + durable_execution: durableExecution + ? { + mode: execution.mode, + submit_path: execution.submitPath, + status_path_template: execution.statusPathTemplate, + poll_after_ms: execution.pollAfterMs, + retention_hours: execution.retentionHours, + } + : null, error, }; } @@ -562,52 +601,218 @@ function sanitizedTransportDiagnostic(error) { return causeCode === null ? errorName : `${errorName}/${causeCode}`; } -async function executeResearch({ endpoint, token, timeoutSeconds, requestId, input }) { - let response; +function resolveDurableEndpoints(endpoint, probe) { + const durable = probe?.durable_execution; + if (durable === null || durable === undefined) { + throw new Error('Internal tool catalog did not provide durable research endpoints.'); + } + const execution = new URL(endpoint); + const submit = new URL(durable.submit_path, execution); + if (submit.origin !== execution.origin + || submit.username !== '' + || submit.password !== '' + || submit.search !== '' + || submit.hash !== '') { + throw new Error('Internal tool catalog returned an unsafe durable submission endpoint.'); + } + return { + submit: submit.toString(), + status(runId) { + const path = durable.status_path_template.replace('{runId}', encodeURIComponent(runId)); + const status = new URL(path, execution); + if (status.origin !== execution.origin + || status.username !== '' + || status.password !== '' + || status.search !== '' + || status.hash !== '') { + throw new Error('Internal tool catalog returned an unsafe durable status endpoint.'); + } + return status.toString(); + }, + pollAfterMs: durable.poll_after_ms, + }; +} + +function retryableHttpStatus(status) { + return [404, 408, 425, 429, 502, 503, 504].includes(status); +} + +function retryDelayMs(attempt, requestedMs = null) { + if (Number.isSafeInteger(requestedMs) && requestedMs >= 250 && requestedMs <= 10_000) { + return requestedMs; + } + return Math.min(5_000, 500 * (2 ** Math.min(attempt, 4))); +} + +async function waitForRetry(delayMs, deadlineMs) { + const remainingMs = deadlineMs - Date.now(); + if (remainingMs <= 0) return; + await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, remainingMs))); +} + +function boundedServerMessage(response, payload) { + const serverMessage = payload?.error?.message; + if (typeof serverMessage === 'string' && serverMessage.length <= 500) { + return serverMessage; + } + return response.ok + ? 'Internal tool returned a failed execution envelope.' + : `Internal tool returned HTTP ${response.status}.`; +} + +async function fetchDurableJson({ url, method, token, requestId, body, deadlineMs }) { + const remainingMs = deadlineMs - Date.now(); + if (remainingMs <= 0) return { kind: 'deadline' }; try { - response = await fetch(endpoint, { - method: 'POST', + const response = await fetch(url, { + method, headers: { authorization: `Bearer ${token}`, - 'content-type': 'application/json', + ...(body === undefined ? {} : { 'content-type': 'application/json' }), 'x-request-id': requestId, }, - body: JSON.stringify({ tool: 'research.run', input }), - signal: AbortSignal.timeout(timeoutSeconds * 1000), + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(Math.max(1, Math.min(15_000, remainingMs))), }); + let payload; + try { + payload = await response.json(); + } catch { + return { kind: 'invalid-json', response }; + } + return { kind: 'response', response, payload }; } catch (error) { - const diagnostic = sanitizedTransportDiagnostic(error); - const message = error instanceof Error - && (error.name === 'TimeoutError' || error.name === 'AbortError') - ? `Internal research timed out after ${timeoutSeconds} seconds (request ${requestId}; transport ${diagnostic}).` - : `Internal research connection closed before completion (request ${requestId}; transport ${diagnostic}). Verify the internal tool catalog and long-run heartbeat deployment, then retry.`; - throw new Error(message); + return { + kind: 'transport', + diagnostic: sanitizedTransportDiagnostic(error), + }; } +} - let payload; - try { - payload = await response.json(); - } catch { - if (response.status === 404) { - throw new Error('Internal tool returned HTTP 404 with invalid JSON. The deployed /internal/tools route may be missing; run --check --probe to verify the catalog.'); - } - if (response.ok) { - throw new Error(`Internal research response stream ended before a valid JSON result was received (request ${requestId}). Verify the long-run heartbeat deployment, then retry.`); +async function submitDurableResearch({ + endpoint, + token, + requestId, + input, + deadlineMs, +}) { + let attempt = 0; + let lastDiagnostic = 'UnknownError'; + while (Date.now() < deadlineMs) { + const outcome = await fetchDurableJson({ + url: endpoint, + method: 'POST', + token, + requestId, + body: { tool: 'research.run', input }, + deadlineMs, + }); + if (outcome.kind === 'response') { + const { response, payload } = outcome; + const run = payload?.run; + const validRun = payload?.ok === true + && run !== null + && typeof run === 'object' + && typeof run.id === 'string' + && /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(run.id) + && run.requestId === requestId + && ['queued', 'running', 'completed', 'failed'].includes(run.status); + if ((response.status === 202 || response.status === 200) && validRun) { + return run; + } + if (!retryableHttpStatus(response.status)) { + throw new Error(boundedServerMessage(response, payload)); + } + lastDiagnostic = `HTTP_${response.status}`; + } else if (outcome.kind === 'invalid-json') { + if (!retryableHttpStatus(outcome.response.status)) { + throw new Error(`Internal durable submission returned HTTP ${outcome.response.status} with invalid JSON.`); + } + lastDiagnostic = `HTTP_${outcome.response.status}_INVALID_JSON`; + } else if (outcome.kind === 'transport') { + lastDiagnostic = outcome.diagnostic; } - throw new Error(`Internal tool returned HTTP ${response.status} with invalid JSON.`); + await waitForRetry(retryDelayMs(attempt), deadlineMs); + attempt += 1; } + throw new Error( + `Internal research submission timed out (request ${requestId}; last transport ${lastDiagnostic}). Re-run with --request-id ${requestId} to resume the same durable run.`, + ); +} - if (!response.ok || payload?.ok !== true) { - const serverMessage = payload?.error?.message; - const message = typeof serverMessage === 'string' && serverMessage.length <= 500 - ? serverMessage - : response.ok - ? 'Internal tool returned a failed execution envelope.' - : `Internal tool returned HTTP ${response.status}.`; - throw new Error(message); +async function pollDurableResearch({ + endpoint, + token, + requestId, + runId, + pollAfterMs, + deadlineMs, +}) { + let attempt = 0; + let lastDiagnostic = 'UnknownError'; + while (Date.now() < deadlineMs) { + const outcome = await fetchDurableJson({ + url: endpoint, + method: 'GET', + token, + requestId, + deadlineMs, + }); + if (outcome.kind === 'response') { + const { response, payload } = outcome; + if (response.status === 200 && payload?.ok === true) { + return validateResult(payload); + } + if (response.status === 202 + && payload?.ok === true + && payload?.run?.id === runId + && ['queued', 'running'].includes(payload.run.status)) { + await waitForRetry( + retryDelayMs(attempt, payload.run.pollAfterMs ?? pollAfterMs), + deadlineMs, + ); + attempt = 0; + continue; + } + if (!retryableHttpStatus(response.status)) { + throw new Error(boundedServerMessage(response, payload)); + } + lastDiagnostic = `HTTP_${response.status}`; + } else if (outcome.kind === 'invalid-json') { + if (!retryableHttpStatus(outcome.response.status)) { + throw new Error(`Internal durable status returned HTTP ${outcome.response.status} with invalid JSON.`); + } + lastDiagnostic = `HTTP_${outcome.response.status}_INVALID_JSON`; + } else if (outcome.kind === 'transport') { + lastDiagnostic = outcome.diagnostic; + } + await waitForRetry(retryDelayMs(attempt), deadlineMs); + attempt += 1; } + throw new Error( + `Internal research timed out before its durable result was ready (request ${requestId}; run ${runId}; last transport ${lastDiagnostic}). Re-run with --request-id ${requestId} to resume.`, + ); +} - return validateResult(payload); +async function executeResearch({ endpoint, token, timeoutSeconds, requestId, input, probe }) { + const durable = resolveDurableEndpoints(endpoint, probe); + const deadlineMs = Date.now() + timeoutSeconds * 1000; + const run = await submitDurableResearch({ + endpoint: durable.submit, + token, + requestId, + input, + deadlineMs, + }); + const result = await pollDurableResearch({ + endpoint: durable.status(run.id), + token, + requestId, + runId: run.id, + pollAfterMs: durable.pollAfterMs, + deadlineMs, + }); + return { result, runId: run.id }; } async function main() { @@ -638,6 +843,7 @@ async function main() { const profileRef = parseProfileRef(values.profile); const profileMode = parseProfileMode(values['profile-mode']); const asOf = parseAsOf(values['as-of']); + const requestedRequestId = parseRequestId(values['request-id']); if (values.probe === true && values.check !== true && values['dry-run'] !== true) { throw new Error('--probe is supported only with --check or --dry-run.'); } @@ -677,6 +883,7 @@ async function main() { requested_profile_ref: profileRef ?? null, requested_profile_mode: profileMode ?? null, requested_as_of: asOf ?? null, + requested_request_id: requestedRequestId ?? null, timeout_seconds: timeoutSeconds, agent_token_source: auth.source, agent_token_present: auth.token !== null, @@ -718,15 +925,22 @@ async function main() { throw new Error('--output, --metadata, and --trusted-output must be different paths.'); } + const liveProbe = probe + ?? await probeInternalCatalog({ endpoint, token: auth.token, timeoutSeconds }); + if (!liveProbe.ready) { + throw new Error(`Preflight failed: ${liveProbe.error ?? 'durable internal tool catalog probe failed'}.`); + } + const brief = await readBrief(values); - const requestId = randomUUID(); + const requestId = requestedRequestId ?? randomUUID(); const startedAt = new Date(); const startedMonotonic = process.hrtime.bigint(); - const result = await executeResearch({ + const execution = await executeResearch({ endpoint, token: auth.token, timeoutSeconds, requestId, + probe: liveProbe, input: { brief, kind, @@ -740,6 +954,7 @@ async function main() { ...(asOf !== undefined ? { asOf } : {}), }, }); + const result = execution.result; if (profileRef !== undefined && result.privateManifest === undefined) { throw new Error('Profile research completed without its required trusted-local private manifest.'); } @@ -754,6 +969,8 @@ async function main() { mode: 'internal', package_version: check.package_version, request_id: requestId, + durable_run_id: execution.runId, + execution_protocol: liveProbe.protocol_version, started_at: startedAt.toISOString(), finished_at: finishedAt.toISOString(), duration_ms: Math.round(durationMs), @@ -814,6 +1031,7 @@ async function main() { metadata: metadataPath, trusted_output: trustedOutputPath, request_id: requestId, + durable_run_id: execution.runId, kind: metadata.kind, engine: metadata.engine, cost_usd: metadata.cost_usd, diff --git a/test/internal-runner-preflight.test.ts b/test/internal-runner-preflight.test.ts index d4f6895..45e66d5 100644 --- a/test/internal-runner-preflight.test.ts +++ b/test/internal-runner-preflight.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { spawn, spawnSync } from 'node:child_process'; import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; -import { createServer } from 'node:http'; +import { createServer, type ServerResponse } from 'node:http'; import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; @@ -12,6 +12,26 @@ const runner = path.join( packageRoot, 'skills/run-braintied-research/scripts/run-internal-research.mjs', ); +const DURABLE_RUN_ID = '22222222-2222-4222-8222-222222222222'; + +function sendDurableCatalog(response: ServerResponse): void { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + ok: true, + protocolVersion: '2', + tools: [{ + name: 'research.run', + version: '2', + execution: { + mode: 'durable-polling', + submitPath: '/internal/tools/runs', + statusPathTemplate: '/internal/tools/runs/{runId}', + pollAfterMs: 250, + retentionHours: 24, + }, + }], + })); +} function runRunner(args: string[], env: NodeJS.ProcessEnv): Promise<{ status: number | null; @@ -157,19 +177,37 @@ test('profile runner separates public Markdown, reference metadata, and trusted- const trustedOutputPath = path.join(temporaryDirectory, 'trusted.json'); const server = createServer(async (request, response) => { - let requestBody = ''; - for await (const chunk of request) requestBody += chunk.toString(); - const parsed = JSON.parse(requestBody) as { - input: { - profileRef: string; - profileMode: string; - asOf: string; + if (request.method === 'GET' && request.url === '/internal/tools') { + sendDurableCatalog(response); + return; + } + if (request.method === 'POST' && request.url === '/internal/tools/runs') { + let requestBody = ''; + for await (const chunk of request) requestBody += chunk.toString(); + const parsed = JSON.parse(requestBody) as { + input: { + profileRef: string; + profileMode: string; + asOf: string; + }; }; - }; - assert.equal(parsed.input.profileRef, 'web-design-intelligence@1'); - assert.equal(parsed.input.profileMode, 'snapshot'); - assert.equal(parsed.input.asOf, '2026-07-22'); - + assert.equal(parsed.input.profileRef, 'web-design-intelligence@1'); + assert.equal(parsed.input.profileMode, 'snapshot'); + assert.equal(parsed.input.asOf, '2026-07-22'); + response.writeHead(202, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + ok: true, + run: { + id: DURABLE_RUN_ID, + requestId: request.headers['x-request-id'], + status: 'queued', + pollAfterMs: 250, + }, + })); + return; + } + assert.equal(request.method, 'GET'); + assert.equal(request.url, `/internal/tools/runs/${DURABLE_RUN_ID}`); response.writeHead(200, { 'content-type': 'application/json' }); response.end(JSON.stringify({ ok: true, @@ -349,7 +387,26 @@ test('profile runner rejects unknown private-text fields in a server manifest', const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'braintied-profile-leak-')); t.after(async () => { await rm(temporaryDirectory, { recursive: true, force: true }); }); - const server = createServer((_request, response) => { + const server = createServer((request, response) => { + if (request.method === 'GET' && request.url === '/internal/tools') { + sendDurableCatalog(response); + return; + } + if (request.method === 'POST' && request.url === '/internal/tools/runs') { + response.writeHead(202, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + ok: true, + run: { + id: DURABLE_RUN_ID, + requestId: request.headers['x-request-id'], + status: 'queued', + pollAfterMs: 250, + }, + })); + return; + } + assert.equal(request.method, 'GET'); + assert.equal(request.url, `/internal/tools/runs/${DURABLE_RUN_ID}`); response.writeHead(200, { 'content-type': 'application/json' }); response.end(JSON.stringify({ ok: true, @@ -430,24 +487,39 @@ test('profile runner rejects unknown private-text fields in a server manifest', assert.equal(result.stderr.includes('PRIVATE-MUST-NOT-BE-WRITTEN'), false); }); -test('live internal runner accepts whitespace heartbeats before the final JSON envelope', async (t) => { +test('live internal runner reattaches to a durable run after submission transport loss', async (t) => { const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'braintied-heartbeat-')); t.after(async () => { await rm(temporaryDirectory, { recursive: true, force: true }); }); const reportPath = path.join(temporaryDirectory, 'report.md'); const metadataPath = path.join(temporaryDirectory, 'metadata.json'); + let submissionAttempts = 0; + let statusReads = 0; + let acceptedRequestId: string | undefined; const server = createServer(async (request, response) => { - assert.equal(request.url, '/internal/tools/execute'); assert.equal(request.headers.authorization, 'Bearer sat_heartbeat_test'); - let requestBody = ''; - for await (const chunk of request) requestBody += chunk.toString(); - const parsed = JSON.parse(requestBody) as { tool: string; input: { kind: string } }; - assert.equal(parsed.tool, 'research.run'); - assert.equal(parsed.input.kind, 'quick'); - - response.writeHead(200, { 'content-type': 'application/json; charset=UTF-8' }); - response.write(' \n'); - setTimeout(() => { + if (request.method === 'GET' && request.url === '/internal/tools') { + sendDurableCatalog(response); + return; + } + if (request.method === 'GET' + && request.url === `/internal/tools/runs/${DURABLE_RUN_ID}`) { + statusReads += 1; + response.writeHead(statusReads === 1 ? 202 : 200, { + 'content-type': 'application/json', + }); + if (statusReads === 1) { + response.end(JSON.stringify({ + ok: true, + run: { + id: DURABLE_RUN_ID, + requestId: acceptedRequestId, + status: 'running', + pollAfterMs: 250, + }, + })); + return; + } response.end(JSON.stringify({ ok: true, tool: 'research.run', @@ -455,13 +527,13 @@ test('live internal runner accepts whitespace heartbeats before the final JSON e kind: 'quick', engine: 'pipeline', report: { - title: 'Heartbeat result', - executive_summary: 'The connection stayed open.', - full_markdown: '# Heartbeat result\n\nThe connection stayed open.', + title: 'Durable result', + executive_summary: 'The run survived a lost submission response.', + full_markdown: '# Durable result\n\nThe run survived a lost submission response.', sections: [], bibliography: [], gaps: [], - word_count: 6, + word_count: 9, }, grounding: { ratio: 0, @@ -475,9 +547,42 @@ test('live internal runner accepts whitespace heartbeats before the final JSON e quoteCount: 0, briefSha256: 'a'.repeat(64), }, - meta: { requestId: 'heartbeat-request', durationMs: 20 }, + meta: { + requestId: acceptedRequestId, + runId: DURABLE_RUN_ID, + durationMs: 20, + durable: true, + }, })); - }, 20); + return; + } + + assert.equal(request.method, 'POST'); + assert.equal(request.url, '/internal/tools/runs'); + let requestBody = ''; + for await (const chunk of request) requestBody += chunk.toString(); + const parsed = JSON.parse(requestBody) as { tool: string; input: { kind: string } }; + assert.equal(parsed.tool, 'research.run'); + assert.equal(parsed.input.kind, 'quick'); + submissionAttempts += 1; + const requestId = request.headers['x-request-id']; + assert.equal(typeof requestId, 'string'); + if (acceptedRequestId === undefined) acceptedRequestId = requestId; + assert.equal(requestId, acceptedRequestId); + if (submissionAttempts === 1) { + request.socket.destroy(); + return; + } + response.writeHead(202, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + ok: true, + run: { + id: DURABLE_RUN_ID, + requestId, + status: 'queued', + pollAfterMs: 250, + }, + })); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); t.after(() => { server.close(); }); @@ -496,17 +601,27 @@ test('live internal runner accepts whitespace heartbeats before the final JSON e ], { ...process.env, BRAINTIED_AGENT_TOKEN: 'sat_heartbeat_test' }); assert.equal(result.status, 0, result.stderr); - assert.match(await readFile(reportPath, 'utf8'), /Heartbeat result/); + assert.match(await readFile(reportPath, 'utf8'), /Durable result/); const metadata = JSON.parse(await readFile(metadataPath, 'utf8')) as { timeout_seconds: number; + durable_run_id: string; + execution_protocol: string; }; assert.equal(metadata.timeout_seconds, 5); + assert.equal(metadata.durable_run_id, DURABLE_RUN_ID); + assert.equal(metadata.execution_protocol, '2'); + assert.equal(submissionAttempts, 2); + assert.equal(statusReads, 2); }); test('live internal runner preserves a sanitized transport diagnostic and request ID', async (t) => { const temporaryDirectory = await mkdtemp(path.join(tmpdir(), 'braintied-transport-')); t.after(async () => { await rm(temporaryDirectory, { recursive: true, force: true }); }); - const server = createServer((request) => { + const server = createServer((request, response) => { + if (request.method === 'GET' && request.url === '/internal/tools') { + sendDurableCatalog(response); + return; + } request.socket.destroy(); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); @@ -520,14 +635,16 @@ test('live internal runner preserves a sanitized transport diagnostic and reques '--max-cost-usd', '0.25', '--endpoint', `http://127.0.0.1:${address.port}/internal/tools/execute`, '--timeout-seconds', '5', + '--request-id', 'request-transport-1', '--output', path.join(temporaryDirectory, 'report.md'), '--metadata', path.join(temporaryDirectory, 'metadata.json'), '--allow-external', ], { ...process.env, BRAINTIED_AGENT_TOKEN: 'sat_transport_test' }); assert.equal(result.status, 1); - assert.match(result.stderr, /connection closed before completion/); - assert.match(result.stderr, /request [a-f0-9-]{36}; transport TypeError(?:\/[A-Z0-9_]+)?/); + assert.match(result.stderr, /submission timed out/); + assert.match(result.stderr, /request request-transport-1; last transport TypeError(?:\/[A-Z0-9_]+)?/); + assert.match(result.stderr, /--request-id request-transport-1/); assert.equal(result.stderr.includes('sat_transport_test'), false); }); @@ -552,11 +669,42 @@ test('internal preflight fails closed when Agent Auth is unavailable', () => { test('authenticated catalog probe confirms research.run is deployed', async (t) => { const server = createServer((request, response) => { assert.equal(request.headers.authorization, 'Bearer sat_probe_test'); + sendDurableCatalog(response); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.close(); }); + const address = server.address(); + assert.ok(address !== null && typeof address === 'object'); + + const result = await runRunner([ + '--check', '--probe', + '--kind', 'quick', + '--max-cost-usd', '1', + '--endpoint', `http://127.0.0.1:${address.port}/internal/tools/execute`, + ], { ...process.env, BRAINTIED_AGENT_TOKEN: 'sat_probe_test' }); + + assert.equal(result.status, 0, result.stderr); + const check = JSON.parse(result.stdout) as { + ready: boolean; + probe: { + http_status: number; + research_run_available: boolean; + durable_execution: { mode: string }; + }; + }; + assert.equal(check.ready, true); + assert.equal(check.probe.http_status, 200); + assert.equal(check.probe.research_run_available, true); + assert.equal(check.probe.durable_execution.mode, 'durable-polling'); +}); + +test('catalog probe rejects legacy streaming-only execution', async (t) => { + const server = createServer((_request, response) => { response.writeHead(200, { 'content-type': 'application/json' }); response.end(JSON.stringify({ ok: true, protocolVersion: '1', - tools: [{ name: 'research.run' }], + tools: [{ name: 'research.run', version: '1' }], })); }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); @@ -571,14 +719,13 @@ test('authenticated catalog probe confirms research.run is deployed', async (t) '--endpoint', `http://127.0.0.1:${address.port}/internal/tools/execute`, ], { ...process.env, BRAINTIED_AGENT_TOKEN: 'sat_probe_test' }); - assert.equal(result.status, 0, result.stderr); + assert.equal(result.status, 2, result.stderr); const check = JSON.parse(result.stdout) as { ready: boolean; - probe: { http_status: number; research_run_available: boolean }; + probe: { error: string }; }; - assert.equal(check.ready, true); - assert.equal(check.probe.http_status, 200); - assert.equal(check.probe.research_run_available, true); + assert.equal(check.ready, false); + assert.match(check.probe.error, /does not advertise durable research execution/); }); test('catalog probe fails readiness when deployment returns 404 HTML', async (t) => {