From 3e4571aeb6061a6cf6671567777ab39013e586a7 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 15 Jul 2026 13:37:28 -0600 Subject: [PATCH] feat(preflight): deploy-time secret-liveness probes + bin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI cannot hold production secrets, so a dead secret is invisible until the first live request fails. On 2026-07-15 four secrets were simultaneously dead in one production day — a dead SANDBOX_API_KEY, a stale SANDBOX_API_URL, and a dead LiteLLM router key + URL — each present in `wrangler secret list` yet invalid against its live endpoint, with nothing checking liveness anywhere. Bind the check at DEPLOY time instead: a product declares probes built from its real env, and `agent-app-preflight` runs them as a step before deploy, failing with a message that names exactly which secret to rotate. - `/preflight`: `runPreflight(probes)` → per-probe verdict + latency + overall pass/fail (any critical probe fails → the run fails; probes default critical). - Standard builders (explicit config, read nothing global): `routerChatProbe` (cheap POST /chat/completions, max_tokens 1 — 200 live, 401/403 dead key, 503 upstream-down), `sandboxAuthProbe` (GET /v1/sandboxes?limit=1), `httpHeadProbe`. Failures classify dead-key vs stale-url vs upstream-down. - `bin/preflight.mjs` reads `preflight.config.mjs` from cwd, prints a table, exits 1 on failure / 2 on config error. - Example config wires legal-agent's four incident probes. Server-only: not added to the browser-safe manifest. --- AGENTS.md | 1 + bin/preflight.mjs | 47 ++++ examples/preflight.config.mjs | 55 +++++ package.json | 9 + src/index.ts | 1 + src/preflight/index.test.ts | 261 ++++++++++++++++++++ src/preflight/index.ts | 441 ++++++++++++++++++++++++++++++++++ tsup.config.ts | 1 + 8 files changed, 816 insertions(+) create mode 100755 bin/preflight.mjs create mode 100644 examples/preflight.config.mjs create mode 100644 src/preflight/index.test.ts create mode 100644 src/preflight/index.ts diff --git a/AGENTS.md b/AGENTS.md index a01db61..75514d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,7 @@ Corollary — **extend, never duplicate.** Before writing anything that complete | `/chat-routes` | the assembled server chat vertical (#188 Phase 1): `createChatTurnRoutes()` — body parse/validate → injected `authorize` seam → producer seam → turn-buffer tap wired BY DEFAULT → NDJSON `Response` + replay endpoint + composed `/interactions` answer endpoints + `/chat-store` persistence (user row on send, assistant row with parts/usage on completion); `createSandboxChatProducer` (raw sandbox events → client vocabulary + persisted parts + usage receipt, non-renderable-ask auto-decline); `createUploadRoute` (multipart → inline `data:` part ≤700 KiB, else base64 write through a structural sandbox sink + path-ref part — the >1 MiB gateway cap makes the two-step mandatory); import-free `./wire` contract (`ChatTurnRequestPayload`, inline-part byte gate) re-exported via `/web-react`'s chat-stream | peer `@tangle-network/agent-runtime` (`handleChatTurn` IS the turn engine — zero loop logic here; subpath-only, not in the root barrel); composes `/stream`, `/chat-store`, `/interactions`, `/web`. Reference assembly: `examples/chat-app.md` | | `/tangle` | app-registration consent URL + cached broker-token provider | structural `TangleAppsClient` (from agent-integrations) | | `/billing` | per-workspace budget-capped key manager (mint/rotate/rollover/usage) | structural tcloud provisioner + store + crypto seams | +| `/preflight` | deploy-time secret-liveness probes: `runPreflight(probes)` → per-probe verdict + latency + overall pass/fail (any critical fail → fail); standard builders `routerChatProbe`/`sandboxAuthProbe`/`httpHeadProbe` (explicit config, read nothing global); `formatPreflightReport` table + the `agent-app-preflight` bin reading `preflight.config.mjs`. Binds at DEPLOY time (the one place with real secrets — CI can't hold them); failures name the exact secret to rotate | — (server-only; product declares probes from `process.env`) | | `/missions` | durable multi-step mission orchestration: guarded status/step machine + cursor + cost ledger over a `MissionStorePort` (CAS updates → typed conflict, opaque `extras` insert passthrough for product columns), idempotent plan engine (cached-done short-circuit, cursor reconciliation, retryable-vs-deterministic failure, detached-session polling), budget/classification/volume gates that park as `waiting_approval`, `:::mission` parser, client-safe live-event reducer + the canonical `StepAgentActivity` per-step delegated-run lane (`step.updated` snapshot, latest-wins) | — (substrate-free; product supplies storage, `SandboxDispatch`, approvals port, `classifyStep`) | | `/trace` | flow observability: FlowSpan/FlowTrace + ASCII waterfall/histogram renderers; mission trace bridge (`createMissionTraceContext`/`childSpanContext`/`traceEnv` — 32-hex/16-hex ids + the `TRACE_ID`/`PARENT_SPAN_ID` env pair agent-runtime's `readTraceContextFromEnv` inherits); delegation→FlowSpan converters (`delegationActivityToFlowSpans`, `loopTraceEventsToFlowSpans` over a structural `LoopTraceEventLike`, `composeMissionFlowTrace`, `stepActivityFlowTrace`) | — (pure data; id formats byte-match agent-runtime's OTLP export, no import) | | `/web-react` | shared chat-shell + observability components: ModelPicker/EffortPicker/ChatMessages/RunDrillIn, `MissionActivityLane` (per-step delegated-run sub-rows → web waterfall), `AgentActivityPanel` (cross-context delegation surface over a `fetchActivity(cursor)` data port, missionRef link slot), `FlowWaterfall` + pure `waterfallLayout`/`mergeActivityPages` helpers, `InteractionQuestionCard`/`InteractionPlanCard` + `useChatInteractions`/`createInteractionAnswerSubmitter` (the client half of `/interactions`) | react peer; renders `/missions` lanes via `/trace` converters and `/interactions` asks via its contract | diff --git a/bin/preflight.mjs b/bin/preflight.mjs new file mode 100755 index 0000000..8279c99 --- /dev/null +++ b/bin/preflight.mjs @@ -0,0 +1,47 @@ +#!/usr/bin/env node +/** + * `agent-app-preflight` — run a product's declared secret-liveness probes as a + * DEPLOY step and fail the deploy if any critical secret is dead. + * + * Reads `preflight.config.mjs` from the current working directory (override + * with `PREFLIGHT_CONFIG`). That file default-exports the probes, built from + * `process.env` at load time — the deploy already has the real secrets in its + * environment, which is the one place they can be probed for liveness (CI + * cannot hold them). Exit 0 when every critical probe is live, 1 when one is + * dead (deploy fails), 2 on a config/usage error. + * + * Wire it as a step BEFORE `wrangler deploy`: + * agent-app-preflight + */ +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +import { formatPreflightReport, runPreflight } from '../dist/preflight/index.js' + +const configFile = process.env.PREFLIGHT_CONFIG ?? 'preflight.config.mjs' + +async function loadProbes() { + const path = resolve(process.cwd(), configFile) + if (!existsSync(path)) { + console.error(`preflight: no config at ${path}`) + console.error( + `Create ${configFile} default-exporting probes built from process.env — see '@tangle-network/agent-app/preflight'.`, + ) + process.exit(2) + } + const mod = await import(pathToFileURL(path).href) + const exported = mod.default ?? mod.probes + const resolved = typeof exported === 'function' ? await exported() : exported + const probes = Array.isArray(resolved) ? resolved : resolved?.probes + if (!Array.isArray(probes) || probes.length === 0) { + console.error(`preflight: ${configFile} must default-export a non-empty array of probes (or { probes }).`) + process.exit(2) + } + return probes +} + +const probes = await loadProbes() +const report = await runPreflight(probes) +console.log(formatPreflightReport(report)) +process.exit(report.ok ? 0 : 1) diff --git a/examples/preflight.config.mjs b/examples/preflight.config.mjs new file mode 100644 index 0000000..02cdd18 --- /dev/null +++ b/examples/preflight.config.mjs @@ -0,0 +1,55 @@ +/** + * Example `preflight.config.mjs` — legal-agent's four secret-liveness probes. + * + * Copy this to a product's repo root. The deploy workflow runs + * `agent-app-preflight` (added by this package's `bin`) as a step before + * `wrangler deploy`; a dead secret fails the deploy with a message naming + * exactly which secret to rotate. + * + * Each probe below guards a secret from the 2026-07-15 incident — four secrets + * dead in one production day, all present in `wrangler secret list`, none + * checked for liveness anywhere: + * 1. routerChatProbe → LITELLM_API_KEY (dead key) + LITELLM_BASE_URL (dead url) + * 2. sandboxAuthProbe → SANDBOX_API_KEY (dead key) + SANDBOX_API_URL (stale url) + * 3. httpHeadProbe → TANGLE_PLATFORM_URL reachability (stale platform url) + * 4. httpHeadProbe → session-gateway reachability (non-critical: warn only) + * + * The config reads nothing but `process.env`, so the same file runs in the + * deploy environment, a smoke test, or a local check. + */ +import { httpHeadProbe, routerChatProbe, sandboxAuthProbe } from '@tangle-network/agent-app/preflight' + +const env = process.env + +const trimSlash = (url) => (url ?? '').replace(/\/+$/, '') + +export default [ + routerChatProbe({ + name: 'router-chat', + baseUrl: env.LITELLM_BASE_URL, + apiKey: env.LITELLM_API_KEY, + model: env.PREFLIGHT_ROUTER_MODEL ?? 'gpt-4o-mini', + keySecret: 'LITELLM_API_KEY', + urlSecret: 'LITELLM_BASE_URL', + }), + sandboxAuthProbe({ + name: 'sandbox-auth', + baseUrl: env.SANDBOX_API_URL, + apiKey: env.SANDBOX_API_KEY, + keySecret: 'SANDBOX_API_KEY', + urlSecret: 'SANDBOX_API_URL', + }), + httpHeadProbe({ + name: 'tangle-platform', + url: `${trimSlash(env.TANGLE_PLATFORM_URL)}/health`, + urlSecret: 'TANGLE_PLATFORM_URL', + }), + httpHeadProbe({ + name: 'session-gateway', + url: `${trimSlash(env.SESSION_GATEWAY_URL)}/health`, + urlSecret: 'SESSION_GATEWAY_URL', + // A degraded gateway should warn the operator, not block a deploy that fixes + // other secrets — the interactive stream reconnects when it recovers. + critical: false, + }), +] diff --git a/package.json b/package.json index b823ef2..ddc5214 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,12 @@ "types": "./dist/index.d.ts", "files": [ "dist", + "bin", ".claude/skills" ], + "bin": { + "agent-app-preflight": "./bin/preflight.mjs" + }, "exports": { ".": { "types": "./dist/index.d.ts", @@ -138,6 +142,11 @@ "import": "./dist/billing/index.js", "default": "./dist/billing/index.js" }, + "./preflight": { + "types": "./dist/preflight/index.d.ts", + "import": "./dist/preflight/index.js", + "default": "./dist/preflight/index.js" + }, "./chat-store": { "types": "./dist/chat-store/index.d.ts", "import": "./dist/chat-store/index.js", diff --git a/src/index.ts b/src/index.ts index b08d802..5b7d581 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,7 @@ export * from './harness/index' export * from './config/index' export * from './preset-cloudflare/index' export * from './billing/index' +export * from './preflight/index' // `/chat-store`'s drizzle factory + store stay subpath-only (they import the // optional drizzle-orm peer at module top); its pure pieces — the stored // `parts` vocabulary, title derivation, input error — are safe here. diff --git a/src/preflight/index.test.ts b/src/preflight/index.test.ts new file mode 100644 index 0000000..50f55c7 --- /dev/null +++ b/src/preflight/index.test.ts @@ -0,0 +1,261 @@ +import { describe, expect, it } from 'vitest' + +import { + formatPreflightReport, + httpHeadProbe, + routerChatProbe, + runPreflight, + sandboxAuthProbe, + type PreflightProbe, +} from './index' + +interface CapturedRequest { + url: string + method?: string + headers?: Record + body?: string +} + +/** A fake `fetch` returning a real `Response` with the given status/body, and + * recording the request it received. */ +function fakeFetch(status: number, body = ''): { fetch: typeof fetch; calls: CapturedRequest[] } { + const calls: CapturedRequest[] = [] + const fetchImpl = (async (url: unknown, init?: RequestInit) => { + calls.push({ + url: String(url), + method: init?.method, + headers: init?.headers as Record | undefined, + body: init?.body as string | undefined, + }) + // 204/205/304 forbid a body; mirror real HTTP so the fake never throws. + const nullBody = status === 204 || status === 205 || status === 304 + return new Response(nullBody ? null : body, { status }) + }) as unknown as typeof fetch + return { fetch: fetchImpl, calls } +} + +/** A fake `fetch` that throws — a DNS / connection-refused style failure. */ +function throwingFetch(message: string): typeof fetch { + return (async () => { + throw new Error(message) + }) as unknown as typeof fetch +} + +/** A fake `fetch` that hangs until the abort signal fires (drives the real + * `AbortSignal.timeout` path). */ +const hangingFetch = ((_url: unknown, init?: RequestInit) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal!.reason)) + })) as unknown as typeof fetch + +describe('routerChatProbe', () => { + it('maps 200 to ok and issues one cheap POST /chat/completions', async () => { + const { fetch, calls } = fakeFetch(200, '{"choices":[]}') + const probe = routerChatProbe({ + baseUrl: 'https://router.example.com/', + apiKey: 'sk-test', + model: 'gpt-4o-mini', + fetchImpl: fetch, + }) + const result = await probe.run() + expect(result.ok).toBe(true) + expect(calls).toHaveLength(1) + expect(calls[0]!.url).toBe('https://router.example.com/chat/completions') + expect(calls[0]!.method).toBe('POST') + expect(calls[0]!.headers?.Authorization).toBe('Bearer sk-test') + const parsed = JSON.parse(calls[0]!.body!) + expect(parsed.max_tokens).toBe(1) + expect(parsed.model).toBe('gpt-4o-mini') + }) + + it('maps 401 to a dead-key failure naming the secret to rotate', async () => { + const { fetch } = fakeFetch(401, 'invalid api key') + const probe = routerChatProbe({ + baseUrl: 'https://router.example.com', + apiKey: 'sk-dead', + model: 'gpt-4o-mini', + keySecret: 'LITELLM_API_KEY', + fetchImpl: fetch, + }) + const result = await probe.run() + expect(result.ok).toBe(false) + expect(result.detail).toContain('DEAD KEY') + expect(result.detail).toContain('LITELLM_API_KEY') + }) + + it('maps 503 to upstream-down and explicitly says NOT to rotate the key', async () => { + const { fetch } = fakeFetch(503, 'service unavailable') + const probe = routerChatProbe({ + baseUrl: 'https://router.example.com', + apiKey: 'sk-live', + model: 'gpt-4o-mini', + keySecret: 'LITELLM_API_KEY', + fetchImpl: fetch, + }) + const result = await probe.run() + expect(result.ok).toBe(false) + expect(result.detail).toContain('UPSTREAM DOWN') + expect(result.detail).toContain('do NOT rotate') + }) + + it('maps a timeout to a failure naming the URL secret to check', async () => { + const probe = routerChatProbe({ + baseUrl: 'https://router.example.com', + apiKey: 'sk-live', + model: 'gpt-4o-mini', + urlSecret: 'LITELLM_BASE_URL', + timeoutMs: 20, + fetchImpl: hangingFetch, + }) + const result = await probe.run() + expect(result.ok).toBe(false) + expect(result.detail).toContain('TIMEOUT') + expect(result.detail).toContain('LITELLM_BASE_URL') + }) + + it('maps a connection failure to unreachable naming the URL secret', async () => { + const probe = routerChatProbe({ + baseUrl: 'https://router.example.com', + apiKey: 'sk-live', + model: 'gpt-4o-mini', + urlSecret: 'LITELLM_BASE_URL', + fetchImpl: throwingFetch('ECONNREFUSED'), + }) + const result = await probe.run() + expect(result.ok).toBe(false) + expect(result.detail).toContain('UNREACHABLE') + expect(result.detail).toContain('LITELLM_BASE_URL') + expect(result.detail).toContain('ECONNREFUSED') + }) +}) + +describe('sandboxAuthProbe', () => { + it('maps 200 to ok and hits GET /v1/sandboxes?limit=1 with a bearer', async () => { + const { fetch, calls } = fakeFetch(200, '{"sandboxes":[]}') + const probe = sandboxAuthProbe({ baseUrl: 'https://sandbox.example.com', apiKey: 'sk-box', fetchImpl: fetch }) + const result = await probe.run() + expect(result.ok).toBe(true) + expect(calls[0]!.url).toBe('https://sandbox.example.com/v1/sandboxes?limit=1') + expect(calls[0]!.method).toBe('GET') + expect(calls[0]!.headers?.Authorization).toBe('Bearer sk-box') + }) + + it('maps 401 to a dead-key failure naming the secret', async () => { + const { fetch } = fakeFetch(401) + const probe = sandboxAuthProbe({ + baseUrl: 'https://sandbox.example.com', + apiKey: 'sk-dead', + keySecret: 'SANDBOX_API_KEY', + fetchImpl: fetch, + }) + const result = await probe.run() + expect(result.ok).toBe(false) + expect(result.detail).toContain('DEAD KEY') + expect(result.detail).toContain('SANDBOX_API_KEY') + }) +}) + +describe('httpHeadProbe', () => { + it('accepts any 2xx/3xx by default', async () => { + const { fetch, calls } = fakeFetch(204) + const probe = httpHeadProbe({ name: 'platform', url: 'https://platform.example.com/health', fetchImpl: fetch }) + const result = await probe.run() + expect(result.ok).toBe(true) + expect(calls[0]!.method).toBe('HEAD') + }) + + it('fails a 500 and names the URL secret to check', async () => { + const { fetch } = fakeFetch(500) + const probe = httpHeadProbe({ + name: 'platform', + url: 'https://platform.example.com/health', + urlSecret: 'TANGLE_PLATFORM_URL', + fetchImpl: fetch, + }) + const result = await probe.run() + expect(result.ok).toBe(false) + expect(result.detail).toContain('UNEXPECTED 500') + expect(result.detail).toContain('TANGLE_PLATFORM_URL') + }) + + it('honours an exact expectStatus', async () => { + const okProbe = httpHeadProbe({ name: 'p', url: 'https://x/', expectStatus: 200, fetchImpl: fakeFetch(200).fetch }) + expect((await okProbe.run()).ok).toBe(true) + const badProbe = httpHeadProbe({ name: 'p', url: 'https://x/', expectStatus: 200, fetchImpl: fakeFetch(301).fetch }) + expect((await badProbe.run()).ok).toBe(false) + }) + + it('honours an expectStatus list', async () => { + const probe = httpHeadProbe({ name: 'p', url: 'https://x/', expectStatus: [200, 405], fetchImpl: fakeFetch(405).fetch }) + expect((await probe.run()).ok).toBe(true) + }) +}) + +describe('runPreflight aggregation', () => { + const passing = (name: string): PreflightProbe => ({ name, run: async () => ({ ok: true, detail: 'ok' }) }) + const failing = (name: string, critical?: boolean): PreflightProbe => ({ + name, + critical, + run: async () => ({ ok: false, detail: 'dead' }), + }) + + it('passes when all probes pass and records per-probe latency', async () => { + const report = await runPreflight([passing('a'), passing('b')]) + expect(report.ok).toBe(true) + expect(report.passed).toBe(2) + expect(report.failed).toBe(0) + expect(report.criticalFailures).toBe(0) + for (const p of report.probes) expect(p.latencyMs).toBeGreaterThanOrEqual(0) + }) + + it('fails overall when a critical probe fails (probes default critical)', async () => { + const report = await runPreflight([passing('a'), failing('b')]) + expect(report.ok).toBe(false) + expect(report.criticalFailures).toBe(1) + const b = report.probes.find((p) => p.name === 'b')! + expect(b.critical).toBe(true) + }) + + it('stays green when only a non-critical probe fails (WARN, not blocking)', async () => { + const report = await runPreflight([passing('a'), failing('b', false)]) + expect(report.ok).toBe(true) + expect(report.failed).toBe(1) + expect(report.criticalFailures).toBe(0) + }) + + it('catches a probe that throws and treats it as a failure', async () => { + const boom: PreflightProbe = { + name: 'boom', + run: async () => { + throw new Error('kaboom') + }, + } + const report = await runPreflight([boom]) + expect(report.ok).toBe(false) + const v = report.probes[0]! + expect(v.ok).toBe(false) + expect(v.detail).toContain('probe threw') + expect(v.detail).toContain('kaboom') + }) +}) + +describe('formatPreflightReport', () => { + it('renders a table and, on failure, names the dead probes + a rotate hint', async () => { + const report = await runPreflight([ + { name: 'router-chat', run: async () => ({ ok: false, detail: 'DEAD KEY — rotate LITELLM_API_KEY' }) }, + { name: 'platform', critical: false, run: async () => ({ ok: false, detail: 'UNREACHABLE' }) }, + ]) + const text = formatPreflightReport(report) + expect(text).toContain('STATUS') + expect(text).toContain('FAIL') + expect(text).toContain('WARN') + expect(text).toContain('LITELLM_API_KEY') + expect(text).toContain('Preflight FAILED') + expect(text).toContain('router-chat') + }) + + it('renders a PASSED verdict when all critical probes are live', async () => { + const report = await runPreflight([{ name: 'ok', run: async () => ({ ok: true }) }]) + expect(formatPreflightReport(report)).toContain('Preflight PASSED') + }) +}) diff --git a/src/preflight/index.ts b/src/preflight/index.ts new file mode 100644 index 0000000..1787c79 --- /dev/null +++ b/src/preflight/index.ts @@ -0,0 +1,441 @@ +/** + * `/preflight` — deploy-time secret-liveness probes. + * + * WHY THIS EXISTS: on 2026-07-15 four secrets were simultaneously dead in one + * production day — a dead `SANDBOX_API_KEY`, a stale `SANDBOX_API_URL`, and a + * dead LiteLLM router key + URL. Each one was present in `wrangler secret list` + * (so nothing looked wrong) yet invalid against its live endpoint, and nothing + * anywhere checked liveness. CI cannot hold production secrets, so this binds + * at DEPLOY time instead: a product declares a handful of probes built from its + * real env, the deploy workflow runs `agent-app-preflight` as a step, and a + * dead secret fails the deploy with a message that names exactly which secret + * to rotate. + * + * A probe is `{ name, run, critical? }`; `run()` returns `{ ok, detail? }`. + * The standard builders (`routerChatProbe`, `sandboxAuthProbe`, `httpHeadProbe`) + * each take explicit config — they read nothing global — so the same probe runs + * identically in a deploy step, a test, or a local check. `runPreflight` fans + * the probes out, times each, and folds them into a pass/fail report: any + * failed CRITICAL probe fails the whole run (probes are critical by default). + * + * Server-only: probes carry live API keys and hit live endpoints. This subpath + * must never reach a browser bundle. + */ + +/** One probe's outcome. `detail` should name the secret to rotate on failure. */ +export interface PreflightProbeResult { + ok: boolean + detail?: string +} + +/** + * A liveness probe. `run` performs one cheap live call and maps the result to + * `{ ok, detail }`. `critical` defaults to `true` — a failed critical probe + * fails the whole preflight (and the deploy). + */ +export interface PreflightProbe { + name: string + run: () => Promise + critical?: boolean +} + +/** Per-probe verdict enriched with the resolved criticality and measured latency. */ +export interface PreflightProbeVerdict { + name: string + ok: boolean + critical: boolean + latencyMs: number + detail?: string +} + +/** Aggregate of every probe verdict plus the overall pass/fail decision. */ +export interface PreflightReport { + /** `false` if any critical probe failed. */ + ok: boolean + probes: PreflightProbeVerdict[] + passed: number + failed: number + criticalFailures: number + durationMs: number +} + +/** Deploy-time deadline for a single probe. Cold upstreams are slow; a dead + * endpoint should still fail fast, so 10s is the ceiling, not the target. */ +const DEFAULT_TIMEOUT_MS = 10_000 + +function nowMs(): number { + return typeof performance !== 'undefined' ? performance.now() : Date.now() +} + +function isAbortLike(err: unknown): boolean { + return err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError') +} + +/** Strip bearer tokens / key material before an upstream string is surfaced in + * a report (deploy logs are not always private). */ +function sanitizeUpstreamMessage(input: unknown): string { + const message = input instanceof Error ? input.message : String(input) + return message + .replace(/Bearer\s+[^\s]+/gi, 'Bearer [redacted]') + .replace(/\b(?:sk|pk|tc)[_-][A-Za-z0-9_-]{8,}\b/g, '[redacted-key]') +} + +function snippet(body: string): string { + const trimmed = body.trim() + if (!trimmed) return '' + const clipped = trimmed.length > 180 ? `${trimmed.slice(0, 180)}…` : trimmed + return `: ${sanitizeUpstreamMessage(clipped)}` +} + +type ProbeOutcome = + | { kind: 'status'; status: number; bodyText: string } + | { kind: 'timeout'; timeoutMs: number } + | { kind: 'network'; message: string } + +interface HttpProbeCall { + fetchImpl: typeof fetch + url: string + method: string + headers?: Record + body?: string + timeoutMs: number +} + +/** One live HTTP call, folded to a probe outcome. Never throws: a timeout, a + * DNS/connection failure, and any thrown error all become an outcome so the + * probe can classify them into an actionable detail. */ +async function runHttp(call: HttpProbeCall): Promise { + let response: Response + try { + response = await call.fetchImpl(call.url, { + method: call.method, + headers: call.headers, + body: call.body, + signal: AbortSignal.timeout(call.timeoutMs), + }) + } catch (err) { + if (isAbortLike(err)) return { kind: 'timeout', timeoutMs: call.timeoutMs } + return { kind: 'network', message: sanitizeUpstreamMessage(err) } + } + let bodyText = '' + try { + bodyText = await response.text() + } catch { + bodyText = '' + } + return { kind: 'status', status: response.status, bodyText } +} + +interface AuthedClassifyContext { + /** Full endpoint reached, for the message. */ + endpoint: string + /** How to name the API-key secret when the endpoint reports auth failure. */ + keyLabel: string + /** How to name the URL secret when the endpoint is unreachable. */ + urlLabel: string +} + +/** + * Shared classification for an authed liveness endpoint (router, sandbox): + * 2xx → live; 401/403 → the KEY is dead, name it; 503 → the UPSTREAM is down, + * the key still looks valid, don't rotate; timeout / unreachable → the URL is + * likely stale, name it; anything else → an unexpected status with a snippet. + */ +function classifyAuthed(outcome: ProbeOutcome, ctx: AuthedClassifyContext): PreflightProbeResult { + switch (outcome.kind) { + case 'status': { + const { status, bodyText } = outcome + if (status >= 200 && status < 300) return { ok: true, detail: `${status} OK` } + if (status === 401 || status === 403) { + return { + ok: false, + detail: `DEAD KEY — ${ctx.endpoint} returned ${status}; rotate ${ctx.keyLabel}`, + } + } + if (status === 503) { + return { + ok: false, + detail: `UPSTREAM DOWN — ${ctx.endpoint} returned 503; ${ctx.keyLabel} still looks valid, retry or check the provider (do NOT rotate)`, + } + } + return { ok: false, detail: `UNEXPECTED ${status} from ${ctx.endpoint}${snippet(bodyText)}` } + } + case 'timeout': + return { + ok: false, + detail: `TIMEOUT after ${outcome.timeoutMs}ms reaching ${ctx.endpoint} — check ${ctx.urlLabel}`, + } + case 'network': + return { + ok: false, + detail: `UNREACHABLE ${ctx.endpoint} (${outcome.message}) — check ${ctx.urlLabel}`, + } + } +} + +function trimTrailingSlash(url: string): string { + return url.replace(/\/+$/, '') +} + +// --- Standard probe builders -------------------------------------------------- + +export interface RouterChatProbeConfig { + /** LLM router base URL (LiteLLM / OpenAI-compatible), e.g. `https://router…`. */ + baseUrl: string + apiKey: string + /** A cheap model id available on the router. */ + model: string + /** Probe name in the report. Default `'router-chat'`. */ + name?: string + /** Default `true`. */ + critical?: boolean + /** Env-var name of the API key, named verbatim in a dead-key failure. */ + keySecret?: string + /** Env-var name of the base URL, named verbatim in an unreachable failure. */ + urlSecret?: string + /** Per-probe deadline. Default 10s. */ + timeoutMs?: number + /** Injection seam for tests; defaults to global `fetch`. */ + fetchImpl?: typeof fetch +} + +/** + * Probe an OpenAI-compatible LLM router with one cheap `POST /chat/completions` + * (`max_tokens: 1`). 200 → live; 401/403 → dead router key; 503 → upstream + * provider down (key still valid); timeout / unreachable → check the router URL. + */ +export function routerChatProbe(config: RouterChatProbeConfig): PreflightProbe { + const keyLabel = config.keySecret ?? 'the router API key' + const urlLabel = config.urlSecret ?? 'the router base URL' + return { + name: config.name ?? 'router-chat', + critical: config.critical, + run: async () => { + const base = trimTrailingSlash(config.baseUrl) + const endpoint = `${base}/chat/completions` + const outcome = await runHttp({ + fetchImpl: config.fetchImpl ?? fetch, + url: endpoint, + method: 'POST', + headers: { + Authorization: `Bearer ${config.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: config.model, + messages: [{ role: 'user', content: 'ping' }], + max_tokens: 1, + }), + timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }) + return classifyAuthed(outcome, { endpoint, keyLabel, urlLabel }) + }, + } +} + +export interface SandboxAuthProbeConfig { + /** Sandbox API base URL. */ + baseUrl: string + apiKey: string + /** Probe name in the report. Default `'sandbox-auth'`. */ + name?: string + /** Default `true`. */ + critical?: boolean + /** Env-var name of the API key, named verbatim in a dead-key failure. */ + keySecret?: string + /** Env-var name of the base URL, named verbatim in an unreachable failure. */ + urlSecret?: string + /** Per-probe deadline. Default 10s. */ + timeoutMs?: number + /** Injection seam for tests; defaults to global `fetch`. */ + fetchImpl?: typeof fetch +} + +/** + * Probe the sandbox API with a cheap authed `GET /v1/sandboxes?limit=1`. + * 200 → live; 401/403 → dead sandbox key; 503 → sandbox platform down (key + * still valid); timeout / unreachable → check the sandbox URL. + */ +export function sandboxAuthProbe(config: SandboxAuthProbeConfig): PreflightProbe { + const keyLabel = config.keySecret ?? 'the sandbox API key' + const urlLabel = config.urlSecret ?? 'the sandbox base URL' + return { + name: config.name ?? 'sandbox-auth', + critical: config.critical, + run: async () => { + const base = trimTrailingSlash(config.baseUrl) + const endpoint = `${base}/v1/sandboxes?limit=1` + const outcome = await runHttp({ + fetchImpl: config.fetchImpl ?? fetch, + url: endpoint, + method: 'GET', + headers: { Authorization: `Bearer ${config.apiKey}` }, + timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }) + return classifyAuthed(outcome, { endpoint, keyLabel, urlLabel }) + }, + } +} + +export interface HttpHeadProbeConfig { + /** Probe name in the report. */ + name: string + /** URL to `HEAD`. */ + url: string + /** + * Accepted status(es). A single number requires an exact match; an array + * requires membership. Omitted → any 2xx/3xx (the host is up and the path + * resolves) counts as live. + */ + expectStatus?: number | number[] + /** Default `true`. */ + critical?: boolean + /** Env-var name of the URL, named verbatim in a failure. */ + urlSecret?: string + /** Per-probe deadline. Default 10s. */ + timeoutMs?: number + /** Injection seam for tests; defaults to global `fetch`. */ + fetchImpl?: typeof fetch +} + +function statusMatches(status: number, expect?: number | number[]): boolean { + if (expect === undefined) return status >= 200 && status < 400 + if (Array.isArray(expect)) return expect.includes(status) + return status === expect +} + +function describeExpected(expect?: number | number[]): string { + if (expect === undefined) return '2xx/3xx' + if (Array.isArray(expect)) return expect.join(' or ') + return String(expect) +} + +/** + * Probe a plain reachability endpoint (e.g. a platform base URL) with a `HEAD`. + * Confirms the URL is live and resolving — the class of failure behind a stale + * platform URL that still sits in the secret store. + */ +export function httpHeadProbe(config: HttpHeadProbeConfig): PreflightProbe { + const urlLabel = config.urlSecret ?? `the URL for ${config.name}` + return { + name: config.name, + critical: config.critical, + run: async () => { + const outcome = await runHttp({ + fetchImpl: config.fetchImpl ?? fetch, + url: config.url, + method: 'HEAD', + timeoutMs: config.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }) + switch (outcome.kind) { + case 'status': { + if (statusMatches(outcome.status, config.expectStatus)) { + return { ok: true, detail: `${outcome.status} OK` } + } + return { + ok: false, + detail: `UNEXPECTED ${outcome.status} from ${config.url} (expected ${describeExpected(config.expectStatus)}) — check ${urlLabel}`, + } + } + case 'timeout': + return { + ok: false, + detail: `TIMEOUT after ${outcome.timeoutMs}ms reaching ${config.url} — check ${urlLabel}`, + } + case 'network': + return { + ok: false, + detail: `UNREACHABLE ${config.url} (${outcome.message}) — check ${urlLabel}`, + } + } + }, + } +} + +// --- Runner + report ---------------------------------------------------------- + +async function runOne(probe: PreflightProbe): Promise { + const critical = probe.critical ?? true + const start = nowMs() + try { + const result = await probe.run() + return { + name: probe.name, + ok: result.ok, + critical, + latencyMs: Math.round(nowMs() - start), + detail: result.detail, + } + } catch (err) { + return { + name: probe.name, + ok: false, + critical, + latencyMs: Math.round(nowMs() - start), + detail: `probe threw: ${sanitizeUpstreamMessage(err)}`, + } + } +} + +/** + * Run every probe (concurrently), time each, and fold into a report. The run + * fails (`ok: false`) iff a critical probe fails; a failed non-critical probe + * is a warning that does not block the deploy. + */ +export async function runPreflight(probes: PreflightProbe[]): Promise { + const start = nowMs() + const verdicts = await Promise.all(probes.map(runOne)) + const failed = verdicts.filter((v) => !v.ok) + const criticalFailures = failed.filter((v) => v.critical).length + return { + ok: criticalFailures === 0, + probes: verdicts, + passed: verdicts.length - failed.length, + failed: failed.length, + criticalFailures, + durationMs: Math.round(nowMs() - start), + } +} + +interface FormatRow { + status: string + name: string + latency: string + detail: string +} + +/** Render a report as an aligned, operator-readable table + verdict line. Pure + * (no I/O) so it is trivially testable and reusable by the bin. */ +export function formatPreflightReport(report: PreflightReport): string { + const header: FormatRow = { status: 'STATUS', name: 'PROBE', latency: 'LATENCY', detail: 'DETAIL' } + const rows: FormatRow[] = report.probes.map((p) => ({ + status: p.ok ? 'PASS' : p.critical ? 'FAIL' : 'WARN', + name: p.name, + latency: `${p.latencyMs}ms`, + detail: p.detail ?? '', + })) + const statusW = Math.max(header.status.length, ...rows.map((r) => r.status.length)) + const nameW = Math.max(header.name.length, ...rows.map((r) => r.name.length)) + const latencyW = Math.max(header.latency.length, ...rows.map((r) => r.latency.length)) + const line = (r: FormatRow): string => + `${r.status.padEnd(statusW)} ${r.name.padEnd(nameW)} ${r.latency.padStart(latencyW)} ${r.detail}`.trimEnd() + + const out: string[] = [ + line(header), + `${'-'.repeat(statusW)} ${'-'.repeat(nameW)} ${'-'.repeat(latencyW)} ------`, + ...rows.map(line), + '', + ] + if (report.ok) { + const warn = report.failed > 0 ? ` (${report.failed} non-critical warning(s))` : '' + out.push(`Preflight PASSED — ${report.passed}/${report.probes.length} probe(s) live${warn}`) + } else { + const dead = report.probes + .filter((p) => !p.ok && p.critical) + .map((p) => p.name) + .join(', ') + out.push(`Preflight FAILED — ${report.criticalFailures} critical probe(s) dead: ${dead}`) + out.push('Rotate the secret named in each FAIL row above, then redeploy.') + } + return out.join('\n') +} diff --git a/tsup.config.ts b/tsup.config.ts index dab53f2..9d73cce 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -21,6 +21,7 @@ export default defineConfig({ 'config/index': 'src/config/index.ts', 'preset-cloudflare/index': 'src/preset-cloudflare/index.ts', 'billing/index': 'src/billing/index.ts', + 'preflight/index': 'src/preflight/index.ts', 'chat-store/index': 'src/chat-store/index.ts', 'chat-routes/index': 'src/chat-routes/index.ts', 'crypto/index': 'src/crypto/index.ts',