From b8b444691eb34343373db6e397134d4ab8c460b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Tue, 21 Jul 2026 12:06:11 +0000 Subject: [PATCH 1/9] task: activate Instant runtime recovery work --- .../2026-07-21-instant-runtime-recovery-state-machine.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tasks/{backlog => active}/2026-07-21-instant-runtime-recovery-state-machine.md (100%) diff --git a/tasks/backlog/2026-07-21-instant-runtime-recovery-state-machine.md b/tasks/active/2026-07-21-instant-runtime-recovery-state-machine.md similarity index 100% rename from tasks/backlog/2026-07-21-instant-runtime-recovery-state-machine.md rename to tasks/active/2026-07-21-instant-runtime-recovery-state-machine.md From 824766ceb1e47cc93d7e502f8902807ca04a4a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Tue, 21 Jul 2026 13:53:26 +0000 Subject: [PATCH 2/9] fix: recover Instant sessions across runtime loss --- .claude/skills/env-reference/SKILL.md | 9 +- apps/api/.env.example | 9 +- .../vm-agent-container-active-work.ts | 110 ++ .../vm-agent-container-recovery.ts | 231 ++++ .../vm-agent-container-runtime.ts | 107 ++ .../src/durable-objects/vm-agent-container.ts | 1032 ++++++++++------- apps/api/src/env.ts | 1 + .../api/src/routes/chat-workspace-resolver.ts | 116 +- apps/api/src/routes/chat.ts | 2 +- .../src/routes/workspaces/agent-sessions.ts | 702 ++++++----- apps/api/src/services/node-agent-readiness.ts | 107 ++ apps/api/src/services/node-agent.ts | 183 ++- apps/api/src/services/vm-agent-container.ts | 26 +- .../cf-container-runtime-contract.test.ts | 33 +- .../vm-agent-container-launch-env.test.ts | 1 + .../vm-agent-container-recovery.test.ts | 364 ++++++ .../vm-agent-container-wake-state.test.ts | 212 ++-- .../agent-session-resume-recovery.test.ts | 201 ++++ .../unit/routes/chat-prompt-cancel.test.ts | 154 ++- ...loyment-release-compose-submission.test.ts | 26 +- ...ode-agent-create-workspace-timeout.test.ts | 98 +- .../instant-runtime-recovery-failure.test.ts | 119 ++ ...stant-runtime-recovery-persistence.test.ts | 116 ++ .../instant-runtime-recovery-resolver.test.ts | 102 ++ .../components/project-message-view/index.tsx | 253 ++-- .../runtimeRecoveryMessages.ts | 29 + .../useConnectionRecovery.ts | 93 +- .../useSessionLifecycle.ts | 440 ++++--- apps/web/src/lib/session-utils.ts | 9 +- .../chat/project-message-view-resume.test.tsx | 215 ++-- .../runtimeRecoveryMessages.test.ts | 44 + .../components/useSessionLifecycle.test.ts | 46 +- apps/web/tests/unit/lib/session-utils.test.ts | 106 +- .../docs/docs/reference/configuration.md | 4 + packages/shared/src/constants/status.ts | 2 + packages/shared/src/types/session.ts | 25 +- packages/shared/src/types/workspace.ts | 3 + scripts/deploy/sync-wrangler-config.ts | 3 + scripts/quality/sync-wrangler-config.test.ts | 11 +- ...-instant-runtime-recovery-state-machine.md | 40 +- 40 files changed, 3825 insertions(+), 1559 deletions(-) create mode 100644 apps/api/src/durable-objects/vm-agent-container-active-work.ts create mode 100644 apps/api/src/durable-objects/vm-agent-container-recovery.ts create mode 100644 apps/api/src/durable-objects/vm-agent-container-runtime.ts create mode 100644 apps/api/src/services/node-agent-readiness.ts create mode 100644 apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts create mode 100644 apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts create mode 100644 apps/api/tests/workers/instant-runtime-recovery-failure.test.ts create mode 100644 apps/api/tests/workers/instant-runtime-recovery-persistence.test.ts create mode 100644 apps/api/tests/workers/instant-runtime-recovery-resolver.test.ts create mode 100644 apps/web/src/components/project-message-view/runtimeRecoveryMessages.ts create mode 100644 apps/web/tests/unit/components/runtimeRecoveryMessages.test.ts diff --git a/.claude/skills/env-reference/SKILL.md b/.claude/skills/env-reference/SKILL.md index c1c35cddd..1e3442ea4 100644 --- a/.claude/skills/env-reference/SKILL.md +++ b/.claude/skills/env-reference/SKILL.md @@ -63,10 +63,15 @@ See `apps/api/.env.example` for the full list. Key variables: - `WRANGLER_PORT` — Local dev port (default: 8787) - `BASE_DOMAIN` — Set automatically by sync scripts - `CF_CONTAINER_ENABLED` — Enables Cloudflare Container instant-session runtime in generated deployment envs (default: `true`; set `false` to force VM runtime) -- `CF_CONTAINER_SLEEP_AFTER` — Container idle sleep duration for instant-session runtime (default: `10m`) +- `CF_CONTAINER_SLEEP_AFTER` — Container idle sleep duration for instant-session runtime (default: `1h`) +- `CF_CONTAINER_ACTIVE_WORK_MAX_MS` — Defensive maximum active-work keepalive duration (default: `7200000`) +- `CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS` — Active-work keepalive renewal interval (default: `300000`) - `CF_CONTAINER_VM_AGENT_PORT` — vm-agent standalone HTTP port inside the raw container (default: `8080`) - `CF_CONTAINER_PORT_READY_TIMEOUT_MS` — Max wait for vm-agent port readiness (default: `30000`) -- `$1 +- `CF_CONTAINER_WAKE_TIMEOUT_MS` — Max wait for launch, snapshot restore, and request readiness (default: `120000`) +- `CF_CONTAINER_RECOVERY_MAX_ATTEMPTS` — Max restore attempts before terminal status reconciliation (default: `2`) +- `CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS` — Synchronous workspace creation and clone budget (default: `120000`) +- `CF_CONTAINER_CLONE_FILTER` — Git partial-clone filter (default: `blob:none`; `off` disables partial clone) - `SESSION_SNAPSHOT_TTL_DAYS` — Retention for hibernated session snapshots; deployment also provisions matching R2 prefix expiration (default: `7`) - `SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES` — Maximum bytes accepted for one snapshot artifact (default: `104857600`) - `SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES` — Per-file threshold before snapshot content is visibly skipped (default: `52428800`) diff --git a/apps/api/.env.example b/apps/api/.env.example index 78a72cfd2..fdb7c1d07 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -74,9 +74,16 @@ BASE_DOMAIN=workspaces.example.com # Generated deployment environments default this to true. Set false to force # profile/runtime resolution back to cloud VMs. # CF_CONTAINER_ENABLED=true -# CF_CONTAINER_SLEEP_AFTER=10m +# CF_CONTAINER_SLEEP_AFTER=1h +# Maximum active-work keepalive lifetime and renewal interval. +# CF_CONTAINER_ACTIVE_WORK_MAX_MS=7200000 +# CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS=300000 # CF_CONTAINER_VM_AGENT_PORT=8080 # CF_CONTAINER_PORT_READY_TIMEOUT_MS=30000 +# Budget for a sleeping/recovering runtime to launch, restore, and accept work. +# CF_CONTAINER_WAKE_TIMEOUT_MS=120000 +# Maximum snapshot restore attempts before terminal status reconciliation. +# CF_CONTAINER_RECOVERY_MAX_ATTEMPTS=2 # CF_CONTAINER_WORKSPACE_BASE_DIR=/workspaces # Budget for the synchronous standalone create-workspace request (includes the # repository clone inside the container). Default 120000 (120s). diff --git a/apps/api/src/durable-objects/vm-agent-container-active-work.ts b/apps/api/src/durable-objects/vm-agent-container-active-work.ts new file mode 100644 index 000000000..cd50c61f9 --- /dev/null +++ b/apps/api/src/durable-objects/vm-agent-container-active-work.ts @@ -0,0 +1,110 @@ +import { log } from '../lib/logger'; + +export type ActiveWorkStatus = 'active' | 'ended' | 'expired'; + +export interface ActiveWorkState { + status: ActiveWorkStatus; + nodeId: string; + workspaceId: string; + agentSessionId: string; + reason: string; + activeSince: number; + lastRenewedAt: number; + deadlineAt: number; + endedAt?: number; + endReason?: string; +} + +export interface ActiveWorkRuntime { + storage: DurableObjectStorage; + activeWorkMaxMs: number; + renewIntervalMs: number; + renewActivityTimeout: () => void; + replaceSchedule: (delayMs: number) => Promise; + clearSchedule: () => Promise; +} + +export const ACTIVE_WORK_KEY = 'activeWork'; + +export async function startActiveWork( + runtime: ActiveWorkRuntime, + nodeId: string, + input: { workspaceId: string; agentSessionId: string; reason: string } +): Promise { + const now = Date.now(); + const activeWork: ActiveWorkState = { + status: 'active', + nodeId, + workspaceId: input.workspaceId, + agentSessionId: input.agentSessionId, + reason: input.reason, + activeSince: now, + lastRenewedAt: now, + deadlineAt: now + runtime.activeWorkMaxMs, + }; + runtime.renewActivityTimeout(); + await runtime.storage.put(ACTIVE_WORK_KEY, activeWork); + await runtime.replaceSchedule(runtime.renewIntervalMs); + log.info('vm_agent_container_active_work_started', { + nodeId, + workspaceId: input.workspaceId, + agentSessionId: input.agentSessionId, + reason: input.reason, + activeSince: new Date(now).toISOString(), + deadlineAt: new Date(activeWork.deadlineAt).toISOString(), + }); +} + +export async function endActiveWork(runtime: ActiveWorkRuntime, reason: string): Promise { + const activeWork = await runtime.storage.get(ACTIVE_WORK_KEY); + if (!activeWork || activeWork.status !== 'active') { + await runtime.clearSchedule(); + return; + } + const now = Date.now(); + await runtime.storage.put(ACTIVE_WORK_KEY, { + ...activeWork, + status: 'ended', + endedAt: now, + endReason: reason, + } satisfies ActiveWorkState); + await runtime.clearSchedule(); + log.info('vm_agent_container_active_work_ended', { + nodeId: activeWork.nodeId, + workspaceId: activeWork.workspaceId, + agentSessionId: activeWork.agentSessionId, + reason, + activeSince: new Date(activeWork.activeSince).toISOString(), + lastRenewedAt: new Date(activeWork.lastRenewedAt).toISOString(), + endedAt: new Date(now).toISOString(), + }); +} + +export async function renewActiveWork(runtime: ActiveWorkRuntime): Promise { + const activeWork = await runtime.storage.get(ACTIVE_WORK_KEY); + if (!activeWork || activeWork.status !== 'active') { + await runtime.clearSchedule(); + return; + } + const now = Date.now(); + if (now >= activeWork.deadlineAt) { + await runtime.storage.put(ACTIVE_WORK_KEY, { + ...activeWork, + status: 'expired', + endedAt: now, + endReason: 'keepalive_deadline_exceeded', + } satisfies ActiveWorkState); + await runtime.clearSchedule(); + log.warn('vm_agent_container_active_work_keepalive_expired', { + nodeId: activeWork.nodeId, + workspaceId: activeWork.workspaceId, + agentSessionId: activeWork.agentSessionId, + deadlineAt: new Date(activeWork.deadlineAt).toISOString(), + }); + return; + } + + runtime.renewActivityTimeout(); + await runtime.storage.put(ACTIVE_WORK_KEY, { ...activeWork, lastRenewedAt: now }); + await runtime.replaceSchedule(runtime.renewIntervalMs); +} diff --git a/apps/api/src/durable-objects/vm-agent-container-recovery.ts b/apps/api/src/durable-objects/vm-agent-container-recovery.ts new file mode 100644 index 000000000..e18c62889 --- /dev/null +++ b/apps/api/src/durable-objects/vm-agent-container-recovery.ts @@ -0,0 +1,231 @@ +import { and, desc, eq, inArray } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/d1'; + +import * as schema from '../db/schema'; +import type { Env } from '../env'; +import { log } from '../lib/logger'; +import { ulid } from '../lib/ulid'; +import * as projectDataService from '../services/project-data'; + +export const RUNTIME_RECOVERING_MESSAGE = + 'Instant session interrupted; restoring the last safe checkpoint.'; +export const RUNTIME_REQUEST_INTERRUPTED_MESSAGE = + 'Your message is saved, but the Instant runtime changed while it was being sent. It was not replayed automatically. Wait for restore to finish, then send it again.'; +export const RUNTIME_RECOVERY_DEGRADED_MESSAGE = + 'The Instant session could not restore its last safe checkpoint. Your transcript and partial output are still available.'; +export const RUNTIME_STOPPED_MESSAGE = 'This Instant session was stopped and cannot be resumed.'; + +export type RuntimeRecoveryCode = + | 'RUNTIME_RECOVERING' + | 'RUNTIME_REQUEST_INTERRUPTED' + | 'RUNTIME_RECOVERY_DEGRADED' + | 'RUNTIME_STOPPED'; + +export type RuntimeRecoveryPhase = 'pending' | 'waking' | 'restoring' | 'degraded' | 'exhausted'; + +export type RuntimeRecoveryTrigger = 'idle' | 'stop' | 'error' | 'request'; + +export type RuntimeRecoveryCause = + | { kind: 'idle_sleep' } + | { kind: 'container_stop'; reason: 'exit' | 'runtime_signal'; exitCode: number } + | { kind: 'container_error'; errorName: string } + | { kind: 'transport_interrupted'; errorName: string } + | { kind: 'missing_session_host'; httpStatus: number }; + +export interface RuntimeRecoveryState { + version: 1; + phase: RuntimeRecoveryPhase; + trigger: RuntimeRecoveryTrigger; + cause: RuntimeRecoveryCause; + attempts: number; + promptDisposition: 'none' | 'manual_retry'; + agentSessionId: string | null; + startedAt: number; + updatedAt: number; + lastFailure?: { + kind: 'launch' | 'restore_http' | 'restore_status' | 'unexpected'; + httpStatus?: number; + }; +} + +export interface RuntimeRecoveryTarget { + nodeId: string; + workspaceId: string; + projectId: string; + chatSessionId: string; + agentSessionId: string; +} + +export interface RuntimeRecoveryContext { + userId: string; + chatSessionId: string; + agentSessionId: string; + agentType: string | null; +} + +const ACTIVE_TASK_STATUSES = ['in_progress', 'delegated', 'awaiting_followup'] as const; + +export async function loadRuntimeRecoveryContext( + env: Env, + input: { workspaceId: string; preferredAgentSessionId?: string | null } +): Promise { + const db = drizzle(env.DATABASE, { schema }); + const workspace = await db + .select({ + userId: schema.workspaces.userId, + chatSessionId: schema.workspaces.chatSessionId, + }) + .from(schema.workspaces) + .where(eq(schema.workspaces.id, input.workspaceId)) + .get(); + if (!workspace?.chatSessionId) return null; + + const agentSession = await db + .select({ id: schema.agentSessions.id, agentType: schema.agentSessions.agentType }) + .from(schema.agentSessions) + .where( + input.preferredAgentSessionId + ? and( + eq(schema.agentSessions.workspaceId, input.workspaceId), + eq(schema.agentSessions.id, input.preferredAgentSessionId) + ) + : eq(schema.agentSessions.workspaceId, input.workspaceId) + ) + .orderBy(desc(schema.agentSessions.updatedAt)) + .get(); + if (!agentSession) return null; + + return { + userId: workspace.userId, + chatSessionId: workspace.chatSessionId, + agentSessionId: agentSession.id, + agentType: agentSession.agentType, + }; +} + +export async function persistRuntimeRecovering( + env: Env, + target: RuntimeRecoveryTarget +): Promise { + const now = new Date().toISOString(); + await env.DATABASE.batch([ + env.DATABASE.prepare( + `UPDATE nodes + SET status = 'recovery', health_status = 'unhealthy', error_message = ?, updated_at = ? + WHERE id = ?` + ).bind(RUNTIME_RECOVERING_MESSAGE, now, target.nodeId), + env.DATABASE.prepare( + `UPDATE workspaces SET status = 'recovery', error_message = ?, updated_at = ? WHERE id = ?` + ).bind(RUNTIME_RECOVERING_MESSAGE, now, target.workspaceId), + env.DATABASE.prepare( + `UPDATE agent_sessions + SET status = 'recovery', stopped_at = NULL, error_message = ?, updated_at = ? + WHERE id = ?` + ).bind(RUNTIME_RECOVERING_MESSAGE, now, target.agentSessionId), + ]); +} + +export async function persistRuntimeRecovered( + env: Env, + target: RuntimeRecoveryTarget, + promptDisposition: RuntimeRecoveryState['promptDisposition'] +): Promise { + const now = new Date().toISOString(); + const agentMessage = + promptDisposition === 'manual_retry' ? RUNTIME_REQUEST_INTERRUPTED_MESSAGE : null; + await env.DATABASE.batch([ + env.DATABASE.prepare( + `UPDATE nodes + SET status = 'running', health_status = 'healthy', error_message = NULL, updated_at = ? + WHERE id = ?` + ).bind(now, target.nodeId), + env.DATABASE.prepare( + `UPDATE workspaces SET status = 'running', error_message = NULL, updated_at = ? WHERE id = ?` + ).bind(now, target.workspaceId), + env.DATABASE.prepare( + `UPDATE agent_sessions + SET status = 'running', stopped_at = NULL, error_message = ?, updated_at = ? + WHERE id = ?` + ).bind(agentMessage, now, target.agentSessionId), + ]); +} + +export async function persistRuntimeRecoveryFailed( + env: Env, + target: RuntimeRecoveryTarget +): Promise { + const db = drizzle(env.DATABASE, { schema }); + const now = new Date().toISOString(); + const task = await db + .select({ id: schema.tasks.id }) + .from(schema.tasks) + .where( + and( + eq(schema.tasks.workspaceId, target.workspaceId), + inArray(schema.tasks.status, [...ACTIVE_TASK_STATUSES]) + ) + ) + .orderBy(desc(schema.tasks.updatedAt)) + .get(); + + const statements: D1PreparedStatement[] = [ + env.DATABASE.prepare( + `UPDATE nodes + SET status = 'error', health_status = 'unhealthy', error_message = ?, updated_at = ? + WHERE id = ?` + ).bind(RUNTIME_RECOVERY_DEGRADED_MESSAGE, now, target.nodeId), + env.DATABASE.prepare( + `UPDATE workspaces SET status = 'error', error_message = ?, updated_at = ? WHERE id = ?` + ).bind(RUNTIME_RECOVERY_DEGRADED_MESSAGE, now, target.workspaceId), + env.DATABASE.prepare( + `UPDATE agent_sessions + SET status = 'error', stopped_at = ?, error_message = ?, updated_at = ? + WHERE id = ?` + ).bind(now, RUNTIME_RECOVERY_DEGRADED_MESSAGE, now, target.agentSessionId), + ]; + + if (task) { + statements.push( + env.DATABASE.prepare( + `INSERT INTO task_status_events + (id, task_id, from_status, to_status, actor_type, actor_id, reason, created_at) + SELECT ?, id, status, 'failed', 'system', ?, ?, ? + FROM tasks + WHERE id = ? AND status IN ('in_progress', 'delegated', 'awaiting_followup')` + ).bind(ulid(), target.nodeId, 'Instant runtime recovery exhausted', now, task.id), + env.DATABASE.prepare( + `UPDATE tasks + SET status = 'failed', execution_step = NULL, error_message = ?, updated_at = ? + WHERE id = ? AND status IN ('in_progress', 'delegated', 'awaiting_followup')` + ).bind(RUNTIME_RECOVERY_DEGRADED_MESSAGE, now, task.id) + ); + } + + await env.DATABASE.batch(statements); + + await projectDataService + .transitionAcpSession(env, target.projectId, target.agentSessionId, 'failed', { + actorType: 'system', + actorId: target.nodeId, + reason: 'Instant runtime recovery exhausted', + errorMessage: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + workspaceId: target.workspaceId, + nodeId: target.nodeId, + }) + .catch((error) => { + log.warn('vm_agent_container_recovery.acp_reconcile_failed', { + nodeId: target.nodeId, + workspaceId: target.workspaceId, + error, + }); + }); + await projectDataService + .failSession(env, target.projectId, target.chatSessionId, RUNTIME_RECOVERY_DEGRADED_MESSAGE) + .catch((error) => { + log.warn('vm_agent_container_recovery.chat_reconcile_failed', { + nodeId: target.nodeId, + workspaceId: target.workspaceId, + error, + }); + }); +} diff --git a/apps/api/src/durable-objects/vm-agent-container-runtime.ts b/apps/api/src/durable-objects/vm-agent-container-runtime.ts new file mode 100644 index 000000000..8565c4f0f --- /dev/null +++ b/apps/api/src/durable-objects/vm-agent-container-runtime.ts @@ -0,0 +1,107 @@ +import type { Env } from '../env'; +import { + RUNTIME_RECOVERING_MESSAGE, + RUNTIME_RECOVERY_DEGRADED_MESSAGE, + RUNTIME_REQUEST_INTERRUPTED_MESSAGE, + RUNTIME_STOPPED_MESSAGE, + type RuntimeRecoveryCode, +} from './vm-agent-container-recovery'; + +interface RuntimeRecoveryResponseResult { + code?: RuntimeRecoveryCode; + message?: string; +} + +export interface RuntimeIdentity { + nodeId: string; + workspaceId: string; +} + +export function isMutatingRuntimeRequest(request: Request): boolean { + return !['GET', 'HEAD', 'OPTIONS'].includes(request.method.toUpperCase()); +} + +export function runtimeRecoveryResponse( + code: RuntimeRecoveryCode, + message: string, + status: 409 | 410 | 503 +): Response { + return Response.json({ error: code, message }, { status }); +} + +export function runtimeResultResponse(result: RuntimeRecoveryResponseResult): Response { + if (result.code === 'RUNTIME_STOPPED') { + return runtimeRecoveryResponse(result.code, result.message ?? RUNTIME_STOPPED_MESSAGE, 410); + } + return runtimeRecoveryResponse( + result.code ?? 'RUNTIME_RECOVERY_DEGRADED', + result.message ?? RUNTIME_RECOVERY_DEGRADED_MESSAGE, + result.code === 'RUNTIME_RECOVERING' ? 503 : 409 + ); +} + +export function interruptedRuntimeRequestResponse(request: Request): Response { + if (isMutatingRuntimeRequest(request)) { + return runtimeRecoveryResponse( + 'RUNTIME_REQUEST_INTERRUPTED', + RUNTIME_REQUEST_INTERRUPTED_MESSAGE, + 409 + ); + } + return runtimeRecoveryResponse('RUNTIME_RECOVERING', RUNTIME_RECOVERING_MESSAGE, 503); +} +export async function isMissingSessionHostResponse(response: Response): Promise { + if (response.status !== 404) return false; + const body = await response + .clone() + .text() + .catch(() => ''); + return body.toLowerCase().includes('no active agent session found'); +} + +export function parsePositiveRuntimeSetting(raw: string | undefined, fallback: number): number { + const parsed = raw ? Number.parseInt(raw, 10) : fallback; + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +export async function persistRuntimeSleeping(env: Env, identity: RuntimeIdentity): Promise { + const now = new Date().toISOString(); + await env.DATABASE.batch([ + env.DATABASE.prepare( + `UPDATE nodes + SET status = 'sleeping', health_status = 'unhealthy', error_message = NULL, updated_at = ? + WHERE id = ?` + ).bind(now, identity.nodeId), + env.DATABASE.prepare( + `UPDATE workspaces SET status = 'sleeping', error_message = NULL, updated_at = ? WHERE id = ?` + ).bind(now, identity.workspaceId), + env.DATABASE.prepare( + `UPDATE agent_sessions SET status = 'sleeping', error_message = NULL, updated_at = ? WHERE workspace_id = ?` + ).bind(now, identity.workspaceId), + ]); +} + +export async function persistRuntimeEnded( + env: Env, + identity: RuntimeIdentity, + status: 'stopped' | 'error', + message: string +): Promise { + const now = new Date().toISOString(); + const errorMessage = status === 'stopped' ? null : message; + await env.DATABASE.batch([ + env.DATABASE.prepare( + `UPDATE nodes + SET status = ?, health_status = 'unhealthy', error_message = ?, updated_at = ? + WHERE id = ?` + ).bind(status, errorMessage, now, identity.nodeId), + env.DATABASE.prepare( + `UPDATE workspaces SET status = ?, error_message = ?, updated_at = ? WHERE id = ?` + ).bind(status, errorMessage, now, identity.workspaceId), + env.DATABASE.prepare( + `UPDATE agent_sessions + SET status = ?, stopped_at = ?, error_message = ?, updated_at = ? + WHERE workspace_id = ?` + ).bind(status, now, errorMessage, now, identity.workspaceId), + ]); +} diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index 49a06aef7..b9dbe574c 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -1,15 +1,46 @@ import { Container, switchPort } from '@cloudflare/containers'; -import { desc, eq } from 'drizzle-orm'; -import { drizzle } from 'drizzle-orm/d1'; -import * as schema from '../db/schema'; import type { Env } from '../env'; import { log } from '../lib/logger'; import { signCallbackToken, signNodeCallbackToken, signNodeManagementToken } from '../services/jwt'; +import { + ACTIVE_WORK_KEY, + type ActiveWorkRuntime, + type ActiveWorkState, + endActiveWork, + renewActiveWork, + startActiveWork, +} from './vm-agent-container-active-work'; +import { + loadRuntimeRecoveryContext, + persistRuntimeRecovered, + persistRuntimeRecovering, + persistRuntimeRecoveryFailed, + RUNTIME_RECOVERING_MESSAGE, + RUNTIME_RECOVERY_DEGRADED_MESSAGE, + RUNTIME_REQUEST_INTERRUPTED_MESSAGE, + RUNTIME_STOPPED_MESSAGE, + type RuntimeRecoveryCause, + type RuntimeRecoveryCode, + type RuntimeRecoveryContext, + type RuntimeRecoveryState, + type RuntimeRecoveryTarget, + type RuntimeRecoveryTrigger, +} from './vm-agent-container-recovery'; +import { + interruptedRuntimeRequestResponse as interruptedRequestResponse, + isMissingSessionHostResponse, + isMutatingRuntimeRequest as isMutatingRequest, + persistRuntimeEnded, + persistRuntimeSleeping, + runtimeRecoveryResponse as recoveryResponse, + runtimeResultResponse as resultResponse, +} from './vm-agent-container-runtime'; export const DEFAULT_CF_CONTAINER_SLEEP_AFTER = '1h'; export const DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS = 2 * 60 * 60 * 1000; export const DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS = 5 * 60 * 1000; +export const DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS = 2; export interface VmAgentContainerLaunchConfig { nodeId: string; @@ -27,26 +58,37 @@ export interface VmAgentContainerLaunchSecrets { nodeCallbackToken: string; } -type LifecycleStatus = 'launching' | 'running' | 'stopping' | 'stopped' | 'sleeping' | 'expired' | 'error'; - -type ActiveWorkStatus = 'active' | 'ended' | 'expired'; - -interface ActiveWorkState { - status: ActiveWorkStatus; - nodeId: string; - workspaceId: string; - agentSessionId: string; - reason: string; - activeSince: number; - lastRenewedAt: number; - deadlineAt: number; - endedAt?: number; - endReason?: string; +export interface VmAgentContainerRecoveryResult { + ok: boolean; + status: 'running' | 'recovering' | 'degraded' | 'stopped'; + code?: RuntimeRecoveryCode; + message?: string; } -const ACTIVE_WORK_KEY = 'activeWork'; +type LifecycleStatus = + | 'launching' + | 'running' + | 'stopping' + | 'stopped' + | 'sleeping' + | 'recovering' + | 'waking' + | 'restoring' + | 'degraded' + | 'expired' + | 'error'; + +const RECOVERY_STATE_KEY = 'runtimeRecovery'; const KEEPALIVE_CALLBACK = 'renewActiveWorkKeepalive'; -const WAKE_DEGRADED_RESPONSE = 'Workspace woke with degraded snapshot restore; retry the prompt or fork from transcript history.'; + +function stoppedRecoveryResult(): VmAgentContainerRecoveryResult { + return { + ok: false, + status: 'stopped', + code: 'RUNTIME_STOPPED', + message: RUNTIME_STOPPED_MESSAGE, + }; +} export class VmAgentContainer extends Container { defaultPort = 8080; @@ -54,20 +96,21 @@ export class VmAgentContainer extends Container { sleepAfter = DEFAULT_CF_CONTAINER_SLEEP_AFTER; enableInternet = true; - // Serializes wake-from-snapshot so two concurrent requests to a sleeping - // container do not both launch a fresh container + restore. DO request - // handlers interleave across `await`, so the sleeping-check + wake must run - // as one critical section (see .claude/rules/45). private wakeChain: Promise = Promise.resolve(); + private lifecycleChain: Promise = Promise.resolve(); constructor(ctx: DurableObjectState>, env: Env) { super(ctx, env); - const configuredPort = Number.parseInt(env.CF_CONTAINER_VM_AGENT_PORT || env.SANDBOX_VM_AGENT_PORT || '', 10); + const configuredPort = Number.parseInt( + env.CF_CONTAINER_VM_AGENT_PORT || env.SANDBOX_VM_AGENT_PORT || '', + 10 + ); if (Number.isFinite(configuredPort) && configuredPort > 0) { this.defaultPort = configuredPort; this.requiredPorts = [configuredPort]; } - this.sleepAfter = env.CF_CONTAINER_SLEEP_AFTER || env.SANDBOX_SLEEP_AFTER || DEFAULT_CF_CONTAINER_SLEEP_AFTER; + this.sleepAfter = + env.CF_CONTAINER_SLEEP_AFTER || env.SANDBOX_SLEEP_AFTER || DEFAULT_CF_CONTAINER_SLEEP_AFTER; } async launch( @@ -77,77 +120,131 @@ export class VmAgentContainer extends Container { await this.ctx.storage.put('launchConfig', config); await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus); await this.ctx.storage.delete(ACTIVE_WORK_KEY); + await this.ctx.storage.delete(RECOVERY_STATE_KEY); await this.clearKeepaliveSchedule(); - await this.startAndWaitForPorts({ - ports: config.vmAgentPort, - startOptions: { - envVars: { - NODE_ROLE: 'standalone', - NODE_ID: config.nodeId, - WORKSPACE_ID: config.workspaceId, - PROJECT_ID: config.projectId, - CHAT_SESSION_ID: config.chatSessionId, - CONTROL_PLANE_URL: config.controlPlaneUrl, - CALLBACK_TOKEN: secrets.nodeCallbackToken, - REPOSITORY: config.repository, - BRANCH: config.branch, - WORKSPACE_DIR: config.workspaceDir, - CONTAINER_WORK_DIR: config.workspaceDir, - CONTAINER_MODE: 'false', - PORT_SCAN_ENABLED: 'false', - VM_AGENT_PORT: String(config.vmAgentPort), - VM_AGENT_PROTOCOL: 'http', - COOKIE_SECURE: 'true', - // Operator override for the standalone git partial-clone filter; - // the vm-agent defaults to blob:none when unset. - ...(this.env.CF_CONTAINER_CLONE_FILTER - ? { STANDALONE_CLONE_FILTER: this.env.CF_CONTAINER_CLONE_FILTER } - : {}), - }, - labels: { - nodeId: config.nodeId, - workspaceId: config.workspaceId, - runtime: 'cf-container', - }, - }, - cancellationOptions: { - portReadyTimeoutMS: this.getPortReadyTimeoutMs(), - }, - }); - - await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus); + try { + await this.startRuntime(config, secrets); + await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus); + } catch (error) { + await this.ctx.storage.put('lifecycleStatus', 'error' satisfies LifecycleStatus); + throw error; + } } async proxyHttp(request: Request, port?: number): Promise { - let state = await this.getState(); - const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); - if (lifecycleStatus === 'sleeping') { - const wake = await this.ensureAwake(); - if (!wake.ok) { - return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); + const ready = await this.prepareForRequest(); + if (!ready.ok) return resultResponse(ready); + + const state = await this.getState(); + if (state.status === 'stopped' || state.status === 'stopped_with_code') { + const recovery = await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { kind: 'transport_interrupted', errorName: 'container_stopped' }, + promptDisposition: isMutatingRequest(request) ? 'manual_retry' : 'none', + }); + if (!recovery) { + return recoveryResponse('RUNTIME_STOPPED', RUNTIME_STOPPED_MESSAGE, 410); + } + return interruptedRequestResponse(request); + } + + try { + const response = await this.containerFetch(request, port ?? this.defaultPort); + if (await isMissingSessionHostResponse(response)) { + await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { kind: 'missing_session_host', httpStatus: response.status }, + promptDisposition: isMutatingRequest(request) ? 'manual_retry' : 'none', + }); + return interruptedRequestResponse(request); } - // wakeFromSnapshot launched a fresh container and restored the session. - // Re-read the container state so the stopped-check below reflects the - // now-running container instead of the pre-wake stopped snapshot, - // otherwise a successfully-woken session is wrongly rejected with 410. - state = await this.getState(); + return response; + } catch (error) { + await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { + kind: 'transport_interrupted', + errorName: error instanceof Error ? error.name : 'unknown', + }, + promptDisposition: isMutatingRequest(request) ? 'manual_retry' : 'none', + }); + return interruptedRequestResponse(request); } + } + + override async fetch(request: Request): Promise { + const ready = await this.prepareForRequest(); + if (!ready.ok) return resultResponse(ready); + + const state = await this.getState(); if (state.status === 'stopped' || state.status === 'stopped_with_code') { - return new Response('Container is stopped; create a new instant session.', { status: 410 }); + await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { kind: 'transport_interrupted', errorName: 'container_stopped' }, + promptDisposition: 'none', + }); + return recoveryResponse('RUNTIME_RECOVERING', RUNTIME_RECOVERING_MESSAGE, 503); } - return this.containerFetch(request, port ?? this.defaultPort); + + try { + return await super.fetch(switchPort(request, this.defaultPort)); + } catch (error) { + await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { + kind: 'transport_interrupted', + errorName: error instanceof Error ? error.name : 'unknown', + }, + promptDisposition: 'none', + }); + return recoveryResponse('RUNTIME_RECOVERING', RUNTIME_RECOVERING_MESSAGE, 503); + } + } + + async resumeRuntime(): Promise { + return this.ensureAwake(); + } + + async markRequestInterrupted(input: { + method: string; + errorName: string; + }): Promise { + const mutating = !['GET', 'HEAD', 'OPTIONS'].includes(input.method.toUpperCase()); + const state = await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { kind: 'transport_interrupted', errorName: input.errorName }, + promptDisposition: mutating ? 'manual_retry' : 'none', + }); + if (!state) { + const status = await this.ctx.storage.get('lifecycleStatus'); + if (status === 'stopped' || status === 'stopping') { + return stoppedRecoveryResult(); + } + } + return { + ok: false, + status: 'recovering', + code: mutating ? 'RUNTIME_REQUEST_INTERRUPTED' : 'RUNTIME_RECOVERING', + message: mutating ? RUNTIME_REQUEST_INTERRUPTED_MESSAGE : RUNTIME_RECOVERING_MESSAGE, + }; } async stopForUser(): Promise { await this.markActiveWorkEnded('user_stop'); - await this.ctx.storage.put('lifecycleStatus', 'stopping' satisfies LifecycleStatus); + await this.withLifecycleLock(async () => { + await this.ctx.storage.put('lifecycleStatus', 'stopping' satisfies LifecycleStatus); + await this.ctx.storage.delete(RECOVERY_STATE_KEY); + }); await this.stop(); } async destroyForUser(): Promise { await this.markActiveWorkEnded('user_destroy'); - await this.ctx.storage.put('lifecycleStatus', 'stopping' satisfies LifecycleStatus); + await this.withLifecycleLock(async () => { + await this.ctx.storage.put('lifecycleStatus', 'stopping' satisfies LifecycleStatus); + await this.ctx.storage.delete(RECOVERY_STATE_KEY); + }); await this.destroy(); } @@ -157,131 +254,57 @@ export class VmAgentContainer extends Container { reason: string; }): Promise { const config = await this.ctx.storage.get('launchConfig'); - const now = Date.now(); - const nodeId = config?.nodeId ?? input.workspaceId; - const activeWork: ActiveWorkState = { - status: 'active', - nodeId, - workspaceId: input.workspaceId, - agentSessionId: input.agentSessionId, - reason: input.reason, - activeSince: now, - lastRenewedAt: now, - deadlineAt: now + this.getActiveWorkMaxMs(), - }; - this.renewActivityTimeout(); - await this.ctx.storage.put(ACTIVE_WORK_KEY, activeWork); - await this.replaceKeepaliveSchedule(this.getKeepaliveRenewIntervalMs()); - log.info('vm_agent_container_active_work_started', { - nodeId, - workspaceId: input.workspaceId, - agentSessionId: input.agentSessionId, - reason: input.reason, - activeSince: new Date(now).toISOString(), - deadlineAt: new Date(activeWork.deadlineAt).toISOString(), - }); + await startActiveWork(this.activeWorkRuntime(), config?.nodeId ?? input.workspaceId, input); } async markActiveWorkEnded(reason: string): Promise { - const activeWork = await this.ctx.storage.get(ACTIVE_WORK_KEY); - if (!activeWork || activeWork.status !== 'active') { - await this.clearKeepaliveSchedule(); - return; - } - const now = Date.now(); - await this.ctx.storage.put(ACTIVE_WORK_KEY, { - ...activeWork, - status: 'ended', - endedAt: now, - endReason: reason, - } satisfies ActiveWorkState); - await this.clearKeepaliveSchedule(); - log.info('vm_agent_container_active_work_ended', { - nodeId: activeWork.nodeId, - workspaceId: activeWork.workspaceId, - agentSessionId: activeWork.agentSessionId, - reason, - activeSince: new Date(activeWork.activeSince).toISOString(), - lastRenewedAt: new Date(activeWork.lastRenewedAt).toISOString(), - endedAt: new Date(now).toISOString(), - }); + await endActiveWork(this.activeWorkRuntime(), reason); } async renewActiveWorkKeepalive(): Promise { - const activeWork = await this.ctx.storage.get(ACTIVE_WORK_KEY); - if (!activeWork || activeWork.status !== 'active') { - await this.clearKeepaliveSchedule(); - return; - } - const now = Date.now(); - if (now >= activeWork.deadlineAt) { - await this.ctx.storage.put(ACTIVE_WORK_KEY, { - ...activeWork, - status: 'expired', - endedAt: now, - endReason: 'keepalive_deadline_exceeded', - } satisfies ActiveWorkState); - await this.clearKeepaliveSchedule(); - log.warn('vm_agent_container_active_work_keepalive_expired', { - nodeId: activeWork.nodeId, - workspaceId: activeWork.workspaceId, - agentSessionId: activeWork.agentSessionId, - activeSince: new Date(activeWork.activeSince).toISOString(), - lastRenewedAt: new Date(activeWork.lastRenewedAt).toISOString(), - deadlineAt: new Date(activeWork.deadlineAt).toISOString(), - }); - return; - } - - this.renewActivityTimeout(); - await this.ctx.storage.put(ACTIVE_WORK_KEY, { - ...activeWork, - lastRenewedAt: now, - } satisfies ActiveWorkState); - await this.replaceKeepaliveSchedule(this.getKeepaliveRenewIntervalMs()); - log.debug('vm_agent_container_active_work_keepalive_renewed', { - nodeId: activeWork.nodeId, - workspaceId: activeWork.workspaceId, - agentSessionId: activeWork.agentSessionId, - activeSince: new Date(activeWork.activeSince).toISOString(), - lastRenewedAt: new Date(now).toISOString(), - deadlineAt: new Date(activeWork.deadlineAt).toISOString(), - }); - } - - override async fetch(request: Request): Promise { - let state = await this.getState(); - const lifecycleStatus = await this.ctx.storage.get('lifecycleStatus'); - if (lifecycleStatus === 'sleeping') { - const wake = await this.ensureAwake(); - if (!wake.ok) { - return new Response(wake.message || WAKE_DEGRADED_RESPONSE, { status: 503 }); - } - // Re-read the container state after wake so the stopped-check reflects - // the freshly-launched container, not the pre-wake stopped snapshot. - state = await this.getState(); - } - if (state.status === 'stopped' || state.status === 'stopped_with_code') { - return new Response('Container is stopped; create a new instant session.', { status: 410 }); - } - return super.fetch(switchPort(request, this.defaultPort)); + await renewActiveWork(this.activeWorkRuntime()); } override async onStart(): Promise { - await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus); + log.info('vm_agent_container_runtime_started', {}); } - override async onStop(params: { exitCode: number; reason: 'exit' | 'runtime_signal' }): Promise { + override async onStop(params: { + exitCode: number; + reason: 'exit' | 'runtime_signal'; + }): Promise { const status = await this.ctx.storage.get('lifecycleStatus'); - if (status === 'expired' || status === 'sleeping') { + if ( + status === 'expired' || + status === 'sleeping' || + status === 'recovering' || + status === 'waking' || + status === 'restoring' || + status === 'degraded' || + status === 'stopped' + ) { return; } - const explicitStop = status === 'stopping'; - await this.markRuntimeEnded( - explicitStop ? 'stopped' : 'error', - explicitStop ? 'Container stopped by user request' : `Container stopped: ${params.reason} (${params.exitCode})` - ); - await this.ctx.storage.put('lifecycleStatus', explicitStop ? 'stopped' : 'error'); + if (status === 'stopping') { + await this.markRuntimeEnded('stopped', 'Container stopped by user request'); + await this.ctx.storage.put('lifecycleStatus', 'stopped' satisfies LifecycleStatus); + return; + } + if (status === 'launching') { + await this.markRuntimeEnded('error', 'Instant runtime failed during initial launch.'); + await this.ctx.storage.put('lifecycleStatus', 'error' satisfies LifecycleStatus); + return; + } + + await this.beginUnexpectedRecovery({ + trigger: 'stop', + cause: { + kind: 'container_stop', + reason: params.reason, + exitCode: params.exitCode, + }, + promptDisposition: 'none', + }); } override async onActivityExpired(): Promise { @@ -296,47 +319,89 @@ export class VmAgentContainer extends Container { } override async onError(error: unknown): Promise { - await this.markRuntimeEnded( - 'error', - error instanceof Error ? `Container error: ${error.message}` : `Container error: ${String(error)}` - ); - await this.ctx.storage.put('lifecycleStatus', 'error' satisfies LifecycleStatus); - } - - private getPortReadyTimeoutMs(): number { - const raw = this.env.CF_CONTAINER_PORT_READY_TIMEOUT_MS || this.env.SANDBOX_EXEC_TIMEOUT_MS; - const parsed = raw ? Number.parseInt(raw, 10) : 30_000; - return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000; - } + const status = await this.ctx.storage.get('lifecycleStatus'); + if (status === 'stopping') { + await this.markRuntimeEnded('stopped', 'Container stopped by user request'); + await this.ctx.storage.put('lifecycleStatus', 'stopped' satisfies LifecycleStatus); + return; + } + if (status === 'launching') { + await this.markRuntimeEnded('error', 'Instant runtime failed during initial launch.'); + await this.ctx.storage.put('lifecycleStatus', 'error' satisfies LifecycleStatus); + return; + } + if (status === 'stopped' || status === 'sleeping' || status === 'expired') return; - private getActiveWorkMaxMs(): number { - const raw = this.env.CF_CONTAINER_ACTIVE_WORK_MAX_MS; - const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS; - return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS; + await this.beginUnexpectedRecovery({ + trigger: 'error', + cause: { + kind: 'container_error', + errorName: error instanceof Error ? error.name : 'unknown', + }, + promptDisposition: 'none', + }); } - private getKeepaliveRenewIntervalMs(): number { - const raw = this.env.CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; - const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; - return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; + private async prepareForRequest(): Promise { + const status = await this.ctx.storage.get('lifecycleStatus'); + if (status === 'running' || status === 'launching') { + return { ok: true, status: 'running' }; + } + if (status === 'stopping' || status === 'stopped') { + return stoppedRecoveryResult(); + } + return this.ensureAwake(); } - /** - * Wake a sleeping container exactly once under concurrency. Serializes on - * `wakeChain` and re-reads `lifecycleStatus` inside the critical section, so a - * second request that arrives while the first is waking sees `running` and - * skips a duplicate launch/restore instead of racing it (see rule 45). - */ - private async ensureAwake(): Promise<{ ok: boolean; message?: string }> { + private async ensureAwake(): Promise { const run = this.wakeChain.then(async () => { - const status = await this.ctx.storage.get('lifecycleStatus'); - if (status !== 'sleeping') { - return { ok: true }; + let status = await this.ctx.storage.get('lifecycleStatus'); + if (status === 'running' || status === 'launching') { + return { ok: true, status: 'running' } satisfies VmAgentContainerRecoveryResult; + } + if (status === 'stopping' || status === 'stopped') { + return stoppedRecoveryResult(); + } + + let recovery: RuntimeRecoveryState | null | undefined = + await this.ctx.storage.get(RECOVERY_STATE_KEY); + if (!recovery) { + const trigger: RuntimeRecoveryTrigger = status === 'sleeping' ? 'idle' : 'error'; + const cause: RuntimeRecoveryCause = + status === 'sleeping' + ? { kind: 'idle_sleep' } + : { kind: 'container_error', errorName: 'legacy_terminal_state' }; + recovery = await this.beginUnexpectedRecovery({ + trigger, + cause, + promptDisposition: 'none', + }); + status = await this.ctx.storage.get('lifecycleStatus'); + } + + if (!recovery || status === 'stopped') { + return { + ok: false, + status: 'degraded', + code: 'RUNTIME_RECOVERY_DEGRADED', + message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + } satisfies VmAgentContainerRecoveryResult; + } + + if (recovery.attempts >= this.getRecoveryMaxAttempts()) { + return this.exhaustRecovery(recovery); } - return this.wakeFromSnapshot(); + + recovery = { + ...recovery, + phase: 'waking', + attempts: recovery.attempts + 1, + updatedAt: Date.now(), + }; + await this.ctx.storage.put(RECOVERY_STATE_KEY, recovery); + await this.ctx.storage.put('lifecycleStatus', 'waking' satisfies LifecycleStatus); + return this.wakeFromSnapshot(recovery); }); - // Keep the chain alive even if this wake rejects, so a failure does not - // permanently wedge all future wakes. this.wakeChain = run.then( () => undefined, () => undefined @@ -344,140 +409,336 @@ export class VmAgentContainer extends Container { return run; } - private async wakeFromSnapshot(): Promise<{ ok: boolean; message?: string }> { + private async beginUnexpectedRecovery(input: { + trigger: RuntimeRecoveryTrigger; + cause: RuntimeRecoveryCause; + promptDisposition: RuntimeRecoveryState['promptDisposition']; + }): Promise { + return this.withLifecycleLock(async () => { + const lifecycle = await this.ctx.storage.get('lifecycleStatus'); + if (lifecycle === 'stopping' || lifecycle === 'stopped' || lifecycle === 'expired') { + return null; + } + + const existing = await this.ctx.storage.get(RECOVERY_STATE_KEY); + if (existing) { + if ( + input.promptDisposition === 'manual_retry' && + existing.promptDisposition !== 'manual_retry' + ) { + const promoted = { + ...existing, + promptDisposition: 'manual_retry' as const, + updatedAt: Date.now(), + }; + await this.ctx.storage.put(RECOVERY_STATE_KEY, promoted); + return promoted; + } + return existing; + } + + const config = await this.ctx.storage.get('launchConfig'); + if (!config) return null; + const activeWork = await this.ctx.storage.get(ACTIVE_WORK_KEY); + const context = await loadRuntimeRecoveryContext(this.env, { + workspaceId: config.workspaceId, + preferredAgentSessionId: activeWork?.agentSessionId, + }); + if (!context) return null; + + const now = Date.now(); + const recovery: RuntimeRecoveryState = { + version: 1, + phase: 'pending', + trigger: input.trigger, + cause: input.cause, + attempts: 0, + promptDisposition: input.promptDisposition, + agentSessionId: context.agentSessionId, + startedAt: now, + updatedAt: now, + }; + await this.ctx.storage.put(RECOVERY_STATE_KEY, recovery); + await this.ctx.storage.put('lifecycleStatus', 'recovering' satisfies LifecycleStatus); + await this.clearKeepaliveSchedule(); + await persistRuntimeRecovering(this.env, this.toRecoveryTarget(config, context)); + log.warn('vm_agent_container_recovery_started', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + agentSessionId: context.agentSessionId, + trigger: input.trigger, + cause: input.cause, + promptDisposition: input.promptDisposition, + }); + return recovery; + }); + } + + private async wakeFromSnapshot( + recovery: RuntimeRecoveryState + ): Promise { const config = await this.ctx.storage.get('launchConfig'); - if (!config) { - return { ok: false, message: 'Container launch configuration is unavailable.' }; - } - const db = drizzle(this.env.DATABASE, { schema }); - const workspace = await db - .select({ - userId: schema.workspaces.userId, - chatSessionId: schema.workspaces.chatSessionId, - }) - .from(schema.workspaces) - .where(eq(schema.workspaces.id, config.workspaceId)) - .get(); - if (!workspace?.chatSessionId) { - return { ok: false, message: 'Workspace session metadata is unavailable.' }; - } - const agentSession = await db - .select({ id: schema.agentSessions.id, agentType: schema.agentSessions.agentType }) - .from(schema.agentSessions) - .where(eq(schema.agentSessions.workspaceId, config.workspaceId)) - .orderBy(desc(schema.agentSessions.updatedAt)) - .get(); - if (!agentSession) { - return { ok: false, message: 'Agent session metadata is unavailable.' }; - } + if (!config) return this.degradeRecovery(recovery, 'launch'); - log.info('vm_agent_container_wake_started', { - nodeId: config.nodeId, + const context = await loadRuntimeRecoveryContext(this.env, { workspaceId: config.workspaceId, - chatSessionId: workspace.chatSessionId, - agentSessionId: agentSession.id, + preferredAgentSessionId: recovery.agentSessionId, }); + if (!context) return this.degradeRecovery(recovery, 'unexpected'); + const target = this.toRecoveryTarget(config, context); - await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus); - // The container's CALLBACK_TOKEN must be node-scoped to match the initial - // launch (see launchInstantSession): the vm-agent uses it for node callbacks - // (error/activity/message reporting) which reject workspace-scoped tokens. - // Using a workspace-scoped token here caused restored sessions to accept a - // prompt (200) but silently fail to report the agent's reply back (403 - // "Insufficient token scope"), so no answer appeared after wake. - const callbackToken = await signNodeCallbackToken(config.nodeId, this.env); - await this.launch(config, { nodeCallbackToken: callbackToken }); - - // The fresh container never ran create-workspace, so its workspace-scoped - // runtime.CallbackToken is unset. The message reporter and snapshot - // callbacks require it (they do NOT fall back to the node-scoped token), so - // pass it on the restore request; without it, restored sessions accept a - // prompt but silently discard the agent's reply ("no auth token"). - const workspaceCallbackToken = await signCallbackToken(config.workspaceId, this.env); - const { token } = await signNodeManagementToken(workspace.userId, config.nodeId, config.workspaceId, this.env); - const restoreUrl = new URL(`http://localhost:${config.vmAgentPort}/workspaces/${config.workspaceId}/agent-sessions/${agentSession.id}/restore`); - const restoreResponse = await this.containerFetch( - new Request(restoreUrl.toString(), { - method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - 'X-SAM-Node-Id': config.nodeId, - 'X-SAM-Workspace-Id': config.workspaceId, - }, - body: JSON.stringify({ - chatSessionId: workspace.chatSessionId, - runtime: 'cf-container', - agentType: agentSession.agentType, - workspaceCallbackToken, - }), - }), - config.vmAgentPort - ); - const restoreBody = await restoreResponse.text().catch(() => ''); - if (!restoreResponse.ok) { - await this.markWakeDegraded(config, restoreBody || `restore failed with HTTP ${restoreResponse.status}`); - return { ok: false, message: restoreBody || 'Session restore failed.' }; + try { + const nodeCallbackToken = await signNodeCallbackToken(config.nodeId, this.env); + await this.startRuntime(config, { nodeCallbackToken }); + } catch (error) { + log.warn('vm_agent_container_recovery_launch_failed', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + errorName: error instanceof Error ? error.name : 'unknown', + }); + return this.degradeRecovery(recovery, 'launch', target); } - let restoreStatus = ''; + + const restoring = { ...recovery, phase: 'restoring' as const, updatedAt: Date.now() }; + await this.ctx.storage.put(RECOVERY_STATE_KEY, restoring); + await this.ctx.storage.put('lifecycleStatus', 'restoring' satisfies LifecycleStatus); + try { - const parsed = JSON.parse(restoreBody) as { status?: unknown }; - restoreStatus = typeof parsed.status === 'string' ? parsed.status : ''; - } catch { - // A successful restore must provide an explicit machine-readable status. + const workspaceCallbackToken = await signCallbackToken(config.workspaceId, this.env); + const { token } = await signNodeManagementToken( + context.userId, + config.nodeId, + config.workspaceId, + this.env + ); + const restoreUrl = new URL( + `http://localhost:${config.vmAgentPort}/workspaces/${config.workspaceId}/agent-sessions/${context.agentSessionId}/restore` + ); + const restoreResponse = await this.containerFetch( + new Request(restoreUrl.toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + 'X-SAM-Node-Id': config.nodeId, + 'X-SAM-Workspace-Id': config.workspaceId, + }, + body: JSON.stringify({ + chatSessionId: context.chatSessionId, + runtime: 'cf-container', + agentType: context.agentType, + workspaceCallbackToken, + }), + }), + config.vmAgentPort + ); + const restoreBody = await restoreResponse.text().catch(() => ''); + if (!restoreResponse.ok) { + return this.degradeRecovery(restoring, 'restore_http', target, restoreResponse.status); + } + let restoreStatus = ''; + try { + const parsed = JSON.parse(restoreBody) as { status?: unknown }; + restoreStatus = typeof parsed.status === 'string' ? parsed.status : ''; + } catch { + // An explicit restored status is required before D1 can become running. + } + if (restoreStatus !== 'restored') { + return this.degradeRecovery(restoring, 'restore_status', target); + } + + const completed = await this.withLifecycleLock(async () => { + const lifecycle = await this.ctx.storage.get('lifecycleStatus'); + if (lifecycle === 'stopping' || lifecycle === 'stopped') return false; + await persistRuntimeRecovered(this.env, target, restoring.promptDisposition); + await this.ctx.storage.delete(RECOVERY_STATE_KEY); + await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus); + return true; + }); + if (!completed) { + return stoppedRecoveryResult(); + } + log.info('vm_agent_container_recovery_completed', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + agentSessionId: context.agentSessionId, + attempts: restoring.attempts, + promptDisposition: restoring.promptDisposition, + }); + return { ok: true, status: 'running' }; + } catch (error) { + log.warn('vm_agent_container_recovery_restore_failed', { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + errorName: error instanceof Error ? error.name : 'unknown', + }); + return this.degradeRecovery(restoring, 'unexpected', target); } - if (restoreStatus !== 'restored') { - const message = restoreBody || 'Session restore did not report restored status.'; - await this.markWakeDegraded(config, message); - return { ok: false, message }; + } + + private async degradeRecovery( + recovery: RuntimeRecoveryState, + kind: NonNullable['kind'], + target?: RuntimeRecoveryTarget, + httpStatus?: number + ): Promise { + const degraded: RuntimeRecoveryState = { + ...recovery, + phase: 'degraded', + updatedAt: Date.now(), + lastFailure: { kind, ...(httpStatus === undefined ? {} : { httpStatus }) }, + }; + const applied = await this.withLifecycleLock(async () => { + const lifecycle = await this.ctx.storage.get('lifecycleStatus'); + if (lifecycle === 'stopping' || lifecycle === 'stopped') return false; + await this.ctx.storage.put(RECOVERY_STATE_KEY, degraded); + await this.ctx.storage.put('lifecycleStatus', 'degraded' satisfies LifecycleStatus); + return true; + }); + if (!applied) return stoppedRecoveryResult(); + await this.stop().catch(() => undefined); + + if (degraded.attempts >= this.getRecoveryMaxAttempts()) { + return this.exhaustRecovery(degraded, target); } + return { + ok: false, + status: 'degraded', + code: 'RUNTIME_RECOVERY_DEGRADED', + message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }; + } - const now = new Date().toISOString(); - await db.update(schema.nodes).set({ - status: 'running', - healthStatus: 'healthy', - errorMessage: null, - updatedAt: now, - }).where(eq(schema.nodes.id, config.nodeId)); - await db.update(schema.workspaces).set({ - status: 'running', - errorMessage: null, - updatedAt: now, - }).where(eq(schema.workspaces.id, config.workspaceId)); - await db.update(schema.agentSessions).set({ - status: 'running', - errorMessage: null, - updatedAt: now, - }).where(eq(schema.agentSessions.id, agentSession.id)); - await this.ctx.storage.put('lifecycleStatus', 'running' satisfies LifecycleStatus); - - log.info('vm_agent_container_wake_completed', { - nodeId: config.nodeId, - workspaceId: config.workspaceId, - chatSessionId: workspace.chatSessionId, - agentSessionId: agentSession.id, + private async exhaustRecovery( + recovery: RuntimeRecoveryState, + providedTarget?: RuntimeRecoveryTarget + ): Promise { + const applied = await this.withLifecycleLock(async () => { + const lifecycle = await this.ctx.storage.get('lifecycleStatus'); + if (lifecycle === 'stopping' || lifecycle === 'stopped') return false; + const config = await this.ctx.storage.get('launchConfig'); + let target = providedTarget; + if (!target && config) { + const context = await loadRuntimeRecoveryContext(this.env, { + workspaceId: config.workspaceId, + preferredAgentSessionId: recovery.agentSessionId, + }); + if (context) target = this.toRecoveryTarget(config, context); + } + const exhausted = { ...recovery, phase: 'exhausted' as const, updatedAt: Date.now() }; + await this.ctx.storage.put(RECOVERY_STATE_KEY, exhausted); + await this.ctx.storage.put('lifecycleStatus', 'error' satisfies LifecycleStatus); + if (target) await persistRuntimeRecoveryFailed(this.env, target); + return true; }); - return { ok: true }; - } - - private async markWakeDegraded(config: VmAgentContainerLaunchConfig, message: string): Promise { - const now = new Date().toISOString(); - const db = drizzle(this.env.DATABASE, { schema }); - await db.update(schema.workspaces).set({ - status: 'recovery', - errorMessage: message, - updatedAt: now, - }).where(eq(schema.workspaces.id, config.workspaceId)); - await db.update(schema.agentSessions).set({ - status: 'error', - errorMessage: message, - updatedAt: now, - }).where(eq(schema.agentSessions.workspaceId, config.workspaceId)); - log.warn('vm_agent_container_wake_degraded', { + if (!applied) return stoppedRecoveryResult(); + return { + ok: false, + status: 'degraded', + code: 'RUNTIME_RECOVERY_DEGRADED', + message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }; + } + + private async startRuntime( + config: VmAgentContainerLaunchConfig, + secrets: VmAgentContainerLaunchSecrets + ): Promise { + await this.startAndWaitForPorts({ + ports: config.vmAgentPort, + startOptions: { + envVars: { + NODE_ROLE: 'standalone', + NODE_ID: config.nodeId, + WORKSPACE_ID: config.workspaceId, + PROJECT_ID: config.projectId, + CHAT_SESSION_ID: config.chatSessionId, + CONTROL_PLANE_URL: config.controlPlaneUrl, + CALLBACK_TOKEN: secrets.nodeCallbackToken, + REPOSITORY: config.repository, + BRANCH: config.branch, + WORKSPACE_DIR: config.workspaceDir, + CONTAINER_WORK_DIR: config.workspaceDir, + CONTAINER_MODE: 'false', + PORT_SCAN_ENABLED: 'false', + VM_AGENT_PORT: String(config.vmAgentPort), + VM_AGENT_PROTOCOL: 'http', + COOKIE_SECURE: 'true', + ...(this.env.CF_CONTAINER_CLONE_FILTER + ? { STANDALONE_CLONE_FILTER: this.env.CF_CONTAINER_CLONE_FILTER } + : {}), + }, + labels: { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + runtime: 'cf-container', + }, + }, + cancellationOptions: { portReadyTimeoutMS: this.getPortReadyTimeoutMs() }, + }); + } + + private toRecoveryTarget( + config: VmAgentContainerLaunchConfig, + context: RuntimeRecoveryContext + ): RuntimeRecoveryTarget { + return { nodeId: config.nodeId, workspaceId: config.workspaceId, - message, - }); + projectId: config.projectId, + chatSessionId: context.chatSessionId, + agentSessionId: context.agentSessionId, + }; + } + + private withLifecycleLock(operation: () => Promise): Promise { + const run = this.lifecycleChain.then(operation); + this.lifecycleChain = run.then( + () => undefined, + () => undefined + ); + return run; + } + + private activeWorkRuntime(): ActiveWorkRuntime { + return { + storage: this.ctx.storage, + activeWorkMaxMs: this.getActiveWorkMaxMs(), + renewIntervalMs: this.getKeepaliveRenewIntervalMs(), + renewActivityTimeout: () => this.renewActivityTimeout(), + replaceSchedule: (delayMs) => this.replaceKeepaliveSchedule(delayMs), + clearSchedule: () => this.clearKeepaliveSchedule(), + }; + } + + private getPortReadyTimeoutMs(): number { + const raw = this.env.CF_CONTAINER_PORT_READY_TIMEOUT_MS || this.env.SANDBOX_EXEC_TIMEOUT_MS; + const parsed = raw ? Number.parseInt(raw, 10) : 30_000; + return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000; + } + + private getActiveWorkMaxMs(): number { + const raw = this.env.CF_CONTAINER_ACTIVE_WORK_MAX_MS; + const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS; + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS; + } + + private getKeepaliveRenewIntervalMs(): number { + const raw = this.env.CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; + const parsed = raw + ? Number.parseInt(raw, 10) + : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; + } + + private getRecoveryMaxAttempts(): number { + const raw = this.env.CF_CONTAINER_RECOVERY_MAX_ATTEMPTS; + const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS; + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS; } private async replaceKeepaliveSchedule(delayMs: number): Promise { @@ -491,97 +752,24 @@ export class VmAgentContainer extends Container { private async markRuntimeSleeping(message: string): Promise { const config = await this.ctx.storage.get('launchConfig'); - if (!config) { - return; - } - + if (!config) return; await this.markActiveWorkEnded('container_idle_sleeping'); - - const now = new Date().toISOString(); - const db = drizzle(this.env.DATABASE, { schema }); - - await db - .update(schema.nodes) - .set({ - status: 'sleeping', - healthStatus: 'unhealthy', - errorMessage: null, - updatedAt: now, - }) - .where(eq(schema.nodes.id, config.nodeId)); - - await db - .update(schema.workspaces) - .set({ - status: 'sleeping', - errorMessage: null, - updatedAt: now, - }) - .where(eq(schema.workspaces.id, config.workspaceId)); - - await db - .update(schema.agentSessions) - .set({ - status: 'sleeping', - errorMessage: null, - updatedAt: now, - }) - .where(eq(schema.agentSessions.workspaceId, config.workspaceId)); - + await persistRuntimeSleeping(this.env, config); log.info('vm_agent_container_runtime_sleeping', { nodeId: config.nodeId, workspaceId: config.workspaceId, - status: 'sleeping', message, }); } - private async markRuntimeEnded(status: Exclude, message: string): Promise { + private async markRuntimeEnded(status: 'stopped' | 'error', message: string): Promise { const config = await this.ctx.storage.get('launchConfig'); - if (!config) { - return; - } - - const now = new Date().toISOString(); - const db = drizzle(this.env.DATABASE, { schema }); - const workspaceStatus = status === 'stopped' ? 'stopped' : 'error'; - const agentStatus = status === 'stopped' ? 'stopped' : 'error'; - const nodeStatus = status === 'stopped' ? 'stopped' : 'error'; - - await db - .update(schema.nodes) - .set({ - status: nodeStatus, - healthStatus: 'unhealthy', - errorMessage: status === 'stopped' ? null : message, - updatedAt: now, - }) - .where(eq(schema.nodes.id, config.nodeId)); - - await db - .update(schema.workspaces) - .set({ - status: workspaceStatus, - errorMessage: status === 'stopped' ? null : message, - updatedAt: now, - }) - .where(eq(schema.workspaces.id, config.workspaceId)); - - await db - .update(schema.agentSessions) - .set({ - status: agentStatus, - stoppedAt: now, - errorMessage: status === 'stopped' ? null : message, - updatedAt: now, - }) - .where(eq(schema.agentSessions.workspaceId, config.workspaceId)); - + if (!config) return; + await persistRuntimeEnded(this.env, config, status, message); log.warn('vm_agent_container_runtime_ended', { nodeId: config.nodeId, workspaceId: config.workspaceId, status, - message, }); } } diff --git a/apps/api/src/env.ts b/apps/api/src/env.ts index f0b06806a..e6a395bb8 100644 --- a/apps/api/src/env.ts +++ b/apps/api/src/env.ts @@ -836,6 +836,7 @@ export interface Env extends WebhookTriggerEnv, TaskRecoveryEnv { CF_CONTAINER_VM_AGENT_PORT?: string; // vm-agent standalone HTTP port inside the raw container (default: 8080) CF_CONTAINER_PORT_READY_TIMEOUT_MS?: string; // Max time to wait for vm-agent port readiness (default: 30000) CF_CONTAINER_WAKE_TIMEOUT_MS?: string; // Max time for launch + restore before forwarding a wake request (default: 120000) + CF_CONTAINER_RECOVERY_MAX_ATTEMPTS?: string; // Max snapshot restore attempts before terminal reconciliation (default: 2) CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS?: string; // Max time for the synchronous standalone create-workspace request incl. clone (default: 120000) CF_CONTAINER_CLONE_FILTER?: string; // Git partial-clone filter passed to the container as STANDALONE_CLONE_FILTER (vm-agent default: blob:none; "off" disables) CF_CONTAINER_WORKSPACE_BASE_DIR?: string; // Base checkout dir inside raw container (default: /workspaces) diff --git a/apps/api/src/routes/chat-workspace-resolver.ts b/apps/api/src/routes/chat-workspace-resolver.ts index 928723f52..332f7c1a7 100644 --- a/apps/api/src/routes/chat-workspace-resolver.ts +++ b/apps/api/src/routes/chat-workspace-resolver.ts @@ -1,10 +1,9 @@ /** - * Tenant-scoped resolution of the live workspace that bridges a chat session - * to its running VM. Extracted from routes/chat.ts to keep that file under the - * 800-line limit (.claude/rules/18) and to make the security-critical resolver - * independently testable against real D1. + * Tenant-scoped resolution of the workspace bridging a chat session to its + * runtime. Recovery states are admitted only for cf-container nodes; VM + * sessions preserve their fail-fast dead-node guard. */ -import { and, eq, inArray } from 'drizzle-orm'; +import { and, eq, inArray, or } from 'drizzle-orm'; import type { drizzle } from 'drizzle-orm/d1'; import * as schema from '../db/schema'; @@ -13,32 +12,28 @@ import { errors } from '../middleware/error'; type ChatDb = ReturnType>; +export interface ChatWorkspaceRuntime { + id: string; + nodeId: string | null; + nodeStatus: string | null; + nodeRuntime: string | null; +} + /** - * Resolve the live workspace that bridges a chat session to its running VM, - * scoped to the owning project AND user. - * - * Security: the workspace bridge MUST NOT be resolvable by `chatSessionId` - * alone. Without project + user scoping, any authenticated caller who supplies - * (or guesses/replays) a sessionId belonging to another tenant can drive the - * VM agent for a workspace they do not own — an IDOR in the same class as past - * cross-tenant leaks. We resolve via the narrowest canonical chat-scoped - * identifier (`chatSessionId`) AND enforce ownership in the query WHERE clause - * (see .claude/rules/06 canonical session routing, .claude/rules/11 identity - * validation). A post-query defence-in-depth assertion rejects any row whose - * ownership does not match the caller, guarding against a future WHERE-clause - * regression (typo/refactor/ORM bug) per .claude/rules/28. - * - * Returns null when no matching active workspace exists for this owner. + * Resolve the workspace bridge, scoped to the owning project and user. + * `chatSessionId` alone is never sufficient: retaining project/user predicates + * is the defence against cross-tenant workspace control. */ export async function resolveLiveWorkspaceForSession( db: ChatDb, { projectId, sessionId, userId }: { projectId: string; sessionId: string; userId: string } -): Promise<{ id: string; nodeId: string | null; nodeStatus: string | null } | null> { +): Promise { const [workspace] = await db .select({ id: schema.workspaces.id, nodeId: schema.workspaces.nodeId, nodeStatus: schema.nodes.status, + nodeRuntime: schema.nodes.runtime, userId: schema.workspaces.userId, projectId: schema.workspaces.projectId, }) @@ -49,16 +44,16 @@ export async function resolveLiveWorkspaceForSession( eq(schema.workspaces.chatSessionId, sessionId), eq(schema.workspaces.projectId, projectId), eq(schema.workspaces.userId, userId), - inArray(schema.workspaces.status, ['running', 'recovery', 'sleeping']) + or( + inArray(schema.workspaces.status, ['running', 'recovery', 'sleeping']), + and(eq(schema.nodes.runtime, 'cf-container'), eq(schema.workspaces.status, 'error')) + ) ) ) .limit(1); - if (!workspace) { - return null; - } + if (!workspace) return null; - // Defence-in-depth: reject any row whose ownership doesn't match the caller. if (workspace.userId !== userId || workspace.projectId !== projectId) { log.error('chat: workspace ownership mismatch on session bridge', { sessionId, @@ -72,53 +67,44 @@ export async function resolveLiveWorkspaceForSession( return null; } - return { id: workspace.id, nodeId: workspace.nodeId, nodeStatus: workspace.nodeStatus }; + return { + id: workspace.id, + nodeId: workspace.nodeId, + nodeStatus: workspace.nodeStatus, + nodeRuntime: workspace.nodeRuntime, + }; } /** - * Resolve the live workspace AND its resumable agent session for a chat action - * (e.g. /prompt, /cancel), enforcing tenant scoping at every layer. - * - * Consolidates the shared resolution path used by the /prompt and /cancel - * handlers: workspace bridge resolution (project + user scoped), node-liveness - * guards, and the defence-in-depth user-scoped agent-session lookup (backed by - * idx_agent_sessions_ws_user_status). Throws the same fail-fast errors both - * handlers relied on so callers can forward straight to the VM agent. - * - * @throws notFound when no active workspace/node or running agent session exists - * @throws conflict when the workspace node is no longer running + * Resolve the workspace and resumable agent session for a chat action. A + * cf-container Durable Object owns replacement classification, so recoverable + * D1 states must reach it. Other runtimes still require a running node. */ export async function resolveLiveAgentSessionForChat( db: ChatDb, { projectId, sessionId, userId }: { projectId: string; sessionId: string; userId: string } -): Promise<{ workspace: { id: string; nodeId: string; nodeStatus: string }; agentSession: { id: string } }> { - // Find the workspace linked to this chat session, scoped to the owning - // project + user (see resolveLiveWorkspaceForSession). The node join also - // verifies the node is still active: when a node is destroyed (e.g., after - // task timeout), its DNS record is cleaned up but the workspace may still be - // marked 'running' in D1. Without this check, the request to the VM agent - // would hit the wildcard DNS record and loop back to this Worker (404). +): Promise<{ + workspace: { id: string; nodeId: string; nodeStatus: string; nodeRuntime: string }; + agentSession: { id: string }; +}> { const workspace = await resolveLiveWorkspaceForSession(db, { projectId, sessionId, userId }); - - if (!workspace || !workspace.nodeId) { + if (!workspace?.nodeId || !workspace.nodeStatus || !workspace.nodeRuntime) { throw errors.notFound('No active workspace found for this session'); } - // Verify the node is still reachable — prevents requests to destroyed VMs - // whose DNS records no longer exist (would loop back via wildcard DNS). - // D1 nodes.status uses 'running' for healthy nodes (not 'active'/'warm', which are DO states). - // A request routed to a sleeping cf-container is the wake signal: the - // container proxy launches a fresh runtime and restores its latest snapshot - // before forwarding the request. Preserve the fail-fast guard for every - // other non-running node state. - if (workspace.nodeStatus !== 'running' && workspace.nodeStatus !== 'sleeping') { + const isContainer = workspace.nodeRuntime === 'cf-container'; + const nodeIsReachable = isContainer + ? ['running', 'sleeping', 'recovery', 'error'].includes(workspace.nodeStatus) + : workspace.nodeStatus === 'running'; + if (!nodeIsReachable) { throw errors.conflict( 'The workspace node is no longer running. Start a new chat to create a fresh workspace.' ); } - // Find the running agent session on that workspace, scoped to the user - // for defence-in-depth (uses idx_agent_sessions_ws_user_status composite index). + const agentStatuses = isContainer + ? ['running', 'sleeping', 'recovery', 'error'] + : ['running', 'sleeping']; const [agentSession] = await db .select({ id: schema.agentSessions.id }) .from(schema.agentSessions) @@ -126,14 +112,20 @@ export async function resolveLiveAgentSessionForChat( and( eq(schema.agentSessions.workspaceId, workspace.id), eq(schema.agentSessions.userId, userId), - inArray(schema.agentSessions.status, ['running', 'sleeping']) + inArray(schema.agentSessions.status, agentStatuses) ) ) .limit(1); - if (!agentSession) { - throw errors.notFound('No running agent session found'); - } + if (!agentSession) throw errors.notFound('No running agent session found'); - return { workspace: { id: workspace.id, nodeId: workspace.nodeId, nodeStatus: workspace.nodeStatus }, agentSession }; + return { + workspace: { + id: workspace.id, + nodeId: workspace.nodeId, + nodeStatus: workspace.nodeStatus, + nodeRuntime: workspace.nodeRuntime, + }, + agentSession, + }; } diff --git a/apps/api/src/routes/chat.ts b/apps/api/src/routes/chat.ts index e88682a4a..839fcf6d1 100644 --- a/apps/api/src/routes/chat.ts +++ b/apps/api/src/routes/chat.ts @@ -541,7 +541,7 @@ chatRoutes.post('/:sessionId/prompt', async (c) => { c.env, userId, undefined, - workspace.nodeStatus === 'sleeping' + workspace.nodeRuntime === 'cf-container' && workspace.nodeStatus !== 'running' ? { requestTimeoutMs: getCfContainerWakeTimeoutMs(c.env) } : undefined ); diff --git a/apps/api/src/routes/workspaces/agent-sessions.ts b/apps/api/src/routes/workspaces/agent-sessions.ts index 69c14246a..9bb7689ce 100644 --- a/apps/api/src/routes/workspaces/agent-sessions.ts +++ b/apps/api/src/routes/workspaces/agent-sessions.ts @@ -1,6 +1,4 @@ -import type { - AgentSession, -} from '@simple-agent-manager/shared'; +import type { AgentSession } from '@simple-agent-manager/shared'; import { and, desc, eq } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/d1'; import { Hono } from 'hono'; @@ -11,8 +9,8 @@ import { log } from '../../lib/logger'; import { toAgentSessionResponse } from '../../lib/mappers'; import { parsePositiveInt } from '../../lib/route-helpers'; import { ulid } from '../../lib/ulid'; -import { getUserId, requireApproved,requireAuth } from '../../middleware/auth'; -import { errors } from '../../middleware/error'; +import { getUserId, requireApproved, requireAuth } from '../../middleware/auth'; +import { AppError, errors } from '../../middleware/error'; import { CreateAgentSessionSchema, jsonValidator, UpdateAgentSessionSchema } from '../../schemas'; import { getRuntimeLimits } from '../../services/limits'; import { generateMcpToken, revokeMcpToken, storeMcpToken } from '../../services/mcp-token'; @@ -22,8 +20,9 @@ import { stopAgentSessionOnNode, suspendAgentSessionOnNode, } from '../../services/node-agent'; +import { resumeVmAgentContainer } from '../../services/vm-agent-container'; import { requireRepositoryOwnerAccess } from '../projects/_helpers'; -import { assertNodeOperational,getOwnedNode, getOwnedWorkspace } from './_helpers'; +import { assertNodeOperational, getOwnedNode, getOwnedWorkspace } from './_helpers'; const agentSessionRoutes = new Hono<{ Bindings: Env }>(); @@ -73,362 +72,431 @@ agentSessionRoutes.get('/:id/agent-sessions', requireAuth(), requireApproved(), return c.json(sessions.map(toAgentSessionResponse)); }); -agentSessionRoutes.post('/:id/agent-sessions', requireAuth(), requireApproved(), jsonValidator(CreateAgentSessionSchema), async (c) => { - const userId = getUserId(c); - const workspaceId = c.req.param('id'); - const db = drizzle(c.env.DATABASE, { schema }); - const body = c.req.valid('json'); - const limits = getRuntimeLimits(c.env); - - const workspace = await getOwnedWorkspace(db, workspaceId, userId); - if (!workspace.nodeId) { - throw errors.badRequest('Workspace is not attached to a node'); - } - - const node = await getOwnedNode(db, workspace.nodeId, userId); - assertNodeOperational(node, 'create agent session'); - await requireWorkspaceAgentGitHubAccess(c.env, db, workspace, userId); - - const existingRunning = await db - .select({ id: schema.agentSessions.id }) - .from(schema.agentSessions) - .where( - and( - eq(schema.agentSessions.workspaceId, workspace.id), - eq(schema.agentSessions.userId, userId), - eq(schema.agentSessions.status, 'running') - ) - ); +agentSessionRoutes.post( + '/:id/agent-sessions', + requireAuth(), + requireApproved(), + jsonValidator(CreateAgentSessionSchema), + async (c) => { + const userId = getUserId(c); + const workspaceId = c.req.param('id'); + const db = drizzle(c.env.DATABASE, { schema }); + const body = c.req.valid('json'); + const limits = getRuntimeLimits(c.env); + + const workspace = await getOwnedWorkspace(db, workspaceId, userId); + if (!workspace.nodeId) { + throw errors.badRequest('Workspace is not attached to a node'); + } - if (existingRunning.length >= limits.maxAgentSessionsPerWorkspace) { - throw errors.badRequest( - `Maximum ${limits.maxAgentSessionsPerWorkspace} agent sessions per workspace` - ); - } + const node = await getOwnedNode(db, workspace.nodeId, userId); + assertNodeOperational(node, 'create agent session'); + await requireWorkspaceAgentGitHubAccess(c.env, db, workspace, userId); + + const existingRunning = await db + .select({ id: schema.agentSessions.id }) + .from(schema.agentSessions) + .where( + and( + eq(schema.agentSessions.workspaceId, workspace.id), + eq(schema.agentSessions.userId, userId), + eq(schema.agentSessions.status, 'running') + ) + ); - const sessionId = ulid(); - const now = new Date().toISOString(); - - await db.insert(schema.agentSessions).values({ - id: sessionId, - workspaceId: workspace.id, - userId, - status: 'running', - label: body.label?.trim() || null, - agentType: body.agentType?.trim() || null, - worktreePath: body.worktreePath?.trim() || null, - createdAt: now, - updatedAt: now, - }); - - let mcpToken: string | null = null; - try { - if (workspace.projectId) { - mcpToken = generateMcpToken(); - await storeMcpToken( - c.env.KV, - mcpToken, - { - // Empty taskId for direct project-chat sessions — only task-runner dispatched - // sessions have a real task row. Setting sessionId as taskId was wrong because - // MCP tools query tasks by this ID and would get "Task not found". Empty string - // is falsy so tools guarding on !tokenData.taskId correctly reject early. - taskId: '', - contextType: workspace.chatSessionId ? 'conversation' : 'direct-workspace', - taskMode: workspace.chatSessionId ? 'conversation' : undefined, - projectId: workspace.projectId, - userId, - workspaceId: workspace.id, - chatSessionId: workspace.chatSessionId ?? undefined, - agentSessionId: sessionId, - createdAt: new Date().toISOString(), - }, - c.env, + if (existingRunning.length >= limits.maxAgentSessionsPerWorkspace) { + throw errors.badRequest( + `Maximum ${limits.maxAgentSessionsPerWorkspace} agent sessions per workspace` ); } - await createAgentSessionOnNode( - workspace.nodeId, - workspace.id, - sessionId, - body.label?.trim() || null, - c.env, + const sessionId = ulid(); + const now = new Date().toISOString(); + + await db.insert(schema.agentSessions).values({ + id: sessionId, + workspaceId: workspace.id, userId, - workspace.chatSessionId, - workspace.projectId, - mcpToken - ? { - url: `https://api.${c.env.BASE_DOMAIN}/mcp`, - token: mcpToken, - } - : undefined, - ); - } catch (err) { - if (mcpToken) { - await revokeMcpToken(c.env.KV, mcpToken).catch((revokeErr) => { - log.warn('agent_session.mcp_token_revoke_failed', { - sessionId, - workspaceId: workspace.id, - error: revokeErr instanceof Error ? revokeErr.message : String(revokeErr), - }); - }); - } + status: 'running', + label: body.label?.trim() || null, + agentType: body.agentType?.trim() || null, + worktreePath: body.worktreePath?.trim() || null, + createdAt: now, + updatedAt: now, + }); - await db - .update(schema.agentSessions) - .set({ - status: 'error', - errorMessage: err instanceof Error ? err.message : 'Failed to create agent session', - updatedAt: new Date().toISOString(), - }) - .where(eq(schema.agentSessions.id, sessionId)); + let mcpToken: string | null = null; + try { + if (workspace.projectId) { + mcpToken = generateMcpToken(); + await storeMcpToken( + c.env.KV, + mcpToken, + { + // Empty taskId for direct project-chat sessions — only task-runner dispatched + // sessions have a real task row. Setting sessionId as taskId was wrong because + // MCP tools query tasks by this ID and would get "Task not found". Empty string + // is falsy so tools guarding on !tokenData.taskId correctly reject early. + taskId: '', + contextType: workspace.chatSessionId ? 'conversation' : 'direct-workspace', + taskMode: workspace.chatSessionId ? 'conversation' : undefined, + projectId: workspace.projectId, + userId, + workspaceId: workspace.id, + chatSessionId: workspace.chatSessionId ?? undefined, + agentSessionId: sessionId, + createdAt: new Date().toISOString(), + }, + c.env + ); + } - throw errors.internal('Failed to create agent session on node'); - } + await createAgentSessionOnNode( + workspace.nodeId, + workspace.id, + sessionId, + body.label?.trim() || null, + c.env, + userId, + workspace.chatSessionId, + workspace.projectId, + mcpToken + ? { + url: `https://api.${c.env.BASE_DOMAIN}/mcp`, + token: mcpToken, + } + : undefined + ); + } catch (err) { + if (mcpToken) { + await revokeMcpToken(c.env.KV, mcpToken).catch((revokeErr) => { + log.warn('agent_session.mcp_token_revoke_failed', { + sessionId, + workspaceId: workspace.id, + error: revokeErr instanceof Error ? revokeErr.message : String(revokeErr), + }); + }); + } - const rows = await db - .select() - .from(schema.agentSessions) - .where(eq(schema.agentSessions.id, sessionId)) - .limit(1); + await db + .update(schema.agentSessions) + .set({ + status: 'error', + errorMessage: err instanceof Error ? err.message : 'Failed to create agent session', + updatedAt: new Date().toISOString(), + }) + .where(eq(schema.agentSessions.id, sessionId)); - return c.json(toAgentSessionResponse(rows[0]!), 201); -}); + throw errors.internal('Failed to create agent session on node'); + } -agentSessionRoutes.patch('/:id/agent-sessions/:sessionId', requireAuth(), requireApproved(), jsonValidator(UpdateAgentSessionSchema), async (c) => { - const userId = getUserId(c); - const workspaceId = c.req.param('id'); - const sessionId = c.req.param('sessionId'); - const db = drizzle(c.env.DATABASE, { schema }); + const rows = await db + .select() + .from(schema.agentSessions) + .where(eq(schema.agentSessions.id, sessionId)) + .limit(1); - const workspace = await getOwnedWorkspace(db, workspaceId, userId); - if (!workspace) { - throw errors.notFound('Workspace'); + return c.json(toAgentSessionResponse(rows[0]!), 201); } +); + +agentSessionRoutes.patch( + '/:id/agent-sessions/:sessionId', + requireAuth(), + requireApproved(), + jsonValidator(UpdateAgentSessionSchema), + async (c) => { + const userId = getUserId(c); + const workspaceId = c.req.param('id'); + const sessionId = c.req.param('sessionId'); + const db = drizzle(c.env.DATABASE, { schema }); + + const workspace = await getOwnedWorkspace(db, workspaceId, userId); + if (!workspace) { + throw errors.notFound('Workspace'); + } - const body = c.req.valid('json'); - const maxLabelLength = parsePositiveInt(c.env.MAX_AGENT_SESSION_LABEL_LENGTH, 50); - const label = body.label?.trim()?.slice(0, maxLabelLength); - if (!label) { - throw errors.badRequest('Label is required and must be non-empty'); - } + const body = c.req.valid('json'); + const maxLabelLength = parsePositiveInt(c.env.MAX_AGENT_SESSION_LABEL_LENGTH, 50); + const label = body.label?.trim()?.slice(0, maxLabelLength); + if (!label) { + throw errors.badRequest('Label is required and must be non-empty'); + } - const rows = await db - .select() - .from(schema.agentSessions) - .where( - and( - eq(schema.agentSessions.id, sessionId), - eq(schema.agentSessions.workspaceId, workspace.id), - eq(schema.agentSessions.userId, userId) + const rows = await db + .select() + .from(schema.agentSessions) + .where( + and( + eq(schema.agentSessions.id, sessionId), + eq(schema.agentSessions.workspaceId, workspace.id), + eq(schema.agentSessions.userId, userId) + ) ) - ) - .limit(1); - - const session = rows[0]; - if (!session) { - throw errors.notFound('Agent session'); - } + .limit(1); - if (session.status !== 'running') { - throw errors.badRequest('Cannot rename a session that is not running'); - } - - await db - .update(schema.agentSessions) - .set({ - label, - updatedAt: new Date().toISOString(), - }) - .where(eq(schema.agentSessions.id, session.id)); + const session = rows[0]; + if (!session) { + throw errors.notFound('Agent session'); + } - return c.json(toAgentSessionResponse({ ...session, label, updatedAt: new Date().toISOString() })); -}); + if (session.status !== 'running') { + throw errors.badRequest('Cannot rename a session that is not running'); + } -agentSessionRoutes.post('/:id/agent-sessions/:sessionId/stop', requireAuth(), requireApproved(), async (c) => { - const userId = getUserId(c); - const workspaceId = c.req.param('id'); - const sessionId = c.req.param('sessionId'); - const db = drizzle(c.env.DATABASE, { schema }); + await db + .update(schema.agentSessions) + .set({ + label, + updatedAt: new Date().toISOString(), + }) + .where(eq(schema.agentSessions.id, session.id)); - const workspace = await getOwnedWorkspace(db, workspaceId, userId); - if (!workspace.nodeId) { - throw errors.badRequest('Workspace is not attached to a node'); + return c.json( + toAgentSessionResponse({ ...session, label, updatedAt: new Date().toISOString() }) + ); } +); + +agentSessionRoutes.post( + '/:id/agent-sessions/:sessionId/stop', + requireAuth(), + requireApproved(), + async (c) => { + const userId = getUserId(c); + const workspaceId = c.req.param('id'); + const sessionId = c.req.param('sessionId'); + const db = drizzle(c.env.DATABASE, { schema }); + + const workspace = await getOwnedWorkspace(db, workspaceId, userId); + if (!workspace.nodeId) { + throw errors.badRequest('Workspace is not attached to a node'); + } - const rows = await db - .select() - .from(schema.agentSessions) - .where( - and( - eq(schema.agentSessions.id, sessionId), - eq(schema.agentSessions.workspaceId, workspace.id), - eq(schema.agentSessions.userId, userId) + const rows = await db + .select() + .from(schema.agentSessions) + .where( + and( + eq(schema.agentSessions.id, sessionId), + eq(schema.agentSessions.workspaceId, workspace.id), + eq(schema.agentSessions.userId, userId) + ) ) - ) - .limit(1); + .limit(1); - const session = rows[0]; - if (!session) { - throw errors.notFound('Agent session'); - } + const session = rows[0]; + if (!session) { + throw errors.notFound('Agent session'); + } - if (session.status !== 'running') { - // Still attempt VM stop for orphaned sessions whose process may be alive - if (workspace.nodeId) { - try { - await stopAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); - } catch (e) { - log.error('agent_session.orphaned_stop_failed', { sessionId: session.id, workspaceId: workspace.id, nodeId: workspace.nodeId, error: String(e) }); + if (session.status !== 'running') { + // Still attempt VM stop for orphaned sessions whose process may be alive + if (workspace.nodeId) { + try { + await stopAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); + } catch (e) { + log.error('agent_session.orphaned_stop_failed', { + sessionId: session.id, + workspaceId: workspace.id, + nodeId: workspace.nodeId, + error: String(e), + }); + } } + return c.json({ status: session.status }); } - return c.json({ status: session.status }); - } - try { - await stopAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); - } catch (e) { - log.error('agent_session.stop_on_node_failed', { sessionId: session.id, workspaceId: workspace.id, nodeId: workspace.nodeId, error: String(e) }); - } - - await db - .update(schema.agentSessions) - .set({ - status: 'stopped', - stoppedAt: new Date().toISOString(), - errorMessage: null, - updatedAt: new Date().toISOString(), - }) - .where(eq(schema.agentSessions.id, session.id)); - - return c.json({ status: 'stopped' }); -}); + try { + await stopAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); + } catch (e) { + log.error('agent_session.stop_on_node_failed', { + sessionId: session.id, + workspaceId: workspace.id, + nodeId: workspace.nodeId, + error: String(e), + }); + } -agentSessionRoutes.post('/:id/agent-sessions/:sessionId/suspend', requireAuth(), requireApproved(), async (c) => { - const userId = getUserId(c); - const workspaceId = c.req.param('id'); - const sessionId = c.req.param('sessionId'); - const db = drizzle(c.env.DATABASE, { schema }); + await db + .update(schema.agentSessions) + .set({ + status: 'stopped', + stoppedAt: new Date().toISOString(), + errorMessage: null, + updatedAt: new Date().toISOString(), + }) + .where(eq(schema.agentSessions.id, session.id)); - const workspace = await getOwnedWorkspace(db, workspaceId, userId); - if (!workspace.nodeId) { - throw errors.badRequest('Workspace is not attached to a node'); + return c.json({ status: 'stopped' }); } +); + +agentSessionRoutes.post( + '/:id/agent-sessions/:sessionId/suspend', + requireAuth(), + requireApproved(), + async (c) => { + const userId = getUserId(c); + const workspaceId = c.req.param('id'); + const sessionId = c.req.param('sessionId'); + const db = drizzle(c.env.DATABASE, { schema }); + + const workspace = await getOwnedWorkspace(db, workspaceId, userId); + if (!workspace.nodeId) { + throw errors.badRequest('Workspace is not attached to a node'); + } - const rows = await db - .select() - .from(schema.agentSessions) - .where( - and( - eq(schema.agentSessions.id, sessionId), - eq(schema.agentSessions.workspaceId, workspace.id), - eq(schema.agentSessions.userId, userId) + const rows = await db + .select() + .from(schema.agentSessions) + .where( + and( + eq(schema.agentSessions.id, sessionId), + eq(schema.agentSessions.workspaceId, workspace.id), + eq(schema.agentSessions.userId, userId) + ) ) - ) - .limit(1); + .limit(1); - const session = rows[0]; - if (!session) { - throw errors.notFound('Agent session'); - } - - if (session.status !== 'running' && session.status !== 'error') { - throw errors.badRequest(`Session cannot be suspended from status: ${session.status}`); - } - - try { - await suspendAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); - } catch (e) { - log.warn('agent_session.suspend_on_node_failed', { sessionId: session.id, workspaceId: workspace.id, nodeId: workspace.nodeId, error: String(e) }); - } + const session = rows[0]; + if (!session) { + throw errors.notFound('Agent session'); + } - const now = new Date().toISOString(); - await db - .update(schema.agentSessions) - .set({ - status: 'suspended', - suspendedAt: now, - errorMessage: null, - updatedAt: now, - }) - .where(eq(schema.agentSessions.id, session.id)); - - return c.json( - toAgentSessionResponse({ - ...session, - status: 'suspended', - suspendedAt: now, - errorMessage: null, - updatedAt: now, - }) - ); -}); + if (session.status !== 'running' && session.status !== 'error') { + throw errors.badRequest(`Session cannot be suspended from status: ${session.status}`); + } -agentSessionRoutes.post('/:id/agent-sessions/:sessionId/resume', requireAuth(), requireApproved(), async (c) => { - const userId = getUserId(c); - const workspaceId = c.req.param('id'); - const sessionId = c.req.param('sessionId'); - const db = drizzle(c.env.DATABASE, { schema }); + try { + await suspendAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); + } catch (e) { + log.warn('agent_session.suspend_on_node_failed', { + sessionId: session.id, + workspaceId: workspace.id, + nodeId: workspace.nodeId, + error: String(e), + }); + } - const workspace = await getOwnedWorkspace(db, workspaceId, userId); - if (!workspace.nodeId) { - throw errors.badRequest('Workspace is not attached to a node'); + const now = new Date().toISOString(); + await db + .update(schema.agentSessions) + .set({ + status: 'suspended', + suspendedAt: now, + errorMessage: null, + updatedAt: now, + }) + .where(eq(schema.agentSessions.id, session.id)); + + return c.json( + toAgentSessionResponse({ + ...session, + status: 'suspended', + suspendedAt: now, + errorMessage: null, + updatedAt: now, + }) + ); } +); + +agentSessionRoutes.post( + '/:id/agent-sessions/:sessionId/resume', + requireAuth(), + requireApproved(), + async (c) => { + const userId = getUserId(c); + const workspaceId = c.req.param('id'); + const sessionId = c.req.param('sessionId'); + const db = drizzle(c.env.DATABASE, { schema }); + + const workspace = await getOwnedWorkspace(db, workspaceId, userId); + if (!workspace.nodeId) { + throw errors.badRequest('Workspace is not attached to a node'); + } - const rows = await db - .select() - .from(schema.agentSessions) - .where( - and( - eq(schema.agentSessions.id, sessionId), - eq(schema.agentSessions.workspaceId, workspace.id), - eq(schema.agentSessions.userId, userId) + const rows = await db + .select() + .from(schema.agentSessions) + .where( + and( + eq(schema.agentSessions.id, sessionId), + eq(schema.agentSessions.workspaceId, workspace.id), + eq(schema.agentSessions.userId, userId) + ) ) - ) - .limit(1); + .limit(1); - const session = rows[0]; - if (!session) { - throw errors.notFound('Agent session'); - } + const session = rows[0]; + if (!session) { + throw errors.notFound('Agent session'); + } - // Already running -- idempotent - if (session.status === 'running') { - return c.json(toAgentSessionResponse(session)); - } + const node = await getOwnedNode(db, workspace.nodeId, userId); + if ( + node.runtime === 'cf-container' && + (node.status !== 'running' || session.status !== 'running') + ) { + const recovery = await resumeVmAgentContainer(c.env, node.id); + if (!recovery.ok) { + if (!recovery.code || !recovery.message) { + throw errors.internal('Instant session recovery failed.'); + } + const statusCode = + recovery.code === 'RUNTIME_STOPPED' + ? 410 + : recovery.code === 'RUNTIME_RECOVERING' + ? 503 + : 409; + throw new AppError(statusCode, recovery.code, recovery.message); + } + } - // Resume is allowed from suspended, stopped, or error states. - // For suspended sessions, also tell the VM agent to resume. - if (session.status === 'suspended') { - try { - await resumeAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); - } catch (e) { - log.warn('agent_session.resume_on_node_failed', { sessionId: session.id, workspaceId: workspace.id, nodeId: workspace.nodeId, error: String(e) }); + // Already running -- idempotent + if (session.status === 'running') { + return c.json(toAgentSessionResponse(session)); } - } - const now = new Date().toISOString(); - await db - .update(schema.agentSessions) - .set({ - status: 'running', - stoppedAt: null, - suspendedAt: null, - errorMessage: null, - updatedAt: now, - }) - .where(eq(schema.agentSessions.id, session.id)); + // Resume is allowed from suspended, stopped, or error states. + // For suspended sessions, also tell the VM agent to resume. + if (session.status === 'suspended') { + try { + await resumeAgentSessionOnNode(workspace.nodeId, workspace.id, session.id, c.env, userId); + } catch (e) { + log.warn('agent_session.resume_on_node_failed', { + sessionId: session.id, + workspaceId: workspace.id, + nodeId: workspace.nodeId, + error: String(e), + }); + } + } - return c.json( - toAgentSessionResponse({ - ...session, - status: 'running', - stoppedAt: null, - suspendedAt: null, - errorMessage: null, - updatedAt: now, - }) - ); -}); + const now = new Date().toISOString(); + await db + .update(schema.agentSessions) + .set({ + status: 'running', + stoppedAt: null, + suspendedAt: null, + errorMessage: null, + updatedAt: now, + }) + .where(eq(schema.agentSessions.id, session.id)); + + return c.json( + toAgentSessionResponse({ + ...session, + status: 'running', + stoppedAt: null, + suspendedAt: null, + errorMessage: null, + updatedAt: now, + }) + ); + } +); export { agentSessionRoutes }; diff --git a/apps/api/src/services/node-agent-readiness.ts b/apps/api/src/services/node-agent-readiness.ts new file mode 100644 index 000000000..2a6d888af --- /dev/null +++ b/apps/api/src/services/node-agent-readiness.ts @@ -0,0 +1,107 @@ +import type { Env } from '../env'; + +const DEFAULT_NODE_AGENT_READY_TIMEOUT_MS = 900_000; +const DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS = 5000; + +type NodeAgentFetch = ( + nodeId: string, + env: Env, + url: string, + options: RequestInit, + requestTimeoutMs: number +) => Promise; + +export function getNodeBackendBaseUrl(nodeId: string, env: Env): string { + const protocol = env.VM_AGENT_PROTOCOL || 'https'; + const port = env.VM_AGENT_PORT || '8443'; + // Two-level subdomain ({nodeId}.vm.{domain}) bypasses Cloudflare same-zone routing. + // The wildcard Worker route *.{domain}/* matches exactly one subdomain level, + // so {nodeId}.vm.{domain} is NOT intercepted — requests reach the VM directly. + return `${protocol}://${nodeId.toLowerCase()}.vm.${env.BASE_DOMAIN}:${port}`; +} + +export function getNodeAgentReadyTimeoutMs(env: { + NODE_AGENT_READY_TIMEOUT_MS?: string; +}): number { + const parsed = env.NODE_AGENT_READY_TIMEOUT_MS + ? Number.parseInt(env.NODE_AGENT_READY_TIMEOUT_MS, 10) + : DEFAULT_NODE_AGENT_READY_TIMEOUT_MS; + if (!Number.isFinite(parsed) || parsed <= 0) { + return DEFAULT_NODE_AGENT_READY_TIMEOUT_MS; + } + return parsed; +} + +export function getNodeAgentReadyPollIntervalMs(env: { + NODE_AGENT_READY_POLL_INTERVAL_MS?: string; +}): number { + const parsed = env.NODE_AGENT_READY_POLL_INTERVAL_MS + ? Number.parseInt(env.NODE_AGENT_READY_POLL_INTERVAL_MS, 10) + : DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS; + if (!Number.isFinite(parsed) || parsed <= 0) { + return DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS; + } + return parsed; +} + +export async function waitForNodeAgentReadyWith( + fetchNodeAgent: NodeAgentFetch, + nodeId: string, + env: Env +): Promise { + const timeoutMs = getNodeAgentReadyTimeoutMs(env); + const pollIntervalMs = getNodeAgentReadyPollIntervalMs(env); + const healthUrl = `${getNodeBackendBaseUrl(nodeId, env)}/health`; + const deadline = Date.now() + timeoutMs; + + let lastError = ''; + + while (Date.now() < deadline) { + const remainingMs = deadline - Date.now(); + const requestTimeoutMs = Math.max(1, Math.min(pollIntervalMs, remainingMs)); + const controller = new AbortController(); + let timeoutHandle: ReturnType | null = null; + + try { + const requestTimeoutError = `request timeout after ${requestTimeoutMs}ms`; + const response = await Promise.race([ + fetchNodeAgent( + nodeId, + env, + healthUrl, + { method: 'GET', signal: controller.signal }, + requestTimeoutMs + ), + new Promise((_resolve, reject) => { + timeoutHandle = setTimeout(() => { + controller.abort(); + reject(new Error(requestTimeoutError)); + }, requestTimeoutMs); + }), + ]); + if (response.ok) return; + + const responseBody = await response.text().catch(() => ''); + lastError = `HTTP ${response.status}${responseBody ? ` ${responseBody}` : ''}`.trim(); + } catch (err) { + if (err instanceof Error && err.message.startsWith('request timeout after ')) { + lastError = err.message; + } else if (err instanceof Error && err.message.startsWith('Request timed out after ')) { + lastError = err.message.replace('Request timed out after ', 'request timeout after '); + } else if (err instanceof Error && err.name === 'AbortError') { + lastError = `request timeout after ${requestTimeoutMs}ms`; + } else { + lastError = err instanceof Error ? err.message : String(err); + } + } finally { + if (timeoutHandle) clearTimeout(timeoutHandle); + } + + const nextRemainingMs = deadline - Date.now(); + if (nextRemainingMs <= 0) break; + await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, nextRemainingMs))); + } + + const details = lastError ? ` Last error: ${lastError}` : ''; + throw new Error(`Node Agent not reachable at ${healthUrl} within ${timeoutMs}ms.${details}`); +} diff --git a/apps/api/src/services/node-agent.ts b/apps/api/src/services/node-agent.ts index 452046ae4..050d90e4f 100644 --- a/apps/api/src/services/node-agent.ts +++ b/apps/api/src/services/node-agent.ts @@ -2,16 +2,25 @@ import { eq } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/d1'; import * as schema from '../db/schema'; +import type { RuntimeRecoveryCode } from '../durable-objects/vm-agent-container-recovery'; import type { Env } from '../env'; import { expectJsonRecord } from '../lib/runtime-validation'; +import { AppError } from '../middleware/error'; import { fetchWithTimeout, getTimeoutMs } from './fetch-timeout'; import { signNodeManagementToken, signTerminalToken } from './jwt'; +import { + getNodeAgentReadyPollIntervalMs, + getNodeAgentReadyTimeoutMs, + getNodeBackendBaseUrl, + waitForNodeAgentReadyWith, +} from './node-agent-readiness'; import { recordNodeRoutingMetric } from './telemetry'; import { fetchVmAgentContainer, getVmAgentContainerConfig, markVmAgentContainerActiveWorkEndedBestEffort, markVmAgentContainerActiveWorkStarted, + markVmAgentContainerRequestInterrupted, } from './vm-agent-container'; const DEFAULT_NODE_AGENT_REQUEST_TIMEOUT_MS = 30_000; @@ -22,57 +31,53 @@ const DEFAULT_CF_CONTAINER_WAKE_TIMEOUT_MS = 120_000; // the wake/restore budget, which re-runs the same clone. See rule 43. const DEFAULT_CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS = 120_000; -const DEFAULT_NODE_AGENT_READY_TIMEOUT_MS = 900_000; // 15 min — cloud-init takes 8-12 min on Hetzner -const DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS = 5000; - -function getNodeBackendBaseUrl(nodeId: string, env: Env): string { - const protocol = env.VM_AGENT_PROTOCOL || 'https'; - const port = env.VM_AGENT_PORT || '8443'; - // Two-level subdomain ({nodeId}.vm.{domain}) bypasses Cloudflare same-zone routing. - // The wildcard Worker route *.{domain}/* matches exactly one subdomain level, - // so {nodeId}.vm.{domain} is NOT intercepted — requests reach the VM directly. - return `${protocol}://${nodeId.toLowerCase()}.vm.${env.BASE_DOMAIN}:${port}`; -} - interface NodeAgentRequestOptions extends RequestInit { userId: string; workspaceId?: string | null; requestTimeoutMs?: number; } +const RUNTIME_RECOVERY_CODES: ReadonlySet = new Set([ + 'RUNTIME_RECOVERING', + 'RUNTIME_REQUEST_INTERRUPTED', + 'RUNTIME_RECOVERY_DEGRADED', + 'RUNTIME_STOPPED', +]); + +function isRuntimeRecoveryCode(value: unknown): value is RuntimeRecoveryCode { + return typeof value === 'string' && RUNTIME_RECOVERY_CODES.has(value); +} + +function runtimeRecoveryStatus(code: RuntimeRecoveryCode): number { + if (code === 'RUNTIME_STOPPED') return 410; + if (code === 'RUNTIME_RECOVERING') return 503; + return 409; +} + +export class NodeAgentRequestError extends AppError { + constructor(statusCode: number, code: RuntimeRecoveryCode, message: string) { + super(statusCode, code, message); + this.name = 'NodeAgentRequestError'; + } +} + function requestInitWithoutSignal(options: RequestInit): RequestInit { const serializableOptions = { ...options }; delete serializableOptions.signal; return serializableOptions; } -export function getNodeAgentReadyTimeoutMs(env: { NODE_AGENT_READY_TIMEOUT_MS?: string }): number { - const parsed = env.NODE_AGENT_READY_TIMEOUT_MS - ? Number.parseInt(env.NODE_AGENT_READY_TIMEOUT_MS, 10) - : DEFAULT_NODE_AGENT_READY_TIMEOUT_MS; - if (!Number.isFinite(parsed) || parsed <= 0) { - return DEFAULT_NODE_AGENT_READY_TIMEOUT_MS; - } - return parsed; -} +export { getNodeAgentReadyPollIntervalMs, getNodeAgentReadyTimeoutMs }; -export function getNodeAgentReadyPollIntervalMs(env: { - NODE_AGENT_READY_POLL_INTERVAL_MS?: string; +export function getNodeAgentRequestTimeoutMs(env: { + NODE_AGENT_REQUEST_TIMEOUT_MS?: string; }): number { - const parsed = env.NODE_AGENT_READY_POLL_INTERVAL_MS - ? Number.parseInt(env.NODE_AGENT_READY_POLL_INTERVAL_MS, 10) - : DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS; - if (!Number.isFinite(parsed) || parsed <= 0) { - return DEFAULT_NODE_AGENT_READY_POLL_INTERVAL_MS; - } - return parsed; -} - -export function getNodeAgentRequestTimeoutMs(env: { NODE_AGENT_REQUEST_TIMEOUT_MS?: string }): number { return getTimeoutMs(env.NODE_AGENT_REQUEST_TIMEOUT_MS, DEFAULT_NODE_AGENT_REQUEST_TIMEOUT_MS); } -export function getCfContainerWakeTimeoutMs(env: { CF_CONTAINER_WAKE_TIMEOUT_MS?: string }): number { +export function getCfContainerWakeTimeoutMs(env: { + CF_CONTAINER_WAKE_TIMEOUT_MS?: string; +}): number { return getTimeoutMs(env.CF_CONTAINER_WAKE_TIMEOUT_MS, DEFAULT_CF_CONTAINER_WAKE_TIMEOUT_MS); } @@ -86,62 +91,7 @@ export function getCfContainerCreateWorkspaceTimeoutMs(env: { } export async function waitForNodeAgentReady(nodeId: string, env: Env): Promise { - const timeoutMs = getNodeAgentReadyTimeoutMs(env); - const pollIntervalMs = getNodeAgentReadyPollIntervalMs(env); - const baseUrl = getNodeBackendBaseUrl(nodeId, env); - const healthUrl = `${baseUrl}/health`; - const deadline = Date.now() + timeoutMs; - - let lastError = ''; - - while (Date.now() < deadline) { - const remainingMs = deadline - Date.now(); - const requestTimeoutMs = Math.max(1, Math.min(pollIntervalMs, remainingMs)); - const controller = new AbortController(); - let timeoutHandle: ReturnType | null = null; - - try { - const requestTimeoutError = `request timeout after ${requestTimeoutMs}ms`; - const response = await Promise.race([ - fetchNodeAgent(nodeId, env, healthUrl, { method: 'GET', signal: controller.signal }, requestTimeoutMs), - new Promise((_resolve, reject) => { - timeoutHandle = setTimeout(() => { - controller.abort(); - reject(new Error(requestTimeoutError)); - }, requestTimeoutMs); - }), - ]); - if (response.ok) { - return; - } - - const responseBody = await response.text().catch(() => ''); - lastError = `HTTP ${response.status}${responseBody ? ` ${responseBody}` : ''}`.trim(); - } catch (err) { - if (err instanceof Error && err.message.startsWith('request timeout after ')) { - lastError = err.message; - } else if (err instanceof Error && err.message.startsWith('Request timed out after ')) { - lastError = err.message.replace('Request timed out after ', 'request timeout after '); - } else if (err instanceof Error && err.name === 'AbortError') { - lastError = `request timeout after ${requestTimeoutMs}ms`; - } else { - lastError = err instanceof Error ? err.message : String(err); - } - } finally { - if (timeoutHandle) { - clearTimeout(timeoutHandle); - } - } - - const nextRemainingMs = deadline - Date.now(); - if (nextRemainingMs <= 0) { - break; - } - await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, nextRemainingMs))); - } - - const details = lastError ? ` Last error: ${lastError}` : ''; - throw new Error(`Node Agent not reachable at ${healthUrl} within ${timeoutMs}ms.${details}`); + return waitForNodeAgentReadyWith(fetchNodeAgent, nodeId, env); } export async function nodeAgentRequest( @@ -180,7 +130,13 @@ export async function nodeAgentRequest( ); const requestTimeoutMs = options.requestTimeoutMs ?? getNodeAgentRequestTimeoutMs(env); - const response = await fetchNodeAgent(nodeId, env, url, { ...options, headers }, requestTimeoutMs); + const response = await fetchNodeAgent( + nodeId, + env, + url, + { ...options, headers }, + requestTimeoutMs + ); recordNodeRoutingMetric( { @@ -196,6 +152,24 @@ export async function nodeAgentRequest( if (!response.ok) { const body = await response.text().catch(() => ''); + let recoveryPayload: { error?: unknown; message?: unknown } | null = null; + try { + recoveryPayload = JSON.parse(body) as { error?: unknown; message?: unknown }; + } catch { + // Non-recovery Node Agent responses retain the existing generic handling. + } + if ( + recoveryPayload && + isRuntimeRecoveryCode(recoveryPayload.error) && + typeof recoveryPayload.message === 'string' + ) { + throw new NodeAgentRequestError( + response.status, + recoveryPayload.error, + recoveryPayload.message + ); + } + // Detect Worker loop-back: when the vm-{nodeId} DNS record is missing, // the wildcard DNS record routes the request back to this API Worker, // which returns its own 404 format. Provide a clear error instead. @@ -277,6 +251,22 @@ export async function fetchNodeAgent( }), ]); return response; + } catch (error) { + const timedOut = error instanceof Error && error.message.startsWith('Request timed out after '); + if (!timedOut) throw error; + + const recovery = await markVmAgentContainerRequestInterrupted(env, nodeId, { + method: options.method ?? 'GET', + errorName: 'request_timeout', + }).catch(() => null); + if (recovery?.code && recovery.message) { + throw new NodeAgentRequestError( + runtimeRecoveryStatus(recovery.code), + recovery.code, + recovery.message + ); + } + throw error; } finally { if (timeoutHandle) { clearTimeout(timeoutHandle); @@ -553,7 +543,10 @@ export async function sendPromptToAgentOnNode( } } -export { hibernateAgentSessionOnNode, restoreAgentSessionOnNode } from './node-agent-session-snapshots'; +export { + hibernateAgentSessionOnNode, + restoreAgentSessionOnNode, +} from './node-agent-session-snapshots'; /** * Cancel a running prompt on an agent session. @@ -588,7 +581,11 @@ export async function cancelAgentSessionOnNode( const statusMatch = msg.match(/failed:\s*(\d{3})/); const status = statusMatch?.[1] ? parseInt(statusMatch[1], 10) : 500; if (status === 409) { - await markVmAgentContainerActiveWorkEndedBestEffort(env, nodeId, 'cancel_agent_session_no_prompt'); + await markVmAgentContainerActiveWorkEndedBestEffort( + env, + nodeId, + 'cancel_agent_session_no_prompt' + ); } return { success: false, status }; } diff --git a/apps/api/src/services/vm-agent-container.ts b/apps/api/src/services/vm-agent-container.ts index 4b6cc0c8f..fc43dd79e 100644 --- a/apps/api/src/services/vm-agent-container.ts +++ b/apps/api/src/services/vm-agent-container.ts @@ -7,6 +7,7 @@ import { type VmAgentContainer, type VmAgentContainerLaunchConfig, type VmAgentContainerLaunchSecrets, + type VmAgentContainerRecoveryResult, } from '../durable-objects/vm-agent-container'; import type { Env } from '../env'; import { log } from '../lib/logger'; @@ -22,8 +23,12 @@ export interface VmAgentContainerConfig { export function getVmAgentContainerConfig(env: Env): VmAgentContainerConfig { return { enabled: (env.CF_CONTAINER_ENABLED ?? env.SANDBOX_ENABLED) === 'true', - vmAgentPort: Number.parseInt(env.CF_CONTAINER_VM_AGENT_PORT || env.SANDBOX_VM_AGENT_PORT || '8080', 10), - sleepAfter: env.CF_CONTAINER_SLEEP_AFTER || env.SANDBOX_SLEEP_AFTER || DEFAULT_CF_CONTAINER_SLEEP_AFTER, + vmAgentPort: Number.parseInt( + env.CF_CONTAINER_VM_AGENT_PORT || env.SANDBOX_VM_AGENT_PORT || '8080', + 10 + ), + sleepAfter: + env.CF_CONTAINER_SLEEP_AFTER || env.SANDBOX_SLEEP_AFTER || DEFAULT_CF_CONTAINER_SLEEP_AFTER, }; } @@ -73,6 +78,23 @@ export async function fetchVmAgentContainer( return container.proxyHttp(request, port); } +export async function resumeVmAgentContainer( + env: Env, + nodeId: string +): Promise { + const container = getVmAgentContainer(env, nodeId); + return container.resumeRuntime(); +} + +export async function markVmAgentContainerRequestInterrupted( + env: Env, + nodeId: string, + input: { method: string; errorName: string } +): Promise { + const container = getVmAgentContainer(env, nodeId); + return container.markRequestInterrupted(input); +} + export async function destroyVmAgentContainer(env: Env, nodeId: string): Promise { const container = getVmAgentContainer(env, nodeId); await container.destroyForUser(); diff --git a/apps/api/tests/unit/cf-container-runtime-contract.test.ts b/apps/api/tests/unit/cf-container-runtime-contract.test.ts index 2f0024ddf..ec48ffc82 100644 --- a/apps/api/tests/unit/cf-container-runtime-contract.test.ts +++ b/apps/api/tests/unit/cf-container-runtime-contract.test.ts @@ -18,7 +18,11 @@ function readPackage(relPath: string): string { } function getPinnedAgentPackage(agentType: string): string { - const manifest = agentInstallManifest as Array<{ agentType: string; package: string; version: string }>; + const manifest = agentInstallManifest as Array<{ + agentType: string; + package: string; + version: string; + }>; const entry = manifest.find((candidate) => candidate.agentType === agentType); if (!entry) { throw new Error(`Missing agent install manifest entry for ${agentType}`); @@ -118,6 +122,7 @@ describe('cf-container runtime spike contracts', () => { it('keeps active raw container prompt work alive with a bounded renewActivityTimeout loop', () => { const containerDo = read('durable-objects/vm-agent-container.ts'); + const activeWork = read('durable-objects/vm-agent-container-active-work.ts'); const containerService = read('services/vm-agent-container.ts'); const nodeAgent = read('services/node-agent.ts'); const activityCallback = read('routes/projects/agent-activity-callback.ts'); @@ -134,15 +139,15 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain( 'await this.schedule(Math.max(1, Math.ceil(delayMs / 1000)), KEEPALIVE_CALLBACK)' ); - expect(containerDo).toContain("endReason: 'keepalive_deadline_exceeded'"); + expect(activeWork).toContain("endReason: 'keepalive_deadline_exceeded'"); expect(nodeAgent).toContain('markVmAgentContainerActiveWorkStarted(env, nodeId'); expect(nodeAgent).toContain("reason: 'start_agent_session'"); expect(nodeAgent).toContain("reason: 'send_prompt'"); expect(nodeAgent).toContain( "markVmAgentContainerActiveWorkEndedBestEffort(env, nodeId, 'cancel_agent_session')" ); - expect(nodeAgent).toContain( - "markVmAgentContainerActiveWorkEndedBestEffort(env, nodeId, 'cancel_agent_session_no_prompt')" + expect(nodeAgent).toMatch( + /markVmAgentContainerActiveWorkEndedBestEffort\(\s*env,\s*nodeId,\s*'cancel_agent_session_no_prompt'\s*\)/ ); expect(nodeAgent).toContain( "markVmAgentContainerActiveWorkEndedBestEffort(env, nodeId, 'stop_agent_session')" @@ -156,7 +161,7 @@ describe('cf-container runtime spike contracts', () => { const chatResolver = read('routes/chat-workspace-resolver.ts'); expect(containerDo).toContain('override async onStop'); - expect(containerDo).toContain("if (status === 'expired' || status === 'sleeping')"); + expect(containerDo).toContain("status === 'recovering'"); expect(containerDo).toContain('override async onError'); expect(containerDo).toContain('override async onActivityExpired'); expect(containerDo).toContain( @@ -165,10 +170,10 @@ describe('cf-container runtime spike contracts', () => { expect(containerDo).toContain( "await this.ctx.storage.put('lifecycleStatus', 'sleeping' satisfies LifecycleStatus)" ); - expect(containerDo).toContain("status: 'sleeping'"); - expect(containerDo).toContain("if (lifecycleStatus === 'sleeping')"); - expect(containerDo).toContain('const wake = await this.ensureAwake()'); - expect(containerDo).toContain('WAKE_DEGRADED_RESPONSE'); + expect(containerDo).toContain("| 'sleeping'"); + expect(containerDo).toContain("status === 'sleeping' ? 'idle' : 'error'"); + expect(containerDo).toContain('return this.ensureAwake()'); + expect(containerDo).toContain('RUNTIME_RECOVERY_DEGRADED_MESSAGE'); expect(containerDo).toContain( "await this.ctx.storage.put('lifecycleStatus', 'launching' satisfies LifecycleStatus)" ); @@ -179,17 +184,15 @@ describe('cf-container runtime spike contracts', () => { "inArray(schema.workspaces.status, ['running', 'recovery', 'sleeping'])" ); expect(chatResolver).toContain( - "workspace.nodeStatus !== 'running' && workspace.nodeStatus !== 'sleeping'" + "['running', 'sleeping', 'recovery', 'error'].includes(workspace.nodeStatus)" ); - expect(chatResolver).toContain("inArray(schema.agentSessions.status, ['running', 'sleeping'])"); + expect(chatResolver).toContain('inArray(schema.agentSessions.status, agentStatuses)'); expect(chatResolver).not.toContain('The workspace container is asleep.'); - expect(containerDo).toContain( - "return new Response('Container is stopped; create a new instant session.', { status: 410 })" - ); + expect(containerDo).toContain('RUNTIME_STOPPED_MESSAGE'); expect(containerDo).toContain("await this.ctx.storage.put('launchConfig', config)"); expect(containerDo).toContain('nodeCallbackToken: string'); expect(containerDo).toContain('CALLBACK_TOKEN: secrets.nodeCallbackToken'); - expect(containerDo).toContain("status === 'stopped' ? 'stopped' : 'error'"); + expect(containerDo).toContain('persistRuntimeEnded(this.env, config, status, message)'); expect(containerDo).not.toContain( 'Container idle timeout expired; start a new instant session.' ); diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts index e222d96b3..c99c50edd 100644 --- a/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts @@ -39,6 +39,7 @@ function makeFake(env: Record) { const fake = { env, startAndWaitForPorts, + startRuntime: (VmAgentContainer.prototype as unknown as { startRuntime: unknown }).startRuntime, clearKeepaliveSchedule: vi.fn().mockResolvedValue(undefined), getPortReadyTimeoutMs: () => 30_000, ctx: { diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts new file mode 100644 index 000000000..67f45d32f --- /dev/null +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts @@ -0,0 +1,364 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const recoveryMocks = vi.hoisted(() => ({ + loadContext: vi.fn(), + persistRecovering: vi.fn(), + persistRecovered: vi.fn(), + persistFailed: vi.fn(), + signNodeCallbackToken: vi.fn(), + signCallbackToken: vi.fn(), + signNodeManagementToken: vi.fn(), +})); + +vi.mock('../../../src/durable-objects/vm-agent-container-recovery', async (importOriginal) => { + const actual = + await importOriginal< + typeof import('../../../src/durable-objects/vm-agent-container-recovery') + >(); + return { + ...actual, + loadRuntimeRecoveryContext: recoveryMocks.loadContext, + persistRuntimeRecovering: recoveryMocks.persistRecovering, + persistRuntimeRecovered: recoveryMocks.persistRecovered, + persistRuntimeRecoveryFailed: recoveryMocks.persistFailed, + }; +}); + +vi.mock('../../../src/services/jwt', () => ({ + signNodeCallbackToken: recoveryMocks.signNodeCallbackToken, + signCallbackToken: recoveryMocks.signCallbackToken, + signNodeManagementToken: recoveryMocks.signNodeManagementToken, +})); + +import { VmAgentContainer } from '../../../src/durable-objects/vm-agent-container'; +import type { RuntimeRecoveryState } from '../../../src/durable-objects/vm-agent-container-recovery'; + +const launchConfig = { + nodeId: 'node-1', + workspaceId: 'workspace-1', + projectId: 'project-1', + chatSessionId: 'chat-1', + repository: 'owner/repo', + branch: 'main', + workspaceDir: '/workspaces/repo', + controlPlaneUrl: 'https://api.example.test', + vmAgentPort: 8080, +}; + +const runtimeContext = { + userId: 'user-1', + chatSessionId: 'chat-1', + agentSessionId: 'agent-session-1', + agentType: 'codex', +}; + +type PrivateContainer = { + ensureAwake: (this: unknown) => Promise; + beginUnexpectedRecovery: (this: unknown, input: unknown) => Promise; + wakeFromSnapshot: (this: unknown, recovery: RuntimeRecoveryState) => Promise; + degradeRecovery: (this: unknown, ...args: unknown[]) => Promise; + exhaustRecovery: (this: unknown, ...args: unknown[]) => Promise; + toRecoveryTarget: (this: unknown, ...args: unknown[]) => unknown; + withLifecycleLock: (this: unknown, operation: () => Promise) => Promise; + prepareForRequest: (this: unknown) => Promise; +}; + +const privateContainer = VmAgentContainer.prototype as unknown as PrivateContainer; + +function makeStorage(initialLifecycle: string) { + const values = new Map([ + ['lifecycleStatus', initialLifecycle], + ['launchConfig', launchConfig], + ]); + return { + values, + storage: { + get: vi.fn(async (key: string) => values.get(key)), + put: vi.fn(async (key: string, value: unknown) => { + values.set(key, value); + }), + delete: vi.fn(async (key: string) => { + values.delete(key); + }), + }, + }; +} + +function makeRecoveryFake(input?: { + lifecycle?: string; + restoreResponse?: Response; + maxAttempts?: number; +}) { + const { values, storage } = makeStorage(input?.lifecycle ?? 'sleeping'); + const fake = { + env: { CF_CONTAINER_RECOVERY_MAX_ATTEMPTS: String(input?.maxAttempts ?? 2) }, + ctx: { storage }, + wakeChain: Promise.resolve(), + lifecycleChain: Promise.resolve(), + defaultPort: 8080, + ensureAwake: privateContainer.ensureAwake, + beginUnexpectedRecovery: privateContainer.beginUnexpectedRecovery, + wakeFromSnapshot: privateContainer.wakeFromSnapshot, + degradeRecovery: privateContainer.degradeRecovery, + exhaustRecovery: privateContainer.exhaustRecovery, + toRecoveryTarget: privateContainer.toRecoveryTarget, + withLifecycleLock: privateContainer.withLifecycleLock, + prepareForRequest: privateContainer.prepareForRequest, + getRecoveryMaxAttempts: () => input?.maxAttempts ?? 2, + clearKeepaliveSchedule: vi.fn().mockResolvedValue(undefined), + markActiveWorkEnded: vi.fn().mockResolvedValue(undefined), + startRuntime: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + containerFetch: vi + .fn() + .mockResolvedValue( + input?.restoreResponse ?? + Response.json({ status: 'restored', degradation: 'none', skipped: [] }) + ), + getState: vi.fn().mockResolvedValue({ status: 'running' }), + }; + return { fake, values, storage }; +} + +async function callEnsureAwake(fake: unknown) { + return privateContainer.ensureAwake.call(fake) as Promise<{ + ok: boolean; + status: string; + code?: string; + }>; +} + +function callProxyHttp(fake: unknown, request: Request): Promise { + return ( + VmAgentContainer.prototype as unknown as { + proxyHttp: (this: unknown, request: Request) => Promise; + } + ).proxyHttp.call(fake, request); +} + +beforeEach(() => { + vi.clearAllMocks(); + recoveryMocks.loadContext.mockResolvedValue(runtimeContext); + recoveryMocks.persistRecovering.mockResolvedValue(undefined); + recoveryMocks.persistRecovered.mockResolvedValue(undefined); + recoveryMocks.persistFailed.mockResolvedValue(undefined); + recoveryMocks.signNodeCallbackToken.mockResolvedValue('fresh-node-token'); + recoveryMocks.signCallbackToken.mockResolvedValue('fresh-workspace-token'); + recoveryMocks.signNodeManagementToken.mockResolvedValue({ token: 'management-token' }); +}); + +describe('VmAgentContainer snapshot recovery state machine', () => { + it('cold-wakes, reinjects fresh callback tokens, restores, then reconciles running', async () => { + const { fake, values } = makeRecoveryFake(); + + const result = await callEnsureAwake(fake); + + expect(result).toEqual({ ok: true, status: 'running' }); + expect(fake.startRuntime).toHaveBeenCalledWith(launchConfig, { + nodeCallbackToken: 'fresh-node-token', + }); + const restoreRequest = fake.containerFetch.mock.calls[0]?.[0] as Request; + await expect(restoreRequest.json()).resolves.toMatchObject({ + chatSessionId: 'chat-1', + runtime: 'cf-container', + agentType: 'codex', + workspaceCallbackToken: 'fresh-workspace-token', + }); + expect(recoveryMocks.persistRecovering).toHaveBeenCalledWith( + fake.env, + expect.objectContaining({ + nodeId: 'node-1', + workspaceId: 'workspace-1', + agentSessionId: 'agent-session-1', + }) + ); + expect(recoveryMocks.persistRecovered).toHaveBeenCalledWith( + fake.env, + expect.objectContaining({ chatSessionId: 'chat-1' }), + 'none' + ); + expect(values.get('lifecycleStatus')).toBe('running'); + expect(values.has('runtimeRecovery')).toBe(false); + }); + + it('keeps missing snapshots degraded until the bounded attempt is exhausted', async () => { + const { fake, values } = makeRecoveryFake({ + maxAttempts: 1, + restoreResponse: Response.json({ error: 'SNAPSHOT_NOT_FOUND' }, { status: 404 }), + }); + + const result = await callEnsureAwake(fake); + + expect(result).toMatchObject({ + ok: false, + status: 'degraded', + code: 'RUNTIME_RECOVERY_DEGRADED', + }); + expect(recoveryMocks.persistRecovered).not.toHaveBeenCalled(); + expect(recoveryMocks.persistFailed).toHaveBeenCalledTimes(1); + expect(values.get('lifecycleStatus')).toBe('error'); + expect(values.get('runtimeRecovery')).toMatchObject({ + phase: 'exhausted', + lastFailure: { kind: 'restore_http', httpStatus: 404 }, + }); + }); + + it('treats a corrupt successful restore body as degraded, never as true resume', async () => { + const { fake, values } = makeRecoveryFake({ + restoreResponse: new Response('{not-json', { status: 200 }), + }); + + const result = await callEnsureAwake(fake); + + expect(result).toMatchObject({ ok: false, status: 'degraded' }); + expect(recoveryMocks.persistRecovered).not.toHaveBeenCalled(); + expect(recoveryMocks.persistFailed).not.toHaveBeenCalled(); + expect(values.get('runtimeRecovery')).toMatchObject({ + phase: 'degraded', + lastFailure: { kind: 'restore_status' }, + }); + }); + + it('keeps an explicit stop terminal when it crosses an active restore', async () => { + let finishRestore!: (response: Response) => void; + const { fake, values } = makeRecoveryFake(); + fake.containerFetch.mockImplementationOnce( + () => + new Promise((resolve) => { + finishRestore = resolve; + }) + ); + const stopForUser = ( + VmAgentContainer.prototype as unknown as { + stopForUser: (this: unknown) => Promise; + } + ).stopForUser; + + const wake = callEnsureAwake(fake); + await vi.waitFor(() => expect(fake.containerFetch).toHaveBeenCalledTimes(1)); + + await stopForUser.call(fake); + finishRestore( + Response.json({ + status: 'restored', + degradation: 'none', + skipped: [], + }) + ); + + await expect(wake).resolves.toMatchObject({ + ok: false, + status: 'stopped', + code: 'RUNTIME_STOPPED', + }); + expect(recoveryMocks.persistRecovered).not.toHaveBeenCalled(); + expect(values.get('lifecycleStatus')).toBe('stopping'); + expect(values.has('runtimeRecovery')).toBe(false); + }); +}); + +describe('VmAgentContainer replacement classification', () => { + it('classifies duplicate runtime_signal callbacks once without calling them rollout', async () => { + const { fake, values } = makeRecoveryFake({ lifecycle: 'running' }); + const onStop = ( + VmAgentContainer.prototype as unknown as { + onStop: ( + this: unknown, + input: { exitCode: number; reason: 'runtime_signal' } + ) => Promise; + } + ).onStop; + + await onStop.call(fake, { exitCode: 0, reason: 'runtime_signal' }); + await onStop.call(fake, { exitCode: 0, reason: 'runtime_signal' }); + + expect(recoveryMocks.persistRecovering).toHaveBeenCalledTimes(1); + expect(values.get('runtimeRecovery')).toMatchObject({ + trigger: 'stop', + cause: { kind: 'container_stop', reason: 'runtime_signal', exitCode: 0 }, + }); + }); + + it('classifies a true container error generically and preserves recovery eligibility', async () => { + const { fake, values } = makeRecoveryFake({ lifecycle: 'running' }); + const onError = ( + VmAgentContainer.prototype as unknown as { + onError: (this: unknown, error: unknown) => Promise; + } + ).onError; + + await onError.call( + fake, + Object.assign(new Error('sensitive detail'), { name: 'RuntimeError' }) + ); + + expect(values.get('runtimeRecovery')).toMatchObject({ + trigger: 'error', + cause: { kind: 'container_error', errorName: 'RuntimeError' }, + }); + }); + + it('keeps an explicit stop terminal and does not create recovery state', async () => { + const { fake, values } = makeRecoveryFake({ lifecycle: 'stopping' }); + const markRuntimeEnded = vi.fn().mockResolvedValue(undefined); + Object.assign(fake, { markRuntimeEnded }); + const onStop = ( + VmAgentContainer.prototype as unknown as { + onStop: (this: unknown, input: { exitCode: number; reason: 'exit' }) => Promise; + } + ).onStop; + + await onStop.call(fake, { exitCode: 0, reason: 'exit' }); + + expect(markRuntimeEnded).toHaveBeenCalledWith('stopped', 'Container stopped by user request'); + expect(recoveryMocks.persistRecovering).not.toHaveBeenCalled(); + expect(values.get('lifecycleStatus')).toBe('stopped'); + }); + + it('does not replay a prompt whose request crosses replacement', async () => { + const { fake, values } = makeRecoveryFake({ lifecycle: 'running' }); + fake.containerFetch.mockRejectedValueOnce( + Object.assign(new Error('socket reset'), { name: 'TypeError' }) + ); + const request = new Request( + 'http://container/workspaces/workspace-1/agent-sessions/agent-session-1/prompt', + { + method: 'POST', + body: JSON.stringify({ prompt: 'continue' }), + } + ); + + const response = await callProxyHttp(fake, request); + + expect(response.status).toBe(409); + await expect(response.json()).resolves.toMatchObject({ + error: 'RUNTIME_REQUEST_INTERRUPTED', + }); + expect(fake.containerFetch).toHaveBeenCalledTimes(1); + expect(values.get('runtimeRecovery')).toMatchObject({ + trigger: 'request', + promptDisposition: 'manual_retry', + cause: { kind: 'transport_interrupted', errorName: 'TypeError' }, + }); + }); + + it('classifies a missing SessionHost before another prompt can be sent', async () => { + const { fake, values } = makeRecoveryFake({ lifecycle: 'running' }); + fake.containerFetch.mockResolvedValueOnce( + Response.json({ error: 'no active agent session found' }, { status: 404 }) + ); + + const response = await callProxyHttp( + fake, + new Request('http://container/workspaces/workspace-1/agent-sessions/agent-session-1/prompt', { + method: 'POST', + }) + ); + + expect(response.status).toBe(409); + expect(values.get('runtimeRecovery')).toMatchObject({ + promptDisposition: 'manual_retry', + cause: { kind: 'missing_session_host', httpStatus: 404 }, + }); + }); +}); diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts index 2c3028a55..7bdf15cf2 100644 --- a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts @@ -1,141 +1,165 @@ import { describe, expect, it, vi } from 'vitest'; import { VmAgentContainer } from '../../../src/durable-objects/vm-agent-container'; +import type { RuntimeRecoveryState } from '../../../src/durable-objects/vm-agent-container-recovery'; -// Regression test for the restored-session prompt failure: `proxyHttp` read the -// container state ONCE before `wakeFromSnapshot()`, then applied the -// `stopped`/`stopped_with_code` -> 410 guard using that stale pre-wake state. -// A freshly-woken, restored container was therefore rejected with 410 (surfaced -// by the Worker as a generic 500), even though restore succeeded. The fix -// re-reads the container state after a successful wake. +type PrivateContainer = { + prepareForRequest: (this: unknown) => Promise; + ensureAwake: (this: unknown) => Promise; + resultResponse: (this: unknown, result: unknown) => Response; + interruptedRequestResponse: (this: unknown, request: Request) => Response; +}; -interface FakeState { - status: string; +const privateContainer = VmAgentContainer.prototype as unknown as PrivateContainer; + +function callProxyHttp(fake: unknown, request: Request): Promise { + return ( + VmAgentContainer.prototype as unknown as { + proxyHttp: (this: unknown, request: Request, port?: number) => Promise; + } + ).proxyHttp.call(fake, request); } -function makeFake(opts: { - statuses: string[]; // sequence returned by getState() - lifecycleStatus: string; - wakeOk: boolean; +function makeProxyFake(input: { + ready: { ok: boolean; status: string; code?: string; message?: string }; + state: string; + response?: Response; }) { - const getState = vi.fn<[], Promise>(); - for (const s of opts.statuses) { - getState.mockResolvedValueOnce({ status: s }); - } - getState.mockResolvedValue({ status: opts.statuses[opts.statuses.length - 1] }); - - const containerFetch = vi - .fn() - .mockResolvedValue(new Response('proxied', { status: 200 })); - const wakeFromSnapshot = vi - .fn() - .mockResolvedValue(opts.wakeOk ? { ok: true } : { ok: false, message: 'degraded' }); - - const fake = { - getState, + const containerFetch = vi.fn().mockResolvedValue(input.response ?? new Response('proxied')); + const beginUnexpectedRecovery = vi.fn().mockResolvedValue(null); + return { + fake: { + defaultPort: 8080, + prepareForRequest: vi.fn().mockResolvedValue(input.ready), + resultResponse: privateContainer.resultResponse, + interruptedRequestResponse: privateContainer.interruptedRequestResponse, + getState: vi.fn().mockResolvedValue({ status: input.state }), + containerFetch, + beginUnexpectedRecovery, + }, containerFetch, - wakeFromSnapshot, - defaultPort: 8080, - wakeChain: Promise.resolve(), - ensureAwake: (VmAgentContainer.prototype as unknown as { ensureAwake: unknown }) - .ensureAwake, - ctx: { storage: { get: vi.fn().mockResolvedValue(opts.lifecycleStatus) } }, + beginUnexpectedRecovery, }; - return { fake, getState, containerFetch, wakeFromSnapshot }; } -function callProxyHttp(fake: unknown, request: Request): Promise { - return (VmAgentContainer.prototype as unknown as { - proxyHttp: (this: unknown, request: Request, port?: number) => Promise; - }).proxyHttp.call(fake, request); -} - -describe('VmAgentContainer.proxyHttp wake state re-read', () => { - it('proxies the prompt after a successful wake even though the pre-wake state was stopped', async () => { - // Pre-wake getState -> stopped; post-wake getState -> running (fresh container). - const { fake, getState, containerFetch, wakeFromSnapshot } = makeFake({ - statuses: ['stopped', 'running'], - lifecycleStatus: 'sleeping', - wakeOk: true, +describe('VmAgentContainer proxy recovery boundaries', () => { + it('forwards a prompt only after wake/restore reports running', async () => { + const { fake, containerFetch } = makeProxyFake({ + ready: { ok: true, status: 'running' }, + state: 'running', }); - const res = await callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })); + const response = await callProxyHttp( + fake, + new Request('http://container/prompt', { method: 'POST' }) + ); - expect(wakeFromSnapshot).toHaveBeenCalledTimes(1); - // State must be re-read after wake (once before, once after) so the stopped - // guard sees the now-running container. - expect(getState).toHaveBeenCalledTimes(2); - // The request is proxied to the running container, NOT rejected with 410. expect(containerFetch).toHaveBeenCalledTimes(1); - expect(res.status).toBe(200); + expect(response.status).toBe(200); }); - it('returns 503 (not 410/proxy) when wake fails', async () => { - const { fake, containerFetch } = makeFake({ - statuses: ['stopped', 'stopped'], - lifecycleStatus: 'sleeping', - wakeOk: false, + it('returns a sanitized degraded response without forwarding', async () => { + const { fake, containerFetch } = makeProxyFake({ + ready: { + ok: false, + status: 'degraded', + code: 'RUNTIME_RECOVERY_DEGRADED', + message: 'The Instant session could not restore its last safe checkpoint.', + }, + state: 'stopped', }); - const res = await callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })); + const response = await callProxyHttp( + fake, + new Request('http://container/prompt', { method: 'POST' }) + ); - expect(res.status).toBe(503); + expect(response.status).toBe(409); + await expect(response.json()).resolves.toMatchObject({ + error: 'RUNTIME_RECOVERY_DEGRADED', + }); expect(containerFetch).not.toHaveBeenCalled(); }); - it('still returns 410 for a genuinely stopped, non-sleeping container', async () => { - const { fake, containerFetch, wakeFromSnapshot } = makeFake({ - statuses: ['stopped'], - lifecycleStatus: 'running', - wakeOk: true, + it('keeps an explicit stop terminal', async () => { + const { fake, containerFetch } = makeProxyFake({ + ready: { + ok: false, + status: 'stopped', + code: 'RUNTIME_STOPPED', + message: 'This Instant session was stopped and cannot be resumed.', + }, + state: 'stopped', }); - const res = await callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })); + const response = await callProxyHttp( + fake, + new Request('http://container/prompt', { method: 'POST' }) + ); - expect(wakeFromSnapshot).not.toHaveBeenCalled(); + expect(response.status).toBe(410); expect(containerFetch).not.toHaveBeenCalled(); - expect(res.status).toBe(410); }); }); -describe('VmAgentContainer.ensureAwake concurrency (rule 45)', () => { - it('wakes a sleeping container exactly once under two concurrent requests', async () => { - // Shared, mutable container state so the mock models a real wake: the first - // wake flips lifecycleStatus to running, and getState follows it. - const shared = { lifecycle: 'sleeping' as string }; - - const getState = vi.fn(async () => ({ - status: shared.lifecycle === 'running' ? 'running' : 'stopped', - })); - const containerFetch = vi.fn().mockResolvedValue(new Response('proxied', { status: 200 })); +describe('VmAgentContainer cold-wake serialization', () => { + it('launches and restores exactly once for two concurrent follow-ups', async () => { + const shared = { + lifecycle: 'sleeping', + recovery: { + version: 1, + phase: 'pending', + trigger: 'idle', + cause: { kind: 'idle_sleep' }, + attempts: 0, + promptDisposition: 'none', + agentSessionId: 'session-1', + startedAt: Date.now(), + updatedAt: Date.now(), + } satisfies RuntimeRecoveryState, + }; + const storage = { + get: vi.fn(async (key: string) => { + if (key === 'lifecycleStatus') return shared.lifecycle; + if (key === 'runtimeRecovery') return shared.recovery; + return undefined; + }), + put: vi.fn(async (key: string, value: unknown) => { + if (key === 'lifecycleStatus') shared.lifecycle = String(value); + if (key === 'runtimeRecovery') shared.recovery = value as RuntimeRecoveryState; + }), + }; const wakeFromSnapshot = vi.fn(async () => { - // Simulate the async launch+restore so the two requests interleave across - // this await; the second must observe the running state and NOT re-wake. - await new Promise((r) => setTimeout(r, 20)); + await new Promise((resolve) => setTimeout(resolve, 10)); shared.lifecycle = 'running'; - return { ok: true }; + return { ok: true, status: 'running' }; }); - + const containerFetch = vi.fn().mockResolvedValue(new Response('proxied')); const fake = { - getState, - containerFetch, - wakeFromSnapshot, defaultPort: 8080, wakeChain: Promise.resolve(), - ensureAwake: (VmAgentContainer.prototype as unknown as { ensureAwake: unknown }).ensureAwake, - ctx: { storage: { get: vi.fn(async () => shared.lifecycle) } }, + ctx: { storage }, + prepareForRequest: privateContainer.prepareForRequest, + ensureAwake: privateContainer.ensureAwake, + resultResponse: privateContainer.resultResponse, + interruptedRequestResponse: privateContainer.interruptedRequestResponse, + getRecoveryMaxAttempts: () => 2, + wakeFromSnapshot, + beginUnexpectedRecovery: vi.fn(), + getState: vi.fn(async () => ({ + status: shared.lifecycle === 'running' ? 'running' : 'stopped', + })), + containerFetch, }; - const [a, b] = await Promise.all([ + const [first, second] = await Promise.all([ callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })), callProxyHttp(fake, new Request('http://container/prompt', { method: 'POST' })), ]); - // The one-time launch+restore fired exactly once despite two concurrent - // requests; both requests were proxied to the now-running container. expect(wakeFromSnapshot).toHaveBeenCalledTimes(1); expect(containerFetch).toHaveBeenCalledTimes(2); - expect(a.status).toBe(200); - expect(b.status).toBe(200); + expect(first.status).toBe(200); + expect(second.status).toBe(200); }); }); diff --git a/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts b/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts new file mode 100644 index 000000000..0a315f0f3 --- /dev/null +++ b/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts @@ -0,0 +1,201 @@ +import { Hono } from 'hono'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { Env } from '../../../src/env'; +import { AppError } from '../../../src/middleware/error'; + +const mocks = vi.hoisted(() => ({ + getOwnedWorkspace: vi.fn(), + getOwnedNode: vi.fn(), + resumeVmAgentContainer: vi.fn(), + updateSet: vi.fn(), + session: { + id: 'agent-session-1', + workspaceId: 'workspace-1', + userId: 'user-1', + status: 'error', + label: 'Instant', + agentType: 'codex', + agentProfileId: null, + skillId: null, + worktreePath: null, + stoppedAt: null, + suspendedAt: null, + errorMessage: 'Runtime unavailable', + lastPrompt: null, + createdAt: '2026-07-21T00:00:00.000Z', + updatedAt: '2026-07-21T00:00:00.000Z', + }, +})); + +vi.mock('../../../src/auth', () => ({ + createAuth: () => ({ + api: { + getSession: vi.fn().mockResolvedValue({ + user: { + id: 'user-1', + email: 'user@example.com', + name: 'Test User', + role: 'user', + status: 'active', + }, + session: { id: 'auth-session-1', expiresAt: new Date('2030-01-01T00:00:00Z') }, + }), + }, + }), +})); + +vi.mock('../../../src/routes/workspaces/_helpers', () => ({ + assertNodeOperational: vi.fn(), + getOwnedWorkspace: mocks.getOwnedWorkspace, + getOwnedNode: mocks.getOwnedNode, +})); + +vi.mock('../../../src/services/node-agent', () => ({ + createAgentSessionOnNode: vi.fn(), + resumeAgentSessionOnNode: vi.fn(), + stopAgentSessionOnNode: vi.fn(), + suspendAgentSessionOnNode: vi.fn(), +})); + +vi.mock('../../../src/services/vm-agent-container', () => ({ + resumeVmAgentContainer: mocks.resumeVmAgentContainer, +})); + +vi.mock('drizzle-orm/d1', () => ({ + drizzle: () => ({ + select: (fields?: Record) => ({ + from: () => ({ + where: () => + fields && 'value' in fields + ? { get: () => Promise.resolve(undefined) } + : { limit: () => Promise.resolve([mocks.session]) }, + }), + }), + update: () => ({ + set: (values: unknown) => { + mocks.updateSet(values); + return { where: () => Promise.resolve() }; + }, + }), + }), +})); + +async function createTestApp() { + const { agentSessionRoutes } = await import('../../../src/routes/workspaces/agent-sessions'); + const app = new Hono<{ Bindings: Env }>(); + app.route('/api/workspaces', agentSessionRoutes); + app.onError((error, c) => { + if (error instanceof AppError) { + return c.json(error.toJSON(), error.statusCode as 409); + } + return c.json({ error: 'INTERNAL_ERROR', message: 'Internal server error' }, 500); + }); + return app; +} + +describe('agent session Instant recovery route', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.session.status = 'error'; + mocks.session.errorMessage = 'Runtime unavailable'; + mocks.getOwnedWorkspace.mockResolvedValue({ + id: 'workspace-1', + userId: 'user-1', + nodeId: 'node-1', + }); + mocks.getOwnedNode.mockResolvedValue({ + id: 'node-1', + userId: 'user-1', + runtime: 'cf-container', + status: 'error', + }); + mocks.resumeVmAgentContainer.mockResolvedValue({ + ok: true, + status: 'running', + degraded: false, + }); + }); + + it('marks the session running only after the container reports recovery', async () => { + let resolveRecovery!: (value: unknown) => void; + mocks.resumeVmAgentContainer.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveRecovery = resolve; + }) + ); + const app = await createTestApp(); + + const responsePromise = app.request( + '/api/workspaces/workspace-1/agent-sessions/agent-session-1/resume', + { method: 'POST' }, + { DATABASE: {} } as Env + ); + await vi.waitFor(() => + expect(mocks.resumeVmAgentContainer).toHaveBeenCalledWith( + expect.objectContaining({ DATABASE: {} }), + 'node-1' + ) + ); + expect(mocks.updateSet).not.toHaveBeenCalled(); + + resolveRecovery({ ok: true, status: 'running', degraded: false }); + const response = await responsePromise; + + expect(response.status).toBe(200); + expect(mocks.updateSet).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'running', + errorMessage: null, + }) + ); + expect(await response.json()).toEqual(expect.objectContaining({ status: 'running' })); + }); + + it('returns a stable degraded error without falsely marking the session running', async () => { + mocks.resumeVmAgentContainer.mockResolvedValueOnce({ + ok: false, + status: 'degraded', + degraded: true, + code: 'SNAPSHOT_RESTORE_FAILED', + message: + 'The Instant session could not restore its saved state. Start a new session to continue.', + }); + const app = await createTestApp(); + + const response = await app.request( + '/api/workspaces/workspace-1/agent-sessions/agent-session-1/resume', + { method: 'POST' }, + { DATABASE: {} } as Env + ); + + const body = await response.json(); + expect(body).toEqual({ + error: 'SNAPSHOT_RESTORE_FAILED', + message: + 'The Instant session could not restore its saved state. Start a new session to continue.', + }); + expect(mocks.updateSet).not.toHaveBeenCalled(); + }); + + it('preserves an explicit stop as terminal', async () => { + mocks.resumeVmAgentContainer.mockResolvedValueOnce({ + ok: false, + status: 'stopped', + degraded: false, + code: 'RUNTIME_STOPPED', + message: 'This Instant session was stopped and cannot be resumed.', + }); + const app = await createTestApp(); + + const response = await app.request( + '/api/workspaces/workspace-1/agent-sessions/agent-session-1/resume', + { method: 'POST' }, + { DATABASE: {} } as Env + ); + + expect(response.status).toBe(410); + expect(mocks.updateSet).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts b/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts index b8e88bece..18227f513 100644 --- a/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts +++ b/apps/api/tests/unit/routes/chat-prompt-cancel.test.ts @@ -2,6 +2,7 @@ import { Hono } from 'hono'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { Env } from '../../../src/env'; +import { AppError } from '../../../src/middleware/error'; import { chatRoutes } from '../../../src/routes/chat'; import * as projectDataService from '../../../src/services/project-data'; @@ -94,6 +95,7 @@ function setupDrizzle(opts: { id: string; nodeId: string | null; nodeStatus: string | null; + nodeRuntime?: string | null; userId?: string; projectId?: string | null; } | null; @@ -102,7 +104,7 @@ function setupDrizzle(opts: { // Default the workspace owner to the authenticated caller so the // defence-in-depth ownership assertion passes for positive cases. const workspaceRow = opts.workspace - ? { userId: 'user-1', projectId: 'proj-1', ...opts.workspace } + ? { userId: 'user-1', projectId: 'proj-1', nodeRuntime: 'vm', ...opts.workspace } : null; let callCount = 0; const selectMock = vi.fn().mockImplementation(() => { @@ -112,15 +114,17 @@ function setupDrizzle(opts: { from: vi.fn().mockReturnValue({ leftJoin: vi.fn().mockReturnValue({ where: vi.fn().mockReturnValue({ - limit: vi.fn().mockResolvedValue( - currentCall === 1 ? (workspaceRow ? [workspaceRow] : []) : [] - ), + limit: vi + .fn() + .mockResolvedValue(currentCall === 1 ? (workspaceRow ? [workspaceRow] : []) : []), }), }), where: vi.fn().mockReturnValue({ - limit: vi.fn().mockResolvedValue( - currentCall === 2 ? (opts.agentSession ? [opts.agentSession] : []) : [] - ), + limit: vi + .fn() + .mockResolvedValue( + currentCall === 2 ? (opts.agentSession ? [opts.agentSession] : []) : [] + ), }), }), }; @@ -158,7 +162,13 @@ describe('GET /sessions', () => { select: vi.fn().mockReturnValue({ from: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue([ - { id: 'user-1', name: 'Alice', email: 'alice@example.com', image: null, avatarUrl: null }, + { + id: 'user-1', + name: 'Alice', + email: 'alice@example.com', + image: null, + avatarUrl: null, + }, { id: 'user-2', name: 'Bob', email: 'bob@example.com', image: null, avatarUrl: null }, ]), }), @@ -176,15 +186,19 @@ describe('GET /sessions', () => { total: 2, }); - const response = await app.request( - '/api/projects/proj-1/sessions', - { method: 'GET' }, - { DATABASE: {} as D1Database } as Env, - ); + const response = await app.request('/api/projects/proj-1/sessions', { method: 'GET' }, { + DATABASE: {} as D1Database, + } as Env); expect(response.status).toBe(200); expect(projectDataService.listSessions).toHaveBeenCalledWith( - expect.anything(), 'proj-1', null, 20, 0, null, null, + expect.anything(), + 'proj-1', + null, + 20, + 0, + null, + null ); const body = await response.json(); expect(body.sessions).toMatchObject([ @@ -203,12 +217,18 @@ describe('GET /sessions', () => { const response = await app.request( '/api/projects/proj-1/sessions?scope=my', { method: 'GET' }, - { DATABASE: {} as D1Database } as Env, + { DATABASE: {} as D1Database } as Env ); expect(response.status).toBe(200); expect(projectDataService.listSessions).toHaveBeenCalledWith( - expect.anything(), 'proj-1', null, 20, 0, null, 'user-1', + expect.anything(), + 'proj-1', + null, + 20, + 0, + null, + 'user-1' ); }); }); @@ -222,7 +242,7 @@ describe('POST /sessions/:sessionId/prompt', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ content: 'hello agent' }), }, - { DATABASE: {} as D1Database } as Env, + { DATABASE: {} as D1Database } as Env ); } @@ -237,14 +257,25 @@ describe('POST /sessions/:sessionId/prompt', () => { expect(response.status).toBe(200); expect(mocks.sendPromptToAgentOnNode).toHaveBeenCalledWith( - 'node-1', 'ws-1', 'agent-sess-1', - 'hello agent', expect.anything(), 'user-1', undefined, undefined, + 'node-1', + 'ws-1', + 'agent-sess-1', + 'hello agent', + expect.anything(), + 'user-1', + undefined, + undefined ); }); it('uses the extended wake budget for a sleeping session', async () => { setupDrizzle({ - workspace: { id: 'ws-1', nodeId: 'node-1', nodeStatus: 'sleeping' }, + workspace: { + id: 'ws-1', + nodeId: 'node-1', + nodeStatus: 'sleeping', + nodeRuntime: 'cf-container', + }, agentSession: { id: 'agent-sess-1' }, }); mocks.sendPromptToAgentOnNode.mockResolvedValue({ ok: true }); @@ -253,12 +284,74 @@ describe('POST /sessions/:sessionId/prompt', () => { expect(response.status).toBe(200); expect(mocks.sendPromptToAgentOnNode).toHaveBeenCalledWith( - 'node-1', 'ws-1', 'agent-sess-1', - 'hello agent', expect.anything(), 'user-1', undefined, - { requestTimeoutMs: 120_000 }, + 'node-1', + 'ws-1', + 'agent-sess-1', + 'hello agent', + expect.anything(), + 'user-1', + undefined, + { requestTimeoutMs: 120_000 } ); }); + it.each(['recovery', 'error'])( + 'routes cf-container %s state to bounded recovery', + async (nodeStatus) => { + setupDrizzle({ + workspace: { + id: 'ws-1', + nodeId: 'node-1', + nodeStatus, + nodeRuntime: 'cf-container', + }, + agentSession: { id: 'agent-sess-1' }, + }); + mocks.sendPromptToAgentOnNode.mockResolvedValue({ ok: true }); + + const response = await postPrompt(); + + expect(response.status).toBe(200); + expect(mocks.sendPromptToAgentOnNode).toHaveBeenCalledWith( + 'node-1', + 'ws-1', + 'agent-sess-1', + 'hello agent', + expect.anything(), + 'user-1', + undefined, + { requestTimeoutMs: 120_000 } + ); + } + ); + + it('returns the stable sanitized interruption error without replaying', async () => { + setupDrizzle({ + workspace: { + id: 'ws-1', + nodeId: 'node-1', + nodeStatus: 'running', + nodeRuntime: 'cf-container', + }, + agentSession: { id: 'agent-sess-1' }, + }); + mocks.sendPromptToAgentOnNode.mockRejectedValue( + new AppError( + 409, + 'RUNTIME_REQUEST_INTERRUPTED', + 'Your message is saved, but it was not replayed automatically.' + ) + ); + + const response = await postPrompt(); + + expect(response.status).toBe(409); + await expect(response.json()).resolves.toEqual({ + error: 'RUNTIME_REQUEST_INTERRUPTED', + message: 'Your message is saved, but it was not replayed automatically.', + }); + }); + it('returns 404 when no active workspace is found', async () => { setupDrizzle({ workspace: null, agentSession: null }); @@ -385,11 +478,9 @@ describe('POST /sessions/:sessionId/prompt', () => { describe('POST /sessions/:sessionId/cancel', () => { function postCancel() { - return app.request( - '/api/projects/proj-1/sessions/chat-1/cancel', - { method: 'POST' }, - { DATABASE: {} as D1Database } as Env, - ); + return app.request('/api/projects/proj-1/sessions/chat-1/cancel', { method: 'POST' }, { + DATABASE: {} as D1Database, + } as Env); } it('cancels a running prompt successfully', async () => { @@ -406,8 +497,11 @@ describe('POST /sessions/:sessionId/cancel', () => { expect(body.status).toBe('cancelled'); expect(body.message).toBe('Prompt cancel signal sent'); expect(mocks.cancelAgentSessionOnNode).toHaveBeenCalledWith( - 'node-1', 'ws-1', 'agent-sess-1', - expect.anything(), 'user-1', + 'node-1', + 'ws-1', + 'agent-sess-1', + expect.anything(), + 'user-1' ); }); diff --git a/apps/api/tests/unit/routes/deployment-release-compose-submission.test.ts b/apps/api/tests/unit/routes/deployment-release-compose-submission.test.ts index 524e8d439..04f9eeaa8 100644 --- a/apps/api/tests/unit/routes/deployment-release-compose-submission.test.ts +++ b/apps/api/tests/unit/routes/deployment-release-compose-submission.test.ts @@ -75,16 +75,22 @@ vi.mock('../../../src/middleware/project-auth', () => ({ requireProjectCapability: vi.fn().mockResolvedValue(undefined), })); -vi.mock('../../../src/middleware/error', () => ({ - errors: { - badRequest: (msg: string) => - Object.assign(new Error(msg), { statusCode: 400, error: 'BAD_REQUEST', message: msg }), - notFound: (msg: string) => - Object.assign(new Error(msg), { statusCode: 404, error: 'NOT_FOUND', message: msg }), - conflict: (msg: string) => - Object.assign(new Error(msg), { statusCode: 409, error: 'CONFLICT', message: msg }), - }, -})); +vi.mock('../../../src/middleware/error', async () => { + const actual = await vi.importActual( + '../../../src/middleware/error' + ); + return { + ...actual, + errors: { + badRequest: (msg: string) => + Object.assign(new Error(msg), { statusCode: 400, error: 'BAD_REQUEST', message: msg }), + notFound: (msg: string) => + Object.assign(new Error(msg), { statusCode: 404, error: 'NOT_FOUND', message: msg }), + conflict: (msg: string) => + Object.assign(new Error(msg), { statusCode: 409, error: 'CONFLICT', message: msg }), + }, + }; +}); vi.mock('drizzle-orm/d1', () => ({ drizzle: () => createMockDb(), diff --git a/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts b/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts index b1a597819..9cfb8220d 100644 --- a/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts +++ b/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts @@ -25,6 +25,7 @@ const mocks = vi.hoisted(() => ({ getVmAgentContainerConfig: vi.fn(), markVmAgentContainerActiveWorkEndedBestEffort: vi.fn(), markVmAgentContainerActiveWorkStarted: vi.fn(), + markVmAgentContainerRequestInterrupted: vi.fn(), }, drizzle: vi.fn(), })); @@ -37,6 +38,8 @@ vi.mock('drizzle-orm/d1', () => ({ drizzle: mocks.drizzle })); import { createWorkspaceOnNode, getCfContainerCreateWorkspaceTimeoutMs, + NodeAgentRequestError, + sendPromptToAgentOnNode, } from '../../../src/services/node-agent'; const cfContainerEnv = { @@ -84,6 +87,7 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { vmAgentPort: 8080, sleepAfter: '10m', }); + mocks.container.markVmAgentContainerRequestInterrupted.mockResolvedValue(null); mocks.drizzle.mockReturnValue({ select: () => ({ from: () => ({ @@ -103,9 +107,15 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { it('times out with the provided budget when the container request stalls', async () => { mocks.container.fetchVmAgentContainer.mockImplementation(() => pendingResponse(60_000)); - const createPromise = createWorkspaceOnNode('node-1', cfContainerEnv, 'user-1', workspacePayload, { - requestTimeoutMs: 50, - }); + const createPromise = createWorkspaceOnNode( + 'node-1', + cfContainerEnv, + 'user-1', + workspacePayload, + { + requestTimeoutMs: 50, + } + ); const rejection = expect(createPromise).rejects.toThrow('Request timed out after 50ms'); await vi.advanceTimersByTimeAsync(60); await rejection; @@ -114,9 +124,15 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { it('completes when the container responds within the provided budget', async () => { mocks.container.fetchVmAgentContainer.mockImplementation(() => pendingResponse(40_000)); - const createPromise = createWorkspaceOnNode('node-1', cfContainerEnv, 'user-1', workspacePayload, { - requestTimeoutMs: 120_000, - }); + const createPromise = createWorkspaceOnNode( + 'node-1', + cfContainerEnv, + 'user-1', + workspacePayload, + { + requestTimeoutMs: 120_000, + } + ); await vi.advanceTimersByTimeAsync(40_500); await expect(createPromise).resolves.toEqual({ workspaceId: 'ws-1' }); }); @@ -124,9 +140,77 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { it('keeps the interactive 30s default when no override is provided', async () => { mocks.container.fetchVmAgentContainer.mockImplementation(() => pendingResponse(60_000)); - const createPromise = createWorkspaceOnNode('node-1', cfContainerEnv, 'user-1', workspacePayload); + const createPromise = createWorkspaceOnNode( + 'node-1', + cfContainerEnv, + 'user-1', + workspacePayload + ); const rejection = expect(createPromise).rejects.toThrow('Request timed out after 30000ms'); await vi.advanceTimersByTimeAsync(30_100); await rejection; }); + it('classifies a timed-out prompt before returning and never replays it', async () => { + mocks.container.fetchVmAgentContainer.mockImplementation(() => pendingResponse(60_000)); + mocks.container.markVmAgentContainerRequestInterrupted.mockResolvedValue({ + ok: false, + status: 'recovering', + code: 'RUNTIME_REQUEST_INTERRUPTED', + message: 'Your message is saved, but it was not replayed automatically.', + }); + + const promptPromise = sendPromptToAgentOnNode( + 'node-1', + 'ws-1', + 'agent-1', + 'continue', + cfContainerEnv, + 'user-1', + 'message-1', + { requestTimeoutMs: 50 } + ).catch((error) => error); + await vi.advanceTimersByTimeAsync(60); + const error = await promptPromise; + + expect(error).toBeInstanceOf(NodeAgentRequestError); + expect(error).toMatchObject({ + statusCode: 409, + error: 'RUNTIME_REQUEST_INTERRUPTED', + }); + expect(mocks.container.fetchVmAgentContainer).toHaveBeenCalledTimes(1); + expect(mocks.container.markVmAgentContainerRequestInterrupted).toHaveBeenCalledWith( + cfContainerEnv, + 'node-1', + { method: 'POST', errorName: 'request_timeout' } + ); + }); + + it('preserves a stable interruption response returned directly by the Durable Object', async () => { + mocks.container.fetchVmAgentContainer.mockResolvedValue( + Response.json( + { + error: 'RUNTIME_REQUEST_INTERRUPTED', + message: 'Your message is saved, but it was not replayed automatically.', + }, + { status: 409 } + ) + ); + + const error = await sendPromptToAgentOnNode( + 'node-1', + 'ws-1', + 'agent-1', + 'continue', + cfContainerEnv, + 'user-1' + ).catch((caught) => caught); + + expect(error).toBeInstanceOf(NodeAgentRequestError); + expect(error).toMatchObject({ + statusCode: 409, + error: 'RUNTIME_REQUEST_INTERRUPTED', + }); + expect(mocks.container.fetchVmAgentContainer).toHaveBeenCalledTimes(1); + expect(mocks.container.markVmAgentContainerRequestInterrupted).not.toHaveBeenCalled(); + }); }); diff --git a/apps/api/tests/workers/instant-runtime-recovery-failure.test.ts b/apps/api/tests/workers/instant-runtime-recovery-failure.test.ts new file mode 100644 index 000000000..cee77d365 --- /dev/null +++ b/apps/api/tests/workers/instant-runtime-recovery-failure.test.ts @@ -0,0 +1,119 @@ +import { env } from 'cloudflare:test'; +import { describe, expect, it } from 'vitest'; + +import { + persistRuntimeRecoveryFailed, + RUNTIME_RECOVERY_DEGRADED_MESSAGE, +} from '../../src/durable-objects/vm-agent-container-recovery'; +import type { Env } from '../../src/env'; +import { + seedInstallation, + seedNode, + seedProject, + seedTask, + seedUser, + seedWorkspace, +} from './helpers/seed-d1'; + +describe('Instant runtime terminal reconciliation with Miniflare D1', () => { + it('fails every related runtime row and the active task after recovery exhaustion', async () => { + const prefix = `runtime-failure-${Date.now()}-${crypto.randomUUID()}`; + const userId = `${prefix}-user`; + const installationId = `${prefix}-installation`; + const projectId = `${prefix}-project`; + const nodeId = `${prefix}-node`; + const workspaceId = `${prefix}-workspace`; + const chatSessionId = `${prefix}-chat`; + const agentSessionId = `${prefix}-agent`; + const taskId = `${prefix}-task`; + + await seedUser(userId); + await seedInstallation(installationId, userId, { + installationIdValue: `${prefix}-external`, + }); + await seedProject(projectId, userId, installationId); + await seedNode(nodeId, userId, { status: 'recovery', healthStatus: 'unhealthy' }); + await env.DATABASE.prepare(`UPDATE nodes SET runtime = 'cf-container' WHERE id = ?`) + .bind(nodeId) + .run(); + await seedWorkspace(workspaceId, nodeId, userId, { + projectId, + status: 'recovery', + chatSessionId, + }); + await env.DATABASE.prepare( + `INSERT INTO agent_sessions + (id, workspace_id, user_id, status, agent_type, created_at, updated_at) + VALUES (?, ?, ?, 'recovery', 'codex', datetime('now'), datetime('now'))` + ) + .bind(agentSessionId, workspaceId, userId) + .run(); + await seedTask(taskId, projectId, userId, { + status: 'in_progress', + workspaceId, + autoProvisionedNodeId: nodeId, + executionStep: 'agent_running', + }); + + await persistRuntimeRecoveryFailed(env as unknown as Env, { + nodeId, + workspaceId, + projectId, + chatSessionId, + agentSessionId, + }); + + const node = await env.DATABASE.prepare( + `SELECT status, health_status, error_message FROM nodes WHERE id = ?` + ) + .bind(nodeId) + .first>(); + const workspace = await env.DATABASE.prepare( + `SELECT status, error_message FROM workspaces WHERE id = ?` + ) + .bind(workspaceId) + .first>(); + const agent = await env.DATABASE.prepare( + `SELECT status, stopped_at, error_message FROM agent_sessions WHERE id = ?` + ) + .bind(agentSessionId) + .first>(); + const task = await env.DATABASE.prepare( + `SELECT status, execution_step, error_message FROM tasks WHERE id = ?` + ) + .bind(taskId) + .first>(); + const event = await env.DATABASE.prepare( + `SELECT from_status, to_status, actor_type, reason + FROM task_status_events WHERE task_id = ? ORDER BY created_at DESC LIMIT 1` + ) + .bind(taskId) + .first>(); + + expect(node).toEqual({ + status: 'error', + health_status: 'unhealthy', + error_message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }); + expect(workspace).toEqual({ + status: 'error', + error_message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }); + expect(agent).toMatchObject({ + status: 'error', + error_message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }); + expect(agent?.stopped_at).toEqual(expect.any(String)); + expect(task).toEqual({ + status: 'failed', + execution_step: null, + error_message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }); + expect(event).toEqual({ + from_status: 'in_progress', + to_status: 'failed', + actor_type: 'system', + reason: 'Instant runtime recovery exhausted', + }); + }); +}); diff --git a/apps/api/tests/workers/instant-runtime-recovery-persistence.test.ts b/apps/api/tests/workers/instant-runtime-recovery-persistence.test.ts new file mode 100644 index 000000000..0ecf9c3ec --- /dev/null +++ b/apps/api/tests/workers/instant-runtime-recovery-persistence.test.ts @@ -0,0 +1,116 @@ +import { env } from 'cloudflare:test'; +import { describe, expect, it } from 'vitest'; + +import { + persistRuntimeRecovered, + persistRuntimeRecovering, + RUNTIME_RECOVERING_MESSAGE, + RUNTIME_REQUEST_INTERRUPTED_MESSAGE, +} from '../../src/durable-objects/vm-agent-container-recovery'; +import type { Env } from '../../src/env'; +import { + seedInstallation, + seedNode, + seedProject, + seedUser, + seedWorkspace, +} from './helpers/seed-d1'; + +describe('Instant runtime status reconciliation with Miniflare D1', () => { + it('moves related rows through recovery to running without losing manual-retry state', async () => { + const prefix = `runtime-persistence-${Date.now()}-${crypto.randomUUID()}`; + const userId = `${prefix}-user`; + const installationId = `${prefix}-installation`; + const projectId = `${prefix}-project`; + const nodeId = `${prefix}-node`; + const workspaceId = `${prefix}-workspace`; + const chatSessionId = `${prefix}-chat`; + const agentSessionId = `${prefix}-agent`; + + await seedUser(userId); + await seedInstallation(installationId, userId, { + installationIdValue: `${prefix}-external`, + }); + await seedProject(projectId, userId, installationId); + await seedNode(nodeId, userId, { status: 'running' }); + await env.DATABASE.prepare(`UPDATE nodes SET runtime = 'cf-container' WHERE id = ?`) + .bind(nodeId) + .run(); + await seedWorkspace(workspaceId, nodeId, userId, { + projectId, + status: 'running', + chatSessionId, + }); + await env.DATABASE.prepare( + `INSERT INTO agent_sessions + (id, workspace_id, user_id, status, agent_type, created_at, updated_at) + VALUES (?, ?, ?, 'running', 'codex', datetime('now'), datetime('now'))` + ) + .bind(agentSessionId, workspaceId, userId) + .run(); + + const bindings = env as unknown as Env; + const target = { nodeId, workspaceId, projectId, chatSessionId, agentSessionId }; + await persistRuntimeRecovering(bindings, target); + + const recoveringNode = await env.DATABASE.prepare( + `SELECT status, health_status, error_message FROM nodes WHERE id = ?` + ) + .bind(nodeId) + .first>(); + const recoveringWorkspace = await env.DATABASE.prepare( + `SELECT status, error_message FROM workspaces WHERE id = ?` + ) + .bind(workspaceId) + .first>(); + const recoveringAgent = await env.DATABASE.prepare( + `SELECT status, error_message FROM agent_sessions WHERE id = ?` + ) + .bind(agentSessionId) + .first>(); + + expect(recoveringNode).toMatchObject({ + status: 'recovery', + health_status: 'unhealthy', + error_message: RUNTIME_RECOVERING_MESSAGE, + }); + expect(recoveringWorkspace).toMatchObject({ + status: 'recovery', + error_message: RUNTIME_RECOVERING_MESSAGE, + }); + expect(recoveringAgent).toMatchObject({ + status: 'recovery', + error_message: RUNTIME_RECOVERING_MESSAGE, + }); + + await persistRuntimeRecovered(bindings, target, 'manual_retry'); + + const recoveredNode = await env.DATABASE.prepare( + `SELECT status, health_status, error_message FROM nodes WHERE id = ?` + ) + .bind(nodeId) + .first>(); + const recoveredWorkspace = await env.DATABASE.prepare( + `SELECT status, error_message FROM workspaces WHERE id = ?` + ) + .bind(workspaceId) + .first>(); + const recoveredAgent = await env.DATABASE.prepare( + `SELECT status, stopped_at, error_message FROM agent_sessions WHERE id = ?` + ) + .bind(agentSessionId) + .first>(); + + expect(recoveredNode).toEqual({ + status: 'running', + health_status: 'healthy', + error_message: null, + }); + expect(recoveredWorkspace).toEqual({ status: 'running', error_message: null }); + expect(recoveredAgent).toEqual({ + status: 'running', + stopped_at: null, + error_message: RUNTIME_REQUEST_INTERRUPTED_MESSAGE, + }); + }); +}); diff --git a/apps/api/tests/workers/instant-runtime-recovery-resolver.test.ts b/apps/api/tests/workers/instant-runtime-recovery-resolver.test.ts new file mode 100644 index 000000000..d796f2311 --- /dev/null +++ b/apps/api/tests/workers/instant-runtime-recovery-resolver.test.ts @@ -0,0 +1,102 @@ +import { env } from 'cloudflare:test'; +import { drizzle } from 'drizzle-orm/d1'; +import { beforeAll, describe, expect, it } from 'vitest'; + +import * as schema from '../../src/db/schema'; +import { + resolveLiveAgentSessionForChat, + resolveLiveWorkspaceForSession, +} from '../../src/routes/chat-workspace-resolver'; +import { + seedInstallation, + seedNode, + seedProject, + seedUser, + seedWorkspace, +} from './helpers/seed-d1'; + +const PREFIX = `instant-recovery-${Date.now()}`; +const USER_ID = `${PREFIX}-user`; +const INSTALLATION_ID = `${PREFIX}-installation`; +const PROJECT_ID = `${PREFIX}-project`; +const CONTAINER_NODE_ID = `${PREFIX}-container-node`; +const VM_NODE_ID = `${PREFIX}-vm-node`; +const CONTAINER_WORKSPACE_ID = `${PREFIX}-container-workspace`; +const VM_WORKSPACE_ID = `${PREFIX}-vm-workspace`; +const CONTAINER_CHAT_ID = `${PREFIX}-container-chat`; +const VM_CHAT_ID = `${PREFIX}-vm-chat`; +const AGENT_SESSION_ID = `${PREFIX}-agent-session`; + +describe('Instant recovery resolver with real D1 state', () => { + beforeAll(async () => { + await seedUser(USER_ID); + await seedInstallation(INSTALLATION_ID, USER_ID, { + installationIdValue: `${PREFIX}-external`, + }); + await seedProject(PROJECT_ID, USER_ID, INSTALLATION_ID); + await seedNode(CONTAINER_NODE_ID, USER_ID, { status: 'error', healthStatus: 'unhealthy' }); + await seedNode(VM_NODE_ID, USER_ID, { status: 'error', healthStatus: 'unhealthy' }); + await env.DATABASE.prepare(`UPDATE nodes SET runtime = 'cf-container' WHERE id = ?`) + .bind(CONTAINER_NODE_ID) + .run(); + await seedWorkspace(CONTAINER_WORKSPACE_ID, CONTAINER_NODE_ID, USER_ID, { + projectId: PROJECT_ID, + status: 'error', + chatSessionId: CONTAINER_CHAT_ID, + }); + await seedWorkspace(VM_WORKSPACE_ID, VM_NODE_ID, USER_ID, { + projectId: PROJECT_ID, + status: 'error', + chatSessionId: VM_CHAT_ID, + }); + await env.DATABASE.prepare( + `INSERT INTO agent_sessions + (id, workspace_id, user_id, status, agent_type, created_at, updated_at) + VALUES (?, ?, ?, 'error', 'codex', datetime('now'), datetime('now'))` + ) + .bind(AGENT_SESSION_ID, CONTAINER_WORKSPACE_ID, USER_ID) + .run(); + }); + + function db() { + return drizzle(env.DATABASE, { schema }); + } + + it('allows a cf-container legacy error row to reach the Durable Object recovery path', async () => { + const resolved = await resolveLiveAgentSessionForChat(db(), { + projectId: PROJECT_ID, + sessionId: CONTAINER_CHAT_ID, + userId: USER_ID, + }); + + expect(resolved).toEqual({ + workspace: { + id: CONTAINER_WORKSPACE_ID, + nodeId: CONTAINER_NODE_ID, + nodeStatus: 'error', + nodeRuntime: 'cf-container', + }, + agentSession: { id: AGENT_SESSION_ID }, + }); + }); + + it('does not weaken the dead-node guard for a VM workspace in error', async () => { + const resolved = await resolveLiveWorkspaceForSession(db(), { + projectId: PROJECT_ID, + sessionId: VM_CHAT_ID, + userId: USER_ID, + }); + + expect(resolved).toBeNull(); + }); + + it('keeps project and user ownership predicates on the recovery path', async () => { + const resolved = await resolveLiveWorkspaceForSession(db(), { + projectId: `${PROJECT_ID}-other`, + sessionId: CONTAINER_CHAT_ID, + userId: USER_ID, + }); + + expect(resolved).toBeNull(); + }); +}); diff --git a/apps/web/src/components/project-message-view/index.tsx b/apps/web/src/components/project-message-view/index.tsx index 464cbbc85..e34e6bbcd 100644 --- a/apps/web/src/components/project-message-view/index.tsx +++ b/apps/web/src/components/project-message-view/index.tsx @@ -6,7 +6,12 @@ * TypewriterText animates the latest assistant message; historical messages * render instantly. */ -import type { ConversationItem, PlanItem, SlashCommand, ToolCallContentItem } from '@simple-agent-manager/acp-client'; +import type { + ConversationItem, + PlanItem, + SlashCommand, + ToolCallContentItem, +} from '@simple-agent-manager/acp-client'; import { mapToolCallContent, PlanModal } from '@simple-agent-manager/acp-client'; import type { AgentProfile } from '@simple-agent-manager/shared'; import { Button, Spinner } from '@simple-agent-manager/ui'; @@ -76,7 +81,15 @@ function useFloatingHeaderHeight(): [(el: HTMLDivElement | null) => void, number /** Floating session header with optional error banner and summary. */ function FloatingHeader({ - projectId, lc, onSessionMutated, onRetry, onFork, onOpenTimeline, sourceContext, onShowHierarchy, containerRef, + projectId, + lc, + onSessionMutated, + onRetry, + onFork, + onOpenTimeline, + sourceContext, + onShowHierarchy, + containerRef, }: { projectId: string; lc: ReturnType; @@ -90,7 +103,7 @@ function FloatingHeader({ }) { if (!lc.session) return null; const initialPromptFallback = !lc.hasMore - ? lc.messages.find((msg) => msg.role === 'user')?.content ?? null + ? (lc.messages.find((msg) => msg.role === 'user')?.content ?? null) : null; const taskStatus = lc.taskEmbed?.status; const hasRecoverableTaskError = Boolean( @@ -149,7 +162,9 @@ function ErrorBanner({ message, recoverable }: { message: string; recoverable: b @@ -172,7 +188,11 @@ function currentPlanToPlanItem(plan: Array<{ content: string; status: string }>) entries: plan.map((e) => ({ content: e.content, priority: 'medium' as const, - status: (e.status === 'completed' ? 'completed' : e.status === 'in_progress' ? 'in_progress' : 'pending') as 'pending' | 'in_progress' | 'completed', + status: (e.status === 'completed' + ? 'completed' + : e.status === 'in_progress' + ? 'in_progress' + : 'pending') as 'pending' | 'in_progress' | 'completed', })), timestamp: Date.now(), }; @@ -192,7 +212,11 @@ const ElapsedTime: FC<{ startedAt: number }> = ({ startedAt }) => { const interval = setInterval(update, 1000); return () => clearInterval(interval); }, [startedAt]); - return ; + return ( + + ); }; interface ProjectMessageViewProps { @@ -279,26 +303,32 @@ export const ProjectMessageView: FC = ({ const [pendingJump, setPendingJump] = useState(null); const [highlightedItemId, setHighlightedItemId] = useState(null); - const scrollAndHighlight = useCallback((itemId: string): boolean => { - const index = itemIndexById.get(itemId); - if (index === undefined) return false; - virtuosoRef.current?.scrollToIndex({ index, behavior: 'smooth', align: 'center' }); - setHighlightedItemId(itemId); - return true; - }, [itemIndexById]); - - const handleTimelineJump = useCallback((target: TimelineJumpTarget) => { - setShowTimeline(false); - // Fast path: exact message already loaded. - if (target.messageId && itemIndexById.has(target.messageId)) { - scrollAndHighlight(target.messageId); - return; - } - // Otherwise resolve via the pending-jump effect, loading older pages toward - // the target timestamp first (no-op when the history is already fully loaded). - setPendingJump(target); - void lc.loadUntil(target.timestamp); - }, [itemIndexById, scrollAndHighlight, lc]); + const scrollAndHighlight = useCallback( + (itemId: string): boolean => { + const index = itemIndexById.get(itemId); + if (index === undefined) return false; + virtuosoRef.current?.scrollToIndex({ index, behavior: 'smooth', align: 'center' }); + setHighlightedItemId(itemId); + return true; + }, + [itemIndexById] + ); + + const handleTimelineJump = useCallback( + (target: TimelineJumpTarget) => { + setShowTimeline(false); + // Fast path: exact message already loaded. + if (target.messageId && itemIndexById.has(target.messageId)) { + scrollAndHighlight(target.messageId); + return; + } + // Otherwise resolve via the pending-jump effect, loading older pages toward + // the target timestamp first (no-op when the history is already fully loaded). + setPendingJump(target); + void lc.loadUntil(target.timestamp); + }, + [itemIndexById, scrollAndHighlight, lc] + ); // Resolve a pending jump once the target (or the nearest message, after // loading settles) is available in the rendered list. @@ -336,10 +366,15 @@ export const ProjectMessageView: FC = ({ const prevMsgCountRef = useRef(0); /** Lazy-load tool content for a compact-mode tool call card. */ - const handleLoadToolContent = useCallback(async (messageId: string): Promise => { - const { content } = await getMessageToolContent(projectId, sessionId, messageId); - return (content as Array<{ type: string } & Record>).map((c) => mapToolCallContent(c)); - }, [projectId, sessionId]); + const handleLoadToolContent = useCallback( + async (messageId: string): Promise => { + const { content } = await getMessageToolContent(projectId, sessionId, messageId); + return (content as Array<{ type: string } & Record>).map((c) => + mapToolCallContent(c) + ); + }, + [projectId, sessionId] + ); // Detect newly added optimistic user messages for fade animation useEffect(() => { @@ -351,7 +386,9 @@ export const ProjectMessageView: FC = ({ if (msg && msg.role === 'user' && msg.id.startsWith('optimistic-')) { animatedUserMsgIds.add(msg.id); // Remove from set after animation completes (max 1.5s + buffer) - setTimeout(() => { animatedUserMsgIds.delete(msg.id); }, 2000); + setTimeout(() => { + animatedUserMsgIds.delete(msg.id); + }, 2000); } } } @@ -368,16 +405,19 @@ export const ProjectMessageView: FC = ({ }, [conversationItems]); const planItem = useMemo( - () => lc.currentPlan && lc.currentPlan.length > 0 ? currentPlanToPlanItem(lc.currentPlan) : null, - [lc.currentPlan], + () => + lc.currentPlan && lc.currentPlan.length > 0 ? currentPlanToPlanItem(lc.currentPlan) : null, + [lc.currentPlan] ); const canWriteSession = lc.session?.isMine !== false; - const sessionOwnerLabel = lc.session?.createdBy?.name?.trim() - || lc.session?.createdBy?.email?.split('@')[0] - || 'the creator'; + const sessionOwnerLabel = + lc.session?.createdBy?.name?.trim() || + lc.session?.createdBy?.email?.split('@')[0] || + 'the creator'; const canArchiveSession = Boolean( onCloseConversation && - (lc.taskEmbed?.taskMode === 'conversation' || (!lc.taskEmbed?.id && lc.session?.status === 'active')) + (lc.taskEmbed?.taskMode === 'conversation' || + (!lc.taskEmbed?.id && lc.session?.status === 'active')) ); // Initial load — only show full spinner when no data exists yet @@ -390,11 +430,7 @@ export const ProjectMessageView: FC = ({ } if (lc.error && !lc.session) { - return ( -
- {lc.error} -
- ); + return
{lc.error}
; } const isActive = lc.sessionState === 'active' || lc.sessionState === 'idle'; @@ -409,35 +445,37 @@ export const ProjectMessageView: FC = ({ )} {/* Connection indicator (DO WebSocket) */} - {lc.sessionState === 'active' && lc.connectionState !== 'connected' && lc.showConnectionBanner && ( - - )} + {lc.sessionState === 'active' && + lc.connectionState !== 'connected' && + lc.showConnectionBanner && ( + + )} {/* Resuming agent banner */} {lc.isResuming && ( -
+
- Resuming agent... - + Waking and restoring Instant session...
)} {/* Resume error banner */} {lc.resumeError && ( -
+
{lc.resumeError}
)} @@ -445,16 +483,46 @@ export const ProjectMessageView: FC = ({ {/* Messages area — virtualized, DO-only */} {conversationItems.length === 0 ? (
- setShowTimeline(true)} sourceContext={sourceContext} onShowHierarchy={onShowHierarchy} containerRef={floatingHeaderRef} /> -
+ setShowTimeline(true)} + sourceContext={sourceContext} + onShowHierarchy={onShowHierarchy} + containerRef={floatingHeaderRef} + /> +
- {lc.sessionState === 'active' ? 'Waiting for messages...' : 'No messages in this session.'} + {lc.sessionState === 'active' + ? 'Waiting for messages...' + : 'No messages in this session.'}
) : ( -
- setShowTimeline(true)} sourceContext={sourceContext} onShowHierarchy={onShowHierarchy} containerRef={floatingHeaderRef} /> +
+ setShowTimeline(true)} + sourceContext={sourceContext} + onShowHierarchy={onShowHierarchy} + containerRef={floatingHeaderRef} + />
= ({ data={conversationItems} firstItemIndex={lc.firstItemIndex} initialTopMostItemIndex={conversationItems.length - 1} - followOutput={(isAtBottom: boolean) => isAtBottom ? 'smooth' : false} + followOutput={(isAtBottom: boolean) => (isAtBottom ? 'smooth' : false)} alignToBottom atBottomThreshold={50} atBottomStateChange={(atBottom) => lc.setShowScrollButton(!atBottom)} overscan={200} itemContent={(index, item) => ( -
+
)} @@ -489,7 +569,12 @@ export const ProjectMessageView: FC = ({
{lc.hasMore && (
-
@@ -538,11 +623,7 @@ export const ProjectMessageView: FC = ({ /> )} {planItem && ( - setShowPlanModal(false)} - /> + setShowPlanModal(false)} /> )} {/* Input area */} @@ -550,15 +631,21 @@ export const ProjectMessageView: FC = ({ { void lc.handleSendFollowUp(); }} - onUploadFiles={(files) => { void lc.handleUploadFiles(files); }} + onSend={() => { + void lc.handleSendFollowUp(); + }} + onUploadFiles={(files) => { + void lc.handleUploadFiles(files); + }} sending={lc.sendingFollowUp} uploading={lc.uploading} - placeholder={lc.agentActivity === 'prompting' || lc.agentActivity === 'responding' - ? 'Agent is working...' - : lc.sessionState === 'idle' - ? 'Send a message to resume the agent...' - : 'Send a message...'} + placeholder={ + lc.agentActivity === 'prompting' || lc.agentActivity === 'responding' + ? 'Agent is working...' + : lc.sessionState === 'idle' + ? 'Send a message to resume the agent...' + : 'Send a message...' + } transcribeApiUrl={lc.transcribeApiUrl} agentProfiles={agentProfiles} slashCommands={slashCommands} @@ -569,9 +656,7 @@ export const ProjectMessageView: FC = ({ )} {lc.sessionState === 'terminated' && (
- - This session has ended. - + This session has ended.
)} diff --git a/apps/web/src/components/project-message-view/runtimeRecoveryMessages.ts b/apps/web/src/components/project-message-view/runtimeRecoveryMessages.ts new file mode 100644 index 000000000..0e43bf4b8 --- /dev/null +++ b/apps/web/src/components/project-message-view/runtimeRecoveryMessages.ts @@ -0,0 +1,29 @@ +import { ApiClientError } from '../../lib/api'; + +const RUNTIME_RECOVERY_CODES = new Set([ + 'RUNTIME_RECOVERING', + 'RUNTIME_REQUEST_INTERRUPTED', + 'RUNTIME_RECOVERY_DEGRADED', + 'RUNTIME_STOPPED', +]); + +export function getRuntimeRecoveryMessage(error: unknown): string | null { + if (!(error instanceof ApiClientError) || !RUNTIME_RECOVERY_CODES.has(error.code)) { + return null; + } + if (error.code === 'RUNTIME_RECOVERING') { + return 'Waking and restoring the Instant session. Wait for restore to finish, then send your message.'; + } + return error.message; +} + +export function getResumeFailureMessage(error: unknown): string { + const runtimeMessage = getRuntimeRecoveryMessage(error); + if (runtimeMessage) return runtimeMessage; + + const message = error instanceof Error ? error.message : String(error); + if (message.includes('404') || /not found/i.test(message)) { + return 'Could not resume agent — workspace may have been cleaned up.'; + } + return 'Could not resume agent — please try again.'; +} diff --git a/apps/web/src/components/project-message-view/useConnectionRecovery.ts b/apps/web/src/components/project-message-view/useConnectionRecovery.ts index 37b37eda5..e517d0eb3 100644 --- a/apps/web/src/components/project-message-view/useConnectionRecovery.ts +++ b/apps/web/src/components/project-message-view/useConnectionRecovery.ts @@ -11,12 +11,9 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import type { ChatConnectionState } from '../../hooks/useChatWebSocket'; import type { ChatSessionResponse } from '../../lib/api'; import { resumeAgentSession, sendFollowUpPrompt } from '../../lib/api'; +import { getResumeFailureMessage, getRuntimeRecoveryMessage } from './runtimeRecoveryMessages'; import type { SessionState } from './types'; -import { - AUTO_RESUME_DELAY_MS, - DEFAULT_IDLE_TIMEOUT_MS, - RECONNECT_BANNER_DELAY_MS, -} from './types'; +import { AUTO_RESUME_DELAY_MS, DEFAULT_IDLE_TIMEOUT_MS, RECONNECT_BANNER_DELAY_MS } from './types'; export interface UseConnectionRecoveryOptions { sessionId: string; @@ -33,11 +30,15 @@ export interface UseConnectionRecoveryResult { resumeError: string | null; showConnectionBanner: boolean; idleCountdownMs: number | null; + clearResumeError: () => void; + reportDeliveryError: (error: unknown) => void; /** Resume an idle session and optionally send a follow-up prompt. */ resumeAndSend: (followUpText?: string) => void; } -export function useConnectionRecovery(opts: UseConnectionRecoveryOptions): UseConnectionRecoveryResult { +export function useConnectionRecovery( + opts: UseConnectionRecoveryOptions +): UseConnectionRecoveryResult { const { sessionId, projectId, @@ -53,6 +54,14 @@ export function useConnectionRecovery(opts: UseConnectionRecoveryOptions): UseCo const [resumeError, setResumeError] = useState(null); const hasAttemptedAutoResumeRef = useRef(false); + const clearResumeError = useCallback(() => setResumeError(null), []); + const reportDeliveryError = useCallback((error: unknown) => { + setResumeError( + getRuntimeRecoveryMessage(error) ?? + 'Your message is saved, but it could not reach the agent. Send it again when the session is available.' + ); + }, []); + // Connection banner debounce const [showConnectionBanner, setShowConnectionBanner] = useState(false); @@ -101,7 +110,8 @@ export function useConnectionRecovery(opts: UseConnectionRecoveryOptions): UseCo !session?.workspaceId || !agentSessionId || hasAttemptedAutoResumeRef.current - ) return; + ) + return; const timer = setTimeout(() => { if (hasAttemptedAutoResumeRef.current) return; @@ -119,18 +129,12 @@ export function useConnectionRecovery(opts: UseConnectionRecoveryOptions): UseCo }) .catch((err) => { setIsResuming(false); - const msg = err instanceof Error ? err.message : String(err); - if (msg.includes('404') || msg.includes('not found') || msg.includes('Not Found')) { - setResumeError('Could not resume agent \u2014 workspace may have been cleaned up.'); - } else { - console.error('Auto-resume failed:', msg); - setResumeError('Could not resume agent \u2014 please try again.'); - } + setResumeError(getResumeFailureMessage(err)); }); }, AUTO_RESUME_DELAY_MS); return () => clearTimeout(timer); - // eslint-disable-next-line react-hooks/exhaustive-deps + // eslint-disable-next-line react-hooks/exhaustive-deps }, [sessionState, isResuming, isProvisioning, session?.workspaceId, agentSessionId]); // Debounced connection banner @@ -144,44 +148,43 @@ export function useConnectionRecovery(opts: UseConnectionRecoveryOptions): UseCo }, [connectionState]); // Resume an idle session and optionally send a follow-up prompt via REST API - const resumeAndSend = useCallback((followUpText?: string) => { - if (!session?.workspaceId || !agentSessionId) return; - hasAttemptedAutoResumeRef.current = true; - setIsResuming(true); - setResumeError(null); + const resumeAndSend = useCallback( + (followUpText?: string) => { + if (!session?.workspaceId || !agentSessionId) return; + hasAttemptedAutoResumeRef.current = true; + setIsResuming(true); + setResumeError(null); - resumeAgentSession(session.workspaceId, agentSessionId) - .then(async () => { - setSession((prev) => { - if (!prev) return prev; - return { ...prev, isIdle: false, agentCompletedAt: null } as ChatSessionResponse; - }); - setIsResuming(false); - if (followUpText) { - try { - await sendFollowUpPrompt(projectId, sessionId, followUpText); - } catch { - // Agent may be offline — message is still persisted via DO + resumeAgentSession(session.workspaceId, agentSessionId) + .then(async () => { + setSession((prev) => { + if (!prev) return prev; + return { ...prev, isIdle: false, agentCompletedAt: null } as ChatSessionResponse; + }); + setIsResuming(false); + if (followUpText) { + try { + await sendFollowUpPrompt(projectId, sessionId, followUpText); + } catch (err) { + reportDeliveryError(err); + } } - } - }) - .catch((err) => { - setIsResuming(false); - const msg = err instanceof Error ? err.message : String(err); - if (msg.includes('404') || msg.includes('not found') || msg.includes('Not Found')) { - setResumeError('Could not resume agent \u2014 workspace may have been cleaned up.'); - } else { - console.error('Agent resume failed:', msg); - setResumeError('Could not resume agent \u2014 please try again.'); - } - }); - }, [session?.workspaceId, agentSessionId, projectId, sessionId, setSession]); + }) + .catch((err) => { + setIsResuming(false); + setResumeError(getResumeFailureMessage(err)); + }); + }, + [session?.workspaceId, agentSessionId, projectId, sessionId, setSession, reportDeliveryError] + ); return { isResuming, resumeError, showConnectionBanner, idleCountdownMs, + clearResumeError, + reportDeliveryError, resumeAndSend, }; } diff --git a/apps/web/src/components/project-message-view/useSessionLifecycle.ts b/apps/web/src/components/project-message-view/useSessionLifecycle.ts index 953f4f9d7..25fa1e7f1 100644 --- a/apps/web/src/components/project-message-view/useSessionLifecycle.ts +++ b/apps/web/src/components/project-message-view/useSessionLifecycle.ts @@ -1,22 +1,51 @@ import type { NodeResponse, WorkspaceResponse } from '@simple-agent-manager/shared'; -import { DEFAULT_CHAT_LOAD_UNTIL_MAX_PAGES, DEFAULT_CHAT_SESSION_MESSAGE_LIMIT, DEFAULT_CHAT_SESSION_MESSAGE_MAX } from '@simple-agent-manager/shared'; +import { + DEFAULT_CHAT_LOAD_UNTIL_MAX_PAGES, + DEFAULT_CHAT_SESSION_MESSAGE_LIMIT, + DEFAULT_CHAT_SESSION_MESSAGE_MAX, +} from '@simple-agent-manager/shared'; import { useCallback, useEffect, useRef, useState } from 'react'; import type { ChatConnectionState } from '../../hooks/useChatWebSocket'; import { useChatWebSocket } from '../../hooks/useChatWebSocket'; import { useTokenRefresh } from '../../hooks/useTokenRefresh'; import { useWorkspacePorts } from '../../hooks/useWorkspacePorts'; -import type { ChatMessageResponse, ChatSessionDetailResponse, ChatSessionResponse, SessionStateSnapshot } from '../../lib/api'; -import { cancelAgentPrompt, getChatSession, getNode, getTerminalToken, getTranscribeApiUrl, getWorkspace, resetIdleTimer, sendFollowUpPrompt, uploadSessionFiles } from '../../lib/api'; +import type { + ChatMessageResponse, + ChatSessionDetailResponse, + ChatSessionResponse, + SessionStateSnapshot, +} from '../../lib/api'; +import { + cancelAgentPrompt, + getChatSession, + getNode, + getTerminalToken, + getTranscribeApiUrl, + getWorkspace, + resetIdleTimer, + sendFollowUpPrompt, + uploadSessionFiles, +} from '../../lib/api'; import { mergeMessages } from '../../lib/merge-messages'; import { isWorkspaceOperational } from '../../lib/workspace-status-utils'; import type { SessionState } from './types'; import type { AgentActivityState } from './types'; -import { CHAT_FALLBACK_POLL_MS, deriveSessionState, IDLE_TIMEOUT_MS, isWorkingActivity, VIRTUAL_START } from './types'; +import { + CHAT_FALLBACK_POLL_MS, + deriveSessionState, + IDLE_TIMEOUT_MS, + isWorkingActivity, + VIRTUAL_START, +} from './types'; import { useActivityVerifyTimer } from './useActivityVerifyTimer'; import { useConnectionRecovery } from './useConnectionRecovery'; -type FilePanelState = { mode: 'browse' | 'view' | 'diff' | 'git-status'; path?: string; line?: number | null } | null; +type FilePanelState = { + mode: 'browse' | 'view' | 'diff' | 'git-status'; + path?: string; + line?: number | null; +} | null; function parsePlanContent(content: string): SessionStateSnapshot['currentPlan'] | null { try { @@ -39,7 +68,9 @@ function hashPlanContent(plan: SessionStateSnapshot['currentPlan']): string { function getPlanFingerprint(state: SessionStateSnapshot | null | undefined): string { if (!state) return 'no-state'; - return state.planUpdatedAt ? `updated:${state.planUpdatedAt}` : `content:${hashPlanContent(state.currentPlan)}`; + return state.planUpdatedAt + ? `updated:${state.planUpdatedAt}` + : `content:${hashPlanContent(state.currentPlan)}`; } export interface UseSessionLifecycleResult { @@ -60,6 +91,7 @@ export interface UseSessionLifecycleResult { uploading: boolean; isResuming: boolean; resumeError: string | null; + clearResumeError: () => void; connectionState: ChatConnectionState; showConnectionBanner: boolean; retryWs: () => void; @@ -89,7 +121,7 @@ export function useSessionLifecycle( projectId: string, sessionId: string, isProvisioning: boolean, - _onSessionMutated?: () => void, + _onSessionMutated?: () => void ): UseSessionLifecycleResult { const [session, setSession] = useState(null); const [taskEmbed, setTaskEmbed] = useState(null); @@ -103,8 +135,12 @@ export function useSessionLifecycle( // can read current state without stale closures. const messagesRef = useRef([]); const hasMoreRef = useRef(false); - useEffect(() => { messagesRef.current = messages; }, [messages]); - useEffect(() => { hasMoreRef.current = hasMore; }, [hasMore]); + useEffect(() => { + messagesRef.current = messages; + }, [messages]); + useEffect(() => { + hasMoreRef.current = hasMore; + }, [hasMore]); const [workspace, setWorkspace] = useState(null); const [node, setNode] = useState(null); @@ -115,7 +151,10 @@ export function useSessionLifecycle( const [agentActivity, setAgentActivity] = useState('idle'); const [currentPlan, setCurrentPlan] = useState(null); const [promptStartedAt, setPromptStartedAt] = useState(null); - const clearActivity = useCallback(() => { setAgentActivity('idle'); setPromptStartedAt(null); }, []); + const clearActivity = useCallback(() => { + setAgentActivity('idle'); + setPromptStartedAt(null); + }, []); const hydratePlan = useCallback((s: SessionStateSnapshot | null | undefined) => { if (!s) return; @@ -131,18 +170,21 @@ export function useSessionLifecycle( onStateSnapshot: hydratePlan, }); - const hydrateState = useCallback((s: SessionStateSnapshot | null | undefined) => { - if (!s) return; - if (isWorkingActivity(s.activity)) { - setAgentActivity(s.activity); - setPromptStartedAt(s.promptStartedAt ?? null); - startVerifyDecayTimer(); - } else { - clearActivity(); - stopVerifyDecayTimer(); - } - hydratePlan(s); - }, [clearActivity, hydratePlan, startVerifyDecayTimer, stopVerifyDecayTimer]); + const hydrateState = useCallback( + (s: SessionStateSnapshot | null | undefined) => { + if (!s) return; + if (isWorkingActivity(s.activity)) { + setAgentActivity(s.activity); + setPromptStartedAt(s.promptStartedAt ?? null); + startVerifyDecayTimer(); + } else { + clearActivity(); + stopVerifyDecayTimer(); + } + hydratePlan(s); + }, + [clearActivity, hydratePlan, startVerifyDecayTimer, stopVerifyDecayTimer] + ); const [filePanel, setFilePanel] = useState(null); @@ -162,59 +204,87 @@ export function useSessionLifecycle( const transcribeApiUrl = getTranscribeApiUrl(); // ── DO WebSocket (sole message source) ── - const { connectionState, wsRef, retry: retryWs } = useChatWebSocket({ + const { + connectionState, + wsRef, + retry: retryWs, + } = useChatWebSocket({ projectId, sessionId, enabled: session?.status === 'active', - onMessage: useCallback((msg: ChatMessageResponse) => { - setMessages((prev) => mergeMessages(prev, [msg], 'append')); + onMessage: useCallback( + (msg: ChatMessageResponse) => { + setMessages((prev) => mergeMessages(prev, [msg], 'append')); - if (msg.role === 'plan' && msg.content) { - const parsed = parsePlanContent(msg.content); - if (parsed) setCurrentPlan(parsed); - } - // Streaming agent output: show 'responding' heuristic, but arm the SHARED - // verify-before-decay timer instead of a blind decay. The blind timer used to - // clobber onAgentActivity's verified timer and flip to idle during long tool calls. - if (msg.role !== 'user') { - setAgentActivity('responding'); - startVerifyDecayTimer(); - } - }, [startVerifyDecayTimer]), + if (msg.role === 'plan' && msg.content) { + const parsed = parsePlanContent(msg.content); + if (parsed) setCurrentPlan(parsed); + } + // Streaming agent output: show 'responding' heuristic, but arm the SHARED + // verify-before-decay timer instead of a blind decay. The blind timer used to + // clobber onAgentActivity's verified timer and flip to idle during long tool calls. + if (msg.role !== 'user') { + setAgentActivity('responding'); + startVerifyDecayTimer(); + } + }, + [startVerifyDecayTimer] + ), onSessionStopped: useCallback(() => { - setSession((prev) => prev ? { ...prev, status: 'stopped' } : prev); + setSession((prev) => (prev ? { ...prev, status: 'stopped' } : prev)); setAgentActivity('idle'); setPromptStartedAt(null); // Stop any pending verify timer so it can't re-arm and flash the bar back on. stopVerifyDecayTimer(); }, [stopVerifyDecayTimer]), - onCatchUp: useCallback((catchUpMessages: ChatMessageResponse[], catchUpSession: ChatSessionResponse, state?: SessionStateSnapshot | null) => { - setSession(catchUpSession); - setMessages((prev) => mergeMessages(prev, catchUpMessages, 'replace')); - hydrateState(state); - }, [hydrateState]), - onAgentCompleted: useCallback((agentCompletedAt: number) => { - setSession((prev) => prev ? { ...prev, agentCompletedAt, isIdle: true } as ChatSessionResponse : prev); - setAgentActivity('idle'); - setPromptStartedAt(null); - // Stop any pending verify timer so it can't re-arm and flash the bar back on. - stopVerifyDecayTimer(); - }, [stopVerifyDecayTimer]), - onAgentActivity: useCallback((activity: 'prompting' | 'idle' | 'recovering' | 'error', promptStartedAt?: number | null) => { - const working = activity === 'prompting' || activity === 'recovering'; - setAgentActivity(working ? activity : 'idle'); - setPromptStartedAt(working ? (promptStartedAt ?? Date.now()) : null); - if (working) { - // Arm the shared verify-before-decay timer (prevents false idle during long tool calls). - startVerifyDecayTimer(); - } else { - // Authoritative idle from the DO: stop any pending verify timer. + onCatchUp: useCallback( + ( + catchUpMessages: ChatMessageResponse[], + catchUpSession: ChatSessionResponse, + state?: SessionStateSnapshot | null + ) => { + setSession(catchUpSession); + setMessages((prev) => mergeMessages(prev, catchUpMessages, 'replace')); + hydrateState(state); + }, + [hydrateState] + ), + onAgentCompleted: useCallback( + (agentCompletedAt: number) => { + setSession((prev) => + prev ? ({ ...prev, agentCompletedAt, isIdle: true } as ChatSessionResponse) : prev + ); + setAgentActivity('idle'); + setPromptStartedAt(null); + // Stop any pending verify timer so it can't re-arm and flash the bar back on. stopVerifyDecayTimer(); - } - }, [startVerifyDecayTimer, stopVerifyDecayTimer]), - onSessionUpdated: useCallback((updates: Partial>) => { - setSession((prev) => prev ? { ...prev, ...updates } : prev); - }, []), + }, + [stopVerifyDecayTimer] + ), + onAgentActivity: useCallback( + ( + activity: 'prompting' | 'idle' | 'recovering' | 'error', + promptStartedAt?: number | null + ) => { + const working = activity === 'prompting' || activity === 'recovering'; + setAgentActivity(working ? activity : 'idle'); + setPromptStartedAt(working ? (promptStartedAt ?? Date.now()) : null); + if (working) { + // Arm the shared verify-before-decay timer (prevents false idle during long tool calls). + startVerifyDecayTimer(); + } else { + // Authoritative idle from the DO: stop any pending verify timer. + stopVerifyDecayTimer(); + } + }, + [startVerifyDecayTimer, stopVerifyDecayTimer] + ), + onSessionUpdated: useCallback( + (updates: Partial>) => { + setSession((prev) => (prev ? { ...prev, ...updates } : prev)); + }, + [] + ), }); // Connection recovery (banner debounce, idle timer, auto-resume) @@ -259,7 +329,9 @@ export function useSessionLifecycle( } }, [projectId, sessionId, hydrateState]); - useEffect(() => { void loadSession(); }, [loadSession]); + useEffect(() => { + void loadSession(); + }, [loadSession]); // Fetch workspace and node details useEffect(() => { @@ -312,7 +384,7 @@ export function useSessionLifecycle( workspace?.url ?? undefined, session?.workspaceId ?? undefined, terminalToken ?? undefined, - isWorkspaceRunning, + isWorkspaceRunning ); // Degraded fallback while the DO WebSocket is unavailable. Connected active @@ -331,9 +403,10 @@ export function useSessionLifecycle( try { // Poll only the most-recent window — mergeReplace preserves the fully // loaded history, so polling must NOT re-fetch the whole conversation. - const data: ChatSessionDetailResponse = await getChatSession( - projectId, sessionId, { signal: abortController.signal, limit: DEFAULT_CHAT_SESSION_MESSAGE_LIMIT }, - ); + const data: ChatSessionDetailResponse = await getChatSession(projectId, sessionId, { + signal: abortController.signal, + limit: DEFAULT_CHAT_SESSION_MESSAGE_LIMIT, + }); if (data.session.id !== sessionId) return; const newLastId = data.messages[data.messages.length - 1]?.id ?? ''; const taskStatus = data.session.task?.status ?? ''; @@ -353,7 +426,9 @@ export function useSessionLifecycle( pollInFlight = false; } }; - const pollInterval = setInterval(() => { void pollActiveSession(); }, CHAT_FALLBACK_POLL_MS); + const pollInterval = setInterval(() => { + void pollActiveSession(); + }, CHAT_FALLBACK_POLL_MS); return () => { clearInterval(pollInterval); @@ -375,7 +450,12 @@ export function useSessionLifecycle( if (result.cleanupAt) { setSession((prev) => { if (!prev) return prev; - return { ...prev, cleanupAt: result.cleanupAt, isIdle: false, agentCompletedAt: null } as ChatSessionResponse; + return { + ...prev, + cleanupAt: result.cleanupAt, + isIdle: false, + agentCompletedAt: null, + } as ChatSessionResponse; }); } }) @@ -384,23 +464,28 @@ export function useSessionLifecycle( // Optimistic user message const optimisticId = `optimistic-${crypto.randomUUID()}`; - setMessages((prev) => [...prev, { - id: optimisticId, - sessionId, - role: 'user', - content: trimmed, - toolMetadata: null, - createdAt: Date.now(), - }]); + setMessages((prev) => [ + ...prev, + { + id: optimisticId, + sessionId, + role: 'user', + content: trimmed, + toolMetadata: null, + createdAt: Date.now(), + }, + ]); // Persist via DO WebSocket if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { - wsRef.current.send(JSON.stringify({ - type: 'message.send', - sessionId, - content: trimmed, - role: 'user', - })); + wsRef.current.send( + JSON.stringify({ + type: 'message.send', + sessionId, + content: trimmed, + role: 'user', + }) + ); } // For idle sessions, resume first then send the prompt @@ -410,8 +495,8 @@ export function useSessionLifecycle( // Forward prompt to the running agent via REST API try { await sendFollowUpPrompt(projectId, sessionId, trimmed); - } catch { - // Agent may be offline — message is still persisted via DO. + } catch (err) { + recovery.reportDeliveryError(err); setAgentActivity('idle'); } } @@ -423,27 +508,33 @@ export function useSessionLifecycle( }; // Upload files - const handleUploadFiles = useCallback(async (files: FileList | File[]) => { - const fileArray = Array.from(files); - if (fileArray.length === 0) return; - setUploading(true); - try { - const result = await uploadSessionFiles(projectId, sessionId, fileArray); - const names = result.files.map((f) => f.name).join(', '); - setMessages((prev) => [...prev, { - id: `optimistic-upload-${crypto.randomUUID()}`, - sessionId, - role: 'user' as const, - content: `Uploaded ${result.files.length} file${result.files.length > 1 ? 's' : ''}: ${names}`, - toolMetadata: null, - createdAt: Date.now(), - }]); - } catch (err) { - console.error('File upload failed:', err); - } finally { - setUploading(false); - } - }, [projectId, sessionId]); + const handleUploadFiles = useCallback( + async (files: FileList | File[]) => { + const fileArray = Array.from(files); + if (fileArray.length === 0) return; + setUploading(true); + try { + const result = await uploadSessionFiles(projectId, sessionId, fileArray); + const names = result.files.map((f) => f.name).join(', '); + setMessages((prev) => [ + ...prev, + { + id: `optimistic-upload-${crypto.randomUUID()}`, + sessionId, + role: 'user' as const, + content: `Uploaded ${result.files.length} file${result.files.length > 1 ? 's' : ''}: ${names}`, + toolMetadata: null, + createdAt: Date.now(), + }, + ]); + } catch (err) { + console.error('File upload failed:', err); + } finally { + setUploading(false); + } + }, + [projectId, sessionId] + ); // Cancel the current in-flight prompt via REST API const cancellingRef = useRef(false); @@ -488,58 +579,93 @@ export function useSessionLifecycle( // Load older pages until a target timestamp is covered (or no more history). // Used by timeline jump-to-message for the rare oversized/guard-trimmed session // where the target predates the loaded window, so a jump never dead-clicks. - const loadUntil = useCallback(async (targetTimestamp: number) => { - let oldest = messagesRef.current[0]?.createdAt ?? Infinity; - if (oldest <= targetTimestamp) return; - let more = hasMoreRef.current; - if (!more) return; - - setLoadingMore(true); - try { - let before: number | undefined = oldest === Infinity ? undefined : oldest; - const accumulated: ChatMessageResponse[] = []; - // Safety bound: never loop unbounded even if the server misreports hasMore. - const maxPages = Number.parseInt( - import.meta.env.VITE_CHAT_LOAD_UNTIL_MAX_PAGES || '', - 10, - ) || DEFAULT_CHAT_LOAD_UNTIL_MAX_PAGES; - let pages = 0; - while (more && oldest > targetTimestamp && pages++ < maxPages) { - const data = await getChatSession(projectId, sessionId, { - before, - limit: DEFAULT_CHAT_SESSION_MESSAGE_LIMIT, - }); - if (data.messages.length === 0) { more = false; break; } - accumulated.unshift(...data.messages); - oldest = data.messages[0]!.createdAt; - before = oldest; - more = data.hasMore; - } - if (accumulated.length > 0) { - setMessages((prev) => { - const merged = mergeMessages(prev, accumulated, 'prepend'); - const actualAdded = merged.length - prev.length; - setFirstItemIndex((fi) => fi - actualAdded); - return merged; - }); + const loadUntil = useCallback( + async (targetTimestamp: number) => { + let oldest = messagesRef.current[0]?.createdAt ?? Infinity; + if (oldest <= targetTimestamp) return; + let more = hasMoreRef.current; + if (!more) return; + + setLoadingMore(true); + try { + let before: number | undefined = oldest === Infinity ? undefined : oldest; + const accumulated: ChatMessageResponse[] = []; + // Safety bound: never loop unbounded even if the server misreports hasMore. + const maxPages = + Number.parseInt(import.meta.env.VITE_CHAT_LOAD_UNTIL_MAX_PAGES || '', 10) || + DEFAULT_CHAT_LOAD_UNTIL_MAX_PAGES; + let pages = 0; + while (more && oldest > targetTimestamp && pages++ < maxPages) { + const data = await getChatSession(projectId, sessionId, { + before, + limit: DEFAULT_CHAT_SESSION_MESSAGE_LIMIT, + }); + if (data.messages.length === 0) { + more = false; + break; + } + accumulated.unshift(...data.messages); + oldest = data.messages[0]!.createdAt; + before = oldest; + more = data.hasMore; + } + if (accumulated.length > 0) { + setMessages((prev) => { + const merged = mergeMessages(prev, accumulated, 'prepend'); + const actualAdded = merged.length - prev.length; + setFirstItemIndex((fi) => fi - actualAdded); + return merged; + }); + } + setHasMore(more); + } finally { + setLoadingMore(false); } - setHasMore(more); - } finally { - setLoadingMore(false); - } - }, [projectId, sessionId]); + }, + [projectId, sessionId] + ); return { - session, messages, hasMore, loading, error, setError, sessionState, taskEmbed, - workspace, node, detectedPorts, - followUp, setFollowUp, sendingFollowUp, uploading, - isResuming: recovery.isResuming, resumeError: recovery.resumeError, - connectionState, showConnectionBanner: recovery.showConnectionBanner, retryWs, - agentActivity, currentPlan, promptStartedAt, - firstItemIndex, showScrollButton, setShowScrollButton, + session, + messages, + hasMore, + loading, + error, + setError, + sessionState, + taskEmbed, + workspace, + node, + detectedPorts, + followUp, + setFollowUp, + sendingFollowUp, + uploading, + isResuming: recovery.isResuming, + resumeError: recovery.resumeError, + clearResumeError: recovery.clearResumeError, + connectionState, + showConnectionBanner: recovery.showConnectionBanner, + retryWs, + agentActivity, + currentPlan, + promptStartedAt, + firstItemIndex, + showScrollButton, + setShowScrollButton, idleCountdownMs: recovery.idleCountdownMs, - filePanel, setFilePanel, handleFileClick, handleOpenFileBrowser, handleOpenGitChanges, - handleCancelPrompt, handleSendFollowUp, handleUploadFiles, - loadMore, loadUntil, loadingMore, transcribeApiUrl, wsRef, + filePanel, + setFilePanel, + handleFileClick, + handleOpenFileBrowser, + handleOpenGitChanges, + handleCancelPrompt, + handleSendFollowUp, + handleUploadFiles, + loadMore, + loadUntil, + loadingMore, + transcribeApiUrl, + wsRef, }; } diff --git a/apps/web/src/lib/session-utils.ts b/apps/web/src/lib/session-utils.ts index 9c34fd31e..ab77ea8c8 100644 --- a/apps/web/src/lib/session-utils.ts +++ b/apps/web/src/lib/session-utils.ts @@ -7,6 +7,11 @@ const ACTIVE_HOST_STATUSES: ReadonlySet = new Set([ 'ready', 'prompting', ]); +const RECOVERABLE_SESSION_STATUSES: ReadonlySet = new Set([ + 'running', + 'recovery', + 'sleeping', +]); /** * Determines if a session should be treated as active (visible in tabs). @@ -14,7 +19,7 @@ const ACTIVE_HOST_STATUSES: ReadonlySet = new Set([ * is still alive despite a non-running status (orphan recovery). */ export function isSessionActive(session: AgentSession): boolean { - if (session.status === 'running') return true; + if (RECOVERABLE_SESSION_STATUSES.has(session.status)) return true; if (!session.hostStatus) return false; return ACTIVE_HOST_STATUSES.has(session.hostStatus); } @@ -24,7 +29,7 @@ export function isSessionActive(session: AgentSession): boolean { * These are "orphaned" sessions -- alive on the VM but previously hidden. */ export function isOrphanedSession(session: AgentSession): boolean { - if (session.status === 'running') return false; + if (RECOVERABLE_SESSION_STATUSES.has(session.status)) return false; if (!session.hostStatus) return false; return ACTIVE_HOST_STATUSES.has(session.hostStatus); } diff --git a/apps/web/tests/unit/components/chat/project-message-view-resume.test.tsx b/apps/web/tests/unit/components/chat/project-message-view-resume.test.tsx index 31c37a4ae..e3677512d 100644 --- a/apps/web/tests/unit/components/chat/project-message-view-resume.test.tsx +++ b/apps/web/tests/unit/components/chat/project-message-view-resume.test.tsx @@ -3,12 +3,12 @@ * * These tests verify: * 1. Follow-up sends trigger auto-resume when agent is idle/suspended - * 2. "Resuming agent..." UI state is shown during resume + * 2. The waking/restoring UI state is shown during resume * 3. Queued messages are flushed when agent becomes active * 4. Resume failures show clear error messages * 5. Idle countdown pauses during resume */ -import { act,fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ProjectMessageView } from '../../../../src/components/project-message-view'; @@ -78,9 +78,17 @@ vi.mock('@simple-agent-manager/acp-client', () => ({ // Mock react-virtuoso vi.mock('react-virtuoso', () => ({ - Virtuoso: ({ data, itemContent }: { data: unknown[]; itemContent: (index: number, item: unknown) => React.ReactNode }) => ( + Virtuoso: ({ + data, + itemContent, + }: { + data: unknown[]; + itemContent: (index: number, item: unknown) => React.ReactNode; + }) => (
- {data?.map((item, i) =>
{itemContent(i, item)}
)} + {data?.map((item, i) => ( +
{itemContent(i, item)}
+ ))}
), })); @@ -127,7 +135,16 @@ describe('ProjectMessageView — auto-resume', () => { // Default: session is idle with workspace and agent session mockGetChatSession.mockResolvedValue({ session: makeSessionResponse({ isIdle: true, agentCompletedAt: Date.now() - 5_000 }), - messages: [{ id: 'msg-1', sessionId: SESSION_ID, role: 'user', content: 'Hello', toolMetadata: null, createdAt: Date.now() - 10_000 }], + messages: [ + { + id: 'msg-1', + sessionId: SESSION_ID, + role: 'user', + content: 'Hello', + toolMetadata: null, + createdAt: Date.now() - 10_000, + }, + ], hasMore: false, }); mockGetWorkspace.mockResolvedValue({ id: WORKSPACE_ID, nodeId: 'node-1' }); @@ -152,19 +169,16 @@ describe('ProjectMessageView — auto-resume', () => { expect(archiveButton).toBeInTheDocument(); fireEvent.click(archiveButton); - const archiveActions = await screen.findAllByRole('button', { name: /^archive conversation$/i }); + const archiveActions = await screen.findAllByRole('button', { + name: /^archive conversation$/i, + }); fireEvent.click(archiveActions[archiveActions.length - 1]); expect(onCloseConversation).toHaveBeenCalledOnce(); }); it('calls resumeAgentSession when sending follow-up to idle session', async () => { - render( - - ); + render(); // Wait for initial load await waitFor(() => { @@ -181,16 +195,11 @@ describe('ProjectMessageView — auto-resume', () => { }); }); - it('shows "Resuming agent..." banner during resume', async () => { + it('shows the waking/restoring banner during resume', async () => { // Make resume hang (never resolve) mockResumeAgentSession.mockReturnValue(new Promise(() => {})); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); @@ -201,19 +210,14 @@ describe('ProjectMessageView — auto-resume', () => { fireEvent.click(screen.getByRole('button', { name: /send/i })); await waitFor(() => { - expect(screen.getByText('Resuming agent...')).toBeInTheDocument(); + expect(screen.getByText('Waking and restoring Instant session...')).toBeInTheDocument(); }); }); it('shows error when resume fails with 404', async () => { mockResumeAgentSession.mockRejectedValue(new Error('404 Not Found')); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); @@ -224,19 +228,16 @@ describe('ProjectMessageView — auto-resume', () => { fireEvent.click(screen.getByRole('button', { name: /send/i })); await waitFor(() => { - expect(screen.getByText(/could not resume agent.*workspace may have been cleaned up/i)).toBeInTheDocument(); + expect( + screen.getByText(/could not resume agent.*workspace may have been cleaned up/i) + ).toBeInTheDocument(); }); }); it('shows generic error when resume fails with non-404 error', async () => { mockResumeAgentSession.mockRejectedValue(new Error('Network timeout')); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); @@ -252,18 +253,13 @@ describe('ProjectMessageView — auto-resume', () => { }); it('shows resuming banner instead of agent offline during resume', async () => { - // When resuming, the "Resuming agent..." banner should be visible + // When resuming, the waking/restoring banner should be visible // and the generic "Agent offline" banner should NOT appear. // For idle sessions, the AgentErrorBanner wouldn't show anyway (guard: sessionState === 'active'), // but the resuming banner IS the intended UX replacement for the disconnect state. mockResumeAgentSession.mockReturnValue(new Promise(() => {})); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); @@ -274,14 +270,13 @@ describe('ProjectMessageView — auto-resume', () => { fireEvent.click(screen.getByRole('button', { name: /send/i })); await waitFor(() => { - expect(screen.getByText('Resuming agent...')).toBeInTheDocument(); + expect(screen.getByText('Waking and restoring Instant session...')).toBeInTheDocument(); }); // Resuming banner is the active indicator — no error/offline banners expect(screen.queryByText(/agent offline/i)).not.toBeInTheDocument(); expect(screen.queryByText(/agent is not connected/i)).not.toBeInTheDocument(); }); - }); describe('ProjectMessageView — auto-resume on page visit', () => { @@ -292,7 +287,16 @@ describe('ProjectMessageView — auto-resume on page visit', () => { // Default: session is idle with workspace and agent session mockGetChatSession.mockResolvedValue({ session: makeSessionResponse({ isIdle: true, agentCompletedAt: Date.now() - 5_000 }), - messages: [{ id: 'msg-1', sessionId: SESSION_ID, role: 'user', content: 'Hello', toolMetadata: null, createdAt: Date.now() - 10_000 }], + messages: [ + { + id: 'msg-1', + sessionId: SESSION_ID, + role: 'user', + content: 'Hello', + toolMetadata: null, + createdAt: Date.now() - 10_000, + }, + ], hasMore: false, }); mockGetWorkspace.mockResolvedValue({ id: WORKSPACE_ID, nodeId: 'node-1' }); @@ -307,12 +311,7 @@ describe('ProjectMessageView — auto-resume on page visit', () => { }); it('auto-resumes idle session after 2s delay without user interaction', async () => { - render( - - ); + render(); // Wait for initial load await waitFor(() => { @@ -323,70 +322,66 @@ describe('ProjectMessageView — auto-resume on page visit', () => { expect(mockResumeAgentSession).not.toHaveBeenCalled(); // Advance past the 2s delay - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); await waitFor(() => { expect(mockResumeAgentSession).toHaveBeenCalledWith(WORKSPACE_ID, AGENT_SESSION_ID); }); }); - it('shows "Resuming agent..." banner during auto-resume', async () => { + it('shows the waking/restoring banner during auto-resume', async () => { // Make resume hang mockResumeAgentSession.mockReturnValue(new Promise(() => {})); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); }); - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); await waitFor(() => { - expect(screen.getByText('Resuming agent...')).toBeInTheDocument(); + expect(screen.getByText('Waking and restoring Instant session...')).toBeInTheDocument(); }); }); it('shows error when auto-resume fails with 404', async () => { mockResumeAgentSession.mockRejectedValue(new Error('404 Not Found')); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); }); - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); await waitFor(() => { - expect(screen.getByText(/could not resume agent.*workspace may have been cleaned up/i)).toBeInTheDocument(); + expect( + screen.getByText(/could not resume agent.*workspace may have been cleaned up/i) + ).toBeInTheDocument(); }); }); it('does not auto-resume during provisioning', async () => { render( - + ); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); }); - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); expect(mockResumeAgentSession).not.toHaveBeenCalled(); }); @@ -399,18 +394,15 @@ describe('ProjectMessageView — auto-resume on page visit', () => { hasMore: false, }); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); }); - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); expect(mockResumeAgentSession).not.toHaveBeenCalled(); }); @@ -418,24 +410,26 @@ describe('ProjectMessageView — auto-resume on page visit', () => { it('shows generic error when auto-resume fails with non-404 error', async () => { mockResumeAgentSession.mockRejectedValue(new Error('Network timeout')); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); }); // Advance timer to trigger the auto-resume, then flush pending promises - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); - await act(async () => { await vi.advanceTimersByTimeAsync(100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); + await act(async () => { + await vi.advanceTimersByTimeAsync(100); + }); - await waitFor(() => { - expect(screen.getByText(/could not resume agent.*please try again/i)).toBeInTheDocument(); - }, { timeout: 3000 }); + await waitFor( + () => { + expect(screen.getByText(/could not resume agent.*please try again/i)).toBeInTheDocument(); + }, + { timeout: 3000 } + ); }); it('resets auto-resume state when session ID changes', async () => { @@ -443,10 +437,7 @@ describe('ProjectMessageView — auto-resume on page visit', () => { mockResumeAgentSession.mockReturnValue(new Promise(() => {})); const { rerender } = render( - + ); await waitFor(() => { @@ -454,35 +445,38 @@ describe('ProjectMessageView — auto-resume on page visit', () => { }); // Trigger auto-resume for first session - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); await waitFor(() => { - expect(screen.getByText('Resuming agent...')).toBeInTheDocument(); + expect(screen.getByText('Waking and restoring Instant session...')).toBeInTheDocument(); }); // Switch to a different session — should reset resume state const NEW_SESSION_ID = 'sess-new'; mockGetChatSession.mockResolvedValue({ - session: makeSessionResponse({ id: NEW_SESSION_ID, isIdle: true, agentCompletedAt: Date.now() - 5_000 }), + session: makeSessionResponse({ + id: NEW_SESSION_ID, + isIdle: true, + agentCompletedAt: Date.now() - 5_000, + }), messages: [], hasMore: false, }); mockResumeAgentSession.mockClear(); mockResumeAgentSession.mockResolvedValue({ id: AGENT_SESSION_ID, status: 'running' }); - rerender( - - ); + rerender(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalledTimes(2); }); // Auto-resume should fire for the new session after the delay - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); await waitFor(() => { expect(mockResumeAgentSession).toHaveBeenCalled(); @@ -492,12 +486,7 @@ describe('ProjectMessageView — auto-resume on page visit', () => { it('does not double-resume when follow-up sent before auto-resume timer fires', async () => { mockResumeAgentSession.mockResolvedValue({ id: AGENT_SESSION_ID, status: 'running' }); - render( - - ); + render(); await waitFor(() => { expect(mockGetChatSession).toHaveBeenCalled(); @@ -514,7 +503,9 @@ describe('ProjectMessageView — auto-resume on page visit', () => { }); // Now advance past the auto-resume timer — it should NOT fire a second resume - await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + await act(async () => { + await vi.advanceTimersByTimeAsync(2100); + }); // Still only 1 call (from the follow-up, not from auto-resume) expect(mockResumeAgentSession).toHaveBeenCalledTimes(1); diff --git a/apps/web/tests/unit/components/runtimeRecoveryMessages.test.ts b/apps/web/tests/unit/components/runtimeRecoveryMessages.test.ts new file mode 100644 index 000000000..2ff95cf83 --- /dev/null +++ b/apps/web/tests/unit/components/runtimeRecoveryMessages.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; + +import { + getResumeFailureMessage, + getRuntimeRecoveryMessage, +} from '../../../src/components/project-message-view/runtimeRecoveryMessages'; +import { ApiClientError } from '../../../src/lib/api'; + +describe('runtime recovery user messages', () => { + it('shows an explicit waking/restoring state', () => { + const error = new ApiClientError( + 'RUNTIME_RECOVERING', + 'Instant session interrupted; restoring the last safe checkpoint.', + 503 + ); + + expect(getRuntimeRecoveryMessage(error)).toBe( + 'Waking and restoring the Instant session. Wait for restore to finish, then send your message.' + ); + }); + + it('preserves the server-sanitized manual-retry and degraded messages', () => { + const interrupted = new ApiClientError( + 'RUNTIME_REQUEST_INTERRUPTED', + 'Your message is saved, but it was not replayed automatically.', + 409 + ); + const degraded = new ApiClientError( + 'RUNTIME_RECOVERY_DEGRADED', + 'The Instant session could not restore its last safe checkpoint. Your transcript is available.', + 409 + ); + + expect(getRuntimeRecoveryMessage(interrupted)).toBe(interrupted.message); + expect(getRuntimeRecoveryMessage(degraded)).toBe(degraded.message); + }); + + it('keeps internal non-recovery errors out of regular-user messaging', () => { + const error = new Error('R2 key snapshots/private-user/home.tar was corrupt'); + + expect(getRuntimeRecoveryMessage(error)).toBeNull(); + expect(getResumeFailureMessage(error)).toBe('Could not resume agent — please try again.'); + }); +}); diff --git a/apps/web/tests/unit/components/useSessionLifecycle.test.ts b/apps/web/tests/unit/components/useSessionLifecycle.test.ts index 48ef4ba23..702368366 100644 --- a/apps/web/tests/unit/components/useSessionLifecycle.test.ts +++ b/apps/web/tests/unit/components/useSessionLifecycle.test.ts @@ -37,7 +37,11 @@ vi.mock('../../../src/lib/api', async (importOriginal) => ({ })); vi.mock('../../../src/hooks/useChatWebSocket', () => ({ - useChatWebSocket: () => ({ connectionState: mocks.connectionState, wsRef: { current: null }, retry: vi.fn() }), + useChatWebSocket: () => ({ + connectionState: mocks.connectionState, + wsRef: { current: null }, + retry: vi.fn(), + }), })); vi.mock('../../../src/hooks/useTokenRefresh', () => ({ useTokenRefresh: () => ({ token: null }), @@ -54,6 +58,8 @@ vi.mock('../../../src/components/project-message-view/useConnectionRecovery', () resumeError: null, showConnectionBanner: false, idleCountdownMs: null, + clearResumeError: vi.fn(), + reportDeliveryError: vi.fn(), resumeAndSend: vi.fn(), }), })); @@ -64,14 +70,36 @@ vi.mock('../../../src/components/project-message-view/types', async (importOrigi import { useSessionLifecycle } from '../../../src/components/project-message-view/useSessionLifecycle'; -type Msg = { id: string; sessionId: string; role: string; content: string; toolMetadata: null; createdAt: number }; +type Msg = { + id: string; + sessionId: string; + role: string; + content: string; + toolMetadata: null; + createdAt: number; +}; function msg(id: string, createdAt: number): Msg { - return { id, sessionId: 'sess-1', role: 'user', content: `m-${id}`, toolMetadata: null, createdAt }; + return { + id, + sessionId: 'sess-1', + role: 'user', + content: `m-${id}`, + toolMetadata: null, + createdAt, + }; } function sessionResponse(status: string) { - return { id: 'sess-1', workspaceId: null, topic: 'T', status, messageCount: 1, createdAt: Date.now(), updatedAt: Date.now() }; + return { + id: 'sess-1', + workspaceId: null, + topic: 'T', + status, + messageCount: 1, + createdAt: Date.now(), + updatedAt: Date.now(), + }; } function detail( @@ -79,7 +107,7 @@ function detail( hasMore: boolean, status = 'stopped', currentPlan: Array<{ content: string; status: string }> | null = null, - planUpdatedAt: number | null = null, + planUpdatedAt: number | null = null ) { return { session: sessionResponse(status), @@ -143,7 +171,9 @@ describe('useSessionLifecycle loading semantics', () => { await waitFor(() => expect(result.current.messages.length).toBe(2)); mocks.getChatSession.mockClear(); - await act(async () => { await result.current.loadUntil(700); }); + await act(async () => { + await result.current.loadUntil(700); + }); // Oldest loaded is 500 <= 700 → nothing to fetch. expect(mocks.getChatSession).not.toHaveBeenCalled(); @@ -155,7 +185,9 @@ describe('useSessionLifecycle loading semantics', () => { await waitFor(() => expect(result.current.messages.length).toBe(1)); mocks.getChatSession.mockClear(); - await act(async () => { await result.current.loadUntil(200); }); + await act(async () => { + await result.current.loadUntil(200); + }); // Target (200) predates the loaded window, but hasMore=false → no fetch. expect(mocks.getChatSession).not.toHaveBeenCalled(); diff --git a/apps/web/tests/unit/lib/session-utils.test.ts b/apps/web/tests/unit/lib/session-utils.test.ts index 17eb54ca0..d2f455696 100644 --- a/apps/web/tests/unit/lib/session-utils.test.ts +++ b/apps/web/tests/unit/lib/session-utils.test.ts @@ -1,11 +1,9 @@ import type { AgentSession } from '@simple-agent-manager/shared'; import { describe, expect, it } from 'vitest'; -import { isOrphanedSession,isSessionActive } from '../../../src/lib/session-utils'; +import { isOrphanedSession, isSessionActive } from '../../../src/lib/session-utils'; -function makeSession( - overrides: Partial = {} -): AgentSession { +function makeSession(overrides: Partial = {}): AgentSession { return { id: 'sess-1', workspaceId: 'ws-1', @@ -21,133 +19,107 @@ describe('isSessionActive', () => { expect(isSessionActive(makeSession({ status: 'running' }))).toBe(true); }); + it.each(['recovery', 'sleeping'] as const)( + 'returns true when status is %s without a live host', + (status) => { + expect(isSessionActive(makeSession({ status }))).toBe(true); + } + ); + it('returns true when status is running even if hostStatus is stopped', () => { - expect( - isSessionActive(makeSession({ status: 'running', hostStatus: 'stopped' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'running', hostStatus: 'stopped' }))).toBe(true); }); it('returns true when status is stopped but hostStatus is ready (orphan)', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: 'ready' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: 'ready' }))).toBe(true); }); it('returns true when status is stopped but hostStatus is prompting', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: 'prompting' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: 'prompting' }))).toBe(true); }); it('returns true when status is stopped but hostStatus is idle', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: 'idle' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: 'idle' }))).toBe(true); }); it('returns true when status is stopped but hostStatus is starting', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: 'starting' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: 'starting' }))).toBe(true); }); it('returns false when status is stopped and hostStatus is stopped', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: 'stopped' })) - ).toBe(false); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: 'stopped' }))).toBe(false); }); it('returns false when status is stopped and hostStatus is error', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: 'error' })) - ).toBe(false); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: 'error' }))).toBe(false); }); it('returns false when status is stopped and hostStatus is null', () => { - expect( - isSessionActive(makeSession({ status: 'stopped', hostStatus: null })) - ).toBe(false); + expect(isSessionActive(makeSession({ status: 'stopped', hostStatus: null }))).toBe(false); }); it('returns false when status is stopped and hostStatus is undefined', () => { - expect( - isSessionActive(makeSession({ status: 'stopped' })) - ).toBe(false); + expect(isSessionActive(makeSession({ status: 'stopped' }))).toBe(false); }); it('returns true when status is error but hostStatus is ready', () => { - expect( - isSessionActive(makeSession({ status: 'error', hostStatus: 'ready' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'error', hostStatus: 'ready' }))).toBe(true); }); // Suspended session tests — isSessionActive returns false for suspended, // but the tab strip includes them separately via explicit status check. it('returns false when status is suspended and no hostStatus', () => { - expect( - isSessionActive(makeSession({ status: 'suspended' })) - ).toBe(false); + expect(isSessionActive(makeSession({ status: 'suspended' }))).toBe(false); }); it('returns false when status is suspended and hostStatus is null', () => { - expect( - isSessionActive(makeSession({ status: 'suspended', hostStatus: null })) - ).toBe(false); + expect(isSessionActive(makeSession({ status: 'suspended', hostStatus: null }))).toBe(false); }); it('returns true when status is suspended but hostStatus is ready (orphan recovery)', () => { - expect( - isSessionActive(makeSession({ status: 'suspended', hostStatus: 'ready' })) - ).toBe(true); + expect(isSessionActive(makeSession({ status: 'suspended', hostStatus: 'ready' }))).toBe(true); }); }); describe('isOrphanedSession', () => { it('returns false when status is running (normal session, not orphaned)', () => { - expect( - isOrphanedSession(makeSession({ status: 'running', hostStatus: 'ready' })) - ).toBe(false); + expect(isOrphanedSession(makeSession({ status: 'running', hostStatus: 'ready' }))).toBe(false); }); + it.each(['recovery', 'sleeping'] as const)( + 'returns false when status is %s because it is intentionally recoverable', + (status) => { + expect(isOrphanedSession(makeSession({ status, hostStatus: 'ready' }))).toBe(false); + } + ); + it('returns true when status is stopped but hostStatus is ready', () => { - expect( - isOrphanedSession(makeSession({ status: 'stopped', hostStatus: 'ready' })) - ).toBe(true); + expect(isOrphanedSession(makeSession({ status: 'stopped', hostStatus: 'ready' }))).toBe(true); }); it('returns true when status is error but hostStatus is prompting', () => { - expect( - isOrphanedSession(makeSession({ status: 'error', hostStatus: 'prompting' })) - ).toBe(true); + expect(isOrphanedSession(makeSession({ status: 'error', hostStatus: 'prompting' }))).toBe(true); }); it('returns false when status is stopped and hostStatus is null', () => { - expect( - isOrphanedSession(makeSession({ status: 'stopped', hostStatus: null })) - ).toBe(false); + expect(isOrphanedSession(makeSession({ status: 'stopped', hostStatus: null }))).toBe(false); }); it('returns false when status is stopped and hostStatus is stopped', () => { - expect( - isOrphanedSession(makeSession({ status: 'stopped', hostStatus: 'stopped' })) - ).toBe(false); + expect(isOrphanedSession(makeSession({ status: 'stopped', hostStatus: 'stopped' }))).toBe( + false + ); }); it('returns false when status is running (regardless of hostStatus)', () => { - expect( - isOrphanedSession(makeSession({ status: 'running' })) - ).toBe(false); + expect(isOrphanedSession(makeSession({ status: 'running' }))).toBe(false); }); it('returns true when status is suspended but hostStatus is ready', () => { - expect( - isOrphanedSession(makeSession({ status: 'suspended', hostStatus: 'ready' })) - ).toBe(true); + expect(isOrphanedSession(makeSession({ status: 'suspended', hostStatus: 'ready' }))).toBe(true); }); it('returns false when status is suspended and hostStatus is null', () => { - expect( - isOrphanedSession(makeSession({ status: 'suspended', hostStatus: null })) - ).toBe(false); + expect(isOrphanedSession(makeSession({ status: 'suspended', hostStatus: null }))).toBe(false); }); }); diff --git a/apps/www/src/content/docs/docs/reference/configuration.md b/apps/www/src/content/docs/docs/reference/configuration.md index e7d35ac9d..6b48a4f09 100644 --- a/apps/www/src/content/docs/docs/reference/configuration.md +++ b/apps/www/src/content/docs/docs/reference/configuration.md @@ -102,7 +102,11 @@ Sandbox runtime surfaces and is not required for guided Codex login. | Variable | Default | Description | | ------------------------------------------ | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `CF_CONTAINER_ENABLED` | `true` | Enables Cloudflare Container instant sessions for matching profiles and zero-config runtime selection. Set `false` to force cloud VM runtime. | +| `CF_CONTAINER_SLEEP_AFTER` | `1h` | Normal inactivity window before an Instant container sleeps. Sleep remains recoverable through the runtime-neutral session snapshot. | +| `CF_CONTAINER_ACTIVE_WORK_MAX_MS` | `7200000` | Defensive maximum lifetime for an active-work keepalive lease. | +| `CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS` | `300000` | Interval used to renew the container activity timeout while prompt work is active. | | `CF_CONTAINER_WAKE_TIMEOUT_MS` | `120000` | Maximum time for a sleeping container to launch, restore its snapshot, and accept the triggering request. | +| `CF_CONTAINER_RECOVERY_MAX_ATTEMPTS` | `2` | Maximum snapshot restore attempts before SAM reconciles the runtime, workspace, agent session, and active task to a visible terminal recovery failure. | | `CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS` | `120000` | Budget for the synchronous instant-session create-workspace request, which includes the repository clone inside the container. | | `CF_CONTAINER_CLONE_FILTER` | `blob:none` | Git partial-clone filter forwarded to instant containers as `STANDALONE_CLONE_FILTER`. Set `off` to force full clones. | | `REQUIRE_APPROVAL` | _(unset)_ | Default signup approval gate. Superadmins can override it at runtime in Admin → Users without redeploying; when no runtime override exists, this value is used. The first genuine human becomes superadmin regardless of this flag — see [First Login & Admin Access](/docs/guides/self-hosting/#first-login--admin-access). | diff --git a/packages/shared/src/constants/status.ts b/packages/shared/src/constants/status.ts index 8e32d7426..5ef9d6b55 100644 --- a/packages/shared/src/constants/status.ts +++ b/packages/shared/src/constants/status.ts @@ -13,6 +13,7 @@ export const STATUS_LABELS: Record = { running: 'Running', recovery: 'Recovery', stopping: 'Stopping', + sleeping: 'Sleeping', stopped: 'Stopped', deleted: 'Deleted', error: 'Error', @@ -25,6 +26,7 @@ export const STATUS_COLORS: Record = { recovery: 'yellow', stopping: 'yellow', stopped: 'gray', + sleeping: 'blue', deleted: 'gray', error: 'red', }; diff --git a/packages/shared/src/types/session.ts b/packages/shared/src/types/session.ts index 2adc9a257..fadc56d5b 100644 --- a/packages/shared/src/types/session.ts +++ b/packages/shared/src/types/session.ts @@ -144,7 +144,13 @@ export interface ProjectWebSocketEvent { // Agent Sessions // ============================================================================= -export type AgentSessionStatus = 'running' | 'suspended' | 'stopped' | 'error'; +export type AgentSessionStatus = + | 'running' + | 'recovery' + | 'sleeping' + | 'suspended' + | 'stopped' + | 'error'; /** Live host status from the VM Agent's SessionHost (more granular than AgentSessionStatus). */ export type AgentHostStatus = 'idle' | 'starting' | 'ready' | 'prompting' | 'error' | 'stopped'; @@ -256,14 +262,15 @@ export const ACP_SESSION_TERMINAL_STATUSES: readonly AcpSessionStatus[] = [ ] as const; /** Valid state machine transitions for ACP sessions. */ -export const ACP_SESSION_VALID_TRANSITIONS: Record = { - pending: ['assigned'], - assigned: ['running', 'failed', 'interrupted'], - running: ['completed', 'failed', 'interrupted'], - completed: [], - failed: [], - interrupted: [], -} as const; +export const ACP_SESSION_VALID_TRANSITIONS: Record = + { + pending: ['assigned'], + assigned: ['running', 'failed', 'interrupted'], + running: ['completed', 'failed', 'interrupted'], + completed: [], + failed: [], + interrupted: [], + } as const; export interface AcpSession { id: string; diff --git a/packages/shared/src/types/workspace.ts b/packages/shared/src/types/workspace.ts index 85df5e3bf..328b3ff93 100644 --- a/packages/shared/src/types/workspace.ts +++ b/packages/shared/src/types/workspace.ts @@ -7,6 +7,8 @@ export type NodeStatus = | 'pending' | 'creating' | 'running' + | 'recovery' + | 'sleeping' | 'stopping' | 'stopped' | 'deleted' @@ -19,6 +21,7 @@ export type WorkspaceStatus = | 'creating' | 'running' | 'recovery' + | 'sleeping' | 'stopping' | 'stopped' | 'deleted' diff --git a/scripts/deploy/sync-wrangler-config.ts b/scripts/deploy/sync-wrangler-config.ts index 34c04a617..684ff9606 100644 --- a/scripts/deploy/sync-wrangler-config.ts +++ b/scripts/deploy/sync-wrangler-config.ts @@ -343,8 +343,11 @@ function getApiWorkerVars( 'HETZNER_BASE_IMAGE', 'CF_CONTAINER_ENABLED', 'CF_CONTAINER_SLEEP_AFTER', + 'CF_CONTAINER_ACTIVE_WORK_MAX_MS', + 'CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS', 'CF_CONTAINER_PORT_READY_TIMEOUT_MS', 'CF_CONTAINER_WAKE_TIMEOUT_MS', + 'CF_CONTAINER_RECOVERY_MAX_ATTEMPTS', 'CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS', 'CF_CONTAINER_CLONE_FILTER', 'CF_CONTAINER_VM_AGENT_PORT', diff --git a/scripts/quality/sync-wrangler-config.test.ts b/scripts/quality/sync-wrangler-config.test.ts index e1e88d7bb..21e3cb674 100644 --- a/scripts/quality/sync-wrangler-config.test.ts +++ b/scripts/quality/sync-wrangler-config.test.ts @@ -110,16 +110,25 @@ describe('sync wrangler config', () => { }); }); - it('passes cf-container clone/create tunables through from process env when set', () => { + it('passes cf-container lifecycle and clone tunables through from process env when set', () => { vi.stubEnv('RESOURCE_PREFIX', 's123abc'); const unset = generateApiWorkerEnv({}, outputs, 'prod', false, false).vars; + expect(unset).not.toHaveProperty('CF_CONTAINER_ACTIVE_WORK_MAX_MS'); + expect(unset).not.toHaveProperty('CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS'); + expect(unset).not.toHaveProperty('CF_CONTAINER_RECOVERY_MAX_ATTEMPTS'); expect(unset).not.toHaveProperty('CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS'); expect(unset).not.toHaveProperty('CF_CONTAINER_CLONE_FILTER'); + vi.stubEnv('CF_CONTAINER_ACTIVE_WORK_MAX_MS', '7200000'); + vi.stubEnv('CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS', '300000'); + vi.stubEnv('CF_CONTAINER_RECOVERY_MAX_ATTEMPTS', '3'); vi.stubEnv('CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS', '180000'); vi.stubEnv('CF_CONTAINER_CLONE_FILTER', 'off'); expect(generateApiWorkerEnv({}, outputs, 'prod', false, false).vars).toMatchObject({ + CF_CONTAINER_ACTIVE_WORK_MAX_MS: '7200000', + CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS: '300000', + CF_CONTAINER_RECOVERY_MAX_ATTEMPTS: '3', CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS: '180000', CF_CONTAINER_CLONE_FILTER: 'off', }); diff --git a/tasks/active/2026-07-21-instant-runtime-recovery-state-machine.md b/tasks/active/2026-07-21-instant-runtime-recovery-state-machine.md index fa73a64d9..88aa32fdc 100644 --- a/tasks/active/2026-07-21-instant-runtime-recovery-state-machine.md +++ b/tasks/active/2026-07-21-instant-runtime-recovery-state-machine.md @@ -60,35 +60,37 @@ Cloudflare stop metadata (`exit` versus `runtime_signal`, exit code, raw hook/er ### Durable runtime lifecycle -- [ ] Add a persisted, typed recovery record to `VmAgentContainer` with phase, trigger, sanitized user disposition, bounded attempt count, interrupted agent-session identity, and detailed admin-only cause metadata. -- [ ] Serialize lifecycle check/transition and wake/restore critical sections across `await` boundaries; make duplicate `onStop`/`onError` callbacks idempotent. -- [ ] Keep idle handback as `sleeping`, explicit stop/destroy as terminal `stopped`, initial launch failure as terminal `error`, and all unexpected loss of an established runtime as generic recoverable state without calling every `runtime_signal` a rollout. -- [ ] Make stopped/error/recovery D1 rows reach the Durable Object recovery path instead of failing a dead-node precheck. -- [ ] Keep the runtime non-running until the fresh container has restored HOME, Git WIP, fresh credentials/callback tokens, and native harness session state. -- [ ] Bound recovery attempts with named environment-backed defaults. Missing, expired, corrupt, or rejected snapshots must degrade visibly and must never become transcript replay presented as a true resume. -- [ ] Catch a transport failure that crosses a request, classify recovery before a late lifecycle callback, preserve the original prompt in the transcript, and return an explicit manual-retry disposition without replaying the ambiguous request. -- [ ] Reconcile node, workspace, agent-session, ACP/chat, and active task state on recovery success and terminal degradation so neither active nor awaiting-follow-up work is stranded. +- [x] Add a persisted, typed recovery record to `VmAgentContainer` with phase, trigger, sanitized user disposition, bounded attempt count, interrupted agent-session identity, and detailed admin-only cause metadata. +- [x] Serialize lifecycle check/transition and wake/restore critical sections across `await` boundaries; make duplicate `onStop`/`onError` callbacks idempotent. +- [x] Keep idle handback as `sleeping`, explicit stop/destroy as terminal `stopped`, initial launch failure as terminal `error`, and all unexpected loss of an established runtime as generic recoverable state without calling every `runtime_signal` a rollout. +- [x] Make stopped/error/recovery D1 rows reach the Durable Object recovery path instead of failing a dead-node precheck. +- [x] Keep the runtime non-running until the fresh container has restored HOME, Git WIP, fresh credentials/callback tokens, and native harness session state. +- [x] Bound recovery attempts with named environment-backed defaults. Missing, expired, corrupt, or rejected snapshots must degrade visibly and must never become transcript replay presented as a true resume. +- [x] Catch a transport failure that crosses a request, classify recovery before a late lifecycle callback, preserve the original prompt in the transcript, and return an explicit manual-retry disposition without replaying the ambiguous request. +- [x] Reconcile node, workspace, agent-session, ACP/chat, and active task state on recovery success and terminal degradation so neither active nor awaiting-follow-up work is stranded. ### Route, service, callback, and UI behavior -- [ ] Replace the agent-session resume route's D1-only rewrite for cf-container sessions with a real bounded wake/restore call; retain the VM suspended-session behavior. -- [ ] Permit tenant-scoped recovery rows in chat resolution while preserving IDOR and destroyed-VM guards. -- [ ] Preserve the extended wake/restore timeout only for cf-container recovery paths and keep clone/create budgets from PR #1639 distinct. -- [ ] Return stable, sanitized error codes/messages for waking, degraded recovery, and ambiguous manual retry. Keep raw snapshot/container details in structured admin logs only. -- [ ] Stop swallowing follow-up delivery errors in project chat. Show waking/restoring progress, tell the user when their message is saved but needs manual retry, and expose degraded recovery without claiming the agent resumed. -- [ ] Verify fresh node-scoped and workspace-scoped callback tokens are minted on every cold runtime and that duplicate/late callbacks cannot regress a recovered or explicitly stopped session. +- [x] Replace the agent-session resume route's D1-only rewrite for cf-container sessions with a real bounded wake/restore call; retain the VM suspended-session behavior. +- [x] Permit tenant-scoped recovery rows in chat resolution while preserving IDOR and destroyed-VM guards. +- [x] Preserve the extended wake/restore timeout only for cf-container recovery paths and keep clone/create budgets from PR #1639 distinct. +- [x] Return stable, sanitized error codes/messages for waking, degraded recovery, and ambiguous manual retry. Keep raw snapshot/container details in structured admin logs only. +- [x] Stop swallowing follow-up delivery errors in project chat. Show waking/restoring progress, tell the user when their message is saved but needs manual retry, and expose degraded recovery without claiming the agent resumed. +- [x] Verify fresh node-scoped and workspace-scoped callback tokens are minted on every cold runtime and that duplicate/late callbacks cannot regress a recovered or explicitly stopped session. ### Tests -- [ ] Add Durable Object unit/Miniflare coverage for idle snapshot handback, cold wake, recovery wake, missing/expired/corrupt snapshot degradation, explicit stop, true unexpected crash, callback ordering/duplication, and terminal status reconciliation. -- [ ] Add a concurrency test proving simultaneous follow-ups launch and restore once and do not duplicate prompt delivery. -- [ ] Add a request-crossing-replacement test proving the interrupted request is classified and returned as manual retry but is never replayed. -- [ ] Add route/service tests for recovery-row resolution, cf-container resume, callback-token reinjection, sanitized regular-user responses, and task/chat status reconciliation. -- [ ] Add web tests for waking/restoring/degraded/manual-retry presentation and preserved optimistic transcript messages. +- [x] Add Durable Object unit/Miniflare coverage for idle snapshot handback, cold wake, recovery wake, missing/expired/corrupt snapshot degradation, explicit stop, true unexpected crash, callback ordering/duplication, and terminal status reconciliation. +- [x] Add a concurrency test proving simultaneous follow-ups launch and restore once and do not duplicate prompt delivery. +- [x] Add a request-crossing-replacement test proving the interrupted request is classified and returned as manual retry but is never replayed. +- [x] Add route/service tests for recovery-row resolution, cf-container resume, callback-token reinjection, sanitized regular-user responses, and task/chat status reconciliation. +- [x] Add web tests for waking/restoring/degraded/manual-retry presentation and preserved optimistic transcript messages. - [ ] Run existing Go snapshot/restore and race suites. Change Go only if a bounded VM-agent consumer or a missing primitive-level invariant is required; otherwise update runtime-neutral persistence idea `01KX4KSXEXQMP41KS34TW9EN01` with the precise VM consumer follow-up. ### Verification and delivery +Local verification gaps are tracked explicitly: the pinned Miniflare/workerd runner segfaults before importing both the new Worker tests and an unchanged worker smoke test; the workspace has no Go toolchain for the VM-agent race suite; and the live Durable Object wall-time gate currently reports two pre-existing staging alarm regressions. These are not treated as passing evidence. + - [ ] Run focused API Worker/Miniflare tests, web tests, the vm-agent Go suite/race detector, migration gates, lint, typecheck, full tests, and build. - [ ] Run `$cloudflare-specialist`, `$go-specialist` if Go changes, `$security-auditor`, `$test-engineer`, `$ui-ux-specialist`, `$constitution-validator`, `$doc-sync-validator`, and mandatory `$task-completion-validator`; address blocking findings. - [ ] Deploy the exact branch head to staging after checking for deployment contention. From 2642f51e3e9811c2192b45fd3d5e529b095b4c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Titsworth-Morin?= Date: Tue, 21 Jul 2026 14:34:09 +0000 Subject: [PATCH 3/9] fix: harden Instant recovery verification --- .../vm-agent-container-recovery.ts | 22 +- .../vm-agent-container-runtime.ts | 83 ++++++ .../src/durable-objects/vm-agent-container.ts | 133 ++++++---- .../src/routes/workspaces/agent-sessions.ts | 2 +- apps/api/src/services/node-agent.ts | 17 +- apps/api/src/services/vm-agent-container.ts | 5 +- .../vm-agent-container-launch-env.test.ts | 2 +- .../vm-agent-container-recovery.test.ts | 103 +++++++- .../vm-agent-container-wake-state.test.ts | 2 +- .../agent-session-resume-recovery.test.ts | 11 +- ...ode-agent-create-workspace-timeout.test.ts | 9 +- .../components/project-message-view/index.tsx | 6 +- .../useConnectionRecovery.ts | 2 +- .../instant-runtime-recovery-audit.spec.ts | 236 ++++++++++++++++++ 14 files changed, 549 insertions(+), 84 deletions(-) create mode 100644 apps/web/tests/playwright/instant-runtime-recovery-audit.spec.ts diff --git a/apps/api/src/durable-objects/vm-agent-container-recovery.ts b/apps/api/src/durable-objects/vm-agent-container-recovery.ts index e18c62889..18df2c7bc 100644 --- a/apps/api/src/durable-objects/vm-agent-container-recovery.ts +++ b/apps/api/src/durable-objects/vm-agent-container-recovery.ts @@ -10,7 +10,7 @@ import * as projectDataService from '../services/project-data'; export const RUNTIME_RECOVERING_MESSAGE = 'Instant session interrupted; restoring the last safe checkpoint.'; export const RUNTIME_REQUEST_INTERRUPTED_MESSAGE = - 'Your message is saved, but the Instant runtime changed while it was being sent. It was not replayed automatically. Wait for restore to finish, then send it again.'; + 'Your message is saved, but delivery was interrupted and its execution outcome is unknown. It was not replayed automatically. After restore finishes, check the transcript and partial output before deciding whether to send it again.'; export const RUNTIME_RECOVERY_DEGRADED_MESSAGE = 'The Instant session could not restore its last safe checkpoint. Your transcript and partial output are still available.'; export const RUNTIME_STOPPED_MESSAGE = 'This Instant session was stopped and cannot be resumed.'; @@ -21,6 +21,13 @@ export type RuntimeRecoveryCode = | 'RUNTIME_RECOVERY_DEGRADED' | 'RUNTIME_STOPPED'; +export function getRuntimeRecoveryMessage(code: RuntimeRecoveryCode): string { + if (code === 'RUNTIME_RECOVERING') return RUNTIME_RECOVERING_MESSAGE; + if (code === 'RUNTIME_REQUEST_INTERRUPTED') return RUNTIME_REQUEST_INTERRUPTED_MESSAGE; + if (code === 'RUNTIME_STOPPED') return RUNTIME_STOPPED_MESSAGE; + return RUNTIME_RECOVERY_DEGRADED_MESSAGE; +} + export type RuntimeRecoveryPhase = 'pending' | 'waking' | 'restoring' | 'degraded' | 'exhausted'; export type RuntimeRecoveryTrigger = 'idle' | 'stop' | 'error' | 'request'; @@ -63,6 +70,19 @@ export interface RuntimeRecoveryContext { agentType: string | null; } +export function toRuntimeRecoveryTarget( + config: { nodeId: string; workspaceId: string; projectId: string }, + context: RuntimeRecoveryContext +): RuntimeRecoveryTarget { + return { + nodeId: config.nodeId, + workspaceId: config.workspaceId, + projectId: config.projectId, + chatSessionId: context.chatSessionId, + agentSessionId: context.agentSessionId, + }; +} + const ACTIVE_TASK_STATUSES = ['in_progress', 'delegated', 'awaiting_followup'] as const; export async function loadRuntimeRecoveryContext( diff --git a/apps/api/src/durable-objects/vm-agent-container-runtime.ts b/apps/api/src/durable-objects/vm-agent-container-runtime.ts index 8565c4f0f..22a157d33 100644 --- a/apps/api/src/durable-objects/vm-agent-container-runtime.ts +++ b/apps/api/src/durable-objects/vm-agent-container-runtime.ts @@ -1,4 +1,6 @@ import type { Env } from '../env'; +import { log } from '../lib/logger'; +import { signNodeManagementToken } from '../services/jwt'; import { RUNTIME_RECOVERING_MESSAGE, RUNTIME_RECOVERY_DEGRADED_MESSAGE, @@ -59,11 +61,92 @@ export async function isMissingSessionHostResponse(response: Response): Promise< return body.toLowerCase().includes('no active agent session found'); } +const LIVE_SESSION_HOST_STATUSES = new Set(['idle', 'starting', 'ready', 'prompting']); + +export async function probeLiveRuntimeSession(input: { + env: Env; + userId: string; + nodeId: string; + workspaceId: string; + agentSessionId: string; + vmAgentPort: number; + containerFetch: (request: Request, port: number) => Promise; +}): Promise { + const { token } = await signNodeManagementToken( + input.userId, + input.nodeId, + input.workspaceId, + input.env + ); + const url = new URL( + `http://localhost:${input.vmAgentPort}/workspaces/${input.workspaceId}/agent-sessions` + ); + const response = await input.containerFetch( + new Request(url.toString(), { + headers: { + Authorization: `Bearer ${token}`, + 'X-SAM-Node-Id': input.nodeId, + 'X-SAM-Workspace-Id': input.workspaceId, + }, + }), + input.vmAgentPort + ); + if (!response.ok) { + log.warn('vm_agent_container_runtime_session_probe_failed', { + nodeId: input.nodeId, + workspaceId: input.workspaceId, + agentSessionId: input.agentSessionId, + status: response.status, + }); + throw new Error(`runtime_session_probe_http_${response.status}`); + } + const body = (await response.json()) as { + sessions?: Array<{ id?: unknown; hostStatus?: unknown }>; + }; + return Boolean( + body.sessions?.some( + (session) => + session.id === input.agentSessionId && + typeof session.hostStatus === 'string' && + LIVE_SESSION_HOST_STATUSES.has(session.hostStatus) + ) + ); +} + export function parsePositiveRuntimeSetting(raw: string | undefined, fallback: number): number { const parsed = raw ? Number.parseInt(raw, 10) : fallback; return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; } +export function resolveRuntimeSettings( + env: Env, + defaults: { + portReadyTimeoutMs: number; + activeWorkMaxMs: number; + keepaliveRenewIntervalMs: number; + recoveryMaxAttempts: number; + } +) { + return { + portReadyTimeoutMs: parsePositiveRuntimeSetting( + env.CF_CONTAINER_PORT_READY_TIMEOUT_MS || env.SANDBOX_EXEC_TIMEOUT_MS, + defaults.portReadyTimeoutMs + ), + activeWorkMaxMs: parsePositiveRuntimeSetting( + env.CF_CONTAINER_ACTIVE_WORK_MAX_MS, + defaults.activeWorkMaxMs + ), + keepaliveRenewIntervalMs: parsePositiveRuntimeSetting( + env.CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS, + defaults.keepaliveRenewIntervalMs + ), + recoveryMaxAttempts: parsePositiveRuntimeSetting( + env.CF_CONTAINER_RECOVERY_MAX_ATTEMPTS, + defaults.recoveryMaxAttempts + ), + }; +} + export async function persistRuntimeSleeping(env: Env, identity: RuntimeIdentity): Promise { const now = new Date().toISOString(); await env.DATABASE.batch([ diff --git a/apps/api/src/durable-objects/vm-agent-container.ts b/apps/api/src/durable-objects/vm-agent-container.ts index b9dbe574c..8d0436f1d 100644 --- a/apps/api/src/durable-objects/vm-agent-container.ts +++ b/apps/api/src/durable-objects/vm-agent-container.ts @@ -22,10 +22,10 @@ import { RUNTIME_STOPPED_MESSAGE, type RuntimeRecoveryCause, type RuntimeRecoveryCode, - type RuntimeRecoveryContext, type RuntimeRecoveryState, type RuntimeRecoveryTarget, type RuntimeRecoveryTrigger, + toRuntimeRecoveryTarget, } from './vm-agent-container-recovery'; import { interruptedRuntimeRequestResponse as interruptedRequestResponse, @@ -33,15 +33,17 @@ import { isMutatingRuntimeRequest as isMutatingRequest, persistRuntimeEnded, persistRuntimeSleeping, + probeLiveRuntimeSession, + resolveRuntimeSettings, runtimeRecoveryResponse as recoveryResponse, runtimeResultResponse as resultResponse, } from './vm-agent-container-runtime'; export const DEFAULT_CF_CONTAINER_SLEEP_AFTER = '1h'; +export const DEFAULT_CF_CONTAINER_PORT_READY_TIMEOUT_MS = 30_000; export const DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS = 2 * 60 * 60 * 1000; export const DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS = 5 * 60 * 1000; export const DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS = 2; - export interface VmAgentContainerLaunchConfig { nodeId: string; workspaceId: string; @@ -53,7 +55,6 @@ export interface VmAgentContainerLaunchConfig { controlPlaneUrl: string; vmAgentPort: number; } - export interface VmAgentContainerLaunchSecrets { nodeCallbackToken: string; } @@ -64,7 +65,6 @@ export interface VmAgentContainerRecoveryResult { code?: RuntimeRecoveryCode; message?: string; } - type LifecycleStatus = | 'launching' | 'running' @@ -80,7 +80,6 @@ type LifecycleStatus = const RECOVERY_STATE_KEY = 'runtimeRecovery'; const KEEPALIVE_CALLBACK = 'renewActiveWorkKeepalive'; - function stoppedRecoveryResult(): VmAgentContainerRecoveryResult { return { ok: false, @@ -202,7 +201,64 @@ export class VmAgentContainer extends Container { } } - async resumeRuntime(): Promise { + async resumeRuntime(agentSessionId?: string): Promise { + const lifecycle = await this.ctx.storage.get('lifecycleStatus'); + if (lifecycle === 'running') { + const config = await this.ctx.storage.get('launchConfig'); + const context = config + ? await loadRuntimeRecoveryContext(this.env, { + workspaceId: config.workspaceId, + preferredAgentSessionId: agentSessionId, + }) + : null; + if (!config || !context) { + return { + ok: false, + status: 'degraded', + code: 'RUNTIME_RECOVERY_DEGRADED', + message: RUNTIME_RECOVERY_DEGRADED_MESSAGE, + }; + } + try { + const live = await probeLiveRuntimeSession({ + env: this.env, + userId: context.userId, + nodeId: config.nodeId, + workspaceId: config.workspaceId, + agentSessionId: context.agentSessionId, + vmAgentPort: config.vmAgentPort, + containerFetch: (request, port) => this.containerFetch(request, port), + }); + if (live) { + const reconciled = await this.withLifecycleLock(async () => { + const current = await this.ctx.storage.get('lifecycleStatus'); + if (current !== 'running') return false; + await persistRuntimeRecovered( + this.env, + toRuntimeRecoveryTarget(config, context), + 'none' + ); + return true; + }); + if (reconciled) return { ok: true, status: 'running' }; + return this.ensureAwake(); + } + await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { kind: 'missing_session_host', httpStatus: 404 }, + promptDisposition: 'none', + }); + } catch (error) { + await this.beginUnexpectedRecovery({ + trigger: 'request', + cause: { + kind: 'transport_interrupted', + errorName: error instanceof Error ? error.name : 'unknown', + }, + promptDisposition: 'none', + }); + } + } return this.ensureAwake(); } @@ -388,7 +444,7 @@ export class VmAgentContainer extends Container { } satisfies VmAgentContainerRecoveryResult; } - if (recovery.attempts >= this.getRecoveryMaxAttempts()) { + if (recovery.attempts >= this.getRuntimeSettings().recoveryMaxAttempts) { return this.exhaustRecovery(recovery); } @@ -461,7 +517,7 @@ export class VmAgentContainer extends Container { await this.ctx.storage.put(RECOVERY_STATE_KEY, recovery); await this.ctx.storage.put('lifecycleStatus', 'recovering' satisfies LifecycleStatus); await this.clearKeepaliveSchedule(); - await persistRuntimeRecovering(this.env, this.toRecoveryTarget(config, context)); + await persistRuntimeRecovering(this.env, toRuntimeRecoveryTarget(config, context)); log.warn('vm_agent_container_recovery_started', { nodeId: config.nodeId, workspaceId: config.workspaceId, @@ -485,7 +541,7 @@ export class VmAgentContainer extends Container { preferredAgentSessionId: recovery.agentSessionId, }); if (!context) return this.degradeRecovery(recovery, 'unexpected'); - const target = this.toRecoveryTarget(config, context); + const target = toRuntimeRecoveryTarget(config, context); try { const nodeCallbackToken = await signNodeCallbackToken(config.nodeId, this.env); @@ -598,7 +654,7 @@ export class VmAgentContainer extends Container { if (!applied) return stoppedRecoveryResult(); await this.stop().catch(() => undefined); - if (degraded.attempts >= this.getRecoveryMaxAttempts()) { + if (degraded.attempts >= this.getRuntimeSettings().recoveryMaxAttempts) { return this.exhaustRecovery(degraded, target); } return { @@ -623,7 +679,7 @@ export class VmAgentContainer extends Container { workspaceId: config.workspaceId, preferredAgentSessionId: recovery.agentSessionId, }); - if (context) target = this.toRecoveryTarget(config, context); + if (context) target = toRuntimeRecoveryTarget(config, context); } const exhausted = { ...recovery, phase: 'exhausted' as const, updatedAt: Date.now() }; await this.ctx.storage.put(RECOVERY_STATE_KEY, exhausted); @@ -674,23 +730,10 @@ export class VmAgentContainer extends Container { runtime: 'cf-container', }, }, - cancellationOptions: { portReadyTimeoutMS: this.getPortReadyTimeoutMs() }, + cancellationOptions: { portReadyTimeoutMS: this.getRuntimeSettings().portReadyTimeoutMs }, }); } - private toRecoveryTarget( - config: VmAgentContainerLaunchConfig, - context: RuntimeRecoveryContext - ): RuntimeRecoveryTarget { - return { - nodeId: config.nodeId, - workspaceId: config.workspaceId, - projectId: config.projectId, - chatSessionId: context.chatSessionId, - agentSessionId: context.agentSessionId, - }; - } - private withLifecycleLock(operation: () => Promise): Promise { const run = this.lifecycleChain.then(operation); this.lifecycleChain = run.then( @@ -701,44 +744,24 @@ export class VmAgentContainer extends Container { } private activeWorkRuntime(): ActiveWorkRuntime { + const settings = this.getRuntimeSettings(); return { storage: this.ctx.storage, - activeWorkMaxMs: this.getActiveWorkMaxMs(), - renewIntervalMs: this.getKeepaliveRenewIntervalMs(), + activeWorkMaxMs: settings.activeWorkMaxMs, + renewIntervalMs: settings.keepaliveRenewIntervalMs, renewActivityTimeout: () => this.renewActivityTimeout(), replaceSchedule: (delayMs) => this.replaceKeepaliveSchedule(delayMs), clearSchedule: () => this.clearKeepaliveSchedule(), }; } - private getPortReadyTimeoutMs(): number { - const raw = this.env.CF_CONTAINER_PORT_READY_TIMEOUT_MS || this.env.SANDBOX_EXEC_TIMEOUT_MS; - const parsed = raw ? Number.parseInt(raw, 10) : 30_000; - return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000; - } - - private getActiveWorkMaxMs(): number { - const raw = this.env.CF_CONTAINER_ACTIVE_WORK_MAX_MS; - const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS; - return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS; - } - - private getKeepaliveRenewIntervalMs(): number { - const raw = this.env.CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; - const parsed = raw - ? Number.parseInt(raw, 10) - : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; - return Number.isFinite(parsed) && parsed > 0 - ? parsed - : DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS; - } - - private getRecoveryMaxAttempts(): number { - const raw = this.env.CF_CONTAINER_RECOVERY_MAX_ATTEMPTS; - const parsed = raw ? Number.parseInt(raw, 10) : DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS; - return Number.isFinite(parsed) && parsed > 0 - ? parsed - : DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS; + private getRuntimeSettings() { + return resolveRuntimeSettings(this.env, { + portReadyTimeoutMs: DEFAULT_CF_CONTAINER_PORT_READY_TIMEOUT_MS, + activeWorkMaxMs: DEFAULT_CF_CONTAINER_ACTIVE_WORK_MAX_MS, + keepaliveRenewIntervalMs: DEFAULT_CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS, + recoveryMaxAttempts: DEFAULT_CF_CONTAINER_RECOVERY_MAX_ATTEMPTS, + }); } private async replaceKeepaliveSchedule(delayMs: number): Promise { diff --git a/apps/api/src/routes/workspaces/agent-sessions.ts b/apps/api/src/routes/workspaces/agent-sessions.ts index 9bb7689ce..e8a81a8ac 100644 --- a/apps/api/src/routes/workspaces/agent-sessions.ts +++ b/apps/api/src/routes/workspaces/agent-sessions.ts @@ -439,7 +439,7 @@ agentSessionRoutes.post( node.runtime === 'cf-container' && (node.status !== 'running' || session.status !== 'running') ) { - const recovery = await resumeVmAgentContainer(c.env, node.id); + const recovery = await resumeVmAgentContainer(c.env, node.id, session.id); if (!recovery.ok) { if (!recovery.code || !recovery.message) { throw errors.internal('Instant session recovery failed.'); diff --git a/apps/api/src/services/node-agent.ts b/apps/api/src/services/node-agent.ts index 050d90e4f..33d18b111 100644 --- a/apps/api/src/services/node-agent.ts +++ b/apps/api/src/services/node-agent.ts @@ -2,7 +2,10 @@ import { eq } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/d1'; import * as schema from '../db/schema'; -import type { RuntimeRecoveryCode } from '../durable-objects/vm-agent-container-recovery'; +import { + getRuntimeRecoveryMessage, + type RuntimeRecoveryCode, +} from '../durable-objects/vm-agent-container-recovery'; import type { Env } from '../env'; import { expectJsonRecord } from '../lib/runtime-validation'; import { AppError } from '../middleware/error'; @@ -158,15 +161,11 @@ export async function nodeAgentRequest( } catch { // Non-recovery Node Agent responses retain the existing generic handling. } - if ( - recoveryPayload && - isRuntimeRecoveryCode(recoveryPayload.error) && - typeof recoveryPayload.message === 'string' - ) { + if (recoveryPayload && isRuntimeRecoveryCode(recoveryPayload.error)) { throw new NodeAgentRequestError( - response.status, + runtimeRecoveryStatus(recoveryPayload.error), recoveryPayload.error, - recoveryPayload.message + getRuntimeRecoveryMessage(recoveryPayload.error) ); } @@ -263,7 +262,7 @@ export async function fetchNodeAgent( throw new NodeAgentRequestError( runtimeRecoveryStatus(recovery.code), recovery.code, - recovery.message + getRuntimeRecoveryMessage(recovery.code) ); } throw error; diff --git a/apps/api/src/services/vm-agent-container.ts b/apps/api/src/services/vm-agent-container.ts index fc43dd79e..4977e5ba0 100644 --- a/apps/api/src/services/vm-agent-container.ts +++ b/apps/api/src/services/vm-agent-container.ts @@ -80,10 +80,11 @@ export async function fetchVmAgentContainer( export async function resumeVmAgentContainer( env: Env, - nodeId: string + nodeId: string, + agentSessionId: string ): Promise { const container = getVmAgentContainer(env, nodeId); - return container.resumeRuntime(); + return container.resumeRuntime(agentSessionId); } export async function markVmAgentContainerRequestInterrupted( diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts index c99c50edd..927cc2713 100644 --- a/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-launch-env.test.ts @@ -41,7 +41,7 @@ function makeFake(env: Record) { startAndWaitForPorts, startRuntime: (VmAgentContainer.prototype as unknown as { startRuntime: unknown }).startRuntime, clearKeepaliveSchedule: vi.fn().mockResolvedValue(undefined), - getPortReadyTimeoutMs: () => 30_000, + getRuntimeSettings: () => ({ portReadyTimeoutMs: 30_000 }), ctx: { storage: { put: vi.fn().mockResolvedValue(undefined), diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts index 67f45d32f..b1861c8e3 100644 --- a/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-recovery.test.ts @@ -58,7 +58,6 @@ type PrivateContainer = { wakeFromSnapshot: (this: unknown, recovery: RuntimeRecoveryState) => Promise; degradeRecovery: (this: unknown, ...args: unknown[]) => Promise; exhaustRecovery: (this: unknown, ...args: unknown[]) => Promise; - toRecoveryTarget: (this: unknown, ...args: unknown[]) => unknown; withLifecycleLock: (this: unknown, operation: () => Promise) => Promise; prepareForRequest: (this: unknown) => Promise; }; @@ -101,10 +100,14 @@ function makeRecoveryFake(input?: { wakeFromSnapshot: privateContainer.wakeFromSnapshot, degradeRecovery: privateContainer.degradeRecovery, exhaustRecovery: privateContainer.exhaustRecovery, - toRecoveryTarget: privateContainer.toRecoveryTarget, withLifecycleLock: privateContainer.withLifecycleLock, prepareForRequest: privateContainer.prepareForRequest, - getRecoveryMaxAttempts: () => input?.maxAttempts ?? 2, + getRuntimeSettings: () => ({ + portReadyTimeoutMs: 30_000, + activeWorkMaxMs: 2 * 60 * 60 * 1000, + keepaliveRenewIntervalMs: 5 * 60 * 1000, + recoveryMaxAttempts: input?.maxAttempts ?? 2, + }), clearKeepaliveSchedule: vi.fn().mockResolvedValue(undefined), markActiveWorkEnded: vi.fn().mockResolvedValue(undefined), startRuntime: vi.fn().mockResolvedValue(undefined), @@ -128,6 +131,17 @@ async function callEnsureAwake(fake: unknown) { }>; } +async function callResumeRuntime(fake: unknown) { + return ( + VmAgentContainer.prototype as unknown as { + resumeRuntime: ( + this: unknown, + agentSessionId?: string + ) => Promise<{ ok: boolean; status: string; code?: string }>; + } + ).resumeRuntime.call(fake, 'agent-session-1'); +} + function callProxyHttp(fake: unknown, request: Request): Promise { return ( VmAgentContainer.prototype as unknown as { @@ -148,6 +162,89 @@ beforeEach(() => { }); describe('VmAgentContainer snapshot recovery state machine', () => { + it('reconciles stale D1 state only after proving the target SessionHost is live', async () => { + const { fake } = makeRecoveryFake({ lifecycle: 'running' }); + fake.containerFetch.mockResolvedValueOnce( + Response.json({ + sessions: [{ id: 'agent-session-1', status: 'running', hostStatus: 'ready' }], + }) + ); + + const result = await callResumeRuntime(fake); + + expect(result).toEqual({ ok: true, status: 'running' }); + expect(recoveryMocks.loadContext).toHaveBeenCalledWith(fake.env, { + workspaceId: 'workspace-1', + preferredAgentSessionId: 'agent-session-1', + }); + expect(recoveryMocks.signNodeManagementToken).toHaveBeenCalledWith( + 'user-1', + 'node-1', + 'workspace-1', + fake.env + ); + const probeRequest = fake.containerFetch.mock.calls[0]?.[0] as Request; + expect(probeRequest.headers.get('Authorization')).toBe('Bearer management-token'); + expect(probeRequest.headers.get('X-SAM-Workspace-Id')).toBe('workspace-1'); + expect(fake.startRuntime).not.toHaveBeenCalled(); + expect(recoveryMocks.persistRecovered).toHaveBeenCalledWith( + fake.env, + expect.objectContaining({ agentSessionId: 'agent-session-1' }), + 'none' + ); + }); + + it('restores instead of falsely resuming when D1 says recovery but SessionHost is missing', async () => { + const { fake, values } = makeRecoveryFake({ lifecycle: 'running' }); + fake.containerFetch + .mockResolvedValueOnce( + Response.json({ sessions: [{ id: 'agent-session-1', status: 'running' }] }) + ) + .mockResolvedValueOnce( + Response.json({ status: 'restored', degradation: 'none', skipped: [] }) + ); + + const result = await callResumeRuntime(fake); + + expect(result).toEqual({ ok: true, status: 'running' }); + expect(fake.startRuntime).toHaveBeenCalledTimes(1); + expect(recoveryMocks.persistRecovering).toHaveBeenCalledTimes(1); + expect(values.has('runtimeRecovery')).toBe(false); + }); + + it('does not reconcile running when an explicit stop crosses the SessionHost probe', async () => { + let finishProbe!: (response: Response) => void; + const { fake, values } = makeRecoveryFake({ lifecycle: 'running' }); + fake.containerFetch.mockImplementationOnce( + () => + new Promise((resolve) => { + finishProbe = resolve; + }) + ); + const stopForUser = ( + VmAgentContainer.prototype as unknown as { + stopForUser: (this: unknown) => Promise; + } + ).stopForUser; + + const resume = callResumeRuntime(fake); + await vi.waitFor(() => expect(fake.containerFetch).toHaveBeenCalledTimes(1)); + await stopForUser.call(fake); + finishProbe( + Response.json({ + sessions: [{ id: 'agent-session-1', status: 'running', hostStatus: 'ready' }], + }) + ); + + await expect(resume).resolves.toMatchObject({ + ok: false, + status: 'stopped', + code: 'RUNTIME_STOPPED', + }); + expect(recoveryMocks.persistRecovered).not.toHaveBeenCalled(); + expect(values.get('lifecycleStatus')).toBe('stopping'); + }); + it('cold-wakes, reinjects fresh callback tokens, restores, then reconciles running', async () => { const { fake, values } = makeRecoveryFake(); diff --git a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts index 7bdf15cf2..b44b7a8be 100644 --- a/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts +++ b/apps/api/tests/unit/durable-objects/vm-agent-container-wake-state.test.ts @@ -143,7 +143,7 @@ describe('VmAgentContainer cold-wake serialization', () => { ensureAwake: privateContainer.ensureAwake, resultResponse: privateContainer.resultResponse, interruptedRequestResponse: privateContainer.interruptedRequestResponse, - getRecoveryMaxAttempts: () => 2, + getRuntimeSettings: () => ({ recoveryMaxAttempts: 2 }), wakeFromSnapshot, beginUnexpectedRecovery: vi.fn(), getState: vi.fn(async () => ({ diff --git a/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts b/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts index 0a315f0f3..bd2140fdb 100644 --- a/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts +++ b/apps/api/tests/unit/routes/agent-session-resume-recovery.test.ts @@ -135,7 +135,8 @@ describe('agent session Instant recovery route', () => { await vi.waitFor(() => expect(mocks.resumeVmAgentContainer).toHaveBeenCalledWith( expect.objectContaining({ DATABASE: {} }), - 'node-1' + 'node-1', + 'agent-session-1' ) ); expect(mocks.updateSet).not.toHaveBeenCalled(); @@ -158,9 +159,9 @@ describe('agent session Instant recovery route', () => { ok: false, status: 'degraded', degraded: true, - code: 'SNAPSHOT_RESTORE_FAILED', + code: 'RUNTIME_RECOVERY_DEGRADED', message: - 'The Instant session could not restore its saved state. Start a new session to continue.', + 'The Instant session could not restore its last safe checkpoint. Your transcript and partial output are still available.', }); const app = await createTestApp(); @@ -172,9 +173,9 @@ describe('agent session Instant recovery route', () => { const body = await response.json(); expect(body).toEqual({ - error: 'SNAPSHOT_RESTORE_FAILED', + error: 'RUNTIME_RECOVERY_DEGRADED', message: - 'The Instant session could not restore its saved state. Start a new session to continue.', + 'The Instant session could not restore its last safe checkpoint. Your transcript and partial output are still available.', }); expect(mocks.updateSet).not.toHaveBeenCalled(); }); diff --git a/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts b/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts index 9cfb8220d..0a4aa54aa 100644 --- a/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts +++ b/apps/api/tests/unit/services/node-agent-create-workspace-timeout.test.ts @@ -156,7 +156,7 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { ok: false, status: 'recovering', code: 'RUNTIME_REQUEST_INTERRUPTED', - message: 'Your message is saved, but it was not replayed automatically.', + message: 'internal transport detail: bearer should-not-leak', }); const promptPromise = sendPromptToAgentOnNode( @@ -176,6 +176,7 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { expect(error).toMatchObject({ statusCode: 409, error: 'RUNTIME_REQUEST_INTERRUPTED', + message: expect.stringContaining('execution outcome is unknown'), }); expect(mocks.container.fetchVmAgentContainer).toHaveBeenCalledTimes(1); expect(mocks.container.markVmAgentContainerRequestInterrupted).toHaveBeenCalledWith( @@ -190,9 +191,9 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { Response.json( { error: 'RUNTIME_REQUEST_INTERRUPTED', - message: 'Your message is saved, but it was not replayed automatically.', + message: 'internal transport detail: bearer should-not-leak', }, - { status: 409 } + { status: 500 } ) ); @@ -209,7 +210,9 @@ describe('createWorkspaceOnNode cf-container timeout plumbing', () => { expect(error).toMatchObject({ statusCode: 409, error: 'RUNTIME_REQUEST_INTERRUPTED', + message: expect.stringContaining('execution outcome is unknown'), }); + expect(error.message).not.toContain('bearer should-not-leak'); expect(mocks.container.fetchVmAgentContainer).toHaveBeenCalledTimes(1); expect(mocks.container.markVmAgentContainerRequestInterrupted).not.toHaveBeenCalled(); }); diff --git a/apps/web/src/components/project-message-view/index.tsx b/apps/web/src/components/project-message-view/index.tsx index e34e6bbcd..8ff455966 100644 --- a/apps/web/src/components/project-message-view/index.tsx +++ b/apps/web/src/components/project-message-view/index.tsx @@ -469,10 +469,12 @@ export const ProjectMessageView: FC = ({ role="alert" className="flex items-center gap-2 px-4 py-2 bg-danger-tint border-b border-border-default text-danger text-xs" > - {lc.resumeError} + + {lc.resumeError} +