fix: recover Instant sessions across runtime loss

.claude/rules/02-quality-gates.md

@@ -91,6 +91,7 @@ Ask: "What test, if it existed before the breaking change was introduced, would
 - **If the bug was a missing propagation** (value set in A but never forwarded to B), write a test that constructs the real lifecycle (A then B) and asserts the value arrives.
 - **If the bug involves streamed UI data that is later reconstructed from durable storage**, write a parity regression test for the persisted representation, not only the live stream. The test MUST include a partial/status-only update event and assert omitted fields do not clear previously visible metadata. See the retained incident lesson in this rule.
 - **If the bug involves lifecycle control across a runtime boundary** (agent/session/workspace/node stop, cancel, retry, replacement, suspend, or resume), the regression test MUST assert the runtime command is invoked before accepting the terminal state or dispatching replacement work. Database state changes and successful JSON responses are insufficient; the test must prove the external agent/node/workspace control side effect.
+- **If runtime liveness is represented in more than one control plane** (for example D1, a session Durable Object, and a runtime Durable Object), timeout and cleanup tests MUST cross those boundaries with deliberately stale secondary state. A heartbeat timeout or sweep may terminalize work only after the runtime owner reports a conclusively terminal lifecycle; sleep, wake, restore, replacement, probe failure, and unknown state are inconclusive. Use one shared lifecycle classifier for every cleanup path so a stale replica cannot strand recoverable work.
 - **If the bug involves shell or process execution lifecycle** (process groups, child processes, cancellation, timeout, or cleanup after command completion), the regression test MUST cover the success path as well as failure/cancellation paths and prove spawned children are not left alive after the tool or command returns.
 - **If the bug involves a utility LLM call through a provider-compatible API**, the regression test MUST assert the exact provider payload controls that make the response contract reliable, not just the returned parsed text. For reasoning-capable models, this includes any explicit thinking/reasoning-disable parameters or response-format controls required for the utility to receive text in the field it reads.
 - **For utility-model defaults and capability changes**, validate the exact production-shaped payload against the real provider before merge, record the accepted and rejected field matrix without prompts or credentials, and test omission of redundant nullable fields. Non-2xx handling must preserve bounded sanitized provider codes/parameter names so later schema drift is diagnosable without logging raw response bodies.

.claude/skills/env-reference/SKILL.md

@@ -63,10 +63,16 @@ See `apps/api/.env.example` for the full list. Key variables:
 - `WRANGLER_PORT` — Local dev port (default: 8787)
 - `BASE_DOMAIN` — Set automatically by sync scripts
 - `CF_CONTAINER_ENABLED` — Enables Cloudflare Container instant-session runtime in generated deployment envs (default: `true`; set `false` to force VM runtime)
-- `CF_CONTAINER_SLEEP_AFTER` — Container idle sleep duration for instant-session runtime (default: `10m`)
+- `CF_CONTAINER_SLEEP_AFTER` — Container idle sleep duration for instant-session runtime (default: `1h`)
+- `CF_CONTAINER_ACTIVE_WORK_MAX_MS` — Defensive maximum active-work keepalive duration (default: `7200000`)
+- `CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS` — Active-work keepalive renewal interval (default: `300000`)
 - `CF_CONTAINER_VM_AGENT_PORT` — vm-agent standalone HTTP port inside the raw container (default: `8080`)
 - `CF_CONTAINER_PORT_READY_TIMEOUT_MS` — Max wait for vm-agent port readiness (default: `30000`)
-- `$1
+- `CF_CONTAINER_WAKE_TIMEOUT_MS` — Max wait for launch, snapshot restore, and request readiness (default: `120000`)
+- `CF_CONTAINER_RECOVERY_MAX_ATTEMPTS` — Max restore attempts before terminal status reconciliation (default: `2`)
+- `INSTANT_STALE_CALLBACK_MARGIN_MS` — Freshness margin for rejecting destructive callbacks from superseded Instant containers (default: `60000`)
+- `CF_CONTAINER_CREATE_WORKSPACE_TIMEOUT_MS` — Synchronous workspace creation and clone budget (default: `120000`)
+- `CF_CONTAINER_CLONE_FILTER` — Git partial-clone filter (default: `blob:none`; `off` disables partial clone)
 - `SESSION_SNAPSHOT_TTL_DAYS` — Retention for hibernated session snapshots; deployment also provisions matching R2 prefix expiration (default: `7`)
 - `SESSION_SNAPSHOT_TOTAL_BUDGET_BYTES` — Maximum bytes accepted for one snapshot artifact (default: `104857600`)
 - `SESSION_SNAPSHOT_ENTRY_THRESHOLD_BYTES` — Per-file threshold before snapshot content is visibly skipped (default: `52428800`)

apps/api/.env.example

@@ -74,9 +74,19 @@ BASE_DOMAIN=workspaces.example.com
 # Generated deployment environments default this to true. Set false to force
 # profile/runtime resolution back to cloud VMs.
 # CF_CONTAINER_ENABLED=true
-# CF_CONTAINER_SLEEP_AFTER=10m
+# CF_CONTAINER_SLEEP_AFTER=1h
+# Maximum active-work keepalive lifetime and renewal interval.
+# CF_CONTAINER_ACTIVE_WORK_MAX_MS=7200000
+# CF_CONTAINER_KEEPALIVE_RENEW_INTERVAL_MS=300000
 # CF_CONTAINER_VM_AGENT_PORT=8080
 # CF_CONTAINER_PORT_READY_TIMEOUT_MS=30000
+# Budget for a sleeping/recovering runtime to launch, restore, and accept work.
+# CF_CONTAINER_WAKE_TIMEOUT_MS=120000
+# Maximum snapshot restore attempts before terminal status reconciliation.
+# CF_CONTAINER_RECOVERY_MAX_ATTEMPTS=2
+# Freshness margin for rejecting destructive (error/failed) callbacks from a
+# superseded Instant container generation after a completed recovery.
+# INSTANT_STALE_CALLBACK_MARGIN_MS=60000
 # CF_CONTAINER_WORKSPACE_BASE_DIR=/workspaces
 # Budget for the synchronous standalone create-workspace request (includes the
 # repository clone inside the container). Default 120000 (120s).

apps/api/src/db/migrations/0099_tasks_workspace_id_index.sql

@@ -0,0 +1,8 @@
+-- Index tasks by workspace_id for the mass-outage Instant recovery hot path.
+-- persistRuntimeRecoveryFailed and other workspace-scoped task lookups run
+-- `SELECT ... FROM tasks WHERE workspace_id = ? AND status IN (...)`, which
+-- previously required a full table scan. Partial (workspace_id IS NOT NULL)
+-- because most tasks never bind a workspace; the predicate still serves every
+-- `workspace_id = ?` lookup since the bound value is always non-null.
+CREATE INDEX IF NOT EXISTS idx_tasks_workspace_id
+  ON tasks(workspace_id) WHERE workspace_id IS NOT NULL;

apps/api/src/db/schema.ts

@@ -909,6 +909,12 @@ export const tasks = sqliteTable(
     chatSessionIdUnique: uniqueIndex('idx_tasks_chat_session_id_unique')
       .on(table.chatSessionId)
       .where(sql`chat_session_id IS NOT NULL`),
+    // Supports the `WHERE workspace_id = ? AND status IN (...)` lookups on the
+    // mass-outage recovery hot path (persistRuntimeRecoveryFailed) and other
+    // workspace-scoped task queries. Partial: most tasks never bind a workspace.
+    workspaceIdIdx: index('idx_tasks_workspace_id')
+      .on(table.workspaceId)
+      .where(sql`workspace_id IS NOT NULL`),
     triggerExecutionIdIdx: index('idx_tasks_trigger_execution_id')
       .on(table.triggerExecutionId)
       .where(sql`trigger_execution_id IS NOT NULL`),

apps/api/src/durable-objects/project-data/acp-sessions.ts

@@ -396,7 +396,16 @@ export function checkHeartbeatTimeouts(
     reason?: string | null;
     errorMessage?: string;
     metadata?: Record<string, unknown> | null;
-  }) => Promise<void>
+  }) => Promise<unknown>,
+  options: {
+    shouldDeferTimeout?: (session: {
+      id: string;
+      chatSessionId: string;
+      workspaceId: string | null;
+      nodeId: string | null;
+      lastHeartbeatAt: number | null;
+    }) => Promise<{ defer: boolean; reason: string }>;
+  } = {}
 ): Promise<Array<{ sessionId: string; workspaceId: string | null }>> {
   const detectionWindow = parseInt(
     env.ACP_SESSION_DETECTION_WINDOW_MS || String(ACP_SESSION_DEFAULTS.DETECTION_WINDOW_MS),
@@ -419,6 +428,28 @@ export function checkHeartbeatTimeouts(
   const timedOut: Array<{ sessionId: string; workspaceId: string | null }> = [];
   const promises = staleSessions.map(async (session) => {
     try {
+      if (options.shouldDeferTimeout) {
+        try {
+          const decision = await options.shouldDeferTimeout(session);
+          if (decision.defer) {
+            log.info('acp_session.heartbeat_timeout_deferred', {
+              sessionId: session.id,
+              workspaceId: session.workspaceId,
+              nodeId: session.nodeId,
+              reason: decision.reason,
+            });
+            return;
+          }
+        } catch (err) {
+          log.warn('acp_session.heartbeat_timeout_policy_failed', {
+            sessionId: session.id,
+            workspaceId: session.workspaceId,
+            nodeId: session.nodeId,
+            error: err instanceof Error ? err.message : String(err),
+          });
+          return;
+        }
+      }
       await transitionFn(session.id, 'interrupted', {
         actorType: 'alarm', reason: 'Heartbeat timeout exceeded detection window',
         errorMessage: `Heartbeat timeout: last heartbeat at ${session.lastHeartbeatAt}, cutoff was ${cutoff}`,
@@ -476,7 +507,8 @@ export function computeHeartbeatAlarmTime(sql: SqlStorage, env: Env): number | n
   const earliestHeartbeat = parseMinEarliest(earliestRow, 'acp_sessions.earliest_heartbeat');
   if (earliestHeartbeat === null) return null;
 
-  return earliestHeartbeat + detectionWindow;
+  const dueAt = earliestHeartbeat + detectionWindow;
+  return dueAt <= Date.now() ? Date.now() + detectionWindow : dueAt;
 }
 
 /**

apps/api/src/durable-objects/project-data/index.ts

@@ -30,6 +30,7 @@ import * as missionState from './missions';
 import * as policies from './policies';
 import * as reconciliation from './reconciliation';
 import { parseCountCnt, parseMaxLatest, parseMetaValue } from './row-schemas';
+import { checkRuntimeHeartbeatTimeouts } from './runtime-heartbeat-policy';
 import * as sessionState from './session-state';
 import * as sessionSummarySync from './session-summary-sync';
 import * as sessions from './sessions';
@@ -341,9 +342,8 @@ export class ProjectData extends DurableObject<Env> {
   // --- DO Alarm Handler ---
 
   async alarm(): Promise<void> {
-    const timedOut = await acpSessions.checkHeartbeatTimeouts(this.sql, this.env, async (sessionId, toStatus, opts) => {
-      await this.transitionAcpSession(sessionId, toStatus, opts);
-    });
+    const timedOut = await checkRuntimeHeartbeatTimeouts(
+      this.sql, this.env, this.transitionAcpSession.bind(this));
 
     // For conversation-mode sessions, couple agent death to workspace death.
     // Stop workspaces whose ACP sessions timed out to prevent zombie state.

apps/api/src/durable-objects/project-data/runtime-heartbeat-policy.ts

@@ -0,0 +1,79 @@
+import { DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS } from '@simple-agent-manager/shared';
+
+import type { VmAgentContainer } from '../vm-agent-container';
+import { isVmAgentContainerLifecycleTerminal } from '../vm-agent-container-lifecycle';
+import { checkHeartbeatTimeouts } from './acp-sessions';
+import type { Env } from './types';
+
+interface HeartbeatTimeoutCandidate {
+  workspaceId: string | null;
+  nodeId: string | null;
+}
+
+const TERMINAL_WORKSPACE_STATUSES = new Set(['stopping', 'stopped', 'error', 'deleted']);
+
+export function checkRuntimeHeartbeatTimeouts(
+  sql: SqlStorage,
+  env: Env,
+  transitionFn: Parameters<typeof checkHeartbeatTimeouts>[2]
+) {
+  return checkHeartbeatTimeouts(sql, env, transitionFn, {
+    shouldDeferTimeout: (session) => shouldDeferRuntimeHeartbeatTimeout(env, session),
+  });
+}
+
+function probeTimeoutMs(env: Env): number {
+  const configured = Number.parseInt(env.TASK_LIVENESS_PROBE_TIMEOUT_MS ?? '', 10);
+  return Number.isFinite(configured) && configured > 0
+    ? configured
+    : DEFAULT_TASK_LIVENESS_PROBE_TIMEOUT_MS;
+}
+
+export async function shouldDeferRuntimeHeartbeatTimeout(
+  env: Env,
+  candidate: HeartbeatTimeoutCandidate
+): Promise<{ defer: boolean; reason: string }> {
+  if (!candidate.workspaceId || !candidate.nodeId) {
+    return { defer: false, reason: 'runtime_identity_incomplete' };
+  }
+
+  const row = await env.DATABASE.prepare(
+    `SELECT w.status AS workspace_status, n.runtime AS node_runtime
+     FROM workspaces w
+     LEFT JOIN nodes n ON n.id = w.node_id
+     WHERE w.id = ? AND w.node_id = ?
+     LIMIT 1`
+  )
+    .bind(candidate.workspaceId, candidate.nodeId)
+    .first<{ workspace_status: string; node_runtime: string | null }>();
+  if (!row || row.node_runtime !== 'cf-container') {
+    return { defer: false, reason: row ? 'non_container_runtime' : 'workspace_missing' };
+  }
+  if (TERMINAL_WORKSPACE_STATUSES.has(row.workspace_status)) {
+    return { defer: false, reason: `workspace_${row.workspace_status}` };
+  }
+  if (!env.VM_AGENT_CONTAINER) {
+    return { defer: true, reason: 'cf_container_lifecycle_binding_unavailable' };
+  }
+
+  const id = env.VM_AGENT_CONTAINER.idFromName(candidate.nodeId.toLowerCase());
+  const stub = env.VM_AGENT_CONTAINER.get(id) as DurableObjectStub<VmAgentContainer>;
+  const timeoutMs = probeTimeoutMs(env);
+  const timeout = Symbol('container_lifecycle_probe_timeout');
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const lifecycle = await Promise.race([
+    stub.inspectLifecycle(),
+    new Promise<typeof timeout>((resolve) => {
+      timer = setTimeout(() => resolve(timeout), timeoutMs);
+    }),
+  ]).finally(() => {
+    if (timer) clearTimeout(timer);
+  });
+  if (lifecycle === timeout) {
+    return { defer: true, reason: 'cf_container_lifecycle_timeout' };
+  }
+  return {
+    defer: !isVmAgentContainerLifecycleTerminal(lifecycle.status),
+    reason: `cf_container_${lifecycle.status ?? 'unknown'}`,
+  };
+}

apps/api/src/durable-objects/project-data/types.ts

@@ -2,8 +2,12 @@
  * Shared types and utilities for ProjectData DO modules.
  */
 
+import type { VmAgentContainer } from '../vm-agent-container';
+
 export type Env = {
   DATABASE: D1Database;
+  VM_AGENT_CONTAINER?: DurableObjectNamespace<VmAgentContainer>;
+  TASK_LIVENESS_PROBE_TIMEOUT_MS?: string;
   BASE_DOMAIN?: string;
   DO_SUMMARY_SYNC_DEBOUNCE_MS?: string;
   MAX_SESSIONS_PER_PROJECT?: string;

apps/api/src/durable-objects/vm-agent-container-active-work.ts

@@ -0,0 +1,120 @@
+import { log } from '../lib/logger';
+
+export type ActiveWorkStatus = 'active' | 'ended' | 'expired';
+
+export interface ActiveWorkState {
+  status: ActiveWorkStatus;
+  nodeId: string;
+  workspaceId: string;
+  agentSessionId: string;
+  reason: string;
+  activeSince: number;
+  lastRenewedAt: number;
+  deadlineAt: number;
+  endedAt?: number;
+  endReason?: string;
+}
+
+export interface ActiveWorkRuntime {
+  storage: DurableObjectStorage;
+  activeWorkMaxMs: number;
+  renewIntervalMs: number;
+  renewActivityTimeout: () => void;
+  replaceSchedule: (delayMs: number) => Promise<void>;
+  clearSchedule: () => Promise<void>;
+}
+
+export const ACTIVE_WORK_KEY = 'activeWork';
+
+// Concurrency note: these read-check-write helpers on ACTIVE_WORK_KEY are
+// deliberately NOT serialized under the DO's lifecycleChain/wakeChain mutexes.
+// Active-work state is advisory keepalive bookkeeping (it only extends the
+// container's idle deadline), so a lost interleaving self-heals on the next
+// keepalive tick or prompt: the worst case is one extra/short renewal window,
+// never an incorrect lifecycle transition. Folding these under the lifecycle
+// mutex would needlessly serialize the frequent keepalive schedule against
+// stop/wake/recovery critical sections. The authoritative lifecycle writes
+// (lifecycleStatus, runtimeRecovery) remain mutex-protected in the DO.
+
+export async function startActiveWork(
+  runtime: ActiveWorkRuntime,
+  nodeId: string,
+  input: { workspaceId: string; agentSessionId: string; reason: string }
+): Promise<void> {
+  const now = Date.now();
+  const activeWork: ActiveWorkState = {
+    status: 'active',
+    nodeId,
+    workspaceId: input.workspaceId,
+    agentSessionId: input.agentSessionId,
+    reason: input.reason,
+    activeSince: now,
+    lastRenewedAt: now,
+    deadlineAt: now + runtime.activeWorkMaxMs,
+  };
+  runtime.renewActivityTimeout();
+  await runtime.storage.put(ACTIVE_WORK_KEY, activeWork);
+  await runtime.replaceSchedule(runtime.renewIntervalMs);
+  log.info('vm_agent_container_active_work_started', {
+    nodeId,
+    workspaceId: input.workspaceId,
+    agentSessionId: input.agentSessionId,
+    reason: input.reason,
+    activeSince: new Date(now).toISOString(),
+    deadlineAt: new Date(activeWork.deadlineAt).toISOString(),
+  });
+}
+
+export async function endActiveWork(runtime: ActiveWorkRuntime, reason: string): Promise<void> {
+  const activeWork = await runtime.storage.get<ActiveWorkState>(ACTIVE_WORK_KEY);
+  if (!activeWork || activeWork.status !== 'active') {
+    await runtime.clearSchedule();
+    return;
+  }
+  const now = Date.now();
+  await runtime.storage.put(ACTIVE_WORK_KEY, {
+    ...activeWork,
+    status: 'ended',
+    endedAt: now,
+    endReason: reason,
+  } satisfies ActiveWorkState);
+  await runtime.clearSchedule();
+  log.info('vm_agent_container_active_work_ended', {
+    nodeId: activeWork.nodeId,
+    workspaceId: activeWork.workspaceId,
+    agentSessionId: activeWork.agentSessionId,
+    reason,
+    activeSince: new Date(activeWork.activeSince).toISOString(),
+    lastRenewedAt: new Date(activeWork.lastRenewedAt).toISOString(),
+    endedAt: new Date(now).toISOString(),
+  });
+}
+
+export async function renewActiveWork(runtime: ActiveWorkRuntime): Promise<void> {
+  const activeWork = await runtime.storage.get<ActiveWorkState>(ACTIVE_WORK_KEY);
+  if (!activeWork || activeWork.status !== 'active') {
+    await runtime.clearSchedule();
+    return;
+  }
+  const now = Date.now();
+  if (now >= activeWork.deadlineAt) {
+    await runtime.storage.put(ACTIVE_WORK_KEY, {
+      ...activeWork,
+      status: 'expired',
+      endedAt: now,
+      endReason: 'keepalive_deadline_exceeded',
+    } satisfies ActiveWorkState);
+    await runtime.clearSchedule();
+    log.warn('vm_agent_container_active_work_keepalive_expired', {
+      nodeId: activeWork.nodeId,
+      workspaceId: activeWork.workspaceId,
+      agentSessionId: activeWork.agentSessionId,
+      deadlineAt: new Date(activeWork.deadlineAt).toISOString(),
+    });
+    return;
+  }
+
+  runtime.renewActivityTimeout();
+  await runtime.storage.put(ACTIVE_WORK_KEY, { ...activeWork, lastRenewedAt: now });
+  await runtime.replaceSchedule(runtime.renewIntervalMs);
+}