fix: preserve Durable Object migration compatibility

.claude/rules/07-env-and-urls.md

@@ -57,18 +57,29 @@ Examples:
 Environment-specific sections (`[env.staging]`, `[env.production]`) are NOT checked into the repository. They are generated dynamically at deploy time by `scripts/deploy/sync-wrangler-config.ts`, which:
 
 1. Reads Pulumi stack outputs for dynamic bindings (D1 IDs, KV IDs, R2 bucket names)
-2. Copies static bindings from the top-level config (Durable Objects, AI, migrations)
+2. Copies static Durable Object and AI bindings, then resolves Durable Object migrations against the target Worker's deployed tag
 3. Derives worker names from `DEPLOYMENT_CONFIG` in `scripts/deploy/config.ts`
 4. Conditionally adds `tail_consumers` (only if the tail worker already exists)
 
 ### Required action when adding a new binding
 
 Add the binding to the **top-level section of `wrangler.toml` only**. The sync script handles the rest.
 
-- **Static bindings** (Durable Objects, AI, migrations): Copied verbatim from top-level to generated env sections.
+- **Static bindings** (Durable Objects, AI): Copied verbatim from top-level to generated env sections.
+- **Durable Object migrations**: The applied prefix is copied verbatim; only pending legacy `new_classes` creates are emitted as `new_sqlite_classes`.
 - **Dynamic bindings** (D1, KV, R2): Generated from Pulumi outputs with correct resource IDs per environment.
 - **Derived bindings** (worker name, routes, tail_consumers): Computed from `DEPLOYMENT_CONFIG` naming conventions.
 
+### Durable Object migration safety
+
+- Applied migration tags are immutable history. Never rewrite an applied `new_classes`, rename, delete, or transfer entry to change storage or behavior.
+- New Durable Object namespaces MUST use `new_sqlite_classes`.
+- The sync script MUST confirm whether the target Worker is absent or read its deployed `migration_tag` before generating an environment.
+- A missing, unreadable, duplicated, or unknown migration tag MUST fail the deployment preflight. Never assume an ambiguous Worker is a clean install; Wrangler can otherwise submit the full local history. The probe retries transient failures a bounded number of times (`DO_MIGRATION_STATE_PROBE_ATTEMPTS`, `DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS`) before failing closed.
+- The `[[migrations]]` array is append-only with sequential `v1..vN` tags. The resolver (and Wrangler) treat array position relative to the deployed tag as the applied/pending boundary, so inserting or reordering entries silently corrupts that boundary. The compatibility test suite enforces the sequence and runs in the `Validate Deploy Scripts` CI job (`scripts/quality/do-migration-compatibility.test.ts`).
+- Wrangler resolves `migrations` via its `inheritable()` config path: if `env.*.migrations` were ever omitted, Wrangler silently falls back to the top-level (legacy) array with NO "not inherited by environments" warning — the deploy grep-guard does not protect this field. The generated-environment test pins that `env.*.migrations` is always emitted.
+- Test both a clean bootstrap and an existing deployment at the latest historical tag whenever migration generation changes. Miniflare alone does not exercise the remote migration contract.
+
 The CI quality check (`pnpm quality:wrangler-bindings`) verifies:
 
 1. No `[env.*]` sections exist in checked-in `wrangler.toml` files

.github/workflows/ci.yml

@@ -311,6 +311,11 @@ jobs:
           npx tsx --check scripts/deploy/sync-wrangler-config.ts
           npx tsx --check scripts/deploy/generate-keys.ts
 
+      # Deploy/quality script behavior — includes the Durable Object migration
+      # compatibility regression suite (clean install vs applied history).
+      - name: Run deploy and quality script tests
+        run: pnpm quality:scripts:test
+
       - name: Validate wrangler binding consistency
         run: pnpm quality:wrangler-bindings
 

.github/workflows/deploy-reusable.yml

@@ -357,6 +357,8 @@ jobs:
           REQUIRE_APPROVAL: ${{ vars.REQUIRE_APPROVAL }}
           HETZNER_BASE_IMAGE: ${{ vars.HETZNER_BASE_IMAGE }}
           ARTIFACTS_BINDING_ENABLED: ${{ vars.ARTIFACTS_BINDING_ENABLED }}
+          DO_MIGRATION_STATE_PROBE_ATTEMPTS: ${{ vars.DO_MIGRATION_STATE_PROBE_ATTEMPTS }}
+          DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS: ${{ vars.DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS }}
           CF_CONTAINER_ENABLED: ${{ vars.CF_CONTAINER_ENABLED }}
           CF_CONTAINER_SLEEP_AFTER: ${{ vars.CF_CONTAINER_SLEEP_AFTER }}
           CF_CONTAINER_PORT_READY_TIMEOUT_MS: ${{ vars.CF_CONTAINER_PORT_READY_TIMEOUT_MS }}
@@ -668,6 +670,8 @@ jobs:
           REQUIRE_APPROVAL: ${{ vars.REQUIRE_APPROVAL }}
           HETZNER_BASE_IMAGE: ${{ vars.HETZNER_BASE_IMAGE }}
           ARTIFACTS_BINDING_ENABLED: ${{ vars.ARTIFACTS_BINDING_ENABLED }}
+          DO_MIGRATION_STATE_PROBE_ATTEMPTS: ${{ vars.DO_MIGRATION_STATE_PROBE_ATTEMPTS }}
+          DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS: ${{ vars.DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS }}
 
       - name: Re-deploy API Worker (with tail_consumers)
         if: ${{ inputs.dry_run != true && steps.first_deploy.outputs.is_first == 'true' }}

CLAUDE.md

@@ -145,7 +145,7 @@ Full env var reference: use the `env-reference` skill or see `apps/api/.env.exam
 
 ## Wrangler Binding Rule (CRITICAL)
 
-Environment-specific `[env.*]` sections are NOT checked into the repository. They are generated at deploy time by `scripts/deploy/sync-wrangler-config.ts` from Pulumi outputs + the top-level config. When adding ANY new binding to `wrangler.toml`, add it to the **top-level section only**. The sync script copies static bindings (Durable Objects, AI, migrations) and generates dynamic bindings (D1, KV, R2, worker name, routes, tail_consumers) automatically. The CI quality check (`pnpm quality:wrangler-bindings`) verifies that no env sections are committed and that required binding types are present at the top level. See `.claude/rules/07-env-and-urls.md` for details.
+Environment-specific `[env.*]` sections are NOT checked into the repository. They are generated at deploy time by `scripts/deploy/sync-wrangler-config.ts` from Pulumi outputs + the top-level config. When adding ANY new binding to `wrangler.toml`, add it to the **top-level section only**. The sync script copies static bindings (Durable Objects, AI), resolves Durable Object migrations against the target Worker's deployed migration tag, and generates dynamic bindings (D1, KV, R2, worker name, routes, tail_consumers) automatically. The CI quality check (`pnpm quality:wrangler-bindings`) verifies that no env sections are committed and that required binding types are present at the top level. See `.claude/rules/07-env-and-urls.md` for details.
 
 ## Architecture Principles
 

apps/www/src/content/docs/docs/guides/self-hosting.mdx

@@ -215,6 +215,16 @@ The workflow:
 5. Builds and uploads VM Agent binaries
 6. Runs health check
 
+Before deploying the API Worker, the workflow reads its applied Durable Object migration tag. A fresh installation creates every namespace with SQLite storage; an existing installation retains its already-applied namespace history and storage backends. This is automatic—do not edit historical migration entries or create namespaces manually.
+
+If migration state cannot be read or does not match the checked-in history, deployment stops before Wrangler runs rather than risking migration replay or namespace replacement.
+
+If deployment stops with a Durable Object migration-state error:
+
+- **Errors mentioning listing lag or a Worker "created moments ago" are transient.** The workflow already retries the state probe a few times (tune with the `DO_MIGRATION_STATE_PROBE_ATTEMPTS` and `DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS` GitHub repository variables); re-running the deployment is safe and resumes cleanly.
+- **"tag is not present in the checked-in history"** means `apps/api/wrangler.toml` lost migration entries that were already applied to your Worker — usually an upgrade merge that dropped fork-local `[[migrations]]` entries. Restore the missing entries so the deployed tag appears in the history, then redeploy.
+- **Never delete the API Worker to recover.** Deleting a Worker destroys every Durable Object namespace behind it — all chat sessions, messages, and task state. There is no undo.
+
 ## Verification
 
 After deployment completes:

package.json

@@ -43,7 +43,7 @@
     "quality:do-migration-safety": "tsx scripts/quality/check-do-migration-safety.ts",
     "quality:source-contract-tests": "tsx scripts/quality/check-source-contract-tests.ts",
     "quality:specialist-review": "tsx scripts/quality/check-specialist-review-evidence.ts",
-    "quality:specialist-review:test": "npx vitest run --config scripts/quality/vitest.config.ts",
+    "quality:scripts:test": "vitest run --config scripts/quality/vitest.config.ts",
     "quality:do-wall-time": "tsx scripts/quality/check-do-wall-time.ts",
     "quality:observability-noise": "tsx scripts/quality/check-observability-noise.ts",
     "test:infra": "pnpm --filter @simple-agent-manager/infra test",

scripts/deploy/durable-object-migrations.ts

@@ -0,0 +1,247 @@
+/**
+ * Durable Object migration compatibility resolution.
+ *
+ * Reads the target API Worker's deployed migration state from Cloudflare and
+ * resolves the checked-in migration history against it: applied entries are
+ * preserved exactly, and only pending legacy `new_classes` creates are emitted
+ * as `new_sqlite_classes` (new Cloudflare accounts cannot create KV-backed
+ * Durable Object namespaces — error 10099, issue #1614).
+ *
+ * Every ambiguous state fails closed: Wrangler treats an unknown deployed tag
+ * as a reason to submit the entire local migration history, which can replay
+ * immutable namespace history.
+ */
+
+import type { MigrationEntry } from './types.js';
+
+const CLOUDFLARE_API_BASE_URL = 'https://api.cloudflare.com/client/v4';
+const WORKERS_LIST_PAGE_SIZE = 1000;
+
+/**
+ * The migration-state probe runs immediately after `wrangler deploy` on the
+ * two-pass first install, where the scripts listing can briefly lag the
+ * just-created Worker. The probe is read-only and idempotent, so transient
+ * failures are retried a bounded number of times before failing the deploy.
+ */
+const DEFAULT_MIGRATION_STATE_PROBE_ATTEMPTS = 3;
+const DEFAULT_MIGRATION_STATE_PROBE_RETRY_DELAY_MS = 2000;
+
+const NEVER_DELETE_WORKER_GUIDANCE =
+  'Never delete a Worker that has served SAM traffic to recover from this: deleting it destroys all Durable Object data.';
+
+function requireCloudflareApiToken(operation: string): string {
+  const apiToken = process.env.CF_API_TOKEN || process.env.CLOUDFLARE_API_TOKEN;
+  if (!apiToken) {
+    throw new Error(`CF_API_TOKEN or CLOUDFLARE_API_TOKEN is required to ${operation}`);
+  }
+  return apiToken;
+}
+
+function readBoundedIntEnv(name: string, fallback: number, minimum: number): number {
+  const raw = process.env[name];
+  if (!raw) {
+    return fallback;
+  }
+  const parsed = Number.parseInt(raw, 10);
+  if (!Number.isSafeInteger(parsed) || parsed < minimum) {
+    console.warn(`  ${name}="${raw}" is not an integer >= ${minimum}; using default ${fallback}`);
+    return fallback;
+  }
+  return parsed;
+}
+
+async function fetchCloudflareMigrationState(
+  url: string,
+  apiToken: string,
+  operation: string
+): Promise<Response> {
+  try {
+    return await fetch(url, { headers: { Authorization: `Bearer ${apiToken}` } });
+  } catch {
+    throw new Error(`${operation}: Cloudflare API request failed`);
+  }
+}
+
+/**
+ * Single read of the deployed migration state. Returns null only when
+ * Cloudflare's exact Worker settings endpoint confirms that the target Worker
+ * does not exist. For an existing Worker, reads the latest applied Durable
+ * Object migration tag from Workers script metadata.
+ *
+ * SECURITY: the settings endpoint's 200 response body contains plaintext
+ * values for every `plain_text` binding, including the live SETUP_TOKEN.
+ * Only the HTTP status may be inspected here — never read or log the body.
+ */
+async function probeDeployedWorkerMigrationTag(
+  accountId: string,
+  workerName: string,
+  apiToken: string
+): Promise<string | null> {
+  const scriptsUrl =
+    `${CLOUDFLARE_API_BASE_URL}/accounts/${encodeURIComponent(accountId)}` +
+    `/workers/scripts?per_page=${WORKERS_LIST_PAGE_SIZE}`;
+  const listOperation = `Failed to list Workers while reading migration state for "${workerName}"`;
+  const scriptsResponse = await fetchCloudflareMigrationState(scriptsUrl, apiToken, listOperation);
+  if (!scriptsResponse.ok) {
+    throw new Error(`${listOperation} (HTTP ${scriptsResponse.status})`);
+  }
+
+  let payload: unknown;
+  try {
+    payload = await scriptsResponse.json();
+  } catch {
+    throw new Error(`${listOperation}: Cloudflare returned an invalid JSON response`);
+  }
+  if (
+    typeof payload !== 'object' ||
+    payload === null ||
+    !('success' in payload) ||
+    payload.success !== true ||
+    !('result' in payload) ||
+    !Array.isArray(payload.result)
+  ) {
+    throw new Error(`${listOperation}: Cloudflare returned an invalid response`);
+  }
+
+  const worker = payload.result.find(
+    (candidate): candidate is Record<string, unknown> =>
+      typeof candidate === 'object' &&
+      candidate !== null &&
+      'id' in candidate &&
+      candidate.id === workerName
+  );
+  if (!worker) {
+    const readOperation = `Failed to read Durable Object migration state for Worker "${workerName}"`;
+    const settingsUrl =
+      `${CLOUDFLARE_API_BASE_URL}/accounts/${encodeURIComponent(accountId)}` +
+      `/workers/scripts/${encodeURIComponent(workerName)}/settings`;
+    const settingsResponse = await fetchCloudflareMigrationState(
+      settingsUrl,
+      apiToken,
+      readOperation
+    );
+
+    if (settingsResponse.status === 404) {
+      return null;
+    }
+    if (!settingsResponse.ok) {
+      throw new Error(`${readOperation} (HTTP ${settingsResponse.status})`);
+    }
+    throw new Error(
+      `Worker "${workerName}" exists but is absent from the Workers scripts listing; ` +
+        `refusing to assume clean migration state. This can be transient listing lag ` +
+        `immediately after the Worker was created — re-running the deployment is safe.`
+    );
+  }
+  if (typeof worker.migration_tag !== 'string' || worker.migration_tag.length === 0) {
+    throw new Error(
+      `Existing Worker "${workerName}" has no migration_tag; refusing to replay Durable Object migrations. ` +
+        `If the Worker was created moments ago this can be propagation lag — re-running the deployment is safe. ` +
+        `Otherwise this Worker was not created by SAM's deploy pipeline and must be investigated manually. ` +
+        NEVER_DELETE_WORKER_GUIDANCE
+    );
+  }
+  return worker.migration_tag;
+}
+
+/**
+ * Reads the deployed Durable Object migration tag with bounded retries.
+ *
+ * This intentionally fails closed on incomplete or unreadable state. Wrangler
+ * otherwise treats an unknown deployed tag as a reason to submit every local
+ * migration, which can replay immutable namespace history. All probe failures
+ * are retried (the probe is read-only and idempotent); a missing API token is
+ * a deterministic configuration error and fails immediately.
+ */
+export async function getDeployedWorkerMigrationTag(
+  accountId: string,
+  workerName: string
+): Promise<string | null> {
+  const apiToken = requireCloudflareApiToken('read Durable Object migration state');
+  const attempts = readBoundedIntEnv(
+    'DO_MIGRATION_STATE_PROBE_ATTEMPTS',
+    DEFAULT_MIGRATION_STATE_PROBE_ATTEMPTS,
+    1
+  );
+  const retryDelayMs = readBoundedIntEnv(
+    'DO_MIGRATION_STATE_PROBE_RETRY_DELAY_MS',
+    DEFAULT_MIGRATION_STATE_PROBE_RETRY_DELAY_MS,
+    0
+  );
+
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await probeDeployedWorkerMigrationTag(accountId, workerName, apiToken);
+    } catch (error) {
+      lastError = error;
+      if (attempt < attempts) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.warn(
+          `  Durable Object migration state probe attempt ${attempt}/${attempts} failed: ${message}`
+        );
+        console.warn(`  Retrying in ${retryDelayMs}ms...`);
+        await new Promise((resolveDelay) => setTimeout(resolveDelay, retryDelayMs));
+      }
+    }
+  }
+  throw lastError;
+}
+
+/**
+ * Preserve applied migration entries exactly, while ensuring every pending
+ * namespace create uses Cloudflare's supported SQLite storage backend.
+ */
+export function resolveDurableObjectMigrations(
+  migrations: MigrationEntry[] | undefined,
+  deployedMigrationTag: string | null
+): MigrationEntry[] | undefined {
+  if (!migrations || migrations.length === 0) {
+    if (deployedMigrationTag !== null) {
+      throw new Error(
+        `Deployed Durable Object migration tag "${deployedMigrationTag}" cannot be reconciled because the checked-in migration history is empty. ` +
+          `Restore the [[migrations]] entries in apps/api/wrangler.toml so the deployed tag appears in the history, then redeploy. ` +
+          NEVER_DELETE_WORKER_GUIDANCE
+      );
+    }
+    return undefined;
+  }
+
+  const duplicateTag = migrations.find(
+    (migration, index) =>
+      migrations.findIndex((candidate) => candidate.tag === migration.tag) !== index
+  )?.tag;
+  if (duplicateTag) {
+    throw new Error(`Durable Object migration tag "${duplicateTag}" is duplicated`);
+  }
+
+  const appliedIndex =
+    deployedMigrationTag === null
+      ? -1
+      : migrations.findIndex((migration) => migration.tag === deployedMigrationTag);
+  if (deployedMigrationTag !== null && appliedIndex === -1) {
+    throw new Error(
+      `Deployed Durable Object migration tag "${deployedMigrationTag}" is not present in the checked-in history; refusing to replay migrations. ` +
+        `The checked-in wrangler.toml is missing migration entries that were already applied to this Worker ` +
+        `(often an upgrade merge that dropped fork-local migrations). Restore the missing entries so the ` +
+        `deployed tag appears in the history, then redeploy. ` +
+        NEVER_DELETE_WORKER_GUIDANCE
+    );
+  }
+
+  return migrations.map((migration, index) => {
+    if (index <= appliedIndex || !migration.new_classes?.length) {
+      return migration;
+    }
+
+    const {
+      new_classes: legacyClasses,
+      new_sqlite_classes: sqliteClasses,
+      ...unchanged
+    } = migration;
+    return {
+      ...unchanged,
+      new_sqlite_classes: [...(sqliteClasses ?? []), ...legacyClasses],
+    };
+  });
+}