fix(cdn): serialize builds per project to stop concurrent-build corruption

.contentrain/content/system/error-messages/en.json

@@ -46,6 +46,7 @@
   "branches.permanent_branch": "The contentrain branch is permanent and cannot be rejected.",
   "branches.reject_forbidden": "You do not have permission to reject branches in this project.",
   "cdn.build_failed": "Failed to create build record: {detail}",
+  "cdn.build_in_progress": "A CDN build is already in progress for this project. Wait for it to finish before starting another.",
   "cdn.content_not_found": "The requested content was not found in CDN storage.",
   "cdn.key_expired": "This API key has expired. Please generate a new one in project settings.",
   "cdn.key_invalid": "Invalid or missing API key. Please check your API key and try again.",

server/api/webhooks/github.post.ts

@@ -83,39 +83,34 @@ export default defineEventHandler(async (event) => {
         })
         if (pushDiff.action === 'skip') continue
 
-        // Create build record + execute async
+        // Claim the single in-flight slot + execute async. A null claim means
+        // a build is already running for this project — skip; that build's
+        // catch-up (runCDNBuild) chases this push's commit when it finishes, so
+        // nothing is stranded.
         const build = await db.createCDNBuild({
           projectId: proj.id as string,
           triggerType: 'webhook',
           commitSha,
           branch: targetBranch,
         })
 
-        if (build) {
-          executeCDNBuild({
+        if (build?.id) {
+          const buildId = build.id as string
+          runCDNBuild({
+            db,
             projectId: proj.id as string,
-            buildId: build.id as string,
+            buildId,
             git,
             cdn,
             contentRoot,
             commitSha,
             branch: targetBranch,
             changedPaths: pushDiff.changedPaths,
             fullRebuild: pushDiff.fullRebuild,
-          }).then(async (result) => {
-            await db.updateCDNBuild(build.id as string, {
-              status: result.error ? 'failed' : 'success',
-              file_count: result.filesUploaded,
-              total_size_bytes: result.totalSizeBytes,
-              changed_models: result.changedModels,
-              build_duration_ms: result.durationMs,
-              error_message: result.error ?? null,
-              completed_at: new Date().toISOString(),
-            })
           }).catch(async (err: unknown) => {
             // Ensure build never stays stuck in 'building'
             const msg = err instanceof Error ? err.message : 'Build failed'
-            await db.updateCDNBuild(build.id as string, {
+            await db.updateCDNBuild(buildId, {
               status: 'failed',
               error_message: msg,
               completed_at: new Date().toISOString(),

server/api/workspaces/[workspaceId]/projects/[projectId]/cdn/builds/trigger.post.ts

@@ -45,7 +45,9 @@ export default defineEventHandler(async (event) => {
   }
   catch { /* use 'manual' */ }
 
-  // Create build record
+  // Claim the single in-flight slot. Null → a build is already running for
+  // this project; a manual rebuild must not race it (its full-rebuild cleanup
+  // would fight the in-flight build's uploads), so surface a clear 409.
   const build = await db.createCDNBuild({
     projectId,
     triggerType: 'manual',
@@ -54,14 +56,17 @@ export default defineEventHandler(async (event) => {
   })
 
   if (!build?.id)
-    throw createError({ statusCode: 500, message: errorMessage('cdn.build_failed', { detail: 'unknown' }) })
+    throw createError({ statusCode: 409, message: errorMessage('cdn.build_in_progress') })
 
   // SSE stream for progress
   const eventStream = createEventStream(event)
 
   const processBuild = async () => {
     try {
-      const result = await executeCDNBuild({
+      // runCDNBuild persists the build row's result and runs the mid-build
+      // catch-up (chasing any push that landed while we built).
+      const result = await runCDNBuild({
+        db,
         projectId,
         buildId: build.id as string,
         git,
@@ -75,17 +80,6 @@ export default defineEventHandler(async (event) => {
         },
       })
 
-      // Update build record
-      await db.updateCDNBuild(build.id as string, {
-        status: result.error ? 'failed' : 'success',
-        file_count: result.filesUploaded,
-        total_size_bytes: result.totalSizeBytes,
-        changed_models: result.changedModels,
-        build_duration_ms: result.durationMs,
-        error_message: result.error ?? null,
-        completed_at: new Date().toISOString(),
-      })
-
       // Emit webhook event (fire-and-forget)
       emitWebhookEvent(projectId, workspaceId, 'cdn.build_complete', {
         buildId: build.id,

server/providers/database.ts

@@ -585,12 +585,20 @@ export interface DatabaseProvider {
   // CDN BUILDS
   // ═══════════════════════════════════════════════════
 
+  /**
+   * Atomically claim the single in-flight build slot for a project and create
+   * the build row (status 'building'). Returns the new row, or `null` when a
+   * build is already in flight for this project — callers must then skip or
+   * defer rather than start a concurrent build that would race on CDN storage.
+   * Slots held by dead builds are reclaimed after a staleness window. Backed by
+   * the `claim_cdn_build` SQL function (per-project advisory lock).
+   */
   createCDNBuild: (input: {
     projectId: string
     triggerType: string
     commitSha?: string
     branch?: string
-  }) => Promise<DatabaseRow>
+  }) => Promise<DatabaseRow | null>
   updateCDNBuild: (buildId: string, updates: Record<string, unknown>) => Promise<void>
   listCDNBuilds: (projectId: string, options?: PaginationOptions & { sort?: string }) => Promise<DatabaseRow[]>
 

server/providers/postgres-db/cdn.ts

@@ -180,22 +180,20 @@ export function cdnMethods(): CDNMethods {
 
     async createCDNBuild(input) {
       try {
-        const row = await getAdmin()
-          .insertInto('cdn_builds')
-          .values({
-            project_id: input.projectId,
-            trigger_type: input.triggerType,
-            commit_sha: (input.commitSha ?? null) as never,
-            branch: (input.branch ?? null) as never,
-            status: 'building',
-          })
-          .returning('id')
-          .executeTakeFirst()
-
-        if (!row)
-          throw createError({ statusCode: 500, message: 'Invalid database response' })
+        // Atomic single-in-flight claim (per-project advisory lock + stale
+        // reclaim). Returns the new id, or NULL when a build is already running.
+        const outcome = await sql<{ result: string | null }>`
+          SELECT public.claim_cdn_build(
+            p_project_id => ${input.projectId}::uuid,
+            p_trigger_type => ${input.triggerType},
+            p_commit_sha => ${input.commitSha ?? null},
+            p_branch => ${input.branch ?? null},
+            p_stale_seconds => 900
+          ) AS result
+        `.execute(getAdmin())
 
-        return row as DatabaseRow
+        const id = outcome.rows[0]?.result ?? null
+        return id ? ({ id } as DatabaseRow) : null
       }
       catch (error) {
         throwDbError(error)

server/providers/supabase-db/cdn.ts

@@ -140,20 +140,19 @@ export function cdnMethods(): CDNMethods {
     },
 
     async createCDNBuild(input) {
-      const { data, error } = await getAdmin()
-        .from('cdn_builds')
-        .insert({
-          project_id: input.projectId,
-          trigger_type: input.triggerType,
-          commit_sha: input.commitSha ?? null,
-          branch: input.branch ?? null,
-          status: 'building',
-        })
-        .select('id')
-        .single()
+      // Atomic single-in-flight claim (per-project advisory lock + stale
+      // reclaim). Returns the new id, or NULL when a build is already running.
+      const { data, error } = await getAdmin().rpc('claim_cdn_build', {
+        p_project_id: input.projectId,
+        p_trigger_type: input.triggerType,
+        p_commit_sha: input.commitSha ?? null,
+        p_branch: input.branch ?? null,
+        p_stale_seconds: 900,
+      })
 
       if (error) throw createError({ statusCode: 500, message: error.message })
-      return data as DatabaseRow
+      const id = (data as string | null) ?? null
+      return id ? ({ id } as DatabaseRow) : null
     },
 
     async updateCDNBuild(buildId, updates) {

server/utils/cdn-build-runner.ts

@@ -0,0 +1,140 @@
+/**
+ * CDN build runner — the shared completion path for both build triggers
+ * (the GitHub push webhook and the manual "Rebuild now" endpoint).
+ *
+ * It runs one build, records the result on its `cdn_builds` row, and then
+ * performs a bounded "catch-up": if the target branch advanced WHILE the build
+ * was running (a push landed mid-build — its own webhook was turned away by the
+ * single-in-flight claim), it claims and runs one more build at the new HEAD so
+ * that mid-build pushes are never stranded. Catch-up builds are full rebuilds
+ * (always correct) and are safe because the claim serializes them — only one
+ * build per project runs at a time, so no destructive cleanup ever races
+ * another build's uploads.
+ *
+ * The claim (DatabaseProvider.createCDNBuild) is what guarantees the mutual
+ * exclusion across instances; this runner only decides WHETHER a follow-up is
+ * warranted and drives it.
+ */
+import type { DatabaseProvider } from '../providers/database'
+import type { GitProvider } from '../providers/git'
+import type { CDNProvider } from '../providers/cdn'
+import type { BuildProgressEvent, BuildResult } from './cdn-builder'
+import { executeCDNBuild } from './cdn-builder'
+
+// Hard ceiling on catch-up chaining so a branch under continuous pushes can't
+// spin builds forever. Each iteration still reflects the latest HEAD, so the
+// CDN converges; this only bounds how many times a single trigger chases it.
+const MAX_CATCHUP_BUILDS = 5
+
+export interface RunCDNBuildInput {
+  db: DatabaseProvider
+  projectId: string
+  buildId: string
+  git: GitProvider
+  cdn: CDNProvider
+  contentRoot: string
+  commitSha: string
+  branch: string
+  changedPaths?: string[]
+  fullRebuild?: boolean
+  /** Progress sink for the initial (user-facing) build only — catch-up builds run silently. */
+  onProgress?: (event: BuildProgressEvent) => void
+}
+
+/** Current head SHA of `branch`, or null if it can't be resolved. */
+async function resolveBranchHead(git: GitProvider, branch: string): Promise<string | null> {
+  try {
+    const branches = await git.listBranches()
+    return branches.find(b => b.name === branch)?.sha ?? null
+  }
+  catch {
+    return null
+  }
+}
+
+/** Execute a build, persist its result row, then chase any mid-build push. */
+export async function runCDNBuild(input: RunCDNBuildInput): Promise<BuildResult> {
+  const { db, projectId, git, cdn, contentRoot, branch } = input
+
+  const result = await executeCDNBuild({
+    projectId,
+    buildId: input.buildId,
+    git,
+    cdn,
+    contentRoot,
+    commitSha: input.commitSha,
+    branch,
+    changedPaths: input.changedPaths,
+    fullRebuild: input.fullRebuild,
+    onProgress: input.onProgress,
+  })
+
+  await db.updateCDNBuild(input.buildId, {
+    status: result.error ? 'failed' : 'success',
+    file_count: result.filesUploaded,
+    total_size_bytes: result.totalSizeBytes,
+    changed_models: result.changedModels,
+    build_duration_ms: result.durationMs,
+    error_message: result.error ?? null,
+    completed_at: new Date().toISOString(),
+  })
+
+  // Catch-up only after a clean build — a failed build is retried by the next
+  // push, not by chasing HEAD on top of a broken state.
+  if (!result.error)
+    await catchUp({ db, projectId, git, cdn, contentRoot, branch, builtSha: input.commitSha, depth: 0 })
+
+  return result
+}
+
+async function catchUp(args: {
+  db: DatabaseProvider
+  projectId: string
+  git: GitProvider
+  cdn: CDNProvider
+  contentRoot: string
+  branch: string
+  builtSha: string
+  depth: number
+}): Promise<void> {
+  if (args.depth >= MAX_CATCHUP_BUILDS) return
+
+  const head = await resolveBranchHead(args.git, args.branch)
+  // Branch unchanged (or unresolvable) → nothing stranded.
+  if (!head || head === args.builtSha) return
+
+  // Claim the in-flight slot for the follow-up. Null → another trigger already
+  // grabbed it (e.g. the mid-build push's own retry, or a concurrent instance);
+  // that build will chase HEAD, so we stop here.
+  const build = await args.db.createCDNBuild({
+    projectId: args.projectId,
+    triggerType: 'webhook',
+    commitSha: head,
+    branch: args.branch,
+  })
+  if (!build?.id) return
+
+  const result = await executeCDNBuild({
+    projectId: args.projectId,
+    buildId: build.id as string,
+    git: args.git,
+    cdn: args.cdn,
+    contentRoot: args.contentRoot,
+    commitSha: head,
+    branch: args.branch,
+    fullRebuild: true,
+  })
+
+  await args.db.updateCDNBuild(build.id as string, {
+    status: result.error ? 'failed' : 'success',
+    file_count: result.filesUploaded,
+    total_size_bytes: result.totalSizeBytes,
+    changed_models: result.changedModels,
+    build_duration_ms: result.durationMs,
+    error_message: result.error ?? null,
+    completed_at: new Date().toISOString(),
+  })
+
+  if (!result.error)
+    await catchUp({ ...args, builtSha: head, depth: args.depth + 1 })
+}

server/utils/cdn-builder.ts

@@ -231,6 +231,30 @@ export async function executeCDNBuild(options: BuildOptions): Promise<BuildResul
 
     changedModelIds.push(...targetModels.map(m => m.id))
 
+    // A selective build whose diff touches no content models (a pure code
+    // push: only app/, server/, etc.) must be a true no-op. Uploading the
+    // manifest here would advance `_manifest.json.commitSha` to the code
+    // commit while the bundle block below is skipped (targetModels empty) —
+    // leaving `_manifest.json.commitSha` ahead of every `_bundle/*.json`.
+    // Consumers that key content freshness off the manifest then read stale/
+    // empty content until a full rebuild re-aligns them. The manifest tracks
+    // the CONTENT version, so a content-less push must not bump it.
+    // fullRebuild (manual trigger) and config/model-def changes never reach
+    // here: the former skips the `else` branch above, the latter make
+    // getAffectedModels non-empty.
+    if (!options.fullRebuild && options.changedPaths?.length && targetModels.length === 0) {
+      return {
+        projectId,
+        buildId,
+        commitSha,
+        filesUploaded: 0,
+        filesDeleted: 0,
+        totalSizeBytes: 0,
+        changedModels: [],
+        durationMs: Date.now() - start,
+      }
+    }
+
     // 4. Upload manifest
     progress({ phase: 'upload', message: 'Uploading manifest...', current: 0, total: targetModels.length })
     const manifest = {