Harden release CI retries and setup tests

.github/workflows/release.yml

@@ -659,22 +659,73 @@ jobs:
           - **Full version extraction:** ${{ inputs.full_versions }}
           - **Workflow run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 
+          gh_retry() {
+            local attempt=1
+            local max_attempts=4
+            local delay=5
+            local label="$1 ${2:-} ${3:-}"
+            local status=0
+            while true; do
+              if "$@"; then
+                return 0
+              else
+                status=$?
+              fi
+              if [ "$attempt" -ge "$max_attempts" ]; then
+                return "$status"
+              fi
+              echo "::warning::GitHub release command failed with exit $status (attempt $attempt/$max_attempts): ${label}; retrying in ${delay}s"
+              sleep "$delay"
+              attempt=$((attempt + 1))
+              delay=$((delay * 2))
+            done
+          }
+
+          ensure_release_exists() {
+            if gh release view "$VERSION" >/dev/null 2>&1; then
+              return 0
+            fi
+
+            local attempt=1
+            local max_attempts=4
+            local delay=5
+            while true; do
+              if gh release create "$VERSION" --title "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"; then
+                return 0
+              fi
+              # GitHub can return a transient 5xx after creating the release.
+              # Treat "now visible" as success so reruns and in-step retries are idempotent.
+              if gh release view "$VERSION" >/dev/null 2>&1; then
+                echo "::warning::Release $VERSION exists after a failed create response; continuing with asset upload."
+                return 0
+              fi
+              if [ "$attempt" -ge "$max_attempts" ]; then
+                return 1
+              fi
+              echo "::warning::GitHub release create failed (attempt $attempt/$max_attempts); retrying in ${delay}s"
+              sleep "$delay"
+              attempt=$((attempt + 1))
+              delay=$((delay * 2))
+            done
+          }
+
           if [ "$REPUBLISH_ASSETS" = "true" ]; then
             # Create release if it doesn't exist, then reupload assets with clobber.
-            gh release create "$VERSION" --title "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}" 2>/dev/null || true
-            gh release upload "$VERSION" dist/*.zip dist/ros-help.db.gz --clobber
-            gh release edit "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"
+            ensure_release_exists
+            gh_retry gh release upload "$VERSION" dist/*.zip dist/ros-help.db.gz --clobber
+            gh_retry gh release edit "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"
             echo "✓ Release $VERSION updated"
           elif gh release view "$VERSION" >/dev/null 2>&1; then
             # A previous run may have created the GitHub Release and then failed
             # at npm publish. The npm preflight above rejects already-published
             # package versions, so this retry path is safe to clobber assets.
-            gh release upload "$VERSION" dist/*.zip dist/ros-help.db.gz --clobber
-            gh release edit "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"
+            gh_retry gh release upload "$VERSION" dist/*.zip dist/ros-help.db.gz --clobber
+            gh_retry gh release edit "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"
             echo "✓ Release $VERSION updated before npm publish retry"
           else
-            gh release create "$VERSION" dist/*.zip dist/ros-help.db.gz \
-              --title "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"
+            ensure_release_exists
+            gh_retry gh release upload "$VERSION" dist/*.zip dist/ros-help.db.gz --clobber
+            gh_retry gh release edit "$VERSION" --notes "$NOTES" "${PRERELEASE_FLAGS[@]}"
             echo "✓ Release $VERSION created"
           fi
 

src/release.test.ts

@@ -28,6 +28,14 @@ function getPhonyBlock(makefile: string): string {
   return makefile.slice(phonyStart, phonyEnd);
 }
 
+function getWorkflowShellFunction(workflow: string, name: string): string {
+  const start = mustIndex(workflow, `          ${name}() {`);
+  const nextFunction = workflow.indexOf("\n          ensure_release_exists() {", start + 1);
+  const end = nextFunction === -1 ? workflow.indexOf("\n          if [", start + 1) : nextFunction;
+  expect(end).toBeGreaterThan(start);
+  return workflow.slice(start, end).replace(/^ {10}/gm, "");
+}
+
 // ---------------------------------------------------------------------------
 // package.json health
 // ---------------------------------------------------------------------------
@@ -801,6 +809,28 @@ describe("release.yml", () => {
   test("creates GitHub Release", () => {
     const src = readText(".github/workflows/release.yml");
     expect(src).toContain("gh release create");
+    expect(src).toContain("gh_retry()");
+    expect(src).toContain("ensure_release_exists()");
+    expect(src).toContain("GitHub can return a transient 5xx after creating the release");
+  });
+
+  test("gh_retry preserves the failing command exit status after retries", () => {
+    const src = readText(".github/workflows/release.yml");
+    const ghRetry = getWorkflowShellFunction(src, "gh_retry");
+    const proc = Bun.spawnSync(
+      [
+        "bash",
+        "-c",
+        `${ghRetry}
+sleep() { :; }
+failing_command() { return 42; }
+gh_retry failing_command
+exit $?`,
+      ],
+      { stdout: "pipe", stderr: "pipe" },
+    );
+
+    expect(proc.exitCode).toBe(42);
   });
 
   test("republish_assets controls immutable npm skips and release clobbering", () => {
@@ -821,6 +851,9 @@ describe("release.yml", () => {
     expect(republishBranchIdx).toBeLessThan(clobberIdx);
     expect(src).toContain('elif gh release view "$VERSION"');
     expect(src).toContain("updated before npm publish retry");
+    expect(src).toContain("ensure_release_exists");
+    expect(src).toContain("gh_retry gh release upload");
+    expect(src).toContain("gh_retry gh release edit");
 
     expect(src).toContain("if: inputs.republish_assets != true");
     expect(src).toContain("if: inputs.republish_assets == true");

src/setup.test.ts

@@ -34,19 +34,36 @@ const { SCHEMA_VERSION } = await import("./paths.ts");
 
 function writeUsableDb(dbFile: string, releaseTag = "v0.0.0-test"): void {
   const db = new sqlite(dbFile);
-  db.run(`PRAGMA user_version = ${SCHEMA_VERSION};`);
-  db.run("CREATE TABLE pages (id INTEGER PRIMARY KEY, title TEXT);");
-  db.run("CREATE TABLE commands (id INTEGER PRIMARY KEY, path TEXT);");
-  db.run("CREATE TABLE db_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);");
+  let insertPage: ReturnType<typeof db.prepare> | null = null;
+  let insertCmd: ReturnType<typeof db.prepare> | null = null;
 
-  const insertPage = db.prepare("INSERT INTO pages (title) VALUES (?)");
-  for (let i = 0; i < 100; i++) insertPage.run(`page-${i}`);
-
-  const insertCmd = db.prepare("INSERT INTO commands (path) VALUES (?)");
-  for (let i = 0; i < 1000; i++) insertCmd.run(`/cmd/${i}`);
+  try {
+    db.run(`PRAGMA user_version = ${SCHEMA_VERSION};`);
+    db.run("CREATE TABLE pages (id INTEGER PRIMARY KEY, title TEXT);");
+    db.run("CREATE TABLE commands (id INTEGER PRIMARY KEY, path TEXT);");
+    db.run("CREATE TABLE db_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);");
 
-  db.run("INSERT INTO db_meta (key, value) VALUES ('release_tag', ?);", [releaseTag]);
-  db.close();
+    insertPage = db.prepare("INSERT INTO pages (title) VALUES (?)");
+    insertCmd = db.prepare("INSERT INTO commands (path) VALUES (?)");
+
+    // Keep the fixture faithful to the package DB minimums without paying
+    // 1,100 autocommit disk syncs. Shared GitHub runners can otherwise stall long
+    // enough to trip Bun's per-test timeout before the lock/probe assertion runs.
+    db.run("BEGIN");
+    for (let i = 0; i < 100; i++) insertPage.run(`page-${i}`);
+    for (let i = 0; i < 1000; i++) insertCmd.run(`/cmd/${i}`);
+    db.run("INSERT INTO db_meta (key, value) VALUES ('release_tag', ?);", [releaseTag]);
+    db.run("COMMIT");
+  } catch (e) {
+    try {
+      db.run("ROLLBACK");
+    } catch {}
+    throw e;
+  } finally {
+    insertPage?.finalize();
+    insertCmd?.finalize();
+    db.close();
+  }
 }
 
 // ---------------------------------------------------------------------------