refactor: harden SQLite delegate cache and address review feedback

.gitignore

@@ -60,3 +60,5 @@ delegate-cache.json
 
 # Generated local/Vercel-hosted data artifacts
 /public/tally-data/
+/data/build/
+data/avatar-map.json

README.md

@@ -19,9 +19,9 @@ A fork of [TallyZero](https://github.com/withtally/tally-zero) purpose-built for
 - **Dual-governor support** — Core Governor (constitutional) and Treasury Governor (funding)
 - **Full lifecycle tracking** — Tracks proposals through all stages: voting → L2 timelock → L1 challenge period → L1 timelock → retryable tickets → final execution
 - **Security Council election support** — View and participate in Security Council member elections
-- **RPC-direct governance data**. Proposals, delegates, lifecycle state, and Snapshot data are fetched directly from the blockchain or from CORS-enabled APIs. The only server-side code is a small first-party proxy used when importing a proposal description from the governance forum (which lacks CORS).
-- **Bundled cache** — Ships with pre-built tracking checkpoints for instant resume without RPC calls
-- **Delegate insights** — Pre-indexed delegate cache with voting power rankings
+- **RPC-direct governance data**. Proposals, delegates, lifecycle state, and Snapshot data are fetched directly from the blockchain or from CORS-enabled APIs. Server-side code is limited to a forum-import proxy and a thin proxy for the SQLite delegate database (see "Data layer" below).
+- **Bundled rank cache** — Ships with a pre-computed gov-tracker delegate rank snapshot for instant lookups without RPC calls
+- **Delegate insights** — Delegate profiles, voting power rankings, and election candidates served from a SQLite-over-HTTP database
 
 ## Tech Stack
 
@@ -33,6 +33,27 @@ A fork of [TallyZero](https://github.com/withtally/tally-zero) purpose-built for
 - @gzeoneth/gov-tracker (proposal lifecycle + delegate indexing)
 - TanStack Table + React Query
 - Radix UI + Shadcn + Tailwind CSS
+- sql.js-httpvfs (browser SQLite over HTTP range requests)
+
+## Data layer
+
+Tally Zero pulls data from three sources, in increasing freshness:
+
+1. **Bundled gov-tracker cache** (`@gzeoneth/gov-tracker/delegate-cache.json`). Pre-computed delegate ranks and snapshot block. Ships with the JS bundle. Used for instant rank lookups without RPC calls.
+2. **SQLite over HTTP** (`/tally-data/tally-zero.sqlite`). A ~200 MB SQLite database hosted on Vercel Blob and queried in the browser via `sql.js-httpvfs`. Only the byte ranges needed for a query are fetched (typically a few hundred KB per page). Used for delegate profiles, election candidates, and address display records.
+3. **On-chain RPC**. Live state. Used for current voting power, vote tallies, and transaction submission.
+
+### Build inputs (not shipped to clients)
+
+- `data/delegates-*.json` (~145 MB). Raw delegate dumps from Tally's API. Read by `pnpm sqlite:build`.
+- `data/avatar-map.json` (gitignored, ~625 KB). Mirrored delegate avatars. Regenerate with `pnpm avatars:upload`. Read by `pnpm sqlite:build`.
+- `data/delegate-index.json`, `data/delegate-labels.json`, `data/election-*-candidates.json`. Additional inputs to the SQLite build.
+
+### Workflows
+
+- `pnpm sqlite:build`. Produces `public/tally-data/tally-zero.sqlite` and `public/tally-data/manifest.json` from the inputs above.
+- `pnpm avatars:upload`. Regenerates `data/avatar-map.json` by mirroring delegate avatars to governance blob storage.
+- `pnpm sqlite:deploy`. Builds (if needed), uploads the SQLite to Vercel Blob, and updates the production env var. Run this when delegate or election data changes.
 
 ## Getting Started
 

app/elections/ElectionPageClient.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import dynamic from "next/dynamic";
-import { useMemo } from "react";
+import { useState } from "react";
 
 import { Skeleton } from "@/components/ui/Skeleton";
 import {
@@ -36,9 +36,10 @@ interface ElectionPageClientProps {
 export default function ElectionPageClient({
   initialDisplayRecords,
 }: ElectionPageClientProps) {
-  useMemo(() => {
+  useState(() => {
     primeAddressDisplayRecordCache(initialDisplayRecords);
-  }, [initialDisplayRecords]);
+    return null;
+  });
 
   return <ElectionContainer />;
 }

app/tally-data/tally-zero.sqlite/route.test.ts

@@ -1,24 +1,37 @@
-import { describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 
-import { GET, HEAD, parseRangeHeader } from "./route";
+import { serverManifest } from "@/lib/tally-data/manifest-server";
 
-const DB_SIZE_BYTES = 878346240;
+import { GET, HEAD, parseRangeHeader, runtime } from "./route";
+
+const DB_SIZE_BYTES = serverManifest.sizeBytes;
 
 describe("tally SQLite route", () => {
+  describe("module exports", () => {
+    it("runs on the nodejs runtime", () => {
+      expect(runtime).toBe("nodejs");
+    });
+  });
+
   describe("HEAD", () => {
-    it("advertises byte serving metadata", async () => {
+    it("returns 502 when upstream HEAD throws", async () => {
       const fetchMock = vi
         .spyOn(globalThis, "fetch")
         .mockRejectedValueOnce(new Error("offline"));
       const response = await HEAD();
 
-      expect(response.status).toBe(200);
-      expect(response.headers.get("accept-ranges")).toBe("bytes");
-      expect(response.headers.get("content-length")).toBe(
-        String(DB_SIZE_BYTES)
-      );
-      expect(response.headers.get("content-encoding")).toBe("identity");
-      expect(response.headers.get("cache-control")).toContain("no-transform");
+      expect(response.status).toBe(502);
+
+      fetchMock.mockRestore();
+    });
+
+    it("returns 502 when upstream HEAD is not ok", async () => {
+      const fetchMock = vi
+        .spyOn(globalThis, "fetch")
+        .mockResolvedValueOnce(new Response(null, { status: 503 }));
+      const response = await HEAD();
+
+      expect(response.status).toBe(502);
 
       fetchMock.mockRestore();
     });
@@ -40,11 +53,60 @@ describe("tally SQLite route", () => {
         method: "HEAD",
       });
       expect(response.status).toBe(200);
+      expect(response.headers.get("accept-ranges")).toBe("bytes");
+      expect(response.headers.get("content-length")).toBe(
+        String(DB_SIZE_BYTES)
+      );
+      expect(response.headers.get("content-encoding")).toBe("identity");
       expect(response.headers.get("etag")).toBe('"sqlite-db-v1"');
       expect(response.headers.get("last-modified")).toBe(
         "Thu, 30 Apr 2026 12:50:38 GMT"
       );
-      expect(response.headers.get("cache-control")).toContain("immutable");
+
+      const cacheControl = response.headers.get("cache-control");
+      expect(cacheControl).toContain("public");
+      expect(cacheControl).toContain("max-age=300");
+      expect(cacheControl).toContain("s-maxage=31536000");
+      expect(cacheControl).toContain("stale-while-revalidate=86400");
+      expect(cacheControl).toContain("immutable");
+
+      fetchMock.mockRestore();
+    });
+  });
+
+  describe("getBlobUrl env-var guard", () => {
+    beforeEach(() => {
+      vi.stubEnv("GOVERNANCE_DATA_SQLITE_BLOB_URL", "");
+      vi.stubEnv("TALLY_DATA_SQLITE_BLOB_URL", "");
+    });
+
+    afterEach(() => {
+      vi.unstubAllEnvs();
+    });
+
+    it("returns 503 on GET when no env var is set in production", async () => {
+      vi.stubEnv("NODE_ENV", "production");
+
+      const response = await GET(
+        new Request("https://example.test/db", {
+          headers: { range: "bytes=0-3" },
+        })
+      );
+
+      expect(response.status).toBe(503);
+    });
+
+    it("falls back to the default blob URL outside production", async () => {
+      vi.stubEnv("NODE_ENV", "development");
+      const fetchMock = vi
+        .spyOn(globalThis, "fetch")
+        .mockResolvedValueOnce(new Response(null, { status: 200 }));
+
+      const response = await HEAD();
+      expect(response.status).toBe(200);
+      expect(fetchMock).toHaveBeenCalledWith(expect.any(String), {
+        method: "HEAD",
+      });
 
       fetchMock.mockRestore();
     });

app/tally-data/tally-zero.sqlite/route.ts

@@ -1,6 +1,13 @@
+// The route segment is `tally-zero.sqlite` (a binary-looking path) because
+// sql.js-httpvfs needs a stable URL that resembles a file path. This route
+// proxies range requests to a Vercel Blob, hiding the team-specific blob
+// hostname so blob rotation is an env-var swap, not a frontend redeploy.
+
+import { serverManifest } from "@/lib/tally-data/manifest-server";
+
 const DEFAULT_BLOB_URL =
   "https://epodj1k6qull8rb3.public.blob.vercel-storage.com/governance-data/delegates.sqlite";
-const DB_SIZE_BYTES = 878346240;
+const DB_SIZE_BYTES = serverManifest.sizeBytes;
 const MAX_RANGE_BYTES = 4 * 1024 * 1024;
 const UPSTREAM_CACHE_HEADERS = ["etag", "last-modified"] as const;
 
@@ -14,23 +21,31 @@ type ParsedRange =
       status: 400 | 416;
     };
 
+// Runtime is `nodejs` rather than `edge` because Next.js' edge dev server
+// has been observed to override the explicit `Content-Length` we set on the
+// HEAD response (recomputing it from the empty body), which breaks
+// sql.js-httpvfs file-size discovery. Vercel's edge cache still serves
+// `s-maxage` cached ranges regardless of the function runtime, so the
+// caching benefit is preserved.
 export const runtime = "nodejs";
-export const dynamic = "force-dynamic";
 
 function getBlobUrl(): string {
-  return (
+  const envUrl =
     process.env.GOVERNANCE_DATA_SQLITE_BLOB_URL ??
-    process.env.TALLY_DATA_SQLITE_BLOB_URL ??
-    DEFAULT_BLOB_URL
-  );
+    process.env.TALLY_DATA_SQLITE_BLOB_URL;
+  if (envUrl) return envUrl;
+  if (process.env.NODE_ENV === "production") {
+    throw new Error("GOVERNANCE_DATA_SQLITE_BLOB_URL is not set in production");
+  }
+  return DEFAULT_BLOB_URL;
 }
 
 function headersForResponse(extraHeaders?: HeadersInit): Headers {
   const headers = new Headers();
   headers.set("Accept-Ranges", "bytes");
   headers.set(
     "Cache-Control",
-    "public, max-age=31536000, immutable, no-transform"
+    "public, max-age=300, s-maxage=31536000, stale-while-revalidate=86400, immutable"
   );
   headers.set("Content-Encoding", "identity");
   headers.set("Content-Type", "application/octet-stream");
@@ -97,14 +112,10 @@ export async function HEAD(): Promise<Response> {
         ),
       });
     }
+    return new Response(null, { status: 502 });
   } catch {
-    // Static byte-serving metadata is enough for sql.js-httpvfs to proceed.
+    return new Response(null, { status: 502 });
   }
-
-  return new Response(null, {
-    status: 200,
-    headers: headersForResponse(),
-  });
 }
 
 export async function GET(request: Request): Promise<Response> {
@@ -121,9 +132,23 @@ export async function GET(request: Request): Promise<Response> {
     return rangeNotSatisfiable();
   }
 
-  const blobResponse = await fetch(getBlobUrl(), {
-    headers: { range: range.header },
-  });
+  let blobUrl: string;
+  try {
+    blobUrl = getBlobUrl();
+  } catch (err) {
+    console.error("[tally-data] blob URL unavailable:", err);
+    return new Response("Blob URL not configured", { status: 503 });
+  }
+
+  let blobResponse: Response;
+  try {
+    blobResponse = await fetch(blobUrl, {
+      headers: { range: range.header },
+    });
+  } catch (err) {
+    console.error("[tally-data] upstream fetch failed:", err);
+    return new Response(null, { status: 502 });
+  }
 
   if (!blobResponse.ok && blobResponse.status !== 206) {
     return new Response(blobResponse.body, {

components/container/delegate/DelegatesTable.tsx

@@ -19,9 +19,9 @@ import {
 import { buildShuffleMap, sortByOrderMap } from "@/lib/collection-utils";
 import {
   getDelegateSummaries,
+  type TallyDelegateListItem,
   type TallyDelegateSummary,
-} from "@/lib/delegate-cache";
-import type { DelegateInfo } from "@/types/delegate";
+} from "@/lib/delegate-data";
 import {
   ColumnFiltersState,
   SortingState,
@@ -38,7 +38,7 @@ import {
 import { useEffect, useMemo, useRef, useState } from "react";
 
 export interface DelegatesTableProps {
-  delegates: DelegateInfo[];
+  delegates: TallyDelegateListItem[];
   totalVotingPower: string;
   isLoading: boolean;
   error: Error | null;
@@ -50,22 +50,29 @@ export interface DelegatesTableProps {
   onVisibleRowsChange: (addresses: string[]) => void;
 }
 
-type DelegateWithSummary = DelegateInfo & Partial<TallyDelegateSummary>;
-
 function getRowDelegateSummary(
-  delegate: DelegateInfo
+  delegate: TallyDelegateListItem
 ): TallyDelegateSummary | null {
-  const row = delegate as DelegateWithSummary;
-  if (!row.displayName && !row.name && !row.ens && !row.picture) return null;
+  if (
+    !delegate.displayName &&
+    !delegate.name &&
+    !delegate.ens &&
+    !delegate.picture
+  )
+    return null;
 
   return {
-    address: row.address,
-    ens: row.ens ?? null,
-    name: row.name ?? null,
-    picture: row.picture ?? null,
-    knownLabel: row.knownLabel ?? null,
+    address: delegate.address,
+    ens: delegate.ens ?? null,
+    name: delegate.name ?? null,
+    picture: delegate.picture ?? null,
+    knownLabel: delegate.knownLabel ?? null,
     displayName:
-      row.displayName ?? row.knownLabel ?? row.name ?? row.ens ?? null,
+      delegate.displayName ??
+      delegate.knownLabel ??
+      delegate.name ??
+      delegate.ens ??
+      null,
   };
 }
 
@@ -143,7 +150,7 @@ export function DelegatesTable({
   );
 
   // eslint-disable-next-line react-hooks/incompatible-library
-  const table = useReactTable<DelegateInfo>({
+  const table = useReactTable<TallyDelegateListItem>({
     data: sortedDelegates,
     columns,
     state: {

components/delegate/DelegateProfile.test.tsx

@@ -2,7 +2,7 @@ import type { ReactNode } from "react";
 import { renderToStaticMarkup } from "react-dom/server";
 import { describe, expect, it, vi } from "vitest";
 
-import type { TallyDelegateProfile } from "@/lib/delegate-cache";
+import type { TallyDelegateProfile } from "@/lib/delegate-data";
 
 import { DelegateProfile, getDelegateDisplayName } from "./DelegateProfile";
 

components/delegate/DelegateProfile.tsx

@@ -23,7 +23,7 @@ import {
   CardTitle,
 } from "@/components/ui/Card";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/Tabs";
-import type { TallyDelegateProfile } from "@/lib/delegate-cache";
+import type { TallyDelegateProfile } from "@/lib/delegate-data";
 import { getAddressExplorerUrl } from "@/lib/explorer-utils";
 import { formatVotingPower, shortenAddress } from "@/lib/format-utils";
 import { proposalSanitizeSchema } from "@/lib/sanitize-schema";

components/delegate/DelegateProfileLoader.tsx

@@ -8,7 +8,7 @@ import { Skeleton } from "@/components/ui/Skeleton";
 import {
   getDelegateProfile,
   type TallyDelegateProfile,
-} from "@/lib/delegate-cache";
+} from "@/lib/delegate-data";
 
 type DelegateProfileState = {
   address: string;

components/delegate/DelegationCard.test.tsx

@@ -57,7 +57,7 @@ vi.mock("wagmi", () => ({
   useWriteContract: mocks.useWriteContract,
 }));
 
-vi.mock("@/lib/delegate-cache", () => ({
+vi.mock("@/lib/delegate-data", () => ({
   useAddressDisplayRecord: mocks.useAddressDisplayRecord,
 }));
 

components/delegate/DelegationCard.tsx

@@ -33,7 +33,7 @@ import { addressesEqual, isValidAddress } from "@/lib/address-utils";
 import {
   type TallyAddressDisplayRecord,
   useAddressDisplayRecord,
-} from "@/lib/delegate-cache";
+} from "@/lib/delegate-data";
 import { getErrorMessage, getSimulationErrorMessage } from "@/lib/error-utils";
 import { formatVotingPower, shortenAddress } from "@/lib/format-utils";