From 34e62299bc09dc60c099cc45b3eb865d9207bb24 Mon Sep 17 00:00:00 2001 From: douglance <4741454+douglance@users.noreply.github.com> Date: Mon, 1 Jun 2026 12:37:44 -0400 Subject: [PATCH] feat: source tally data from governance indexer --- .../[...path]/route.test.ts | 97 ++ app/api/governance-indexer/[...path]/route.ts | 60 + .../tally-zero.sqlite/route.test.ts | 162 --- app/tally-data/tally-zero.sqlite/route.ts | 149 --- hooks/use-multi-governor-search.ts | 22 +- hooks/use-proposal-stages.tsx | 6 +- lib/delegate-cache.ts | 5 +- lib/sqlite-checkpoint-seed.ts | 38 - lib/tally-data/client.ts | 4 +- lib/tally-data/indexer.test.ts | 93 ++ lib/tally-data/indexer.ts | 185 +++ lib/tally-data/sqlite.ts | 1155 ----------------- lib/tally-data/types.ts | 7 +- package.json | 3 - pnpm-lock.yaml | 13 - scripts/build-tally-sqlite.ts | 796 ------------ scripts/deploy-tally-sqlite.ts | 331 ----- 17 files changed, 453 insertions(+), 2673 deletions(-) create mode 100644 app/api/governance-indexer/[...path]/route.test.ts create mode 100644 app/api/governance-indexer/[...path]/route.ts delete mode 100644 app/tally-data/tally-zero.sqlite/route.test.ts delete mode 100644 app/tally-data/tally-zero.sqlite/route.ts delete mode 100644 lib/sqlite-checkpoint-seed.ts create mode 100644 lib/tally-data/indexer.test.ts create mode 100644 lib/tally-data/indexer.ts delete mode 100644 lib/tally-data/sqlite.ts delete mode 100644 scripts/build-tally-sqlite.ts delete mode 100644 scripts/deploy-tally-sqlite.ts diff --git a/app/api/governance-indexer/[...path]/route.test.ts b/app/api/governance-indexer/[...path]/route.test.ts new file mode 100644 index 0000000..83d4871 --- /dev/null +++ b/app/api/governance-indexer/[...path]/route.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { GET } from "./route"; + +function context(path: string[]) { + return { params: Promise.resolve({ path }) }; +} + +describe("governance indexer proxy", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("returns 503 when the indexer URL is not configured", async () => { + vi.stubEnv("GOVERNANCE_INDEXER_URL", ""); + + const response = await GET( + new Request( + "https://example.test/api/governance-indexer/api/tally/stats" + ), + context(["api", "tally", "stats"]) + ); + + expect(response.status).toBe(503); + await expect(response.json()).resolves.toEqual({ + error: "Governance indexer is not configured.", + }); + }); + + it("proxies the encoded path and query string to the configured upstream", async () => { + vi.stubEnv("GOVERNANCE_INDEXER_URL", "https://indexer.example.test/"); + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( + new Response(JSON.stringify({ ok: true }), { + status: 200, + statusText: "OK", + }) + ); + + const response = await GET( + new Request( + "https://example.test/api/governance-indexer/api/tally/delegates?query=dao&limit=10" + ), + context(["api", "tally", "delegates"]) + ); + + expect(response.status).toBe(200); + expect(String(fetchMock.mock.calls[0][0])).toBe( + "https://indexer.example.test/api/tally/delegates?query=dao&limit=10" + ); + expect(fetchMock.mock.calls[0][1]).toMatchObject({ + method: "GET", + headers: { accept: "application/json" }, + }); + expect(response.headers.get("content-type")).toBe("application/json"); + expect(response.headers.get("cache-control")).toBe( + "public, s-maxage=30, stale-while-revalidate=300" + ); + }); + + it("forwards upstream status codes", async () => { + vi.stubEnv("GOVERNANCE_INDEXER_URL", "https://indexer.example.test"); + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( + new Response(JSON.stringify({ error: "missing" }), { + status: 404, + statusText: "Not Found", + }) + ); + + const response = await GET( + new Request( + "https://example.test/api/governance-indexer/api/tally/missing" + ), + context(["api", "tally", "missing"]) + ); + + expect(response.status).toBe(404); + expect(response.statusText).toBe("Not Found"); + }); + + it("returns 502 for upstream failures", async () => { + vi.stubEnv("GOVERNANCE_INDEXER_URL", "https://indexer.example.test"); + vi.spyOn(globalThis, "fetch").mockRejectedValueOnce(new Error("offline")); + + const response = await GET( + new Request( + "https://example.test/api/governance-indexer/api/tally/stats" + ), + context(["api", "tally", "stats"]) + ); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toEqual({ + error: "Indexer upstream error.", + }); + }); +}); diff --git a/app/api/governance-indexer/[...path]/route.ts b/app/api/governance-indexer/[...path]/route.ts new file mode 100644 index 0000000..e90727b --- /dev/null +++ b/app/api/governance-indexer/[...path]/route.ts @@ -0,0 +1,60 @@ +export const dynamic = "force-dynamic"; + +const FETCH_TIMEOUT_MS = 10_000; + +type RouteContext = { + params: Promise<{ path: string[] }>; +}; + +function getIndexerUrl(): string | null { + // eslint-disable-next-line no-process-env + const value = process.env.GOVERNANCE_INDEXER_URL?.trim(); + return value ? value.replace(/\/+$/, "") : null; +} + +export async function GET( + request: Request, + context: RouteContext +): Promise { + const { path } = await context.params; + const indexerUrl = getIndexerUrl(); + if (!indexerUrl) { + return Response.json( + { error: "Governance indexer is not configured." }, + { status: 503 } + ); + } + + const inboundUrl = new URL(request.url); + const encodedPath = path.map(encodeURIComponent).join("/"); + const upstreamUrl = new URL(`${indexerUrl}/${encodedPath}`); + upstreamUrl.search = inboundUrl.search; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + + try { + const upstream = await fetch(upstreamUrl, { + method: "GET", + signal: controller.signal, + headers: { accept: "application/json" }, + }); + + const headers = new Headers(); + headers.set("content-type", "application/json"); + headers.set( + "cache-control", + "public, s-maxage=30, stale-while-revalidate=300" + ); + + return new Response(upstream.body, { + status: upstream.status, + statusText: upstream.statusText, + headers, + }); + } catch { + return Response.json({ error: "Indexer upstream error." }, { status: 502 }); + } finally { + clearTimeout(timer); + } +} diff --git a/app/tally-data/tally-zero.sqlite/route.test.ts b/app/tally-data/tally-zero.sqlite/route.test.ts deleted file mode 100644 index 8e920ed..0000000 --- a/app/tally-data/tally-zero.sqlite/route.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { GET, HEAD, parseRangeHeader } from "./route"; - -const DB_SIZE_BYTES = 882110464; - -describe("tally SQLite route", () => { - describe("HEAD", () => { - it("advertises byte serving metadata", async () => { - const fetchMock = vi - .spyOn(globalThis, "fetch") - .mockRejectedValueOnce(new Error("offline")); - const response = await HEAD(); - - expect(response.status).toBe(200); - expect(response.headers.get("accept-ranges")).toBe("bytes"); - expect(response.headers.get("content-length")).toBe( - String(DB_SIZE_BYTES) - ); - expect(response.headers.get("content-encoding")).toBe("identity"); - expect(response.headers.get("cache-control")).toContain("no-transform"); - - fetchMock.mockRestore(); - }); - - it("forwards upstream cache validators", async () => { - const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( - new Response(null, { - status: 200, - headers: { - etag: '"sqlite-db-v1"', - "last-modified": "Thu, 30 Apr 2026 12:50:38 GMT", - }, - }) - ); - - const response = await HEAD(); - - expect(fetchMock).toHaveBeenCalledWith(expect.any(String), { - method: "HEAD", - }); - expect(response.status).toBe(200); - expect(response.headers.get("etag")).toBe('"sqlite-db-v1"'); - expect(response.headers.get("last-modified")).toBe( - "Thu, 30 Apr 2026 12:50:38 GMT" - ); - expect(response.headers.get("cache-control")).toContain("immutable"); - - fetchMock.mockRestore(); - }); - }); - - describe("parseRangeHeader", () => { - it("accepts single bounded byte ranges", () => { - expect(parseRangeHeader("bytes=0-15")).toEqual({ - isValid: true, - header: "bytes=0-15", - }); - }); - - it("rejects missing ranges as a bad request", () => { - expect(parseRangeHeader(null)).toEqual({ - isValid: false, - status: 400, - }); - }); - - it("rejects malformed and multi ranges", () => { - expect(parseRangeHeader("bytes=0-")).toEqual({ - isValid: false, - status: 416, - }); - expect(parseRangeHeader("bytes=-4096")).toEqual({ - isValid: false, - status: 416, - }); - expect(parseRangeHeader("bytes=0-15,32-63")).toEqual({ - isValid: false, - status: 416, - }); - }); - - it("rejects out-of-bounds ranges", () => { - expect( - parseRangeHeader(`bytes=${DB_SIZE_BYTES}-${DB_SIZE_BYTES}`) - ).toEqual({ - isValid: false, - status: 416, - }); - expect(parseRangeHeader("bytes=100-99")).toEqual({ - isValid: false, - status: 416, - }); - }); - - it("rejects ranges larger than four MiB", () => { - expect(parseRangeHeader("bytes=0-4194304")).toEqual({ - isValid: false, - status: 416, - }); - }); - }); - - describe("GET", () => { - it("rejects full-file requests", async () => { - const response = await GET(new Request("https://example.test/db")); - - expect(response.status).toBe(400); - expect(await response.text()).toBe("Range header is required"); - }); - - it("returns 416 for invalid ranges", async () => { - const response = await GET( - new Request("https://example.test/db", { - headers: { range: "bytes=0-4194304" }, - }) - ); - - expect(response.status).toBe(416); - expect(response.headers.get("content-range")).toBe( - `bytes */${DB_SIZE_BYTES}` - ); - }); - - it("proxies valid ranges to Blob", async () => { - const body = new Uint8Array([1, 2, 3, 4]); - const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( - new Response(body, { - status: 206, - headers: { - "content-length": "4", - "content-range": `bytes 0-3/${DB_SIZE_BYTES}`, - etag: '"sqlite-db-v1"', - "last-modified": "Thu, 30 Apr 2026 12:50:38 GMT", - }, - }) - ); - - const response = await GET( - new Request("https://example.test/db", { - headers: { range: "bytes=0-3" }, - }) - ); - - expect(fetchMock).toHaveBeenCalledWith(expect.any(String), { - headers: { range: "bytes=0-3" }, - }); - expect(response.status).toBe(206); - expect(response.headers.get("content-range")).toBe( - `bytes 0-3/${DB_SIZE_BYTES}` - ); - expect(response.headers.get("content-length")).toBe("4"); - expect(response.headers.get("content-encoding")).toBe("identity"); - expect(response.headers.get("etag")).toBe('"sqlite-db-v1"'); - expect(response.headers.get("last-modified")).toBe( - "Thu, 30 Apr 2026 12:50:38 GMT" - ); - - fetchMock.mockRestore(); - }); - }); -}); diff --git a/app/tally-data/tally-zero.sqlite/route.ts b/app/tally-data/tally-zero.sqlite/route.ts deleted file mode 100644 index 86cc828..0000000 --- a/app/tally-data/tally-zero.sqlite/route.ts +++ /dev/null @@ -1,149 +0,0 @@ -const DEFAULT_BLOB_URL = - "https://epodj1k6qull8rb3.public.blob.vercel-storage.com/governance-data/delegates.sqlite"; -const DB_SIZE_BYTES = 882110464; -const MAX_RANGE_BYTES = 4 * 1024 * 1024; -const UPSTREAM_CACHE_HEADERS = ["etag", "last-modified"] as const; - -type ParsedRange = - | { - isValid: true; - header: string; - } - | { - isValid: false; - status: 400 | 416; - }; - -export const runtime = "nodejs"; -export const dynamic = "force-dynamic"; - -function getBlobUrl(): string { - return ( - process.env.GOVERNANCE_DATA_SQLITE_BLOB_URL ?? - process.env.TALLY_DATA_SQLITE_BLOB_URL ?? - DEFAULT_BLOB_URL - ); -} - -function headersForResponse(extraHeaders?: HeadersInit): Headers { - const headers = new Headers(); - headers.set("Accept-Ranges", "bytes"); - headers.set( - "Cache-Control", - "public, max-age=31536000, immutable, no-transform" - ); - headers.set("Content-Encoding", "identity"); - headers.set("Content-Type", "application/octet-stream"); - headers.set("Content-Length", String(DB_SIZE_BYTES)); - headers.set("Content-Disposition", 'inline; filename="tally-zero.sqlite"'); - if (extraHeaders) { - new Headers(extraHeaders).forEach((value, key) => { - headers.set(key, value); - }); - } - return headers; -} - -function getUpstreamCacheHeaders(upstreamHeaders: Headers): Headers { - const headers = new Headers(); - - for (const headerName of UPSTREAM_CACHE_HEADERS) { - const value = upstreamHeaders.get(headerName); - if (value) headers.set(headerName, value); - } - - return headers; -} - -function rangeNotSatisfiable(): Response { - return new Response(null, { - status: 416, - headers: headersForResponse({ - "Content-Range": `bytes */${DB_SIZE_BYTES}`, - "Content-Length": "0", - }), - }); -} - -export function parseRangeHeader(range: string | null): ParsedRange { - if (!range) return { isValid: false, status: 400 }; - - const match = /^bytes=(\d+)-(\d+)$/.exec(range.trim()); - if (!match) return { isValid: false, status: 416 }; - - const start = Number(match[1]); - const end = Number(match[2]); - if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end)) { - return { isValid: false, status: 416 }; - } - if (start > end || start >= DB_SIZE_BYTES || end >= DB_SIZE_BYTES) { - return { isValid: false, status: 416 }; - } - if (end - start + 1 > MAX_RANGE_BYTES) { - return { isValid: false, status: 416 }; - } - - return { isValid: true, header: `bytes=${start}-${end}` }; -} - -export async function HEAD(): Promise { - try { - const blobResponse = await fetch(getBlobUrl(), { method: "HEAD" }); - if (blobResponse.ok) { - return new Response(null, { - status: 200, - headers: headersForResponse( - getUpstreamCacheHeaders(blobResponse.headers) - ), - }); - } - } catch { - // Static byte-serving metadata is enough for sql.js-httpvfs to proceed. - } - - return new Response(null, { - status: 200, - headers: headersForResponse(), - }); -} - -export async function GET(request: Request): Promise { - const range = parseRangeHeader(request.headers.get("range")); - if (!range.isValid) { - if (range.status === 400) { - return new Response("Range header is required", { - status: 400, - headers: headersForResponse({ - "Content-Length": String("Range header is required".length), - }), - }); - } - return rangeNotSatisfiable(); - } - - const blobResponse = await fetch(getBlobUrl(), { - headers: { range: range.header }, - }); - - if (!blobResponse.ok && blobResponse.status !== 206) { - return new Response(blobResponse.body, { - status: blobResponse.status, - statusText: blobResponse.statusText, - }); - } - - const headers = headersForResponse( - getUpstreamCacheHeaders(blobResponse.headers) - ); - const contentRange = blobResponse.headers.get("content-range"); - const contentLength = blobResponse.headers.get("content-length"); - - if (contentRange) headers.set("Content-Range", contentRange); - if (contentLength) headers.set("Content-Length", contentLength); - - return new Response(blobResponse.body, { - status: blobResponse.status, - statusText: blobResponse.statusText, - headers, - }); -} diff --git a/hooks/use-multi-governor-search.ts b/hooks/use-multi-governor-search.ts index b3ee4c1..3c4ea2e 100644 --- a/hooks/use-multi-governor-search.ts +++ b/hooks/use-multi-governor-search.ts @@ -199,7 +199,7 @@ function upsertProposal( ); } -async function loadSqliteProposalIndexProposals(): Promise { +async function loadIndexedProposalIndexProposals(): Promise { try { const client = getTallyDataClient(); const entries = await client.getProposalsIndex(); @@ -215,7 +215,7 @@ async function loadSqliteProposalIndexProposals(): Promise { proposalFromSqliteIndexEntry(entry, voteSummaries[index]) ); } catch (error) { - debug.search("failed to load SQLite proposals index: %O", error); + debug.search("failed to load indexed proposals: %O", error); return []; } } @@ -257,8 +257,8 @@ export function useMultiGovernorSearch({ setProgress(5); if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); - const [sqliteIndexProposals, sqliteWatermarkBlock] = await Promise.all([ - loadSqliteProposalIndexProposals(), + const [indexedProposals, indexedWatermarkBlock] = await Promise.all([ + loadIndexedProposalIndexProposals(), getDelegateVotesWatermarkBlock().catch(() => 0), ]); @@ -266,16 +266,16 @@ export function useMultiGovernorSearch({ if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); const proposalsByKey = new Map(); - for (const proposal of sqliteIndexProposals) { + for (const proposal of indexedProposals) { upsertProposal(proposalsByKey, proposal); } const cachedCount = proposalsByKey.size; debug.search( - "loaded %d proposals from SQLite index (watermark block %d)", + "loaded %d proposals from indexer (watermark block %d)", cachedCount, - sqliteWatermarkBlock + indexedWatermarkBlock ); const proposalsNeedingRefresh = Array.from( @@ -302,8 +302,8 @@ export function useMultiGovernorSearch({ if (signal?.aborted) throw new DOMException("Aborted", "AbortError"); const rpcStartBlock = - sqliteWatermarkBlock > 0 - ? Math.max(sqliteWatermarkBlock + 1, userStartBlock) + indexedWatermarkBlock > 0 + ? Math.max(indexedWatermarkBlock + 1, userStartBlock) : userStartBlock; let freshCount = 0; @@ -342,7 +342,7 @@ export function useMultiGovernorSearch({ } else { debug.search( "skipping RPC search - watermark %d covers search range", - sqliteWatermarkBlock + indexedWatermarkBlock ); setProgress(80); } @@ -355,7 +355,7 @@ export function useMultiGovernorSearch({ proposals: sortProposals(Array.from(proposalsByKey.values())), cacheInfo: { loaded: cachedCount > 0, - snapshotBlock: sqliteWatermarkBlock, + snapshotBlock: indexedWatermarkBlock, cacheStartBlock: 0, cachedCount, freshCount, diff --git a/hooks/use-proposal-stages.tsx b/hooks/use-proposal-stages.tsx index 2222c75..64c69b5 100644 --- a/hooks/use-proposal-stages.tsx +++ b/hooks/use-proposal-stages.tsx @@ -1,6 +1,7 @@ "use client"; import { getGovernorByAddress } from "@/config/governors"; +import { initializeBundledCache } from "@/lib/bundled-cache-loader"; import { getErrorMessage } from "@/lib/error-utils"; import { getCacheAdapter, trimCachedStages } from "@/lib/gov-tracker-cache"; import { @@ -8,7 +9,6 @@ import { trackerManager, type TrackingSession, } from "@/lib/proposal-tracker-manager"; -import { seedProposalCheckpointFromSqlite } from "@/lib/sqlite-checkpoint-seed"; import { createProposalTracker, getAllStageMetadata, @@ -149,9 +149,7 @@ export function useProposalStages({ // Get cache adapter for zero-RPC resume const cache = getCacheAdapter(); - // Seed this proposal's checkpoint from SQLite so gov-tracker can resume - // from the last recorded stage instead of starting RPC discovery from zero. - await seedProposalCheckpointFromSqlite(cache, creationTxHash); + await initializeBundledCache(cache); // Create tracker using gov-tracker package with cache for resume // Gov-tracker handles checkpoint loading/saving automatically diff --git a/lib/delegate-cache.ts b/lib/delegate-cache.ts index 9084723..2e18edc 100644 --- a/lib/delegate-cache.ts +++ b/lib/delegate-cache.ts @@ -257,10 +257,7 @@ export async function getProposalIndexEntry( } export async function getDelegateVotesWatermarkBlock(): Promise { - const value = await getTallyDataClient().getBuildMetadata( - "delegate_votes_watermark_block" - ); - return value ? Number(value) : 0; + return getTallyDataClient().getDelegateVotesWatermarkBlock(); } export { useAddressDisplayRecord, useAddressDisplayRecords }; diff --git a/lib/sqlite-checkpoint-seed.ts b/lib/sqlite-checkpoint-seed.ts deleted file mode 100644 index b359aec..0000000 --- a/lib/sqlite-checkpoint-seed.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { txHashCacheKey, type CacheAdapter } from "@gzeoneth/gov-tracker"; - -import { debug } from "@/lib/debug"; -import { getTallyDataClient } from "@/lib/tally-data/client"; - -const inFlight = new Map>(); - -export async function seedProposalCheckpointFromSqlite( - cache: CacheAdapter, - creationTxHash: string -): Promise { - if (!creationTxHash) return; - - const key = txHashCacheKey(creationTxHash); - - const existing = inFlight.get(key); - if (existing) return existing; - - const promise = (async () => { - if (await cache.has(key)) return; - - const checkpoint = - await getTallyDataClient().getProposalCheckpoint(creationTxHash); - if (!checkpoint) return; - - if (await cache.has(key)) return; - await cache.set(key, checkpoint); - debug.cache( - "seeded gov-tracker cache from SQLite for %s", - creationTxHash.slice(0, 10) - ); - })().finally(() => { - inFlight.delete(key); - }); - - inFlight.set(key, promise); - return promise; -} diff --git a/lib/tally-data/client.ts b/lib/tally-data/client.ts index 5644d8a..aca710f 100644 --- a/lib/tally-data/client.ts +++ b/lib/tally-data/client.ts @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState } from "react"; -import { SqliteTallyDataClient } from "@/lib/tally-data/sqlite"; +import { IndexerTallyDataClient } from "@/lib/tally-data/indexer"; import type { TallyAddressDisplayRecord, TallyDataClient, @@ -25,7 +25,7 @@ export type AddressDisplayRecordsState = { }; export function getTallyDataClient(): TallyDataClient { - client ??= new SqliteTallyDataClient(); + client ??= new IndexerTallyDataClient(); return client; } diff --git a/lib/tally-data/indexer.test.ts b/lib/tally-data/indexer.test.ts new file mode 100644 index 0000000..5f3c698 --- /dev/null +++ b/lib/tally-data/indexer.test.ts @@ -0,0 +1,93 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { IndexerTallyDataClient } from "@/lib/tally-data/indexer"; + +function mockJsonFetch(body: unknown, status = 200) { + return vi.spyOn(globalThis, "fetch").mockResolvedValueOnce( + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }) + ); +} + +describe("IndexerTallyDataClient", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("routes delegate list requests through the governance indexer proxy", async () => { + const fetchMock = mockJsonFetch({ + delegates: [], + totalVotingPower: "0", + totalSupply: "0", + }); + + await new IndexerTallyDataClient().getDelegateList("42"); + + expect(fetchMock).toHaveBeenCalledWith( + "/api/governance-indexer/api/tally/delegates?minVotingPower=42", + { headers: { accept: "application/json" } } + ); + }); + + it("normalizes, dedupes, and batches address map requests", async () => { + const addresses = Array.from( + { length: 201 }, + (_, index) => `0x${index.toString(16).padStart(40, "0")}` + ); + const requestedBatches: string[][] = []; + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input) => { + const url = new URL(String(input), "https://example.test"); + const batch = (url.searchParams.get("addresses") ?? "").split(","); + requestedBatches.push(batch); + return new Response( + JSON.stringify( + batch.map((address) => ({ + address, + ens: null, + name: null, + picture: null, + knownLabel: null, + displayName: null, + })) + ), + { status: 200, headers: { "content-type": "application/json" } } + ); + }); + + const summaries = await new IndexerTallyDataClient().getDelegateSummaries([ + addresses[0].toUpperCase(), + ...addresses, + "", + ]); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(requestedBatches.map((batch) => batch.length)).toEqual([200, 1]); + expect(summaries.size).toBe(201); + expect(summaries.has(addresses[0])).toBe(true); + }); + + it("fetches the delegate votes watermark from the dedicated endpoint", async () => { + const fetchMock = mockJsonFetch({ blockNumber: 123 }); + + await expect( + new IndexerTallyDataClient().getDelegateVotesWatermarkBlock() + ).resolves.toBe(123); + + expect(fetchMock).toHaveBeenCalledWith( + "/api/governance-indexer/api/tally/delegate-votes-watermark", + { headers: { accept: "application/json" } } + ); + }); + + it("throws when the indexer returns a non-2xx response", async () => { + mockJsonFetch({ error: "bad" }, 500); + + await expect(new IndexerTallyDataClient().getStats()).rejects.toThrow( + "Governance indexer request failed: 500" + ); + }); +}); diff --git a/lib/tally-data/indexer.ts b/lib/tally-data/indexer.ts new file mode 100644 index 0000000..329d66e --- /dev/null +++ b/lib/tally-data/indexer.ts @@ -0,0 +1,185 @@ +"use client"; + +import type { + TallyAddressDisplayRecord, + TallyCandidateSummary, + TallyDataClient, + TallyDataStats, + TallyDelegateListResult, + TallyDelegateProfile, + TallyDelegateSearchResult, + TallyDelegateSummary, + TallyDelegateVote, + TallyElectionCandidate, + TallyProposalIndexEntry, + TallyProposalVoteSummary, + TallyProposalVoteSupport, + TallyProposalVoter, +} from "@/lib/tally-data/types"; + +const ADDRESS_BATCH_SIZE = 200; + +function normalizeAddress(address: string): string { + return address.toLowerCase(); +} + +function uniqueNormalizedAddresses(addresses: string[]): string[] { + return Array.from( + new Set( + addresses.filter(Boolean).map((address) => normalizeAddress(address)) + ) + ); +} + +function mapByAddress( + rows: T[] +): Map { + return new Map(rows.map((row) => [normalizeAddress(row.address), row])); +} + +function queryString(params: Record) { + const search = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) { + if (value !== undefined) search.set(key, String(value)); + } + const value = search.toString(); + return value ? `?${value}` : ""; +} + +async function fetchIndexer(path: string): Promise { + const response = await fetch(`/api/governance-indexer/${path}`, { + headers: { accept: "application/json" }, + }); + if (!response.ok) { + throw new Error(`Governance indexer request failed: ${response.status}`); + } + return (await response.json()) as T; +} + +async function fetchAddressMap( + endpoint: string, + addresses: string[] +): Promise> { + const normalized = uniqueNormalizedAddresses(addresses); + if (normalized.length === 0) return new Map(); + + const rows: T[] = []; + for (let i = 0; i < normalized.length; i += ADDRESS_BATCH_SIZE) { + const batch = normalized.slice(i, i + ADDRESS_BATCH_SIZE); + rows.push( + ...(await fetchIndexer( + `${endpoint}${queryString({ addresses: batch.join(",") })}` + )) + ); + } + + return mapByAddress(rows); +} + +export class IndexerTallyDataClient implements TallyDataClient { + getDelegateList(minVotingPower = "10000000000000000000") { + return fetchIndexer( + `api/tally/delegates${queryString({ minVotingPower })}` + ); + } + + getDelegate(address: string) { + return fetchIndexer( + `api/tally/delegates/${normalizeAddress(address)}` + ); + } + + async getDelegateSummaries( + addresses: string[] + ): Promise> { + return fetchAddressMap( + "api/tally/delegate-summaries", + addresses + ); + } + + searchDelegates(query: string, limit = 1000) { + return fetchIndexer( + `api/tally/delegates${queryString({ query, limit })}` + ); + } + + getCandidate(address: string) { + return fetchIndexer( + `api/tally/candidates/${normalizeAddress(address)}` + ); + } + + async getCandidateSummaries( + addresses: string[] + ): Promise> { + return fetchAddressMap( + "api/tally/candidate-summaries", + addresses + ); + } + + async getAddressDisplayRecords( + addresses: string[] + ): Promise> { + return fetchAddressMap( + "api/tally/address-display-records", + addresses + ); + } + + getDelegateVotes(address: string) { + return fetchIndexer( + `api/tally/delegates/${normalizeAddress(address)}/votes` + ); + } + + getProposalVotes(proposalId: string, governorAddress: string) { + return fetchIndexer( + `api/tally/proposals/${normalizeAddress(governorAddress)}/${proposalId}/votes` + ); + } + + getProposalVoteSummary(proposalId: string, governorAddress: string) { + return fetchIndexer( + `api/tally/proposals/${normalizeAddress( + governorAddress + )}/${proposalId}/vote-summary` + ); + } + + getProposalVotersPage( + proposalId: string, + governorAddress: string, + support: TallyProposalVoteSupport, + offset: number, + limit: number + ) { + return fetchIndexer( + `api/tally/proposals/${normalizeAddress( + governorAddress + )}/${proposalId}/votes${queryString({ support, offset, limit })}` + ); + } + + getProposalsIndex() { + return fetchIndexer("api/tally/proposals"); + } + + getProposalIndexEntry(proposalId: string, governorAddress: string) { + return fetchIndexer( + `api/tally/proposals/${normalizeAddress(governorAddress)}/${proposalId}` + ); + } + + async getDelegateVotesWatermarkBlock(): Promise { + const { blockNumber } = await fetchIndexer<{ blockNumber: number }>( + "api/tally/delegate-votes-watermark" + ); + return blockNumber; + } + + getStats(): Promise { + return fetchIndexer("api/tally/stats"); + } +} diff --git a/lib/tally-data/sqlite.ts b/lib/tally-data/sqlite.ts deleted file mode 100644 index 07be1dd..0000000 --- a/lib/tally-data/sqlite.ts +++ /dev/null @@ -1,1155 +0,0 @@ -"use client"; - -import type { TrackingCheckpoint } from "@gzeoneth/gov-tracker"; -import { - createDbWorker, - type SqliteStats, - type WorkerHttpvfs, -} from "sql.js-httpvfs"; - -import type { - TallyAddressDisplayRecord, - TallyCandidateSummary, - TallyDataClient, - TallyDataStats, - TallyDelegateListItem, - TallyDelegateListResult, - TallyDelegateProfile, - TallyDelegateSearchResult, - TallyDelegateSummary, - TallyDelegateVote, - TallyElectionCandidate, - TallyProposalDelegateVote, - TallyProposalIndexEntry, - TallyProposalVoteSummary, - TallyProposalVoteSupport, - TallyProposalVoter, -} from "@/lib/tally-data/types"; - -const DEFAULT_DB_SCHEMA_VERSION = "delegate-votes-v7"; -const DEFAULT_DB_SIZE_BYTES = 882110464; -const DEFAULT_DB_URL = - // eslint-disable-next-line no-process-env - process.env.NODE_ENV === "development" - ? "/tally-data/db.sqlite" - : "/tally-data/tally-zero.sqlite"; -const DEFAULT_DB_CACHE_BUST = `${DEFAULT_DB_SCHEMA_VERSION}-${DEFAULT_DB_SIZE_BYTES}`; -const DEFAULT_DB_VIRTUAL_FILENAME = `tally-zero-${DEFAULT_DB_CACHE_BUST}.sqlite`; -const DEFAULT_CHUNK_SIZE = 4096; -const MAX_BATCH_SIZE = 800; -const DEFAULT_MIN_VOTING_POWER = "10000000000000000000"; -const ARB_TOTAL_SUPPLY = "10000000000000000000000000000"; -const LOCAL_STORAGE_PREFIX = `tally-zero:sqlite:${DEFAULT_DB_SCHEMA_VERSION}:${DEFAULT_DB_SIZE_BYTES}:`; -const LOCAL_STORAGE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; - -let workerPromise: Promise | null = null; - -type DelegateRow = { - id: string; - address: string; - ens: string | null; - name: string | null; - bio: string | null; - twitter: string | null; - picture: string | null; - votes_count: string; - delegators_count: number; - is_prioritized: number; - labels_json: string; - delegate_eligibility_json: string | null; - statement: string | null; - statement_summary: string | null; - is_seeking_delegation: number | null; - known_label: string | null; -}; - -type DelegateSummaryRow = { - address: string; - ens: string | null; - name: string | null; - picture: string | null; - known_label: string | null; -}; - -type CandidateSummaryRow = { - address: string; - name: string; - title: string | null; - type: string | null; -}; - -type CandidateRow = CandidateSummaryRow & { - twitter: string | null; - representative: string | null; - motivation: string | null; - experience: string | null; - skills_json: string; - projects: string | null; - country: string | null; - registered_at: string | null; -}; - -type DelegateVoteRow = { - proposal_id: string; - governor_address: string; - support: number; - weight: string; - block_number: number; -}; - -type ProposalDelegateVoteRow = DelegateVoteRow & { - voter_lower: string; -}; - -type ProposalVoterRow = ProposalDelegateVoteRow & { - delegate_ens: string | null; - delegate_name: string | null; - delegate_picture: string | null; - delegate_known_label: string | null; - candidate_name: string | null; - candidate_title: string | null; -}; - -type ProposalIndexRow = { - proposal_id: string; - governor_address: string; - snapshot_block: number; - state: string | null; - proposer: string | null; - description: string | null; -}; - -type ProposalVoteSummaryRow = { - support: number; - voter_count: number; - weight_total: string; -}; - -type ProposalVoteWeightRow = { - support: number; - weight: string; -}; - -type ProposalCheckpointRow = { - checkpoint_json: string; -}; - -type BuildMetadataRow = { - value: string; -}; - -type LocalStorageEntry = { - createdAt: number; - value: T; -}; - -type StoredDelegateListItem = [ - address: `0x${string}`, - votingPower: string, - lastChangeBlock: number, - ens: string | null, - name: string | null, - picture: string | null, - knownLabel: string | null, - delegatorsCount: number, - isPrioritized: boolean, -]; - -type StoredDelegateListResult = { - d: StoredDelegateListItem[]; - tvp: string; - ts: string; -}; - -function getDatabaseUrl(): string { - return DEFAULT_DB_URL; -} - -function getWorker(): Promise { - if (typeof window === "undefined") { - throw new Error("The Tally data SQLite adapter can only run in a browser."); - } - - const databaseConfig = { - from: "inline" as const, - virtualFilename: DEFAULT_DB_VIRTUAL_FILENAME, - config: { - serverMode: "full" as const, - requestChunkSize: DEFAULT_CHUNK_SIZE, - cacheBust: DEFAULT_DB_CACHE_BUST, - url: getDatabaseUrl(), - }, - }; - - workerPromise ??= createDbWorker( - [databaseConfig], - new URL("sql.js-httpvfs/dist/sqlite.worker.js", import.meta.url).toString(), - new URL("sql.js-httpvfs/dist/sql-wasm.wasm", import.meta.url).toString() - ); - - return workerPromise; -} - -function normalizeAddress(address: string): string { - return address.toLowerCase(); -} - -function uniqueNormalizedAddresses(addresses: string[]): string[] { - return Array.from( - new Set( - addresses.filter(Boolean).map((address) => normalizeAddress(address)) - ) - ); -} - -function placeholders(count: number): string { - return Array.from({ length: count }, () => "?").join(","); -} - -async function queryRows(sql: string, ...params: unknown[]): Promise { - const worker = await getWorker(); - if (params.length === 0) { - return (await worker.db.query(sql)) as T[]; - } - return (await worker.db.query(sql, params)) as T[]; -} - -function getLocalStorageKey(key: string): string { - return `${LOCAL_STORAGE_PREFIX}${key}`; -} - -function readLocalStorage(key: string): T | null { - if (typeof window === "undefined") return null; - - try { - const raw = window.localStorage.getItem(getLocalStorageKey(key)); - if (!raw) return null; - - const parsed = JSON.parse(raw) as LocalStorageEntry; - if (Date.now() - parsed.createdAt > LOCAL_STORAGE_MAX_AGE_MS) { - window.localStorage.removeItem(getLocalStorageKey(key)); - return null; - } - - return parsed.value; - } catch { - return null; - } -} - -function writeLocalStorage(key: string, value: T): void { - if (typeof window === "undefined") return; - - try { - const entry: LocalStorageEntry = { - createdAt: Date.now(), - value, - }; - window.localStorage.setItem(getLocalStorageKey(key), JSON.stringify(entry)); - } catch { - // localStorage is best-effort; query results are still usable without it. - } -} - -function readDelegateListLocalStorage( - key: string -): TallyDelegateListResult | null { - const stored = readLocalStorage(key); - if (!stored) return null; - - return { - delegates: stored.d.map( - ([ - address, - votingPower, - lastChangeBlock, - ens, - name, - picture, - knownLabel, - delegatorsCount, - isPrioritized, - ]) => ({ - address, - votingPower, - lastChangeBlock, - ens, - name, - picture, - knownLabel, - displayName: knownLabel ?? name ?? ens ?? null, - votesCount: votingPower, - delegatorsCount, - isPrioritized, - }) - ), - totalVotingPower: stored.tvp, - totalSupply: stored.ts, - }; -} - -function writeDelegateListLocalStorage( - key: string, - result: TallyDelegateListResult -): void { - writeLocalStorage(key, { - d: result.delegates.map((delegate) => [ - delegate.address, - delegate.votingPower, - delegate.lastChangeBlock, - delegate.ens, - delegate.name, - delegate.picture, - delegate.knownLabel, - delegate.delegatorsCount, - delegate.isPrioritized, - ]), - tvp: result.totalVotingPower, - ts: result.totalSupply, - }); -} - -async function queryAddressBatches( - addresses: string[], - buildSql: (placeholderSql: string) => string -): Promise { - const normalized = uniqueNormalizedAddresses(addresses); - const rows: T[] = []; - - for (let i = 0; i < normalized.length; i += MAX_BATCH_SIZE) { - const batch = normalized.slice(i, i + MAX_BATCH_SIZE); - if (batch.length === 0) continue; - rows.push( - ...(await queryRows(buildSql(placeholders(batch.length)), ...batch)) - ); - } - - return rows; -} - -function parseJson(value: string | null, fallback: T): T { - if (!value) return fallback; - try { - return JSON.parse(value) as T; - } catch { - return fallback; - } -} - -function toFtsPrefixQuery(query: string): string { - return query - .trim() - .split(/[^\p{L}\p{N}_]+/u) - .filter(Boolean) - .map((term) => `${term.replaceAll('"', '""')}*`) - .join(" "); -} - -function decimalGteSql(columnName: string): string { - return `(length(${columnName}) > length(?) or (length(${columnName}) = length(?) and ${columnName} >= ?))`; -} - -function toDelegate(row: DelegateRow): TallyDelegateProfile { - return { - id: row.id, - votesCount: row.votes_count, - delegatorsCount: row.delegators_count, - isPrioritized: Boolean(row.is_prioritized), - account: { - address: row.address, - ens: row.ens ?? "", - name: row.name ?? "", - bio: row.bio ?? "", - twitter: row.twitter ?? "", - picture: row.picture, - }, - statement: { - statement: row.statement ?? "", - statementSummary: row.statement_summary ?? "", - isSeekingDelegation: Boolean(row.is_seeking_delegation), - }, - labels: parseJson(row.labels_json, []), - delegateEligibility: parseJson( - row.delegate_eligibility_json, - null - ), - knownLabel: row.known_label, - }; -} - -function toDelegateSummary(row: DelegateSummaryRow): TallyDelegateSummary { - const displayName = row.known_label ?? row.name ?? row.ens ?? null; - return { - address: row.address as `0x${string}`, - ens: row.ens, - name: row.name, - picture: row.picture, - knownLabel: row.known_label, - displayName, - }; -} - -function toDelegateListItem( - row: DelegateSummaryRow & { - votes_count: string; - delegators_count: number; - is_prioritized: number; - } -): TallyDelegateListItem { - const summary = toDelegateSummary(row); - return { - ...summary, - address: summary.address as `0x${string}`, - votesCount: row.votes_count, - votingPower: row.votes_count, - delegatorsCount: row.delegators_count, - isPrioritized: Boolean(row.is_prioritized), - lastChangeBlock: 0, - }; -} - -function toDelegateVote(row: DelegateVoteRow): TallyDelegateVote { - return { - proposalId: row.proposal_id, - governorAddress: row.governor_address, - support: row.support as 0 | 1 | 2, - weight: row.weight, - blockNumber: row.block_number, - }; -} - -function toProposalDelegateVote( - row: ProposalDelegateVoteRow -): TallyProposalDelegateVote { - return { - voter: row.voter_lower, - ...toDelegateVote(row), - }; -} - -function toProposalVoterDisplay( - row: ProposalVoterRow -): TallyAddressDisplayRecord { - const address = row.voter_lower; - - if (row.candidate_name) { - return { - address, - label: row.candidate_name, - title: row.candidate_title, - picture: null, - profileUrl: `/elections/contender/${address}`, - source: "candidate", - }; - } - - const delegateLabel = - row.delegate_known_label ?? row.delegate_name ?? row.delegate_ens ?? null; - - if (delegateLabel || row.delegate_picture) { - return { - address, - label: delegateLabel, - title: null, - picture: row.delegate_picture, - profileUrl: null, - source: "delegate", - }; - } - - return { - address, - label: null, - title: null, - picture: null, - profileUrl: null, - source: "address", - }; -} - -function toProposalVoter(row: ProposalVoterRow): TallyProposalVoter { - return { - ...toProposalDelegateVote(row), - display: toProposalVoterDisplay(row), - }; -} - -function toProposalIndexEntry(row: ProposalIndexRow): TallyProposalIndexEntry { - return { - proposalId: row.proposal_id, - governorAddress: row.governor_address, - snapshotBlock: row.snapshot_block, - state: row.state, - proposer: row.proposer, - description: row.description, - }; -} - -function emptyProposalVoteSummary(): TallyProposalVoteSummary { - return { - for: { voterCount: 0, weight: "0" }, - against: { voterCount: 0, weight: "0" }, - abstain: { voterCount: 0, weight: "0" }, - totalCount: 0, - }; -} - -function toProposalVoteSummary( - rows: ProposalVoteSummaryRow[] -): TallyProposalVoteSummary { - const summary = emptyProposalVoteSummary(); - - for (const row of rows) { - const value = { - voterCount: row.voter_count, - weight: row.weight_total, - }; - - if (row.support === 1) summary.for = value; - if (row.support === 0) summary.against = value; - if (row.support === 2) summary.abstain = value; - } - - summary.totalCount = - summary.for.voterCount + - summary.against.voterCount + - summary.abstain.voterCount; - - return summary; -} - -function computeProposalVoteSummaryFromRows( - voters: ProposalVoteWeightRow[] -): TallyProposalVoteSummary { - const summary = emptyProposalVoteSummary(); - - for (const voter of voters) { - const bucket = - voter.support === 1 - ? summary.for - : voter.support === 0 - ? summary.against - : summary.abstain; - - bucket.voterCount += 1; - bucket.weight = (BigInt(bucket.weight) + BigInt(voter.weight)).toString(); - summary.totalCount += 1; - } - - return summary; -} - -function isMissingTableError(err: unknown, tableName: string): boolean { - return ( - err instanceof Error && - err.message.toLowerCase().includes(`no such table: ${tableName}`) - ); -} - -function isMissingColumnError(err: unknown, columnName: string): boolean { - return ( - err instanceof Error && - err.message.toLowerCase().includes(`no such column: ${columnName}`) - ); -} - -function toCandidate(row: CandidateRow): TallyElectionCandidate { - return { - address: row.address, - name: row.name, - title: row.title, - twitter: row.twitter, - type: row.type, - representative: row.representative, - motivation: row.motivation, - experience: row.experience, - skills: parseJson(row.skills_json, null), - projects: row.projects, - country: row.country, - registeredAt: row.registered_at, - }; -} - -function mapByAddress( - rows: T[] -): Map { - const map = new Map(); - for (const row of rows) { - map.set(normalizeAddress(row.address), row); - } - return map; -} - -export class SqliteTallyDataClient implements TallyDataClient { - async getDelegateList( - minVotingPower = DEFAULT_MIN_VOTING_POWER - ): Promise { - const cacheKey = `delegate-list:${minVotingPower}`; - const cached = readDelegateListLocalStorage(cacheKey); - if (cached) return cached; - - const rows = await queryRows< - DelegateSummaryRow & { - votes_count: string; - delegators_count: number; - is_prioritized: number; - } - >( - ` -select - d.address, - d.ens, - d.name, - d.picture, - d.voting_power as votes_count, - d.delegators_count, - d.is_prioritized, - d.known_label -from delegate_list d -where ${decimalGteSql("d.voting_power")} -order by d.rank -`, - minVotingPower, - minVotingPower, - minVotingPower - ); - - const delegates = rows.map(toDelegateListItem); - const result: TallyDelegateListResult = { - delegates, - totalVotingPower: delegates - .reduce( - (sum, delegate) => sum + BigInt(delegate.votingPower), - BigInt(0) - ) - .toString(), - totalSupply: ARB_TOTAL_SUPPLY, - }; - - writeDelegateListLocalStorage(cacheKey, result); - return result; - } - - async getDelegate(address: string): Promise { - const cacheKey = `delegate:${normalizeAddress(address)}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - const rows = await queryRows( - ` -select - d.*, - s.statement, - s.statement_summary, - s.is_seeking_delegation, - l.label as known_label -from delegates d -left join delegate_statements s on s.address_lower = d.address_lower -left join delegate_labels l on l.address_lower = d.address_lower -where d.address_lower = ? -limit 1 -`, - normalizeAddress(address) - ); - - const delegate = rows[0] ? toDelegate(rows[0]) : null; - writeLocalStorage(cacheKey, delegate); - return delegate; - } - - async getDelegateSummaries( - addresses: string[] - ): Promise> { - const normalized = uniqueNormalizedAddresses(addresses); - const cachedRows = new Map(); - const missing: string[] = []; - - for (const address of normalized) { - const cached = readLocalStorage( - `delegate-summary:${address}` - ); - if (cached) { - cachedRows.set(address, cached); - } else { - missing.push(address); - } - } - - const rows = await queryAddressBatches( - missing, - (addressPlaceholders) => ` -select - d.address, - d.ens, - d.name, - d.picture, - l.label as known_label -from delegates d -left join delegate_labels l on l.address_lower = d.address_lower -where d.address_lower in (${addressPlaceholders}) -` - ); - const fetchedRows = rows.map(toDelegateSummary); - for (const row of fetchedRows) { - const address = normalizeAddress(row.address); - cachedRows.set(address, row); - writeLocalStorage(`delegate-summary:${address}`, row); - } - - return cachedRows; - } - - async searchDelegates( - query: string, - limit = 1000 - ): Promise { - const normalizedQuery = query.trim().toLowerCase(); - const ftsQuery = toFtsPrefixQuery(normalizedQuery); - if (!ftsQuery) return []; - const cacheKey = `delegate-search:${normalizedQuery}:${limit}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - const rows = await queryRows< - DelegateSummaryRow & { - votes_count: string; - delegators_count: number; - is_prioritized: number; - } - >( - ` -with metadata_matches(address_lower) as ( - select address_lower - from delegate_search - where delegate_search match ? - union - select address_lower - from delegate_search_substrings - where delegate_search_substrings match ? -) -select - d.address, - d.ens, - d.name, - d.picture, - d.votes_count, - d.delegators_count, - d.is_prioritized, - l.label as known_label -from delegates d -join metadata_matches on metadata_matches.address_lower = d.address_lower -left join delegate_labels l on l.address_lower = d.address_lower -order by d.is_prioritized desc, d.delegators_count desc, d.name collate nocase -limit ? -`, - ftsQuery, - ftsQuery, - limit - ); - - const delegates = rows.map((row) => { - const item = toDelegateListItem(row); - return { - address: item.address, - ens: item.ens, - name: item.name, - picture: item.picture, - knownLabel: item.knownLabel, - displayName: item.displayName, - votesCount: item.votesCount, - delegatorsCount: item.delegatorsCount, - isPrioritized: item.isPrioritized, - }; - }); - writeLocalStorage(cacheKey, delegates); - return delegates; - } - - async getCandidate(address: string): Promise { - const rows = await queryRows( - ` -select - address, - name, - title, - twitter, - type, - representative, - motivation, - experience, - skills_json, - projects, - country, - registered_at -from election_candidates -where address_lower = ? -limit 1 -`, - normalizeAddress(address) - ); - - return rows[0] ? toCandidate(rows[0]) : null; - } - - async getCandidateSummaries( - addresses: string[] - ): Promise> { - const rows = await queryAddressBatches( - addresses, - (addressPlaceholders) => ` -select address, name, title, type -from election_candidates -where address_lower in (${addressPlaceholders}) -` - ); - return mapByAddress(rows); - } - - async getAddressDisplayRecords( - addresses: string[] - ): Promise> { - const normalized = uniqueNormalizedAddresses(addresses); - const [candidates, delegates] = await Promise.all([ - this.getCandidateSummaries(normalized), - this.getDelegateSummaries(normalized), - ]); - const records = new Map(); - - for (const addressLower of normalized) { - const candidate = candidates.get(addressLower); - if (candidate) { - records.set(addressLower, { - address: candidate.address, - label: candidate.name, - title: candidate.title, - picture: null, - profileUrl: `/elections/contender/${candidate.address.toLowerCase()}`, - source: "candidate", - }); - continue; - } - - const delegate = delegates.get(addressLower); - if (delegate?.displayName || delegate?.picture) { - records.set(addressLower, { - address: delegate.address, - label: delegate.displayName, - title: null, - picture: delegate.picture, - profileUrl: null, - source: "delegate", - }); - continue; - } - - records.set(addressLower, { - address: addressLower, - label: null, - title: null, - picture: null, - profileUrl: null, - source: "address", - }); - } - - return records; - } - - async getDelegateVotes(address: string): Promise { - const voterLower = normalizeAddress(address); - const cacheKey = `delegate-votes:${voterLower}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - const rows = await queryRows( - ` -select proposal_id, governor_address, support, weight, block_number -from delegate_votes -where voter_lower = ? -`, - voterLower - ); - - const votes = rows.map(toDelegateVote); - writeLocalStorage(cacheKey, votes); - return votes; - } - - async getProposalVotes( - proposalId: string, - governorAddress: string - ): Promise { - const governorLower = normalizeAddress(governorAddress); - const cacheKey = `proposal-voters:${proposalId}:${governorLower}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - const rows = await queryRows( - ` -select - v.voter_lower, - v.proposal_id, - v.governor_address, - v.support, - v.weight, - v.block_number, - d.ens as delegate_ens, - d.name as delegate_name, - d.picture as delegate_picture, - l.label as delegate_known_label, - c.name as candidate_name, - c.title as candidate_title -from delegate_votes v -left join delegates d on d.address_lower = v.voter_lower -left join delegate_labels l on l.address_lower = v.voter_lower -left join election_candidates c on c.address_lower = v.voter_lower -where v.proposal_id = ? and v.governor_address = ? -order by length(v.weight) desc, v.weight desc -`, - proposalId, - governorLower - ); - - const voters = rows.map(toProposalVoter); - writeLocalStorage(cacheKey, voters); - return voters; - } - - async getProposalVoteSummary( - proposalId: string, - governorAddress: string - ): Promise { - const governorLower = normalizeAddress(governorAddress); - const cacheKey = `proposal-vote-summary:${proposalId}:${governorLower}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - try { - const rows = await queryRows( - ` -select support, voter_count, weight_total -from proposal_vote_summary -where proposal_id = ? and governor_address = ? -`, - proposalId, - governorLower - ); - - const summary = toProposalVoteSummary(rows); - writeLocalStorage(cacheKey, summary); - return summary; - } catch (err) { - if (!isMissingTableError(err, "proposal_vote_summary")) throw err; - - const rows = await queryRows( - ` -select support, weight -from delegate_votes -where proposal_id = ? and governor_address = ? -`, - proposalId, - governorLower - ); - const summary = computeProposalVoteSummaryFromRows(rows); - writeLocalStorage(cacheKey, summary); - return summary; - } - } - - async getProposalVotersPage( - proposalId: string, - governorAddress: string, - support: TallyProposalVoteSupport, - offset: number, - limit: number - ): Promise { - const governorLower = normalizeAddress(governorAddress); - const safeOffset = Math.max(0, Math.trunc(offset)); - const safeLimit = Math.max(0, Math.trunc(limit)); - const cacheKey = `proposal-voters-page:${proposalId}:${governorLower}:${support}:${safeOffset}:${safeLimit}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - if (safeLimit === 0) return []; - - const rows = await queryRows( - ` -select - v.voter_lower, - v.proposal_id, - v.governor_address, - v.support, - v.weight, - v.block_number, - d.ens as delegate_ens, - d.name as delegate_name, - d.picture as delegate_picture, - l.label as delegate_known_label, - c.name as candidate_name, - c.title as candidate_title -from delegate_votes v -left join delegates d on d.address_lower = v.voter_lower -left join delegate_labels l on l.address_lower = v.voter_lower -left join election_candidates c on c.address_lower = v.voter_lower -where v.proposal_id = ? and v.governor_address = ? and v.support = ? -order by length(v.weight) desc, v.weight desc -limit ? offset ? -`, - proposalId, - governorLower, - support, - safeLimit, - safeOffset - ); - - const voters = rows.map(toProposalVoter); - writeLocalStorage(cacheKey, voters); - return voters; - } - - async getProposalsIndex(): Promise { - const cacheKey = `proposals-index`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - let rows: ProposalIndexRow[]; - try { - rows = await queryRows( - `select proposal_id, governor_address, snapshot_block, state, proposer, description from proposals_index` - ); - } catch (err) { - if ( - !isMissingColumnError(err, "state") && - !isMissingColumnError(err, "proposer") && - !isMissingColumnError(err, "description") - ) { - throw err; - } - - rows = await queryRows( - `select proposal_id, governor_address, snapshot_block, null as state, null as proposer, null as description from proposals_index` - ); - } - - const entries = rows.map(toProposalIndexEntry); - writeLocalStorage(cacheKey, entries); - return entries; - } - - async getProposalIndexEntry( - proposalId: string, - governorAddress: string - ): Promise { - const governorLower = normalizeAddress(governorAddress); - const cacheKey = `proposal-index-entry:${proposalId}:${governorLower}`; - const cached = readLocalStorage(cacheKey); - if (cached) return cached; - - let rows: ProposalIndexRow[]; - try { - rows = await queryRows( - ` -select proposal_id, governor_address, snapshot_block, state, proposer, description -from proposals_index -where proposal_id = ? and governor_address = ? -limit 1 -`, - proposalId, - governorLower - ); - } catch (err) { - if ( - !isMissingColumnError(err, "state") && - !isMissingColumnError(err, "proposer") && - !isMissingColumnError(err, "description") - ) { - throw err; - } - - rows = await queryRows( - ` -select proposal_id, governor_address, snapshot_block, null as state, null as proposer, null as description -from proposals_index -where proposal_id = ? and governor_address = ? -limit 1 -`, - proposalId, - governorLower - ); - } - - const entry = rows[0] ? toProposalIndexEntry(rows[0]) : null; - writeLocalStorage(cacheKey, entry); - return entry; - } - - async getProposalCheckpoint( - txHash: string - ): Promise { - const normalized = txHash.toLowerCase(); - const cacheKey = `proposal-checkpoint:${normalized}`; - const cached = readLocalStorage(cacheKey); - if (cached !== null) return cached; - - try { - const rows = await queryRows( - `select checkpoint_json from proposal_checkpoints where tx_hash = ? limit 1`, - normalized - ); - const checkpoint = rows[0] - ? (parseJson( - rows[0].checkpoint_json, - null - ) ?? null) - : null; - writeLocalStorage(cacheKey, checkpoint); - return checkpoint; - } catch (err) { - if (isMissingTableError(err, "proposal_checkpoints")) return null; - throw err; - } - } - - async getBuildMetadata(key: string): Promise { - const cacheKey = `build-metadata:${key}`; - const cached = readLocalStorage(cacheKey); - if (cached !== null) return cached; - - const rows = await queryRows( - `select value from build_metadata where key = ? limit 1`, - key - ); - - const value = rows[0]?.value ?? null; - writeLocalStorage(cacheKey, value); - return value; - } - - async getStats(): Promise { - const worker = await getWorker(); - const rows = (await worker.db.query(` -select - (select count(*) from delegates) as delegates, - (select count(*) from delegate_index) as delegate_index, - (select count(*) from delegate_labels) as delegate_labels, - (select count(*) from delegate_search) as delegate_search, - (select count(*) from election_candidates) as election_candidates -`)) as { - delegates: number; - delegate_index: number; - delegate_labels: number; - delegate_search: number; - election_candidates: number; - }[]; - const httpvfs: SqliteStats | null = await worker.worker.getStats(); - const counts = rows[0]; - - return { - delegates: counts.delegates, - delegateIndex: counts.delegate_index, - delegateLabels: counts.delegate_labels, - delegateSearch: counts.delegate_search, - electionCandidates: counts.election_candidates, - httpvfs, - }; - } -} diff --git a/lib/tally-data/types.ts b/lib/tally-data/types.ts index 3204c22..fd37430 100644 --- a/lib/tally-data/types.ts +++ b/lib/tally-data/types.ts @@ -1,6 +1,4 @@ import type { TallyDelegate } from "@/types/tally-delegate"; -import type { TrackingCheckpoint } from "@gzeoneth/gov-tracker"; -import type { SqliteStats } from "sql.js-httpvfs"; export type TallyDelegateProfile = TallyDelegate & { knownLabel: string | null; @@ -109,7 +107,7 @@ export type TallyDataStats = { delegateLabels: number; delegateSearch: number; electionCandidates: number; - httpvfs: SqliteStats | null; + httpvfs: null; }; export interface TallyDataClient { @@ -150,7 +148,6 @@ export interface TallyDataClient { proposalId: string, governorAddress: string ): Promise; - getProposalCheckpoint(txHash: string): Promise; - getBuildMetadata(key: string): Promise; + getDelegateVotesWatermarkBlock(): Promise; getStats(): Promise; } diff --git a/package.json b/package.json index a9209d1..71b5bea 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,6 @@ "scripts": { "dev": "next dev", "build": "next build", - "sqlite:build": "tsx scripts/build-tally-sqlite.ts", - "sqlite:deploy": "tsx scripts/deploy-tally-sqlite.ts", "vote-history:fetch": "tsx scripts/fetch-vote-history.ts", "avatars:upload": "tsx scripts/upload-governance-avatars.ts", "cache:build:delegates": "npx gov-tracker delegates --output data/delegate-cache.json", @@ -76,7 +74,6 @@ "rehype-sanitize": "^6.0.0", "remark-gfm": "^4.0.1", "sonner": "^1.4.0", - "sql.js-httpvfs": "^0.8.12", "tailwind-merge": "^2.2.1", "tailwindcss-animate": "^1.0.7", "vaul": "^1.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 161faf9..5420182 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -167,9 +167,6 @@ importers: sonner: specifier: ^1.4.0 version: 1.7.4(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - sql.js-httpvfs: - specifier: ^0.8.12 - version: 0.8.12 tailwind-merge: specifier: ^2.2.1 version: 2.6.1 @@ -9582,12 +9579,6 @@ packages: integrity: sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==, } - sql.js-httpvfs@0.8.12: - resolution: - { - integrity: sha512-lcEBc2q0psFRfdCx8Di22oUIkkv5MUIaVO/fGCj/Jjx6YQDKVylQEcjd7NSSbmINHTRwVkm/vWP8uuevT7Rkkw==, - } - stable-hash-x@0.2.0: resolution: { @@ -18690,10 +18681,6 @@ snapshots: dependencies: through: 2.3.8 - sql.js-httpvfs@0.8.12: - dependencies: - comlink: 4.4.2 - stable-hash-x@0.2.0: {} stack-utils@2.0.6: diff --git a/scripts/build-tally-sqlite.ts b/scripts/build-tally-sqlite.ts deleted file mode 100644 index fb5a9f4..0000000 --- a/scripts/build-tally-sqlite.ts +++ /dev/null @@ -1,796 +0,0 @@ -import { spawn } from "node:child_process"; -import { once } from "node:events"; -import fs from "node:fs"; -import { createRequire } from "node:module"; -import path from "node:path"; - -import type { TallyDelegate } from "@/types/tally-delegate"; - -const cjsRequire = createRequire(import.meta.url); - -type GovernorCheckpointInput = { - type: "governor"; - governorAddress: string; - proposalId: string; - creationTxHash: string; -}; - -type BundledCheckpoint = { - input?: { type?: string } & Partial; - [key: string]: unknown; -}; - -type BundledCacheJson = Record; - -type DelegateIndexEntry = { - name: string; - picture: string | null; -}; - -type CandidateData = { - name?: string; - title?: string; - address?: string; - twitter?: string; - type?: string; - representative?: string; - motivation?: string; - experience?: string; - skills?: unknown; - projects?: string; - country?: string; - registered_at?: string; -}; - -type DelegateSearchMetadata = { - name: string | null; - ens: string | null; -}; - -type VoteHistoryFile = { - watermarkBlock: number; - generatedAt: string; - votes: Array<{ - governorAddress: string; - proposalId: string; - voter: string; - support: number; - weight: string; - blockNumber: number; - }>; -}; - -type ProposalsIndexFile = { - watermarkBlock: number; - generatedAt: string; - proposals: Array<{ - governorAddress: string; - proposalId: string; - snapshotBlock: number; - state?: string | number | null; - proposer?: string | null; - description?: string | null; - }>; -}; - -type ProposalVoteSummary = { - proposalId: string; - governorAddress: string; - support: number; - voterCount: number; - weightTotal: bigint; -}; - -type ProposalVoteSummarySource = { - proposalId: string; - governorAddress: string; - support: number; - weight: string; -}; - -const rootDir = process.cwd(); -const outputDir = path.join(rootDir, "public", "tally-data"); -const outputDbFilename = "db.sqlite"; -const outputDbPath = path.join(outputDir, outputDbFilename); -const manifestPath = path.join(outputDir, "manifest.json"); -const avatarMapPath = path.join(rootDir, "data", "avatar-map.json"); - -const delegateFiles = [ - "delegates-1.json", - "delegates-2.json", - "delegates-3.json", -]; -const candidateFiles = [ - "election-candidates.json", - "election-5-candidates.json", -]; -const DEFAULT_MIN_VOTING_POWER = BigInt("10000000000000000000"); -const VALID_PROPOSAL_STATES = [ - "Pending", - "Active", - "Canceled", - "Defeated", - "Succeeded", - "Queued", - "Expired", - "Executed", -] as const; - -function readJson(relativePath: string): T { - return JSON.parse(fs.readFileSync(path.join(rootDir, relativePath), "utf8")); -} - -function readOptionalJson(filePath: string, fallback: T): T { - if (!fs.existsSync(filePath)) return fallback; - return JSON.parse(fs.readFileSync(filePath, "utf8")) as T; -} - -function sqlValue(value: unknown): string { - if (value === null || value === undefined) return "NULL"; - if (typeof value === "boolean") return value ? "1" : "0"; - return `'${String(value).replaceAll("'", "''")}'`; -} - -function sqlJson(value: unknown): string { - return sqlValue(JSON.stringify(value ?? null)); -} - -function toSearchSubstringTerms(values: Array) { - const terms = new Set(); - - for (const value of values) { - for (const token of (value ?? "") - .toLowerCase() - .split(/[^\p{L}\p{N}_]+/u) - .filter((part) => part.length >= 2)) { - for (let index = 0; index <= token.length - 2; index += 1) { - terms.add(token.slice(index)); - } - } - } - - return Array.from(terms).join(" "); -} - -function normalizeProposalState( - state: string | number | null | undefined -): string | null { - if (state === undefined || state === null) return null; - - const normalized = String(state).toLowerCase(); - return ( - VALID_PROPOSAL_STATES.find((value) => value.toLowerCase() === normalized) ?? - null - ); -} - -async function writeSql(stdin: NodeJS.WritableStream, sql: string) { - if (!stdin.write(sql)) { - await once(stdin, "drain"); - } -} - -async function main() { - fs.mkdirSync(outputDir, { recursive: true }); - // Build away from the production-facing path so failed builds preserve it. - const temporaryOutputDir = fs.mkdtempSync(path.join(outputDir, ".db-build-")); - const temporaryDbPath = path.join(temporaryOutputDir, outputDbFilename); - let sqlite: ReturnType | null = null; - - try { - sqlite = spawn("sqlite3", [temporaryDbPath], { - stdio: ["pipe", "inherit", "inherit"], - }); - - if (!sqlite.stdin) { - throw new Error("sqlite3 stdin was not available"); - } - - let delegateCount = 0; - let delegateIndexCount = 0; - let delegateLabelCount = 0; - let delegateSearchCount = 0; - let delegateSearchSubstringCount = 0; - let delegateListCount = 0; - let delegateListAvatarCount = 0; - let candidateCount = 0; - let delegateAvatarCount = 0; - let delegateIndexAvatarCount = 0; - let proposalVoteSummaryCount = 0; - let proposalCheckpointCount = 0; - const avatarMap = readOptionalJson>( - avatarMapPath, - {} - ); - const delegateAddresses = new Set(); - const metadataSearchAddresses = new Set(); - const delegateMetadataByAddressLower = new Map< - string, - DelegateSearchMetadata - >(); - const candidateAddresses = new Set(); - - await writeSql( - sqlite.stdin, - ` -pragma journal_mode = delete; -pragma page_size = 4096; -pragma synchronous = off; -pragma temp_store = memory; - -create table delegates ( - address_lower text primary key, - id text not null, - address text not null, - ens text, - name text, - bio text, - twitter text, - picture text, - votes_count text not null, - delegators_count integer not null, - is_prioritized integer not null, - labels_json text not null, - delegate_eligibility_json text -); - -create table delegate_statements ( - address_lower text primary key, - statement text, - statement_summary text, - is_seeking_delegation integer not null -); - -create table delegate_index ( - address_lower text primary key, - name text not null, - picture text -); - -create table delegate_labels ( - address_lower text primary key, - label text not null -); - -create virtual table delegate_search using fts5( - address_lower unindexed, - address unindexed, - name, - ens, - label, - tokenize = 'unicode61' -); - -create virtual table delegate_search_substrings using fts5( - address_lower unindexed, - terms, - tokenize = 'unicode61' -); - -create table delegate_list ( - rank integer primary key, - address_lower text not null unique, - address text not null, - voting_power text not null, - delegators_count integer not null, - is_prioritized integer not null, - ens text, - name text, - picture text, - known_label text -); - -create table election_candidates ( - address_lower text primary key, - address text not null, - name text not null, - title text, - twitter text, - type text, - representative text, - motivation text, - experience text, - skills_json text not null, - projects text, - country text, - registered_at text -); - -create table delegate_votes ( - voter_lower text not null, - proposal_id text not null, - governor_address text not null, - support integer not null, - weight text not null, - block_number integer not null, - primary key (voter_lower, proposal_id, governor_address) -) without rowid; - -create table proposal_vote_summary ( - proposal_id text not null, - governor_address text not null, - support integer not null, - voter_count integer not null, - weight_total text not null, - primary key (proposal_id, governor_address, support) -) without rowid; - -create table proposals_index ( - proposal_id text not null, - governor_address text not null, - snapshot_block integer not null, - state text, - proposer text, - description text, - primary key (proposal_id, governor_address) -) without rowid; - -create table proposal_checkpoints ( - tx_hash text primary key, - governor_address text not null, - proposal_id text not null, - checkpoint_json text not null -) without rowid; - -create table build_metadata ( - key text primary key, - value text not null -); - -begin; -` - ); - - for (const filename of delegateFiles) { - const delegates = readJson(`data/${filename}`); - for (const delegate of delegates) { - const address = delegate.account.address; - const addressLower = address.toLowerCase(); - const accountEns = delegate.account.ens?.trim(); - const accountName = delegate.account.name?.trim(); - const picture = avatarMap[addressLower] ?? delegate.account.picture; - delegateAddresses.add(addressLower); - delegateMetadataByAddressLower.set(addressLower, { - name: accountName || null, - ens: accountEns || null, - }); - if ( - accountName || - (accountEns && !accountEns.toLowerCase().startsWith("0x")) - ) { - metadataSearchAddresses.add(addressLower); - } - if (BigInt(delegate.votesCount) >= DEFAULT_MIN_VOTING_POWER) { - delegateListCount += 1; - if (picture) { - delegateListAvatarCount += 1; - } - } - - await writeSql( - sqlite.stdin, - `insert or replace into delegates values (${[ - sqlValue(addressLower), - sqlValue(delegate.id), - sqlValue(address), - sqlValue(delegate.account.ens || null), - sqlValue(delegate.account.name || null), - sqlValue(delegate.account.bio || null), - sqlValue(delegate.account.twitter || null), - sqlValue(picture), - sqlValue(delegate.votesCount), - sqlValue(delegate.delegatorsCount), - sqlValue(delegate.isPrioritized), - sqlJson(delegate.labels), - sqlJson(delegate.delegateEligibility), - ].join(",")});\n` - ); - if (picture) { - delegateAvatarCount += 1; - } - - await writeSql( - sqlite.stdin, - `insert or replace into delegate_statements values (${[ - sqlValue(addressLower), - sqlValue(delegate.statement.statement || null), - sqlValue(delegate.statement.statementSummary || null), - sqlValue(delegate.statement.isSeekingDelegation), - ].join(",")});\n` - ); - delegateCount += 1; - } - } - - const delegateIndex = readJson>( - "data/delegate-index.json" - ); - const indexByAddressLower = new Map(); - for (const [address, entry] of Object.entries(delegateIndex)) { - const addressLower = address.toLowerCase(); - const entryWithAvatar = { - ...entry, - picture: avatarMap[addressLower] ?? entry.picture, - }; - indexByAddressLower.set(addressLower, entryWithAvatar); - await writeSql( - sqlite.stdin, - `insert or replace into delegate_index values (${[ - sqlValue(addressLower), - sqlValue(entry.name), - sqlValue(entryWithAvatar.picture), - ].join(",")});\n` - ); - if (entryWithAvatar.picture) { - delegateIndexAvatarCount += 1; - } - delegateIndexCount += 1; - } - - const delegateLabels = readJson<{ - delegates: Record; - }>("data/delegate-labels.json"); - for (const [address, label] of Object.entries(delegateLabels.delegates)) { - await writeSql( - sqlite.stdin, - `insert or replace into delegate_labels values (${[ - sqlValue(address.toLowerCase()), - sqlValue(label), - ].join(",")});\n` - ); - delegateLabelCount += 1; - } - - const labelByAddressLower = new Map( - Object.entries(delegateLabels.delegates).map(([address, label]) => [ - address.toLowerCase(), - label, - ]) - ); - const searchAddresses = new Set([ - ...metadataSearchAddresses, - ...indexByAddressLower.keys(), - ...labelByAddressLower.keys(), - ]); - for (const addressLower of searchAddresses) { - if (!delegateAddresses.has(addressLower)) continue; - - const entry = indexByAddressLower.get(addressLower); - const metadata = delegateMetadataByAddressLower.get(addressLower); - const label = labelByAddressLower.get(addressLower) ?? null; - await writeSql( - sqlite.stdin, - `insert into delegate_search (address_lower, address, name, ens, label) -select ${[ - sqlValue(addressLower), - "address", - `coalesce(${sqlValue(entry?.name ?? null)}, name)`, - "ens", - sqlValue(label), - ].join(",")} -from delegates -where address_lower = ${sqlValue(addressLower)};\n` - ); - - const substringTerms = toSearchSubstringTerms([ - entry?.name ?? metadata?.name, - metadata?.ens, - label, - ]); - if (substringTerms) { - await writeSql( - sqlite.stdin, - `insert into delegate_search_substrings (address_lower, terms) values (${[ - sqlValue(addressLower), - sqlValue(substringTerms), - ].join(",")});\n` - ); - delegateSearchSubstringCount += 1; - } - - delegateSearchCount += 1; - } - - await writeSql( - sqlite.stdin, - ` -insert into delegate_list ( - rank, - address_lower, - address, - voting_power, - delegators_count, - is_prioritized, - ens, - name, - picture, - known_label -) -select - row_number() over (order by length(d.votes_count) desc, d.votes_count desc), - d.address_lower, - d.address, - d.votes_count, - d.delegators_count, - d.is_prioritized, - d.ens, - d.name, - d.picture, - l.label -from delegates d -left join delegate_labels l on l.address_lower = d.address_lower -where length(d.votes_count) > length('10000000000000000000') - or ( - length(d.votes_count) = length('10000000000000000000') - and d.votes_count >= '10000000000000000000' - );\n` - ); - for (const filename of candidateFiles) { - if (!fs.existsSync(path.join(rootDir, "data", filename))) { - continue; - } - const candidates = readJson>( - `data/${filename}` - ); - for (const [key, candidate] of Object.entries(candidates)) { - const address = candidate.address ?? key; - candidateAddresses.add(address.toLowerCase()); - await writeSql( - sqlite.stdin, - `insert or replace into election_candidates values (${[ - sqlValue(address.toLowerCase()), - sqlValue(address), - sqlValue(candidate.name ?? ""), - sqlValue(candidate.title ?? null), - sqlValue(candidate.twitter ?? null), - sqlValue(candidate.type ?? null), - sqlValue(candidate.representative ?? null), - sqlValue(candidate.motivation ?? null), - sqlValue(candidate.experience ?? null), - sqlJson(candidate.skills ?? null), - sqlValue(candidate.projects ?? null), - sqlValue(candidate.country ?? null), - sqlValue(candidate.registered_at ?? null), - ].join(",")});\n` - ); - } - } - candidateCount = candidateAddresses.size; - - const votesFile = readJson("data/votes.json"); - const proposalsIndexFile = readJson( - "data/proposals-index.json" - ); - const votesWatermarkBlock = Math.min( - votesFile.watermarkBlock, - proposalsIndexFile.watermarkBlock - ); - - for (const proposal of proposalsIndexFile.proposals) { - const governorAddress = proposal.governorAddress.toLowerCase(); - const proposalState = normalizeProposalState(proposal.state); - const proposer = proposal.proposer - ? proposal.proposer.toLowerCase() - : null; - const description = proposal.description ?? null; - - await writeSql( - sqlite.stdin, - `insert or replace into proposals_index values (${[ - sqlValue(proposal.proposalId), - sqlValue(governorAddress), - sqlValue(proposal.snapshotBlock), - sqlValue(proposalState), - sqlValue(proposer), - sqlValue(description), - ].join(",")});\n` - ); - } - - const bundledCachePath = cjsRequire.resolve( - "@gzeoneth/gov-tracker/bundled-cache.json" - ); - const bundledCache = JSON.parse( - fs.readFileSync(bundledCachePath, "utf8") - ) as BundledCacheJson; - for (const [key, checkpoint] of Object.entries(bundledCache)) { - if (!key.startsWith("tx:")) continue; - const input = checkpoint.input; - if ( - !input || - input.type !== "governor" || - !input.creationTxHash || - !input.governorAddress || - !input.proposalId - ) { - continue; - } - - const txHash = input.creationTxHash.toLowerCase(); - const governorAddress = input.governorAddress.toLowerCase(); - await writeSql( - sqlite.stdin, - `insert or replace into proposal_checkpoints values (${[ - sqlValue(txHash), - sqlValue(governorAddress), - sqlValue(input.proposalId), - sqlJson(checkpoint), - ].join(",")});\n` - ); - proposalCheckpointCount += 1; - } - - const proposalVoteSummary = new Map(); - const proposalVoteSummarySources = new Map< - string, - ProposalVoteSummarySource - >(); - for (const vote of votesFile.votes) { - const governorAddress = vote.governorAddress.toLowerCase(); - proposalVoteSummarySources.set( - [vote.voter.toLowerCase(), vote.proposalId, governorAddress].join(":"), - { - proposalId: vote.proposalId, - governorAddress, - support: vote.support, - weight: vote.weight, - } - ); - - await writeSql( - sqlite.stdin, - `insert or replace into delegate_votes values (${[ - sqlValue(vote.voter.toLowerCase()), - sqlValue(vote.proposalId), - sqlValue(governorAddress), - sqlValue(vote.support), - sqlValue(vote.weight), - sqlValue(vote.blockNumber), - ].join(",")});\n` - ); - } - - for (const vote of proposalVoteSummarySources.values()) { - const summaryKey = [ - vote.proposalId, - vote.governorAddress, - String(vote.support), - ].join(":"); - const summary = proposalVoteSummary.get(summaryKey); - if (summary) { - summary.voterCount += 1; - summary.weightTotal += BigInt(vote.weight); - } else { - proposalVoteSummary.set(summaryKey, { - proposalId: vote.proposalId, - governorAddress: vote.governorAddress, - support: vote.support, - voterCount: 1, - weightTotal: BigInt(vote.weight), - }); - } - } - - for (const summary of proposalVoteSummary.values()) { - await writeSql( - sqlite.stdin, - `insert or replace into proposal_vote_summary values (${[ - sqlValue(summary.proposalId), - sqlValue(summary.governorAddress), - sqlValue(summary.support), - sqlValue(summary.voterCount), - sqlValue(summary.weightTotal.toString()), - ].join(",")});\n` - ); - proposalVoteSummaryCount += 1; - } - - await writeSql( - sqlite.stdin, - `insert or replace into build_metadata values (${[ - sqlValue("delegate_votes_watermark_block"), - sqlValue(String(votesWatermarkBlock)), - ].join(",")});\n` - ); - - await writeSql( - sqlite.stdin, - ` -commit; - -create index delegates_name_idx on delegates(name collate nocase); -create index delegates_ens_idx on delegates(ens collate nocase); -create index delegates_prioritized_idx on delegates(is_prioritized, delegators_count); -create index delegate_index_name_idx on delegate_index(name collate nocase); -create index delegate_list_voting_power_idx on delegate_list(rank, voting_power); -create index election_candidates_name_idx on election_candidates(name collate nocase); -create index delegate_votes_voter_idx on delegate_votes(voter_lower); -create index delegate_votes_proposal_idx on delegate_votes(proposal_id, governor_address); -create index delegate_votes_proposal_support_weight_idx on delegate_votes(proposal_id, governor_address, support, length(weight) desc, weight desc); -create index proposal_checkpoints_proposal_idx on proposal_checkpoints(proposal_id, governor_address); - -analyze; -vacuum; -` - ); - sqlite.stdin.end(); - - const [code] = await once(sqlite, "exit"); - if (code !== 0) { - throw new Error(`sqlite3 exited with code ${code}`); - } - - const sizeBytes = fs.statSync(temporaryDbPath).size; - fs.renameSync(temporaryDbPath, outputDbPath); - - fs.writeFileSync( - manifestPath, - `${JSON.stringify( - { - version: 1, - generatedAt: new Date().toISOString(), - databaseUrl: "/tally-data/db.sqlite", - pageSize: 4096, - sizeBytes, - tables: { - delegates: delegateCount, - delegateAvatars: delegateAvatarCount, - delegateIndex: delegateIndexCount, - delegateIndexAvatars: delegateIndexAvatarCount, - delegateLabels: delegateLabelCount, - delegateSearch: delegateSearchCount, - delegateSearchSubstrings: delegateSearchSubstringCount, - delegateList: delegateListCount, - delegateListAvatars: delegateListAvatarCount, - electionCandidates: candidateCount, - delegateVotes: votesFile.votes.length, - proposalVoteSummary: proposalVoteSummaryCount, - proposalsIndex: proposalsIndexFile.proposals.length, - proposalCheckpoints: proposalCheckpointCount, - }, - delegateVotesWatermarkBlock: votesWatermarkBlock, - }, - null, - 2 - )}\n` - ); - - console.log( - JSON.stringify( - { - database: path.relative(rootDir, outputDbPath), - manifest: path.relative(rootDir, manifestPath), - sizeBytes, - delegates: delegateCount, - delegateAvatars: delegateAvatarCount, - delegateIndex: delegateIndexCount, - delegateIndexAvatars: delegateIndexAvatarCount, - delegateLabels: delegateLabelCount, - delegateSearch: delegateSearchCount, - delegateSearchSubstrings: delegateSearchSubstringCount, - delegateList: delegateListCount, - delegateListAvatars: delegateListAvatarCount, - electionCandidates: candidateCount, - delegateVotes: votesFile.votes.length, - proposalVoteSummary: proposalVoteSummaryCount, - proposalsIndex: proposalsIndexFile.proposals.length, - proposalCheckpoints: proposalCheckpointCount, - delegateVotesWatermarkBlock: votesWatermarkBlock, - }, - null, - 2 - ) - ); - } finally { - if (sqlite && sqlite.exitCode === null && !sqlite.killed) { - sqlite.stdin?.destroy(); - sqlite.kill(); - } - fs.rmSync(temporaryOutputDir, { force: true, recursive: true }); - } -} - -main().catch((error) => { - console.error(error); - process.exitCode = 1; -}); diff --git a/scripts/deploy-tally-sqlite.ts b/scripts/deploy-tally-sqlite.ts deleted file mode 100644 index 1c8bbf0..0000000 --- a/scripts/deploy-tally-sqlite.ts +++ /dev/null @@ -1,331 +0,0 @@ -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import path from "node:path"; - -type Manifest = { - generatedAt: string; - sizeBytes: number; -}; - -type Options = { - dryRun: boolean; - skipBuild: boolean; - skipEnv: boolean; - pathname?: string; - pathnamesByEnv: Record; - environments: string[]; - envName: string; - scope: string; - sourcePrimary?: string; -}; - -const DEFAULT_PATHNAMES: Record = { - production: "governance-data/delegates.sqlite", - preview: "governance-data/delegates-preview.sqlite", - development: "governance-data/delegates-development.sqlite", -}; - -const rootDir = process.cwd(); -const dbPath = path.join(rootDir, "public", "tally-data", "db.sqlite"); -const manifestPath = path.join( - rootDir, - "public", - "tally-data", - "manifest.json" -); -const routePath = path.join( - rootDir, - "app", - "tally-data", - "tally-zero.sqlite", - "route.ts" -); -const routeTestPath = path.join( - rootDir, - "app", - "tally-data", - "tally-zero.sqlite", - "route.test.ts" -); -const sqliteClientPath = path.join(rootDir, "lib", "tally-data", "sqlite.ts"); - -function parseArgs(argv: string[]): Options { - const options: Options = { - dryRun: false, - skipBuild: false, - skipEnv: false, - pathnamesByEnv: {}, - environments: ["preview", "production", "development"], - envName: "GOVERNANCE_DATA_SQLITE_BLOB_URL", - scope: "offchain-labs", - }; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - - if (arg === "--") { - continue; - } else if (arg === "--dry-run") { - options.dryRun = true; - } else if (arg === "--skip-build") { - options.skipBuild = true; - } else if (arg === "--skip-env") { - options.skipEnv = true; - } else if (arg === "--pathname") { - options.pathname = argv[++index]; - } else if (arg.startsWith("--pathname-")) { - const env = arg.slice("--pathname-".length); - options.pathnamesByEnv[env] = argv[++index]; - } else if (arg === "--env") { - options.environments = argv[++index].split(",").map((env) => env.trim()); - } else if (arg === "--env-name") { - options.envName = argv[++index]; - } else if (arg === "--scope") { - options.scope = argv[++index]; - } else if (arg === "--source-primary") { - options.sourcePrimary = argv[++index]; - } else { - throw new Error(`Unknown argument: ${arg}`); - } - } - - return options; -} - -function resolvePathnameForEnv(env: string, options: Options): string { - if (options.pathname) return options.pathname; - const explicit = options.pathnamesByEnv[env]; - if (explicit) return explicit; - const fallback = DEFAULT_PATHNAMES[env]; - if (fallback) return fallback; - throw new Error( - `No pathname configured for env "${env}". Pass --pathname-${env} .` - ); -} - -function run(command: string, args: string[], options?: { input?: string }) { - const printable = [command, ...args].join(" "); - console.log(`$ ${printable}`); - - const result = spawnSync(command, args, { - cwd: rootDir, - encoding: "utf8", - input: options?.input, - }); - - if (result.stdout) process.stdout.write(result.stdout); - if (result.stderr) process.stderr.write(result.stderr); - if (result.error) throw result.error; - if (result.status !== 0) { - throw new Error(`${printable} exited with status ${result.status}`); - } - - return `${result.stdout}\n${result.stderr}`; -} - -function readManifest(): Manifest { - return JSON.parse(fs.readFileSync(manifestPath, "utf8")) as Manifest; -} - -function parseBlobUrl(output: string): string { - const match = output.match(/https:\/\/\S+\.blob\.vercel-storage\.com\/\S+/); - if (!match) { - throw new Error("Could not find uploaded Blob URL in vercel output."); - } - return match[0].trim(); -} - -function replaceOrThrow( - filePath: string, - pattern: RegExp, - replacement: string -) { - const original = fs.readFileSync(filePath, "utf8"); - if (!pattern.test(original)) { - throw new Error( - `No match found while updating ${path.relative(rootDir, filePath)}` - ); - } - const next = original.replace(pattern, replacement); - if (next !== original) fs.writeFileSync(filePath, next); -} - -function updateSourceConstants( - blobUrl: string | null, - sizeBytes: number, - dryRun: boolean -) { - const updates = [ - path.relative(rootDir, routePath), - path.relative(rootDir, routeTestPath), - path.relative(rootDir, sqliteClientPath), - ]; - - if (blobUrl) { - console.log( - `Updating source constants (URL + size): ${updates.join(", ")}` - ); - } else { - console.log( - `Updating source constants (size only, DEFAULT_BLOB_URL preserved): ${updates.join(", ")}` - ); - } - if (dryRun) return; - - if (blobUrl) { - replaceOrThrow( - routePath, - /const DEFAULT_BLOB_URL =\n "https:\/\/[^"]+";/, - `const DEFAULT_BLOB_URL =\n "${blobUrl}";` - ); - } - replaceOrThrow( - routePath, - /const DB_SIZE_BYTES = \d+;/, - `const DB_SIZE_BYTES = ${sizeBytes};` - ); - replaceOrThrow( - routeTestPath, - /const DB_SIZE_BYTES = \d+;/, - `const DB_SIZE_BYTES = ${sizeBytes};` - ); - replaceOrThrow( - sqliteClientPath, - /const DEFAULT_DB_SIZE_BYTES = \d+;/, - `const DEFAULT_DB_SIZE_BYTES = ${sizeBytes};` - ); -} - -function updateVercelEnv( - envName: string, - uploads: Array<{ env: string; blobUrl: string }>, - scope: string, - dryRun: boolean -) { - for (const { env: environment, blobUrl } of uploads) { - if (dryRun) { - console.log( - `$ printf ${blobUrl} | vercel env update ${envName} ${environment} --yes --scope ${scope}` - ); - continue; - } - const update = spawnSync( - "vercel", - ["env", "update", envName, environment, "--yes", "--scope", scope], - { - cwd: rootDir, - encoding: "utf8", - input: blobUrl, - } - ); - if (update.stdout) process.stdout.write(update.stdout); - if (update.stderr) process.stderr.write(update.stderr); - if (update.error) throw update.error; - if (update.status === 0) continue; - - run("vercel", ["env", "add", envName, environment, "--scope", scope], { - input: blobUrl, - }); - } -} - -function main() { - const options = parseArgs(process.argv.slice(2)); - - run("vercel", ["--version"]); - - if (!options.skipBuild) { - if (options.dryRun) { - console.log("$ pnpm sqlite:build"); - } else { - run("pnpm", ["sqlite:build"]); - } - } - - if (!fs.existsSync(dbPath)) { - throw new Error( - `SQLite database not found: ${path.relative(rootDir, dbPath)}` - ); - } - if (!fs.existsSync(manifestPath)) { - throw new Error( - `SQLite manifest not found: ${path.relative(rootDir, manifestPath)}` - ); - } - - const manifest = readManifest(); - const uploads: Array<{ env: string; pathname: string; blobUrl: string }> = []; - - for (const env of options.environments) { - const pathname = resolvePathnameForEnv(env, options); - const blobArgs = [ - "blob", - "put", - path.relative(rootDir, dbPath), - "--scope", - options.scope, - "--pathname", - pathname, - "--content-type", - "application/octet-stream", - "--cache-control-max-age", - "31536000", - "--access", - "public", - "--allow-overwrite", - "true", - ]; - - let blobUrl = `https://example.invalid/${pathname}`; - if (options.dryRun) { - console.log(`# upload for env=${env}`); - console.log(`$ vercel ${blobArgs.join(" ")}`); - } else { - blobUrl = parseBlobUrl(run("vercel", blobArgs)); - } - uploads.push({ env, pathname, blobUrl }); - } - - const productionUpload = uploads.find( - (upload) => upload.env === "production" - ); - const explicitPrimary = options.sourcePrimary - ? uploads.find((upload) => upload.env === options.sourcePrimary) - : undefined; - if (options.sourcePrimary && !explicitPrimary) { - throw new Error( - `--source-primary "${options.sourcePrimary}" was not in the deployed envs.` - ); - } - - // DEFAULT_BLOB_URL is the production fallback baked into source. Only rewrite - // it when production is part of this deploy (or the caller explicitly opts in - // via --source-primary). Preview-only deploys must not move it. - const urlSource = explicitPrimary ?? productionUpload ?? null; - updateSourceConstants( - urlSource ? urlSource.blobUrl : null, - manifest.sizeBytes, - options.dryRun - ); - - if (!options.skipEnv) { - updateVercelEnv(options.envName, uploads, options.scope, options.dryRun); - } - - console.log( - JSON.stringify( - { - uploads, - defaultBlobUrlUpdatedFromEnv: urlSource?.env ?? null, - sizeBytes: manifest.sizeBytes, - envName: options.skipEnv ? null : options.envName, - environments: options.skipEnv ? [] : options.environments, - scope: options.scope, - }, - null, - 2 - ) - ); -} - -main();