From 51faf0e259d7bef03d2afb19350ccc1bd5641c36 Mon Sep 17 00:00:00 2001 From: Luiz Gustavo Abou Hatem de Liz Date: Tue, 9 Jun 2026 19:16:34 -0300 Subject: [PATCH] fix: exclude stale candidates from unconfirmed /by_uids batch; document backfill behavior (COW-992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevents redundant /by_uids calls for already-expired candidates on every block during post-backfill catch-up. The stale path (account fallback, 500/block) already handles these — excluding them from the unconfirmed SELECT eliminates wasted API quota until the drain completes. Also documents cold-start duration, /ready semantics, and the historical discrete-order gap for integrators in docs/deployment.md. Co-Authored-By: Claude Sonnet 4.6 --- docs/deployment.md | 33 ++++++++++++++++++++++++ src/application/handlers/blockHandler.ts | 10 ++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/docs/deployment.md b/docs/deployment.md index 27159f2..97cc66c 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -118,3 +118,36 @@ On the target machine, you need Docker and DNS configured to point at the contai To tear down: `npx tsx deployment/manage.ts down --env-file deployment/.env` +## Cold-Start and Backfill Behavior + +### Timeline + +A fresh deployment (no prior `ponder_sync` cache) reindexes from the configured start blocks. Expected durations: + +| Phase | Typical duration | Notes | +|-------|-----------------|-------| +| Event backfill | 4–10 hours | Fetches `eth_getLogs` from start block to tip. Bottleneck is RPC throughput; a generous RPC endpoint shortens this. | +| Live-sync catch-up | 5–15 minutes | Block handlers (C1–C5) run at "latest" only. Stale TWAP candidates drain at 500/block. | +| Full data completeness | After live-sync catch-up | All generators have candidates or discrete orders; historical TWAP parts resolved via account fallback. | + +A reindex that reuses an existing `ponder_sync` cache (same chain, same start blocks) skips the event backfill and completes in minutes. + +### `/ready` Semantics + +`GET /ready` returns `200` when Ponder has processed all historical blocks up to the tip and the live indexer is running. It does **not** guarantee that all historical discrete-order data is complete — that depends on the live-sync catch-up phase completing (see above). + +During backfill, `GET /ready` returns `503`. GraphQL queries are still available but data is incomplete (generators and transactions accumulate; discrete orders are absent until live sync starts). + +### Historical Discrete Order Gap + +Block handlers only run during live sync. TWAP parts computed during backfill land in `candidate_discrete_order` with past `validTo` dates. When live sync starts, C2 (CandidateConfirmer) promotes these via the stale sweep path: + +1. Tries `/orders/by_uids` — aged-out UIDs return empty +2. Falls back to `/account/{owner}/orders` for each owner with missed UIDs +3. Promotes with the actual API status (fulfilled/expired/cancelled) instead of defaulting to `expired` + +**Residual gap**: Orders that no longer appear in `/account/{owner}/orders` (beyond the CoW API's retention window) will be recorded as `expired` regardless of their actual fill status. This affects only very old orders for users with a large order history. + +Non-deterministic generators (PerpetualSwap, GoodAfterTime, TradeAboveThreshold, Unknown) are handled by C4 (OwnerBackfill), which calls `/account/{owner}/orders` once at live-sync start and upserts discovered orders directly into `discrete_order`. + + diff --git a/src/application/handlers/blockHandler.ts b/src/application/handlers/blockHandler.ts index dc5c822..e8abfb9 100644 --- a/src/application/handlers/blockHandler.ts +++ b/src/application/handlers/blockHandler.ts @@ -16,7 +16,7 @@ import { ponder } from "ponder:registry"; import { bootstrapRetryQueue, candidateDiscreteOrder, conditionalOrderGenerator, discreteOrder, discreteOrderStatusEnum } from "ponder:schema"; -import { and, asc, eq, inArray, isNull, lte, or, sql } from "ponder"; +import { and, asc, eq, gt, inArray, isNull, lte, or, sql } from "ponder"; import type { Hex } from "viem"; import { COMPOSABLE_COW_ADDRESS_BY_CHAIN_ID, @@ -410,6 +410,10 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { // Promoted candidates are always deleted below — no join needed to filter them. // Skip TWAP parts whose validity window hasn't started (possibleValidAfterTimestamp). + // Also exclude already-expired candidates (validTo in the past) — the stale path + // below handles those via /account/{owner}/orders fallback. Without this filter, + // every block would call /by_uids for all remaining stale UIDs (which always miss), + // wasting API quota until the stale drain completes. const unconfirmed = await context.db.sql .select({ orderUid: candidateDiscreteOrder.orderUid, @@ -428,6 +432,10 @@ ponder.on("CandidateConfirmer:block", async ({ event, context }) => { isNull(candidateDiscreteOrder.possibleValidAfterTimestamp), lte(candidateDiscreteOrder.possibleValidAfterTimestamp, event.block.timestamp), ), + or( + isNull(candidateDiscreteOrder.validTo), + gt(candidateDiscreteOrder.validTo, Number(event.block.timestamp)), + ), ), ) as { orderUid: string;