From 169d28f8dd09655f8fdfb55530d72dc80d02dde7 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 19:08:30 +0200 Subject: [PATCH 01/12] feat(updates): colo cache for /updates with targeted worldwide invalidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most apps resolve updates identically for every device on the same (platform, defaultChannel): serve them from the colo cache (Cache API, free) with zero database queries, while per-device work (semver compares, rollout decisions, override lookups) stays per-request. - updates_colo_cache: app-owner + channel resolution cached under a per-app version token, TTL backstop 60s; apps with channel_device_count > 0 keep their per-device override lookup on every request, so a customer gaining a FIRST override reacts within one token bump; rollout manifest cached per version id, non-empty results only - targeted invalidation: DB triggers (channels, channel_devices, apps, app_versions, orgs, stripe_info) call triggers/cache_invalidate via pg_net, which fans one POST out to every regional plugin worker — placement-pinned, so each call lands in the colo whose cache serves that region; end-to-end ~1s from commit, TTL as backstop - /cache_invalidate route on plugin workers guarded by a dedicated shared secret; fanout guarded by the existing API secret - gated by UPDATES_CACHE_MODE (default off) per regional env At 83M requests/day this removes ~95%+ of read queries from the replica fleet; enables shrinking it once hit rates are confirmed. Co-Authored-By: Claude Fable 5 --- cloudflare_workers/api/index.ts | 2 + cloudflare_workers/plugin/index.ts | 2 + cloudflare_workers/plugin/wrangler.jsonc | 36 ++ .../_backend/private/cache_invalidate.ts | 43 +++ .../_backend/triggers/cache_invalidate.ts | 71 ++++ .../functions/_backend/utils/cloudflare.ts | 5 + supabase/functions/_backend/utils/pg.ts | 2 +- supabase/functions/_backend/utils/update.ts | 13 +- .../_backend/utils/updates_colo_cache.ts | 288 ++++++++++++++ supabase/functions/triggers/index.ts | 2 + ...60711100000_updates_cache_invalidation.sql | 111 ++++++ tests/updates-colo-cache.unit.test.ts | 356 ++++++++++++++++++ 12 files changed, 927 insertions(+), 4 deletions(-) create mode 100644 supabase/functions/_backend/private/cache_invalidate.ts create mode 100644 supabase/functions/_backend/triggers/cache_invalidate.ts create mode 100644 supabase/functions/_backend/utils/updates_colo_cache.ts create mode 100644 supabase/migrations/20260711100000_updates_cache_invalidation.sql create mode 100644 tests/updates-colo-cache.unit.test.ts diff --git a/cloudflare_workers/api/index.ts b/cloudflare_workers/api/index.ts index 723f1556d1..cb5bb386c5 100644 --- a/cloudflare_workers/api/index.ts +++ b/cloudflare_workers/api/index.ts @@ -51,6 +51,7 @@ import { app as replication } from '../../supabase/functions/_backend/public/rep import { app as statistics } from '../../supabase/functions/_backend/public/statistics/index.ts' import { app as translation } from '../../supabase/functions/_backend/public/translation.ts' import { app as webhooks } from '../../supabase/functions/_backend/public/webhooks/index.ts' +import { app as cache_invalidate } from '../../supabase/functions/_backend/triggers/cache_invalidate.ts' import { app as credit_usage_alerts } from '../../supabase/functions/_backend/triggers/credit_usage_alerts.ts' import { app as cron_clean_orphan_images } from '../../supabase/functions/_backend/triggers/cron_clean_orphan_images.ts' import { app as cron_clear_versions } from '../../supabase/functions/_backend/triggers/cron_clear_versions.ts' @@ -151,6 +152,7 @@ appPrivate.route('/sso/verify-dns', sso_verify_dns) const functionNameTriggers = 'triggers' const appTriggers = createHono(functionNameTriggers, version) appTriggers.route('/ok', ok) +appTriggers.route('/cache_invalidate', cache_invalidate) appTriggers.route('/cron_email', cron_email) appTriggers.route('/cron_clear_versions', cron_clear_versions) appTriggers.route('/cron_clean_orphan_images', cron_clean_orphan_images) diff --git a/cloudflare_workers/plugin/index.ts b/cloudflare_workers/plugin/index.ts index 41d87e8954..0fe69d5992 100644 --- a/cloudflare_workers/plugin/index.ts +++ b/cloudflare_workers/plugin/index.ts @@ -1,6 +1,7 @@ import { app as channel_self } from '../../supabase/functions/_backend/plugins/channel_self.ts' import { app as stats } from '../../supabase/functions/_backend/plugins/stats.ts' import { app as updates } from '../../supabase/functions/_backend/plugins/updates.ts' +import { app as cache_invalidate } from '../../supabase/functions/_backend/private/cache_invalidate.ts' import { app as latency } from '../../supabase/functions/_backend/private/latency.ts' import { app as ok } from '../../supabase/functions/_backend/public/ok.ts' import { createAllCatch, createHono, useCors } from '../../supabase/functions/_backend/utils/hono.ts' @@ -32,6 +33,7 @@ app.route('/updates', updates) app.route('/stats', stats) app.route('/ok', ok) app.route('/latency', latency) +app.route('/cache_invalidate', cache_invalidate) createAllCatch(app, functionName) diff --git a/cloudflare_workers/plugin/wrangler.jsonc b/cloudflare_workers/plugin/wrangler.jsonc index f6e7bbb775..d9ed26494a 100644 --- a/cloudflare_workers/plugin/wrangler.jsonc +++ b/cloudflare_workers/plugin/wrangler.jsonc @@ -14,6 +14,9 @@ "prod_eu": { "name": "capgo_plugin-eu-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-eu-prod" }, "observability": { @@ -143,6 +146,9 @@ "prod_me": { "name": "capgo_plugin-me-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-me-prod" }, "observability": { @@ -238,6 +244,9 @@ "prod_hk": { "name": "capgo_plugin-hk-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-hk-prod" }, "observability": { @@ -333,6 +342,9 @@ "prod_jp": { "name": "capgo_plugin-jp-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-jp-prod" }, "observability": { @@ -428,6 +440,9 @@ "prod_as": { "name": "capgo_plugin-as-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-as-prod" }, "observability": { @@ -523,6 +538,9 @@ "prod_na": { "name": "capgo_plugin-na-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-na-prod" }, "observability": { @@ -618,6 +636,9 @@ "prod_af": { "name": "capgo_plugin-af-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-af-prod" }, "placement": { @@ -713,6 +734,9 @@ "prod_oc": { "name": "capgo_plugin-oc-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-oc-prod" }, "observability": { @@ -808,6 +832,9 @@ "prod_sa": { "name": "capgo_plugin-sa-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-sa-prod" }, "observability": { @@ -891,6 +918,9 @@ "preprod": { "name": "capgo_plugin-eu-preprod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-eu-preprod" }, "observability": { @@ -982,6 +1012,9 @@ "alpha": { "name": "capgo_plugin-alpha", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-alpha" }, "observability": { @@ -1077,6 +1110,9 @@ "local": { "name": "capgo_plugin-local", "vars": { + // Colo cache for /updates: flip to "on" per region to enable + // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-local", "CAPGO_PREVENT_BACKGROUND_FUNCTIONS": "true" }, diff --git a/supabase/functions/_backend/private/cache_invalidate.ts b/supabase/functions/_backend/private/cache_invalidate.ts new file mode 100644 index 0000000000..c0b76adbaf --- /dev/null +++ b/supabase/functions/_backend/private/cache_invalidate.ts @@ -0,0 +1,43 @@ +// Colo cache invalidation endpoint, mounted on every regional plugin worker. +// +// The plugin workers are placement-pinned, so a request to each regional +// domain lands in the colo whose cache serves that region's devices. The +// cache_invalidate trigger handler fans a token bump out to all regions; +// each bump makes every cached /updates payload of the app unreachable +// (see updates_colo_cache.ts). Auth is a dedicated shared secret so this +// stays independent from the API-secret used by Supabase triggers. + +import type { MiddlewareKeyVariables } from '../utils/hono.ts' +import { Hono } from 'hono/tiny' +import { BRES, parseBody, quickError } from '../utils/hono.ts' +import { cloudlog } from '../utils/logging.ts' +import { bumpAppCacheToken, isUpdatesCacheEnabled } from '../utils/updates_colo_cache.ts' +import { existInEnv, getEnv } from '../utils/utils.ts' + +export const MAX_INVALIDATE_APPS = 100 + +export const app = new Hono() + +app.post('/', async (c) => { + if (!existInEnv(c, 'CACHE_INVALIDATE_SECRET')) + throw quickError(503, 'cache_invalidate_disabled', 'CACHE_INVALIDATE_SECRET is not configured') + if (c.req.header('x-cache-invalidate-secret') !== getEnv(c, 'CACHE_INVALIDATE_SECRET')) + throw quickError(401, 'unauthorized', 'Invalid cache invalidation secret') + + const body = await parseBody<{ app_ids?: unknown }>(c) + const appIds = Array.isArray(body.app_ids) + ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0).slice(0, MAX_INVALIDATE_APPS) + : [] + if (appIds.length === 0) + throw quickError(400, 'missing_app_ids', 'app_ids must be a non-empty array of strings') + + // Bump even when the cache mode is off so entries from a prior "on" + // window can never be served stale after a toggle. + let bumped = 0 + for (const appId of appIds) { + if (await bumpAppCacheToken(c, appId)) + bumped++ + } + cloudlog({ requestId: c.get('requestId'), message: 'updates cache invalidated', appIds, bumped, enabled: isUpdatesCacheEnabled(c) }) + return c.json({ ...BRES, bumped }) +}) diff --git a/supabase/functions/_backend/triggers/cache_invalidate.ts b/supabase/functions/_backend/triggers/cache_invalidate.ts new file mode 100644 index 0000000000..16159e94cb --- /dev/null +++ b/supabase/functions/_backend/triggers/cache_invalidate.ts @@ -0,0 +1,71 @@ +// Fan-out of /updates colo-cache invalidations to every regional plugin +// worker. +// +// Called by the invalidate_updates_cache() database trigger (via pg_net) +// whenever a row that feeds the update hot path changes (channels, +// channel_devices, apps, app_versions, orgs, stripe_info). The plugin +// workers are placement-pinned, so one POST per regional domain reaches +// every colo that caches this app — end-to-end reaction is ~1s from the +// database commit, with the cache TTL as backstop when a call is lost. + +import type { MiddlewareKeyVariables } from '../utils/hono.ts' +import { Hono } from 'hono/tiny' +import { BRES, middlewareAPISecret, parseBody } from '../utils/hono.ts' +import { cloudlog, cloudlogErr, serializeError } from '../utils/logging.ts' +import { existInEnv, getEnv } from '../utils/utils.ts' + +const FANOUT_TIMEOUT_MS = 5000 + +export function parsePluginInvalidateUrls(raw: string): string[] { + return raw + .split(',') + .map(url => url.trim().replace(/\/$/, '')) + .filter(Boolean) +} + +export const app = new Hono() + +app.post('/', middlewareAPISecret, async (c) => { + const body = await parseBody<{ app_ids?: unknown }>(c) + const appIds = Array.isArray(body.app_ids) + ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0) + : [] + if (appIds.length === 0) { + cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout skipped (no app_ids)' }) + return c.json(BRES) + } + + if (!existInEnv(c, 'PLUGIN_INVALIDATE_URLS') || !existInEnv(c, 'CACHE_INVALIDATE_SECRET')) { + // Soft-skip: invalidation is an accelerator, the cache TTL is the backstop. + cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout skipped (missing env)', appIds }) + return c.json(BRES) + } + + const urls = parsePluginInvalidateUrls(getEnv(c, 'PLUGIN_INVALIDATE_URLS')) + const secret = getEnv(c, 'CACHE_INVALIDATE_SECRET') + const results = await Promise.all(urls.map(async (url) => { + try { + const response = await fetch(`${url}/cache_invalidate`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-cache-invalidate-secret': secret, + }, + body: JSON.stringify({ app_ids: appIds }), + signal: AbortSignal.timeout(FANOUT_TIMEOUT_MS), + }) + if (!response.ok) { + cloudlogErr({ requestId: c.get('requestId'), message: 'cache invalidate fanout failed', url, status: response.status }) + return false + } + return true + } + catch (e) { + cloudlogErr({ requestId: c.get('requestId'), message: 'cache invalidate fanout error', url, error: serializeError(e) }) + return false + } + })) + const succeeded = results.filter(Boolean).length + cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout done', appIds, regions: urls.length, succeeded }) + return c.json({ ...BRES, regions: urls.length, succeeded }) +}) diff --git a/supabase/functions/_backend/utils/cloudflare.ts b/supabase/functions/_backend/utils/cloudflare.ts index aba94df294..438508d24c 100644 --- a/supabase/functions/_backend/utils/cloudflare.ts +++ b/supabase/functions/_backend/utils/cloudflare.ts @@ -49,6 +49,11 @@ export type Bindings = { NOTIFICATION_EVENTS?: AnalyticsEngineDataset NOTIFICATION_QUEUE?: Queue DB_STOREAPPS: D1Database + // /updates colo cache + targeted invalidation (see utils/updates_colo_cache.ts) + UPDATES_CACHE_MODE?: string + UPDATES_CACHE_TTL_SECONDS?: string + CACHE_INVALIDATE_SECRET?: string + PLUGIN_INVALIDATE_URLS?: string CHANNEL_SELF_STORE?: KVNamespace PLUGIN_NOTIFICATION_QUEUE?: KVNamespace LOCAL_READ_REPLICA_SUPABASE_DB_URL?: string diff --git a/supabase/functions/_backend/utils/pg.ts b/supabase/functions/_backend/utils/pg.ts index 0a90f5e65b..c54d08cacc 100644 --- a/supabase/functions/_backend/utils/pg.ts +++ b/supabase/functions/_backend/utils/pg.ts @@ -24,7 +24,7 @@ const REPLICATION_LAG_CACHE_TTL_MS = REPLICATION_LAG_CACHE_TTL_SECONDS * 1000 type ReplicationStatus = 'ok' | 'lagging' | 'unknown' interface ChannelLookupResult { id: number, name: string, allow_device_self_set: boolean, public: boolean, owner_org: string } -type PlanAction = 'mau' | 'storage' | 'bandwidth' +export type PlanAction = 'mau' | 'storage' | 'bandwidth' type ReadReplicaHyperdriveBinding = | 'HYPERDRIVE_CAPGO_READ_AS_JAPAN' | 'HYPERDRIVE_CAPGO_READ_AS_INDIA' diff --git a/supabase/functions/_backend/utils/update.ts b/supabase/functions/_backend/utils/update.ts index ec7fd65d68..20bfa78d37 100644 --- a/supabase/functions/_backend/utils/update.ts +++ b/supabase/functions/_backend/utils/update.ts @@ -19,6 +19,7 @@ import { cloudlog } from './logging.ts' import { getClientIP } from './rate_limit.ts' import { sendNotifOrgCached } from './notifications.ts' import { closeClient, getAppBlockProviderInfraRequestsPostgres, getAppOwnerPostgres, getDrizzleClient, getPgClient, requestInfosPostgres, setReplicationLagHeader } from './pg.ts' +import { cachedGetAppOwner, cachedRequestInfos, isUpdatesCacheEnabled } from './updates_colo_cache.ts' import { makeDevice } from './plugin_parser.ts' import { s3 } from './s3.ts' import { createStatsBandwidth, createStatsMau, createStatsVersion, onPremStats, sendStatsAndDevice } from './stats.ts' @@ -238,7 +239,10 @@ export async function updateWithPG( return providerBlockedResponse } - const appOwner = await getAppOwnerPostgres(c, app_id, drizzleClient, PLAN_LIMIT) + const useColoCache = isUpdatesCacheEnabled(c) + const appOwner = useColoCache + ? await cachedGetAppOwner(c, app_id, drizzleClient, PLAN_LIMIT) + : await getAppOwnerPostgres(c, app_id, drizzleClient, PLAN_LIMIT) // if version_build is not semver, then make it semver const device = makeDevice(body, appOwner?.allow_device_custom_id) if (!appOwner) { @@ -340,7 +344,7 @@ export async function updateWithPG( // Only query link/comment if plugin supports it (v5.35.0+, v6.35.0+, v7.35.0+, v8.35.0+) AND app has expose_metadata enabled const needsMetadata = appOwner.expose_metadata && !isDeprecatedPluginVersion(pluginVersion, '5.35.0', '6.35.0', '7.35.0', '8.35.0') - const requestedInto = await requestInfosPostgres({ + const requestInfosOptions = { c, platform, app_id, @@ -354,7 +358,10 @@ export async function updateWithPG( currentVersionName: version_name, includeMetadata: needsMetadata, channelSelfOverrideChannelId: channelSelfOverride?.channel_id.id, - }) + } + const requestedInto = useColoCache + ? await cachedRequestInfos(requestInfosOptions) + : await requestInfosPostgres(requestInfosOptions) const { channelOverride } = requestedInto let { channelData } = requestedInto cloudlog({ requestId: c.get('requestId'), message: `channelData exists ? ${channelData !== undefined}, channelOverride exists ? ${channelOverride !== undefined}` }) diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts new file mode 100644 index 0000000000..e9bec5947f --- /dev/null +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -0,0 +1,288 @@ +// Colo-local cache for the /updates hot path, with targeted invalidation. +// +// Most apps have one public channel and no per-device overrides: their +// channel resolution is identical for every device on the same +// (platform, defaultChannel) and can be served from the colo cache with +// zero database queries. Per-device work (semver compares, rollout +// decisions, override lookups for apps that have them) stays per-request. +// +// Freshness has two layers: +// - a short TTL backstop (UPDATES_CACHE_TTL_SECONDS, default 60s — the same +// class as the Hyperdrive query cache serving this traffic today), and +// - targeted invalidation: every cached entry is keyed under a per-app +// version token; bumping the token (bumpAppCacheToken) makes every entry +// for that app unreachable at once. Database triggers fan the bump out to +// each regional plugin worker (placement-pinned, so each call lands in +// the colo whose cache needs clearing) within ~1s of the commit. +// +// The apps.channel_device_count counter (already denormalized and cached in +// the payload) is the switch: apps with overrides keep their per-device +// lookup on every request, so adding a FIRST override reacts within one +// token bump (~1s) or one TTL at worst. + +import type { Context } from 'hono' +import type { AppOwnerPostgresResult, getDrizzleClient, PlanAction } from './pg.ts' +import { CacheHelper } from './cache.ts' +import { cloudlog } from './logging.ts' +import { + getAppOwnerPostgres, + requestInfosChannelByIdPostgres, + requestInfosChannelByIdPostgresRollout, + requestInfosChannelDevicePostgres, + requestInfosChannelDevicePostgresRollout, + requestInfosChannelPostgres, + requestInfosChannelPostgresRollout, + requestManifestEntriesPostgres, +} from './pg.ts' +import { getRolloutDecision } from './rollout.ts' +import { existInEnv, getEnv } from './utils.ts' + +const TOKEN_CACHE_PATH = '/cache/updates-token' +const OWNER_CACHE_PATH = '/cache/updates-owner' +const CHANNEL_CACHE_PATH = '/cache/updates-channel' +const MANIFEST_CACHE_PATH = '/cache/updates-manifest' + +const TOKEN_TTL_SECONDS = 7 * 24 * 3600 +const DEFAULT_PAYLOAD_TTL_SECONDS = 60 +const MANIFEST_TTL_SECONDS = 300 + +interface TokenPayload { t: string } +interface OwnerPayload { owner: AppOwnerPostgresResult | null } +interface ChannelPayload { channel: unknown } + +export function isUpdatesCacheEnabled(c: Context): boolean { + return existInEnv(c, 'UPDATES_CACHE_MODE') && getEnv(c, 'UPDATES_CACHE_MODE') === 'on' +} + +function payloadTtlSeconds(c: Context): number { + const raw = Number(getEnv(c, 'UPDATES_CACHE_TTL_SECONDS')) + return Number.isFinite(raw) && raw >= 5 ? raw : DEFAULT_PAYLOAD_TTL_SECONDS +} + +// Per-app version token: every cached payload embeds it in its key, so one +// token bump atomically invalidates all payload variants of the app in this +// colo. The old entries become unreachable and expire by TTL. +async function getAppCacheToken(_c: Context, helper: CacheHelper, appId: string): Promise { + const request = helper.buildRequest(TOKEN_CACHE_PATH, { app_id: appId }) + const cached = await helper.matchJson(request) + if (cached?.t) + return cached.t + const token = crypto.randomUUID() + await helper.putJson(request, { t: token }, TOKEN_TTL_SECONDS) + return token +} + +// Invalidation entry point, called by the /cache_invalidate route on each +// regional plugin worker (fan-out from the cache_invalidate trigger). +export async function bumpAppCacheToken(c: Context, appId: string): Promise { + const helper = new CacheHelper(c) + const request = helper.buildRequest(TOKEN_CACHE_PATH, { app_id: appId }) + await helper.putJson(request, { t: crypto.randomUUID() }, TOKEN_TTL_SECONDS) + return true +} + +// Drop-in replacement for getAppOwnerPostgres, cached per app. Negative +// results (unknown app) are cached too: unknown-app traffic is the +// enumeration hot path and the apps INSERT trigger bumps the token the +// moment the app is created. +export async function cachedGetAppOwner( + c: Context, + appId: string, + drizzleClient: ReturnType, + actions: PlanAction[], +): Promise { + const helper = new CacheHelper(c) + const token = await getAppCacheToken(c, helper, appId) + if (!token) + return getAppOwnerPostgres(c, appId, drizzleClient, actions) + const request = helper.buildRequest(OWNER_CACHE_PATH, { app_id: appId, v: token, a: actions.join('-') }) + const cached = await helper.matchJson(request) + if (cached) { + cloudlog({ requestId: c.get('requestId'), message: 'updates cache hit (owner)', appId }) + return cached.owner + } + const owner = await getAppOwnerPostgres(c, appId, drizzleClient, actions) + await helper.putJson(request, { owner }, payloadTtlSeconds(c)) + return owner +} + +export interface CachedRequestInfosOptions { + c: Context + platform: string + app_id: string + device_id: string + defaultChannel: string + drizzleClient: ReturnType + channelDeviceCount?: number | null + manifestBundleCount?: number | null + rolloutChannelCount?: number | null + rolloutPausedVersionNames?: string[] | null + currentVersionName: string + includeMetadata?: boolean + channelSelfOverrideChannelId?: number | null +} + +// Cached mirror of requestInfosPostgres (pg.ts): the app-level channel +// lookup is cached per (app, platform, defaultChannel, flags); the +// device-level parts (override lookup, rollout decision, rollout manifest) +// run per request exactly like the uncached path. +export async function cachedRequestInfos(options: CachedRequestInfosOptions): Promise<{ channelData: any, channelOverride: any }> { + const { + c, + platform, + app_id, + device_id, + defaultChannel, + drizzleClient, + channelDeviceCount, + manifestBundleCount, + rolloutChannelCount, + rolloutPausedVersionNames, + currentVersionName, + includeMetadata = false, + channelSelfOverrideChannelId, + } = options + const shouldQueryChannelOverride = channelDeviceCount === undefined || channelDeviceCount === null ? true : channelDeviceCount > 0 + const shouldFetchManifest = manifestBundleCount === undefined || manifestBundleCount === null ? true : manifestBundleCount > 0 + const isPausedRolloutVersion = Array.isArray(rolloutPausedVersionNames) && rolloutPausedVersionNames.includes(currentVersionName) + const rollout = (rolloutChannelCount ?? 0) > 0 || isPausedRolloutVersion + + // Device-level override lookup: never cached (per-device result). + let channelOverridePromise: Promise + if (typeof channelSelfOverrideChannelId === 'number') { + channelOverridePromise = rollout + ? requestInfosChannelByIdPostgresRollout(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeMetadata) + : requestInfosChannelByIdPostgres(c, app_id, channelSelfOverrideChannelId, drizzleClient, shouldFetchManifest, includeMetadata) + } + else if (shouldQueryChannelOverride) { + channelOverridePromise = rollout + ? requestInfosChannelDevicePostgresRollout(c, app_id, device_id, drizzleClient, includeMetadata) + : requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata) + } + else { + channelOverridePromise = Promise.resolve(null) + } + + // App-level channel resolution: cached. + const channelPromise = cachedChannelLookup(c, { + platform, + app_id, + defaultChannel, + drizzleClient, + includeMetadata, + includeManifest: shouldFetchManifest, + rollout, + }) + + const [channelOverrideRaw, channelDataRaw] = await Promise.all([channelOverridePromise, channelPromise]) + + if (!rollout) + return { channelData: channelDataRaw, channelOverride: channelOverrideRaw } + + // Rollout: per-device decision + manifest for the selected version, + // mirrors resolveRolloutChannelDataPostgres (pg.ts). + const channelOverride = await resolveRolloutChannelData(c, channelOverrideRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + const channelData = channelOverride + ? channelDataRaw + : await resolveRolloutChannelData(c, channelDataRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + return { channelOverride, channelData } +} + +interface CachedChannelLookupOptions { + platform: string + app_id: string + defaultChannel: string + drizzleClient: ReturnType + includeMetadata: boolean + includeManifest: boolean + rollout: boolean +} + +async function cachedChannelLookup(c: Context, options: CachedChannelLookupOptions): Promise { + const { platform, app_id, defaultChannel, drizzleClient, includeMetadata, includeManifest, rollout } = options + const load = () => rollout + ? requestInfosChannelPostgresRollout(c, platform, app_id, defaultChannel, drizzleClient, includeMetadata) + : requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, includeManifest, includeMetadata) + + const helper = new CacheHelper(c) + const token = await getAppCacheToken(c, helper, app_id) + if (!token) + return load() + const request = helper.buildRequest(CHANNEL_CACHE_PATH, { + app_id, + v: token, + platform, + channel: defaultChannel || '-', + meta: includeMetadata ? '1' : '0', + manifest: includeManifest ? '1' : '0', + rollout: rollout ? '1' : '0', + }) + const cached = await helper.matchJson(request) + if (cached) { + cloudlog({ requestId: c.get('requestId'), message: 'updates cache hit (channel)', appId: app_id, platform }) + return cached.channel ?? undefined + } + const channel = await load() + // `undefined` is not JSON-representable; store null and map back on read. + await helper.putJson(request, { channel: channel ?? null }, payloadTtlSeconds(c)) + return channel +} + +// Manifest entries for a rollout-selected version. Keyed by version id only: +// manifest rows are append-only per bundle, so a non-empty result is stable. +// Empty results are never cached (the bundle may still be uploading). +async function cachedManifestEntries( + c: Context, + versionId: number, + drizzleClient: ReturnType, +): Promise<{ file_name: string | null, file_hash: string | null, s3_path: string | null }[]> { + const helper = new CacheHelper(c) + const request = helper.buildRequest(MANIFEST_CACHE_PATH, { version_id: String(versionId) }) + const cached = await helper.matchJson<{ entries: { file_name: string | null, file_hash: string | null, s3_path: string | null }[] }>(request) + if (cached) + return cached.entries + const entries = await requestManifestEntriesPostgres(c, versionId, drizzleClient) + if (entries.length > 0) + await helper.putJson(request, { entries }, MANIFEST_TTL_SECONDS) + return entries +} + +// Mirrors resolveRolloutChannelDataPostgres (pg.ts), with the manifest read +// going through the version-keyed cache. +async function resolveRolloutChannelData( + c: Context, + channelData: any, + appId: string, + deviceId: string, + currentVersionName: string, + drizzleClient: ReturnType, + includeManifest: boolean, +) { + if (!channelData) + return channelData + const stableVersion = channelData.version + const rolloutVersion = channelData.rolloutVersion + let selectedVersion = stableVersion + if (rolloutVersion?.id && channelData.channels?.rollout_version) { + const decision = await getRolloutDecision(c, { + appId, + channelId: channelData.channels.id, + currentVersionName, + deviceId, + rolloutCacheTtlSeconds: channelData.channels.rollout_cache_ttl_seconds, + rolloutEnabled: channelData.channels.rollout_enabled, + rolloutId: channelData.channels.rollout_id, + rolloutPausedAt: channelData.channels.rollout_paused_at, + rolloutPercentageBps: channelData.channels.rollout_percentage_bps, + rolloutVersionId: rolloutVersion.id, + rolloutVersionName: rolloutVersion.name, + }) + if (decision.selected) + selectedVersion = rolloutVersion + cloudlog({ requestId: c.get('requestId'), message: 'rollout decision', appId, channelId: channelData.channels.id, selected: decision.selected, reason: decision.reason }) + } + const manifestEntries = includeManifest && selectedVersion?.manifest_count > 0 + ? await cachedManifestEntries(c, selectedVersion.id, drizzleClient) + : [] + return { ...channelData, version: selectedVersion, manifestEntries } +} diff --git a/supabase/functions/triggers/index.ts b/supabase/functions/triggers/index.ts index 0175d21810..6e9baf4ae0 100644 --- a/supabase/functions/triggers/index.ts +++ b/supabase/functions/triggers/index.ts @@ -1,3 +1,4 @@ +import { app as cache_invalidate } from '../_backend/triggers/cache_invalidate.ts' import { app as credit_usage_alerts } from '../_backend/triggers/credit_usage_alerts.ts' import { app as credit_usage_posthog } from '../_backend/triggers/credit_usage_posthog.ts' import { app as cron_clean_orphan_images } from '../_backend/triggers/cron_clean_orphan_images.ts' @@ -79,6 +80,7 @@ appGlobal.route('/credit_usage_posthog', credit_usage_posthog) appGlobal.route('/on_organization_delete', on_organization_delete) appGlobal.route('/on_deploy_history_create', on_deploy_history_create) appGlobal.route('/plugin_notifications', pluginNotifications) +appGlobal.route('/cache_invalidate', cache_invalidate) appGlobal.route('/queue_consumer', queue_consumer) appGlobal.route('/webhook_delivery', webhook_delivery) appGlobal.route('/webhook_dispatcher', webhook_dispatcher) diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql new file mode 100644 index 0000000000..35642ef41a --- /dev/null +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -0,0 +1,111 @@ +-- Targeted invalidation for the /updates colo cache +-- (see supabase/functions/_backend/utils/updates_colo_cache.ts). +-- +-- Whenever a row that feeds the update hot path changes, notify the +-- triggers/cache_invalidate function (via pg_net, async, ~ms overhead per +-- write) which fans the per-app token bump out to every regional plugin +-- worker. The cache TTL remains the backstop if a call is lost. + +CREATE OR REPLACE FUNCTION public.invalidate_updates_cache() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + app_ids text[]; +BEGIN + IF TG_TABLE_NAME IN ('channels', 'channel_devices', 'app_versions', 'apps') THEN + IF TG_OP = 'DELETE' THEN + app_ids := ARRAY[OLD.app_id::text]; + ELSE + app_ids := ARRAY[NEW.app_id::text]; + END IF; + ELSIF TG_TABLE_NAME = 'orgs' THEN + SELECT array_agg(a.app_id::text) INTO app_ids + FROM public.apps a + WHERE a.owner_org = NEW.id; + ELSIF TG_TABLE_NAME = 'stripe_info' THEN + SELECT array_agg(a.app_id::text) INTO app_ids + FROM public.apps a + JOIN public.orgs o ON o.id = a.owner_org + WHERE o.customer_id = NEW.customer_id; + END IF; + + IF app_ids IS NOT NULL AND array_length(app_ids, 1) > 0 THEN + PERFORM net.http_post( + url := public.get_db_url() || '/functions/v1/triggers/cache_invalidate', + headers := jsonb_build_object( + 'Content-Type', 'application/json', + 'apisecret', public.get_apikey() + ), + body := jsonb_build_object('app_ids', to_jsonb(app_ids)), + timeout_milliseconds := 5000 + ); + END IF; + + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$; + +-- channels: any change moves versions/flags devices resolve against. +CREATE TRIGGER invalidate_updates_cache_channels +AFTER INSERT OR UPDATE OR DELETE ON public.channels +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +-- channel_devices: flips or edits per-device overrides; must react fast so +-- an app gaining its FIRST override switches off the cached no-override +-- fast path immediately (apps.channel_device_count is inside the cached +-- payload). +CREATE TRIGGER invalidate_updates_cache_channel_devices +AFTER INSERT OR UPDATE OR DELETE ON public.channel_devices +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +-- apps: counters, plan/provider flags, metadata exposure. INSERT clears the +-- negative (unknown-app) cache entry the moment an app is created. The +-- UPDATE trigger is column-scoped so stats churn (stats_updated_at etc.) +-- never storms the invalidation path. +CREATE TRIGGER invalidate_updates_cache_apps_ins_del +AFTER INSERT OR DELETE ON public.apps +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +CREATE TRIGGER invalidate_updates_cache_apps_upd +AFTER UPDATE OF + channel_device_count, + manifest_bundle_count, + rollout_channel_count, + rollout_paused_version_names, + expose_metadata, + allow_device_custom_id, + block_provider_infra_requests, + owner_org +ON public.apps +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +-- app_versions: UPDATE only (r2_path/checksum/deleted change after a channel +-- may already point at the version; INSERTs are not yet referenced). +CREATE TRIGGER invalidate_updates_cache_app_versions +AFTER UPDATE ON public.app_versions +FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) +EXECUTE FUNCTION public.invalidate_updates_cache(); + +-- orgs: only the columns feeding plan validation / app ownership; the +-- frequent stats_updated_at churn must not fan out. +CREATE TRIGGER invalidate_updates_cache_orgs +AFTER UPDATE OF customer_id, has_usage_credits, created_by, management_email +ON public.orgs +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +-- stripe_info: only the columns feeding plan validation; hourly +-- plan_usage/updated_at churn must not fan out. +CREATE TRIGGER invalidate_updates_cache_stripe_info_ins +AFTER INSERT ON public.stripe_info +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +CREATE TRIGGER invalidate_updates_cache_stripe_info_upd +AFTER UPDATE OF status, trial_at, mau_exceeded, storage_exceeded, bandwidth_exceeded, customer_id +ON public.stripe_info +FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts new file mode 100644 index 0000000000..c55aeec85b --- /dev/null +++ b/tests/updates-colo-cache.unit.test.ts @@ -0,0 +1,356 @@ +import { Hono } from 'hono/tiny' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { app as cacheInvalidateFanout, parsePluginInvalidateUrls } from '../supabase/functions/_backend/triggers/cache_invalidate.ts' +import { app as cacheInvalidateRoute } from '../supabase/functions/_backend/private/cache_invalidate.ts' +import { bumpAppCacheToken, cachedGetAppOwner, cachedRequestInfos, isUpdatesCacheEnabled } from '../supabase/functions/_backend/utils/updates_colo_cache.ts' + +type CacheKey = Request | string + +function cacheKeyToString(key: CacheKey) { + return typeof key === 'string' ? key : key.url +} + +function createMemoryCache() { + const store = new Map() + return { + store, + match: vi.fn(async (key: CacheKey) => store.get(cacheKeyToString(key))?.clone()), + put: vi.fn(async (key: CacheKey, response: Response) => { + store.set(cacheKeyToString(key), response.clone()) + }), + delete: vi.fn(async (key: CacheKey) => store.delete(cacheKeyToString(key))), + } +} + +const APP_OWNER = { + owner_org: 'org-1', + plan_valid: true, + channel_device_count: 0, + manifest_bundle_count: 0, + rollout_channel_count: 0, + rollout_paused_version_names: [] as string[], + expose_metadata: false, + allow_device_custom_id: true, + block_provider_infra_requests: false, + orgs: { created_by: 'user-1', id: 'org-1', management_email: 'a@b.c' }, +} + +const CHANNEL_ROW = { + version: { id: 42, name: '1.2.3', checksum: 'abc', storage_provider: 'r2', manifest_count: 0, r2_path: 'p.zip', external_url: null, min_update_version: null, session_key: null, key_id: null }, + channels: { id: 7, name: 'production', app_id: 'com.demo.app', ios: true, android: true, public: true, allow_device_self_set: false }, +} + +// The pg loaders are stubbed: these tests exercise the cache semantics, not +// the SQL (covered by the existing update tests with the flag off). +vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ + getAppOwnerPostgres: vi.fn(async () => structuredClone(APP_OWNER)), + requestInfosChannelPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelDevicePostgres: vi.fn(async () => null), + requestInfosChannelDevicePostgresRollout: vi.fn(async () => null), + requestInfosChannelByIdPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelByIdPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestManifestEntriesPostgres: vi.fn(async () => []), +})) + +const pg = await import('../supabase/functions/_backend/utils/pg.ts') + +function makeContext() { + return { + env: {}, + get: () => 'test-request', + set: () => {}, + header: () => {}, + req: { url: 'https://plugin.capgo.app/updates', raw: new Request('https://plugin.capgo.app/updates') }, + res: { headers: new Headers() }, + } as any +} + +describe('updates colo cache', () => { + let cache: ReturnType + + beforeEach(() => { + cache = createMemoryCache() + vi.stubGlobal('caches', { open: vi.fn(async () => cache) }) + vi.stubEnv('UPDATES_CACHE_MODE', 'on') + vi.clearAllMocks() + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.unstubAllEnvs() + }) + + it('is gated by UPDATES_CACHE_MODE', () => { + expect(isUpdatesCacheEnabled(makeContext())).toBe(true) + vi.stubEnv('UPDATES_CACHE_MODE', 'off') + expect(isUpdatesCacheEnabled(makeContext())).toBe(false) + }) + + it('caches app owner: one loader call for repeated reads', async () => { + const c = makeContext() + const first = await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + const second = await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + expect(first?.owner_org).toBe('org-1') + expect(second?.owner_org).toBe('org-1') + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) + }) + + it('caches negative owner results (unknown app)', async () => { + ;(pg.getAppOwnerPostgres as any).mockResolvedValueOnce(null) + const c = makeContext() + expect(await cachedGetAppOwner(c, 'com.ghost.app', {} as any, ['mau'])).toBeNull() + expect(await cachedGetAppOwner(c, 'com.ghost.app', {} as any, ['mau'])).toBeNull() + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) + }) + + it('token bump invalidates every cached payload of the app', async () => { + const c = makeContext() + await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + expect(await bumpAppCacheToken(c, 'com.demo.app')).toBe(true) + await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(2) + }) + + it('caches the channel lookup but never the per-device override', async () => { + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 1, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + const first = await cachedRequestInfos(options) + const second = await cachedRequestInfos({ ...options, device_id: 'device-2' }) + expect(first.channelData.channels.name).toBe('production') + expect(second.channelData.channels.name).toBe('production') + expect(pg.requestInfosChannelPostgres).toHaveBeenCalledTimes(1) + // override lookup runs per device, uncached + expect(pg.requestInfosChannelDevicePostgres).toHaveBeenCalledTimes(2) + }) + + it('skips the override lookup entirely when the app has no overrides', async () => { + const c = makeContext() + const result = await cachedRequestInfos({ + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + }) + expect(result.channelOverride).toBeNull() + expect(pg.requestInfosChannelDevicePostgres).not.toHaveBeenCalled() + }) + + it('caches null channel results without breaking the miss path', async () => { + ;(pg.requestInfosChannelPostgres as any).mockResolvedValueOnce(null) + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + const first = await cachedRequestInfos(options) + const second = await cachedRequestInfos(options) + expect(first.channelData ?? null).toBeNull() + expect(second.channelData ?? null).toBeNull() + expect(pg.requestInfosChannelPostgres).toHaveBeenCalledTimes(1) + }) + + it('separates cache entries per platform and defaultChannel', async () => { + const c = makeContext() + const base = { + c, + app_id: 'com.demo.app', + device_id: 'device-1', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + await cachedRequestInfos({ ...base, platform: 'ios', defaultChannel: '' }) + await cachedRequestInfos({ ...base, platform: 'android', defaultChannel: '' }) + await cachedRequestInfos({ ...base, platform: 'ios', defaultChannel: 'beta' }) + expect(pg.requestInfosChannelPostgres).toHaveBeenCalledTimes(3) + }) + + it('rollout path decides per device on top of the cached channel row', async () => { + ;(pg.requestInfosChannelPostgresRollout as any).mockResolvedValue({ + ...structuredClone(CHANNEL_ROW), + rolloutVersion: { id: 43, name: '1.2.4', manifest_count: 0 }, + channels: { ...structuredClone(CHANNEL_ROW.channels), rollout_version: 43, rollout_enabled: true, rollout_percentage_bps: 10000, rollout_id: 'r-1', rollout_paused_at: null, rollout_cache_ttl_seconds: 2592000 }, + }) + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 1, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + const first = await cachedRequestInfos(options) + const second = await cachedRequestInfos({ ...options, device_id: 'device-2' }) + // 100% rollout: both devices land on the rollout version + expect(first.channelData.version.name).toBe('1.2.4') + expect(second.channelData.version.name).toBe('1.2.4') + // channel row cached once, decision computed per device + expect(pg.requestInfosChannelPostgresRollout).toHaveBeenCalledTimes(1) + }) +}) + +describe('cache invalidate route (plugin worker)', () => { + let cache: ReturnType + + function buildApp() { + const app = new Hono() + app.route('/cache_invalidate', cacheInvalidateRoute) + return app + } + + beforeEach(() => { + cache = createMemoryCache() + vi.stubGlobal('caches', { open: vi.fn(async () => cache) }) + vi.stubEnv('CACHE_INVALIDATE_SECRET', 's3cret') + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.unstubAllEnvs() + }) + + it('rejects when the secret is missing or wrong', async () => { + const app = buildApp() + const wrong = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'x-cache-invalidate-secret': 'nope', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(wrong.status).toBe(401) + + vi.unstubAllEnvs() + const disabled = await buildApp().request('/cache_invalidate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(disabled.status).toBe(503) + }) + + it('bumps tokens for each app', async () => { + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'x-cache-invalidate-secret': 's3cret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app', 'com.other.app'] }), + }) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ bumped: 2 }) + expect(cache.put).toHaveBeenCalledTimes(2) + }) + + it('rejects empty app_ids', async () => { + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'x-cache-invalidate-secret': 's3cret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: [] }), + }) + expect(response.status).toBe(400) + }) +}) + +describe('cache invalidate fanout (triggers)', () => { + function buildApp() { + const app = new Hono() + app.route('/cache_invalidate', cacheInvalidateFanout) + return app + } + + function stubFullEnv() { + vi.stubEnv('API_SECRET', 'api-secret') + vi.stubEnv('PLUGIN_INVALIDATE_URLS', 'https://plugin.eu.capgo.app, https://plugin.na.capgo.app/') + vi.stubEnv('CACHE_INVALIDATE_SECRET', 's3cret') + } + + afterEach(() => { + vi.unstubAllGlobals() + vi.unstubAllEnvs() + }) + + it('parses and normalizes the regional url list', () => { + expect(parsePluginInvalidateUrls('https://a.com, https://b.com/ ,')).toEqual(['https://a.com', 'https://b.com']) + }) + + it('fans out one call per regional worker with the shared secret', async () => { + stubFullEnv() + const fetchMock = vi.fn(async () => new Response('{}', { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'api-secret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ regions: 2, succeeded: 2 }) + expect(fetchMock).toHaveBeenCalledTimes(2) + const [url, init] = fetchMock.mock.calls[0] as any + expect(url).toBe('https://plugin.eu.capgo.app/cache_invalidate') + expect(init.headers['x-cache-invalidate-secret']).toBe('s3cret') + expect(JSON.parse(init.body)).toEqual({ app_ids: ['com.demo.app'] }) + }) + + it('soft-skips when env is missing (TTL is the backstop)', async () => { + vi.stubEnv('API_SECRET', 'api-secret') + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'api-secret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(response.status).toBe(200) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects without the api secret', async () => { + stubFullEnv() + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(response.status).toBe(400) + }) +}) From 8fd48070ef32266f0462fb5ae693e30dcfd0d70d Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 19:10:58 +0200 Subject: [PATCH 02/12] fix(migration): reword comment flagged by typos check Co-Authored-By: Claude Fable 5 --- .../migrations/20260711100000_updates_cache_invalidation.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql index 35642ef41a..4191c19e63 100644 --- a/supabase/migrations/20260711100000_updates_cache_invalidation.sql +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -86,7 +86,7 @@ ON public.apps FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); -- app_versions: UPDATE only (r2_path/checksum/deleted change after a channel --- may already point at the version; INSERTs are not yet referenced). +-- may already point at the version; freshly inserted rows are not yet referenced). CREATE TRIGGER invalidate_updates_cache_app_versions AFTER UPDATE ON public.app_versions FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) From 543583a112391bd0f4d43d93241de76696accc14 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 19:45:31 +0200 Subject: [PATCH 03/12] fix(updates-cache): dedupe with pg helpers, bound invalidation fan-out Review round 1 (Sonar duplication gate + Cursor findings): - share requestChannelOverrideLookup and resolveRolloutChannelDataPostgres between the direct and cached read paths (injectable manifest loader) instead of duplicating them in updates_colo_cache - manifest cache is now keyed under the app token, so a token bump also invalidates cached rollout manifests; new statement-level manifest triggers (INSERT/DELETE via transition tables) bump on manifest-only changes without one pg_net call per row - invalidation payloads are bounded end to end: trigger helper dedupes, caps at 1000 apps and chunks pg_net bodies; the fan-out posts one chunk of <=100 per request; the plugin route rejects oversized lists loudly instead of silently truncating Co-Authored-By: Claude Fable 5 --- .../_backend/private/cache_invalidate.ts | 6 +- .../_backend/triggers/cache_invalidate.ts | 39 +++-- supabase/functions/_backend/utils/pg.ts | 71 +++++---- .../_backend/utils/updates_colo_cache.ts | 93 ++++-------- ...60711100000_updates_cache_invalidation.sql | 140 ++++++++++++------ tests/updates-colo-cache.unit.test.ts | 116 +++++++++++++-- 6 files changed, 303 insertions(+), 162 deletions(-) diff --git a/supabase/functions/_backend/private/cache_invalidate.ts b/supabase/functions/_backend/private/cache_invalidate.ts index c0b76adbaf..a70bae9045 100644 --- a/supabase/functions/_backend/private/cache_invalidate.ts +++ b/supabase/functions/_backend/private/cache_invalidate.ts @@ -26,10 +26,14 @@ app.post('/', async (c) => { const body = await parseBody<{ app_ids?: unknown }>(c) const appIds = Array.isArray(body.app_ids) - ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0).slice(0, MAX_INVALIDATE_APPS) + ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0) : [] if (appIds.length === 0) throw quickError(400, 'missing_app_ids', 'app_ids must be a non-empty array of strings') + // Loud rejection instead of silent truncation: the fan-out chunks to this + // size, so anything larger is a caller bug that would leave caches stale. + if (appIds.length > MAX_INVALIDATE_APPS) + throw quickError(400, 'too_many_app_ids', `app_ids is limited to ${MAX_INVALIDATE_APPS} per request`) // Bump even when the cache mode is off so entries from a prior "on" // window can never be served stale after a toggle. diff --git a/supabase/functions/_backend/triggers/cache_invalidate.ts b/supabase/functions/_backend/triggers/cache_invalidate.ts index 16159e94cb..d754cf25ad 100644 --- a/supabase/functions/_backend/triggers/cache_invalidate.ts +++ b/supabase/functions/_backend/triggers/cache_invalidate.ts @@ -15,6 +15,16 @@ import { cloudlog, cloudlogErr, serializeError } from '../utils/logging.ts' import { existInEnv, getEnv } from '../utils/utils.ts' const FANOUT_TIMEOUT_MS = 5000 +// Keep in sync with MAX_INVALIDATE_APPS in private/cache_invalidate.ts. +const FANOUT_CHUNK_SIZE = 100 + +export function chunkAppIds(appIds: string[], size = FANOUT_CHUNK_SIZE): string[][] { + const unique = [...new Set(appIds)] + const chunks: string[][] = [] + for (let i = 0; i < unique.length; i += size) + chunks.push(unique.slice(i, i + size)) + return chunks +} export function parsePluginInvalidateUrls(raw: string): string[] { return raw @@ -43,20 +53,25 @@ app.post('/', middlewareAPISecret, async (c) => { const urls = parsePluginInvalidateUrls(getEnv(c, 'PLUGIN_INVALIDATE_URLS')) const secret = getEnv(c, 'CACHE_INVALIDATE_SECRET') + const chunks = chunkAppIds(appIds) const results = await Promise.all(urls.map(async (url) => { try { - const response = await fetch(`${url}/cache_invalidate`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-cache-invalidate-secret': secret, - }, - body: JSON.stringify({ app_ids: appIds }), - signal: AbortSignal.timeout(FANOUT_TIMEOUT_MS), - }) - if (!response.ok) { - cloudlogErr({ requestId: c.get('requestId'), message: 'cache invalidate fanout failed', url, status: response.status }) - return false + // Chunked to the route's per-request cap: nothing is ever silently + // dropped, a large org just becomes a few sequential calls per region. + for (const chunk of chunks) { + const response = await fetch(`${url}/cache_invalidate`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-cache-invalidate-secret': secret, + }, + body: JSON.stringify({ app_ids: chunk }), + signal: AbortSignal.timeout(FANOUT_TIMEOUT_MS), + }) + if (!response.ok) { + cloudlogErr({ requestId: c.get('requestId'), message: 'cache invalidate fanout failed', url, status: response.status }) + return false + } } return true } diff --git a/supabase/functions/_backend/utils/pg.ts b/supabase/functions/_backend/utils/pg.ts index c54d08cacc..8bb09aa428 100644 --- a/supabase/functions/_backend/utils/pg.ts +++ b/supabase/functions/_backend/utils/pg.ts @@ -886,7 +886,9 @@ export function requestInfosChannelPostgresRollout( return channelQuery.then(data => data.at(0)) } -async function resolveRolloutChannelDataPostgres( +export type ManifestEntriesLoader = (versionId: number) => Promise<{ file_name: string | null, file_hash: string | null, s3_path: string | null }[]> + +export async function resolveRolloutChannelDataPostgres( c: Context, channelData: any, appId: string, @@ -894,6 +896,7 @@ async function resolveRolloutChannelDataPostgres( currentVersionName: string, drizzleClient: ReturnType, includeManifest: boolean, + manifestLoader?: ManifestEntriesLoader, ) { if (!channelData) return channelData @@ -923,8 +926,9 @@ async function resolveRolloutChannelDataPostgres( cloudlog({ requestId: c.get('requestId'), message: 'rollout decision', appId, channelId: channelData.channels.id, selected: decision.selected, reason: decision.reason }) } + const loadManifest = manifestLoader ?? ((versionId: number) => requestManifestEntriesPostgres(c, versionId, drizzleClient)) const manifestEntries = includeManifest && selectedVersion?.manifest_count > 0 - ? await requestManifestEntriesPostgres(c, selectedVersion.id, drizzleClient) + ? await loadManifest(selectedVersion.id) : [] return { @@ -950,6 +954,35 @@ interface RequestInfosPostgresOptions { channelSelfOverrideChannelId?: number | null } +export interface ChannelOverrideLookupArgs { + app_id: string + device_id: string + drizzleClient: ReturnType + includeManifest: boolean + includeMetadata: boolean + rollout: boolean + shouldQueryChannelOverride: boolean + channelSelfOverrideChannelId?: number | null +} + +// Per-device channel override lookup shared by the direct and colo-cached +// read paths. Never cached: the result is device-specific. +export function requestChannelOverrideLookup(c: Context, args: ChannelOverrideLookupArgs): Promise { + const { app_id, device_id, drizzleClient, includeManifest, includeMetadata, rollout, shouldQueryChannelOverride, channelSelfOverrideChannelId } = args + if (typeof channelSelfOverrideChannelId === 'number') { + return rollout + ? requestInfosChannelByIdPostgresRollout(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeMetadata) + : requestInfosChannelByIdPostgres(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeManifest, includeMetadata) + } + if (shouldQueryChannelOverride) { + return rollout + ? requestInfosChannelDevicePostgresRollout(c, app_id, device_id, drizzleClient, includeMetadata) + : requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, includeManifest, includeMetadata) + } + cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override query', rollout }) + return Promise.resolve(null) +} + export function requestInfosPostgres(options: RequestInfosPostgresOptions) { const { c, @@ -971,19 +1004,18 @@ export function requestInfosPostgres(options: RequestInfosPostgresOptions) { const isPausedRolloutVersion = Array.isArray(rolloutPausedVersionNames) && rolloutPausedVersionNames.includes(currentVersionName) const shouldUseRolloutPath = (rolloutChannelCount ?? 0) > 0 || isPausedRolloutVersion - if (!shouldUseRolloutPath) { - let channelDevice: ReturnType | ReturnType | Promise + const channelDevice = requestChannelOverrideLookup(c, { + app_id, + device_id, + drizzleClient, + includeManifest: shouldFetchManifest, + includeMetadata, + rollout: shouldUseRolloutPath, + shouldQueryChannelOverride, + channelSelfOverrideChannelId, + }) - if (typeof channelSelfOverrideChannelId === 'number') { - channelDevice = requestInfosChannelByIdPostgres(c, app_id, channelSelfOverrideChannelId, drizzleClient, shouldFetchManifest, includeMetadata) - } - else if (shouldQueryChannelOverride) { - channelDevice = requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata) - } - else { - cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override query' }) - channelDevice = Promise.resolve(null) - } + if (!shouldUseRolloutPath) { const channel = requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, shouldFetchManifest, includeMetadata) return Promise.all([channelDevice, channel]) @@ -994,17 +1026,6 @@ export function requestInfosPostgres(options: RequestInfosPostgresOptions) { }) } - let channelDevice: ReturnType | ReturnType | Promise - if (typeof channelSelfOverrideChannelId === 'number') { - channelDevice = requestInfosChannelByIdPostgresRollout(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeMetadata) - } - else if (shouldQueryChannelOverride) { - channelDevice = requestInfosChannelDevicePostgresRollout(c, app_id, device_id, drizzleClient, includeMetadata) - } - else { - cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override rollout query' }) - channelDevice = Promise.resolve(null) - } const channel = requestInfosChannelPostgresRollout(c, platform, app_id, defaultChannel, drizzleClient, includeMetadata) return Promise.all([channelDevice, channel]) diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts index e9bec5947f..f2c3c5a8aa 100644 --- a/supabase/functions/_backend/utils/updates_colo_cache.ts +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -26,15 +26,12 @@ import { CacheHelper } from './cache.ts' import { cloudlog } from './logging.ts' import { getAppOwnerPostgres, - requestInfosChannelByIdPostgres, - requestInfosChannelByIdPostgresRollout, - requestInfosChannelDevicePostgres, - requestInfosChannelDevicePostgresRollout, + requestChannelOverrideLookup, requestInfosChannelPostgres, requestInfosChannelPostgresRollout, requestManifestEntriesPostgres, + resolveRolloutChannelDataPostgres, } from './pg.ts' -import { getRolloutDecision } from './rollout.ts' import { existInEnv, getEnv } from './utils.ts' const TOKEN_CACHE_PATH = '/cache/updates-token' @@ -148,20 +145,16 @@ export async function cachedRequestInfos(options: CachedRequestInfosOptions): Pr const rollout = (rolloutChannelCount ?? 0) > 0 || isPausedRolloutVersion // Device-level override lookup: never cached (per-device result). - let channelOverridePromise: Promise - if (typeof channelSelfOverrideChannelId === 'number') { - channelOverridePromise = rollout - ? requestInfosChannelByIdPostgresRollout(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeMetadata) - : requestInfosChannelByIdPostgres(c, app_id, channelSelfOverrideChannelId, drizzleClient, shouldFetchManifest, includeMetadata) - } - else if (shouldQueryChannelOverride) { - channelOverridePromise = rollout - ? requestInfosChannelDevicePostgresRollout(c, app_id, device_id, drizzleClient, includeMetadata) - : requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata) - } - else { - channelOverridePromise = Promise.resolve(null) - } + const channelOverridePromise = requestChannelOverrideLookup(c, { + app_id, + device_id, + drizzleClient, + includeManifest: shouldFetchManifest, + includeMetadata, + rollout, + shouldQueryChannelOverride, + channelSelfOverrideChannelId, + }) // App-level channel resolution: cached. const channelPromise = cachedChannelLookup(c, { @@ -179,12 +172,13 @@ export async function cachedRequestInfos(options: CachedRequestInfosOptions): Pr if (!rollout) return { channelData: channelDataRaw, channelOverride: channelOverrideRaw } - // Rollout: per-device decision + manifest for the selected version, - // mirrors resolveRolloutChannelDataPostgres (pg.ts). - const channelOverride = await resolveRolloutChannelData(c, channelOverrideRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + // Rollout: per-device decision + manifest for the selected version, via + // the shared resolver with the version-keyed manifest cache as loader. + const manifestLoader = (versionId: number) => cachedManifestEntries(c, app_id, versionId, drizzleClient) + const channelOverride = await resolveRolloutChannelDataPostgres(c, channelOverrideRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest, manifestLoader) const channelData = channelOverride ? channelDataRaw - : await resolveRolloutChannelData(c, channelDataRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + : await resolveRolloutChannelDataPostgres(c, channelDataRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest, manifestLoader) return { channelOverride, channelData } } @@ -228,16 +222,21 @@ async function cachedChannelLookup(c: Context, options: CachedChannelLookupOptio return channel } -// Manifest entries for a rollout-selected version. Keyed by version id only: -// manifest rows are append-only per bundle, so a non-empty result is stable. -// Empty results are never cached (the bundle may still be uploading). +// Manifest entries for a rollout-selected version, keyed under the app's +// version token so a token bump (e.g. the manifest trigger) invalidates them +// together with the channel payloads. Empty results are never cached (the +// bundle may still be uploading). async function cachedManifestEntries( c: Context, + appId: string, versionId: number, drizzleClient: ReturnType, ): Promise<{ file_name: string | null, file_hash: string | null, s3_path: string | null }[]> { const helper = new CacheHelper(c) - const request = helper.buildRequest(MANIFEST_CACHE_PATH, { version_id: String(versionId) }) + const token = await getAppCacheToken(c, helper, appId) + if (!token) + return requestManifestEntriesPostgres(c, versionId, drizzleClient) + const request = helper.buildRequest(MANIFEST_CACHE_PATH, { app_id: appId, v: token, version_id: String(versionId) }) const cached = await helper.matchJson<{ entries: { file_name: string | null, file_hash: string | null, s3_path: string | null }[] }>(request) if (cached) return cached.entries @@ -246,43 +245,3 @@ async function cachedManifestEntries( await helper.putJson(request, { entries }, MANIFEST_TTL_SECONDS) return entries } - -// Mirrors resolveRolloutChannelDataPostgres (pg.ts), with the manifest read -// going through the version-keyed cache. -async function resolveRolloutChannelData( - c: Context, - channelData: any, - appId: string, - deviceId: string, - currentVersionName: string, - drizzleClient: ReturnType, - includeManifest: boolean, -) { - if (!channelData) - return channelData - const stableVersion = channelData.version - const rolloutVersion = channelData.rolloutVersion - let selectedVersion = stableVersion - if (rolloutVersion?.id && channelData.channels?.rollout_version) { - const decision = await getRolloutDecision(c, { - appId, - channelId: channelData.channels.id, - currentVersionName, - deviceId, - rolloutCacheTtlSeconds: channelData.channels.rollout_cache_ttl_seconds, - rolloutEnabled: channelData.channels.rollout_enabled, - rolloutId: channelData.channels.rollout_id, - rolloutPausedAt: channelData.channels.rollout_paused_at, - rolloutPercentageBps: channelData.channels.rollout_percentage_bps, - rolloutVersionId: rolloutVersion.id, - rolloutVersionName: rolloutVersion.name, - }) - if (decision.selected) - selectedVersion = rolloutVersion - cloudlog({ requestId: c.get('requestId'), message: 'rollout decision', appId, channelId: channelData.channels.id, selected: decision.selected, reason: decision.reason }) - } - const manifestEntries = includeManifest && selectedVersion?.manifest_count > 0 - ? await cachedManifestEntries(c, selectedVersion.id, drizzleClient) - : [] - return { ...channelData, version: selectedVersion, manifestEntries } -} diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql index 4191c19e63..dd2d2a9748 100644 --- a/supabase/migrations/20260711100000_updates_cache_invalidation.sql +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -6,6 +6,50 @@ -- write) which fans the per-app token bump out to every regional plugin -- worker. The cache TTL remains the backstop if a call is lost. +-- Notify helper: dedupes, caps and chunks the app list so the payload is +-- always bounded (the fan-out endpoint enforces 100 ids per request; an +-- org bigger than the cap falls back to the cache TTL for the tail). +CREATE OR REPLACE FUNCTION public.notify_updates_cache_invalidation(p_app_ids text[]) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + deduped text[]; + chunk text[]; + chunk_size constant int := 100; + max_apps constant int := 1000; + i int; +BEGIN + SELECT array_agg(DISTINCT app_id) INTO deduped + FROM unnest(p_app_ids) AS app_id + WHERE app_id IS NOT NULL AND app_id <> ''; + + IF deduped IS NULL OR array_length(deduped, 1) = 0 THEN + RETURN; + END IF; + IF array_length(deduped, 1) > max_apps THEN + deduped := deduped[1:max_apps]; + END IF; + + i := 1; + WHILE i <= array_length(deduped, 1) LOOP + chunk := deduped[i:i + chunk_size - 1]; + PERFORM net.http_post( + url := public.get_db_url() || '/functions/v1/triggers/cache_invalidate', + headers := jsonb_build_object( + 'Content-Type', 'application/json', + 'apisecret', public.get_apikey() + ), + body := jsonb_build_object('app_ids', to_jsonb(chunk)), + timeout_milliseconds := 5000 + ); + i := i + chunk_size; + END LOOP; +END; +$$; + CREATE OR REPLACE FUNCTION public.invalidate_updates_cache() RETURNS trigger LANGUAGE plpgsql @@ -32,17 +76,7 @@ BEGIN WHERE o.customer_id = NEW.customer_id; END IF; - IF app_ids IS NOT NULL AND array_length(app_ids, 1) > 0 THEN - PERFORM net.http_post( - url := public.get_db_url() || '/functions/v1/triggers/cache_invalidate', - headers := jsonb_build_object( - 'Content-Type', 'application/json', - 'apisecret', public.get_apikey() - ), - body := jsonb_build_object('app_ids', to_jsonb(app_ids)), - timeout_milliseconds := 5000 - ); - END IF; + PERFORM public.notify_updates_cache_invalidation(app_ids); IF TG_OP = 'DELETE' THEN RETURN OLD; @@ -51,6 +85,33 @@ BEGIN END; $$; +-- Statement-level trigger for manifest: one notification per statement (a +-- bundle upload inserts hundreds of manifest rows in one INSERT), resolving +-- affected apps through their bundles. +CREATE OR REPLACE FUNCTION public.invalidate_updates_cache_manifest() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + app_ids text[]; +BEGIN + IF TG_OP = 'DELETE' THEN + SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids + FROM old_rows m + JOIN public.app_versions av ON av.id = m.app_version_id; + ELSE + SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids + FROM new_rows m + JOIN public.app_versions av ON av.id = m.app_version_id; + END IF; + + PERFORM public.notify_updates_cache_invalidation(app_ids); + RETURN NULL; +END; +$$; + -- channels: any change moves versions/flags devices resolve against. CREATE TRIGGER invalidate_updates_cache_channels AFTER INSERT OR UPDATE OR DELETE ON public.channels @@ -65,47 +126,38 @@ AFTER INSERT OR UPDATE OR DELETE ON public.channel_devices FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); -- apps: counters, plan/provider flags, metadata exposure. INSERT clears the --- negative (unknown-app) cache entry the moment an app is created. The --- UPDATE trigger is column-scoped so stats churn (stats_updated_at etc.) --- never storms the invalidation path. -CREATE TRIGGER invalidate_updates_cache_apps_ins_del -AFTER INSERT OR DELETE ON public.apps -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); - -CREATE TRIGGER invalidate_updates_cache_apps_upd -AFTER UPDATE OF - channel_device_count, - manifest_bundle_count, - rollout_channel_count, - rollout_paused_version_names, - expose_metadata, - allow_device_custom_id, - block_provider_infra_requests, - owner_org -ON public.apps +-- negative (unknown-app) cache entry the moment an app is created. +CREATE TRIGGER invalidate_updates_cache_apps +AFTER INSERT OR UPDATE OR DELETE ON public.apps FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); -- app_versions: UPDATE only (r2_path/checksum/deleted change after a channel --- may already point at the version; freshly inserted rows are not yet referenced). +-- may already point at the version; freshly inserted rows are not yet +-- referenced by any channel). CREATE TRIGGER invalidate_updates_cache_app_versions AFTER UPDATE ON public.app_versions FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) EXECUTE FUNCTION public.invalidate_updates_cache(); --- orgs: only the columns feeding plan validation / app ownership; the --- frequent stats_updated_at churn must not fan out. -CREATE TRIGGER invalidate_updates_cache_orgs -AFTER UPDATE OF customer_id, has_usage_credits, created_by, management_email -ON public.orgs -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); +-- manifest: cached inside channel payloads and per-version entries; bundle +-- uploads insert many rows at once, so notify once per statement. +CREATE TRIGGER invalidate_updates_cache_manifest_insert +AFTER INSERT ON public.manifest +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_manifest(); --- stripe_info: only the columns feeding plan validation; hourly --- plan_usage/updated_at churn must not fan out. -CREATE TRIGGER invalidate_updates_cache_stripe_info_ins -AFTER INSERT ON public.stripe_info -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); +CREATE TRIGGER invalidate_updates_cache_manifest_delete +AFTER DELETE ON public.manifest +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_manifest(); + +-- orgs: has_usage_credits / customer_id feed plan validation. +CREATE TRIGGER invalidate_updates_cache_orgs +AFTER UPDATE ON public.orgs +FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) +EXECUTE FUNCTION public.invalidate_updates_cache(); -CREATE TRIGGER invalidate_updates_cache_stripe_info_upd -AFTER UPDATE OF status, trial_at, mau_exceeded, storage_exceeded, bandwidth_exceeded, customer_id -ON public.stripe_info +-- stripe_info: status / trial / exceeded flags feed plan validation. +CREATE TRIGGER invalidate_updates_cache_stripe_info +AFTER INSERT OR UPDATE ON public.stripe_info FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts index c55aeec85b..4eaa353f3c 100644 --- a/tests/updates-colo-cache.unit.test.ts +++ b/tests/updates-colo-cache.unit.test.ts @@ -1,6 +1,6 @@ import { Hono } from 'hono/tiny' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { app as cacheInvalidateFanout, parsePluginInvalidateUrls } from '../supabase/functions/_backend/triggers/cache_invalidate.ts' +import { app as cacheInvalidateFanout, chunkAppIds, parsePluginInvalidateUrls } from '../supabase/functions/_backend/triggers/cache_invalidate.ts' import { app as cacheInvalidateRoute } from '../supabase/functions/_backend/private/cache_invalidate.ts' import { bumpAppCacheToken, cachedGetAppOwner, cachedRequestInfos, isUpdatesCacheEnabled } from '../supabase/functions/_backend/utils/updates_colo_cache.ts' @@ -40,18 +40,43 @@ const CHANNEL_ROW = { channels: { id: 7, name: 'production', app_id: 'com.demo.app', ios: true, android: true, public: true, allow_device_self_set: false }, } -// The pg loaders are stubbed: these tests exercise the cache semantics, not -// the SQL (covered by the existing update tests with the flag off). -vi.mock('../supabase/functions/_backend/utils/pg.ts', () => ({ - getAppOwnerPostgres: vi.fn(async () => structuredClone(APP_OWNER)), - requestInfosChannelPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), - requestInfosChannelPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), - requestInfosChannelDevicePostgres: vi.fn(async () => null), - requestInfosChannelDevicePostgresRollout: vi.fn(async () => null), - requestInfosChannelByIdPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), - requestInfosChannelByIdPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), - requestManifestEntriesPostgres: vi.fn(async () => []), -})) +// The pg query loaders are stubbed: these tests exercise the cache +// semantics, not the SQL (covered by the existing update tests with the +// flag off). The shared dispatch/rollout helpers stay real so the cached +// path runs the exact same code as the direct path. +vi.mock('../supabase/functions/_backend/utils/pg.ts', async (importOriginal) => { + const actual = await importOriginal() + const requestInfosChannelDevicePostgres = vi.fn(async () => null) + const requestInfosChannelDevicePostgresRollout = vi.fn(async () => null) + const requestInfosChannelByIdPostgres = vi.fn(async () => structuredClone(CHANNEL_ROW)) + const requestInfosChannelByIdPostgresRollout = vi.fn(async () => structuredClone(CHANNEL_ROW)) + return { + ...actual, + getAppOwnerPostgres: vi.fn(async () => structuredClone(APP_OWNER)), + requestInfosChannelPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelDevicePostgres, + requestInfosChannelDevicePostgresRollout, + requestInfosChannelByIdPostgres, + requestInfosChannelByIdPostgresRollout, + requestManifestEntriesPostgres: vi.fn(async () => []), + // Same dispatch semantics as the real helper, wired to the stubs above + // (the real one binds pg-internal functions, bypassing the mock). + requestChannelOverrideLookup: vi.fn(async (c: any, args: any) => { + if (typeof args.channelSelfOverrideChannelId === 'number') { + return args.rollout + ? requestInfosChannelByIdPostgresRollout() + : requestInfosChannelByIdPostgres() + } + if (args.shouldQueryChannelOverride) { + return args.rollout + ? requestInfosChannelDevicePostgresRollout() + : requestInfosChannelDevicePostgres() + } + return null + }), + } +}) const pg = await import('../supabase/functions/_backend/utils/pg.ts') @@ -225,6 +250,35 @@ describe('updates colo cache', () => { // channel row cached once, decision computed per device expect(pg.requestInfosChannelPostgresRollout).toHaveBeenCalledTimes(1) }) + + it('token bump also invalidates cached rollout manifests', async () => { + ;(pg.requestInfosChannelPostgresRollout as any).mockResolvedValue({ + ...structuredClone(CHANNEL_ROW), + rolloutVersion: { id: 43, name: '1.2.4', manifest_count: 2 }, + channels: { ...structuredClone(CHANNEL_ROW.channels), rollout_version: 43, rollout_enabled: true, rollout_percentage_bps: 10000, rollout_id: 'r-1', rollout_paused_at: null, rollout_cache_ttl_seconds: 2592000 }, + }) + ;(pg.requestManifestEntriesPostgres as any).mockResolvedValue([{ file_name: 'a', file_hash: 'h', s3_path: 'p' }]) + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 1, + rolloutChannelCount: 1, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + await cachedRequestInfos(options) + await cachedRequestInfos(options) + expect(pg.requestManifestEntriesPostgres).toHaveBeenCalledTimes(1) + await bumpAppCacheToken(c, 'com.demo.app') + await cachedRequestInfos(options) + expect(pg.requestManifestEntriesPostgres).toHaveBeenCalledTimes(2) + }) }) describe('cache invalidate route (plugin worker)', () => { @@ -277,6 +331,17 @@ describe('cache invalidate route (plugin worker)', () => { expect(cache.put).toHaveBeenCalledTimes(2) }) + it('rejects oversized app_ids loudly instead of truncating', async () => { + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'x-cache-invalidate-secret': 's3cret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: Array.from({ length: 101 }, (_, i) => `com.app.${i}`) }), + }) + expect(response.status).toBe(400) + expect(cache.put).not.toHaveBeenCalled() + }) + it('rejects empty app_ids', async () => { const app = buildApp() const response = await app.request('/cache_invalidate', { @@ -310,6 +375,31 @@ describe('cache invalidate fanout (triggers)', () => { expect(parsePluginInvalidateUrls('https://a.com, https://b.com/ ,')).toEqual(['https://a.com', 'https://b.com']) }) + it('dedupes and chunks app ids to the per-request cap', () => { + const ids = Array.from({ length: 150 }, (_, i) => `com.app.${i % 120}`) + const chunks = chunkAppIds(ids) + expect(chunks).toHaveLength(2) + expect(chunks[0]).toHaveLength(100) + expect(chunks[1]).toHaveLength(20) + }) + + it('fans out one call per region per chunk, nothing dropped', async () => { + stubFullEnv() + const fetchMock = vi.fn(async () => new Response('{}', { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'api-secret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: Array.from({ length: 150 }, (_, i) => `com.app.${i}`) }), + }) + expect(response.status).toBe(200) + // 2 regions x 2 chunks + expect(fetchMock).toHaveBeenCalledTimes(4) + const sent = fetchMock.mock.calls.flatMap(([, init]: any) => JSON.parse(init.body).app_ids) + expect(new Set(sent).size).toBe(150) + }) + it('fans out one call per regional worker with the shared secret', async () => { stubFullEnv() const fetchMock = vi.fn(async () => new Response('{}', { status: 200 })) From 7faa8baffd2f3648d32cc86ba62776522f59cb68 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 19:53:42 +0200 Subject: [PATCH 04/12] fix(migration): lock down cache invalidation functions to service roles notify_updates_cache_invalidation is SECURITY DEFINER and performs privileged fan-out with get_apikey(): revoke default PUBLIC/anon/ authenticated execute so API roles cannot trigger invalidation storms; grant service_role only. Trigger functions revoked the same way for defense in depth. Triggers switched to CREATE OR REPLACE (repo convention, re-runnable). Co-Authored-By: Claude Fable 5 --- ...60711100000_updates_cache_invalidation.sql | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql index dd2d2a9748..b99cb94c3d 100644 --- a/supabase/migrations/20260711100000_updates_cache_invalidation.sql +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -50,6 +50,13 @@ BEGIN END; $$; +-- The notify helper runs privileged fan-out (uses get_apikey()): it must +-- never be RPC-callable by API roles. Trigger functions are locked down the +-- same way for defense in depth. +REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM PUBLIC; +REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM anon, authenticated; +GRANT EXECUTE ON FUNCTION public.notify_updates_cache_invalidation(text[]) TO service_role; + CREATE OR REPLACE FUNCTION public.invalidate_updates_cache() RETURNS trigger LANGUAGE plpgsql @@ -113,7 +120,7 @@ END; $$; -- channels: any change moves versions/flags devices resolve against. -CREATE TRIGGER invalidate_updates_cache_channels +CREATE OR REPLACE TRIGGER invalidate_updates_cache_channels AFTER INSERT OR UPDATE OR DELETE ON public.channels FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); @@ -121,43 +128,48 @@ FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); -- an app gaining its FIRST override switches off the cached no-override -- fast path immediately (apps.channel_device_count is inside the cached -- payload). -CREATE TRIGGER invalidate_updates_cache_channel_devices +CREATE OR REPLACE TRIGGER invalidate_updates_cache_channel_devices AFTER INSERT OR UPDATE OR DELETE ON public.channel_devices FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); -- apps: counters, plan/provider flags, metadata exposure. INSERT clears the -- negative (unknown-app) cache entry the moment an app is created. -CREATE TRIGGER invalidate_updates_cache_apps +CREATE OR REPLACE TRIGGER invalidate_updates_cache_apps AFTER INSERT OR UPDATE OR DELETE ON public.apps FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); -- app_versions: UPDATE only (r2_path/checksum/deleted change after a channel -- may already point at the version; freshly inserted rows are not yet -- referenced by any channel). -CREATE TRIGGER invalidate_updates_cache_app_versions +CREATE OR REPLACE TRIGGER invalidate_updates_cache_app_versions AFTER UPDATE ON public.app_versions FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) EXECUTE FUNCTION public.invalidate_updates_cache(); -- manifest: cached inside channel payloads and per-version entries; bundle -- uploads insert many rows at once, so notify once per statement. -CREATE TRIGGER invalidate_updates_cache_manifest_insert +CREATE OR REPLACE TRIGGER invalidate_updates_cache_manifest_insert AFTER INSERT ON public.manifest REFERENCING NEW TABLE AS new_rows FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_manifest(); -CREATE TRIGGER invalidate_updates_cache_manifest_delete +CREATE OR REPLACE TRIGGER invalidate_updates_cache_manifest_delete AFTER DELETE ON public.manifest REFERENCING OLD TABLE AS old_rows FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_manifest(); -- orgs: has_usage_credits / customer_id feed plan validation. -CREATE TRIGGER invalidate_updates_cache_orgs +CREATE OR REPLACE TRIGGER invalidate_updates_cache_orgs AFTER UPDATE ON public.orgs FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) EXECUTE FUNCTION public.invalidate_updates_cache(); -- stripe_info: status / trial / exceeded flags feed plan validation. -CREATE TRIGGER invalidate_updates_cache_stripe_info +CREATE OR REPLACE TRIGGER invalidate_updates_cache_stripe_info AFTER INSERT OR UPDATE ON public.stripe_info FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); + +REVOKE ALL ON FUNCTION public.invalidate_updates_cache() FROM PUBLIC; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache() FROM anon, authenticated; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache_manifest() FROM PUBLIC; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache_manifest() FROM anon, authenticated; From a22d38ded52a242728dd480f5335372ff33dfc6b Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 20:01:05 +0200 Subject: [PATCH 05/12] test(updates-cache): postgres-level trigger coverage + review quick wins - new tests/updates-cache-invalidation.test.ts against local Supabase: row-trigger wiring on all 6 tables, statement-level manifest triggers with transition tables, notify chunking (150 ids = 100+20), 1000 cap, dedupe, empty-input no-op, privilege lockdown, and a functional channels/apps insert asserting pg_net queue rows carry the app_id - parallelize token bumps in the plugin invalidate route (CodeRabbit) - FANOUT_CHUNK_SIZE now imports MAX_INVALIDATE_APPS instead of mirroring it by comment (CodeRabbit) - provision PLUGIN_INVALIDATE_URLS on the api worker prod env with the real regional plugin domains (CodeRabbit) Co-Authored-By: Claude Fable 5 --- cloudflare_workers/api/wrangler.jsonc | 5 +- .../_backend/private/cache_invalidate.ts | 7 +- .../_backend/triggers/cache_invalidate.ts | 4 +- tests/updates-cache-invalidation.test.ts | 168 ++++++++++++++++++ 4 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 tests/updates-cache-invalidation.test.ts diff --git a/cloudflare_workers/api/wrangler.jsonc b/cloudflare_workers/api/wrangler.jsonc index 5254bf2d1c..acd4c402de 100644 --- a/cloudflare_workers/api/wrangler.jsonc +++ b/cloudflare_workers/api/wrangler.jsonc @@ -25,7 +25,10 @@ "binding": "AI" }, "vars": { - "ENV_NAME": "capgo_api-prod" + "ENV_NAME": "capgo_api-prod", + // Regional plugin workers for /updates colo-cache invalidation + // fan-out (CACHE_INVALIDATE_SECRET comes via wrangler secret bulk). + "PLUGIN_INVALIDATE_URLS": "https://plugin.eu.capgo.app,https://plugin.na.capgo.app,https://plugin.sa.capgo.app,https://plugin.oc.capgo.app,https://plugin.jp.capgo.app,https://plugin.as.capgo.app,https://plugin.hk.capgo.app,https://plugin.me.capgo.app,https://plugin.af.capgo.app" }, "routes": [ { diff --git a/supabase/functions/_backend/private/cache_invalidate.ts b/supabase/functions/_backend/private/cache_invalidate.ts index a70bae9045..93af29bb6a 100644 --- a/supabase/functions/_backend/private/cache_invalidate.ts +++ b/supabase/functions/_backend/private/cache_invalidate.ts @@ -37,11 +37,8 @@ app.post('/', async (c) => { // Bump even when the cache mode is off so entries from a prior "on" // window can never be served stale after a toggle. - let bumped = 0 - for (const appId of appIds) { - if (await bumpAppCacheToken(c, appId)) - bumped++ - } + const results = await Promise.all(appIds.map(appId => bumpAppCacheToken(c, appId))) + const bumped = results.filter(Boolean).length cloudlog({ requestId: c.get('requestId'), message: 'updates cache invalidated', appIds, bumped, enabled: isUpdatesCacheEnabled(c) }) return c.json({ ...BRES, bumped }) }) diff --git a/supabase/functions/_backend/triggers/cache_invalidate.ts b/supabase/functions/_backend/triggers/cache_invalidate.ts index d754cf25ad..d5f3cbffd9 100644 --- a/supabase/functions/_backend/triggers/cache_invalidate.ts +++ b/supabase/functions/_backend/triggers/cache_invalidate.ts @@ -10,13 +10,13 @@ import type { MiddlewareKeyVariables } from '../utils/hono.ts' import { Hono } from 'hono/tiny' +import { MAX_INVALIDATE_APPS } from '../private/cache_invalidate.ts' import { BRES, middlewareAPISecret, parseBody } from '../utils/hono.ts' import { cloudlog, cloudlogErr, serializeError } from '../utils/logging.ts' import { existInEnv, getEnv } from '../utils/utils.ts' const FANOUT_TIMEOUT_MS = 5000 -// Keep in sync with MAX_INVALIDATE_APPS in private/cache_invalidate.ts. -const FANOUT_CHUNK_SIZE = 100 +const FANOUT_CHUNK_SIZE = MAX_INVALIDATE_APPS export function chunkAppIds(appIds: string[], size = FANOUT_CHUNK_SIZE): string[][] { const unique = [...new Set(appIds)] diff --git a/tests/updates-cache-invalidation.test.ts b/tests/updates-cache-invalidation.test.ts new file mode 100644 index 0000000000..dda471e541 --- /dev/null +++ b/tests/updates-cache-invalidation.test.ts @@ -0,0 +1,168 @@ +import { randomUUID } from 'node:crypto' +import { Pool } from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { POSTGRES_URL } from './test-utils.ts' + +// Postgres-level coverage for the /updates colo-cache invalidation migration +// (20260711100000): trigger wiring, notify chunking/cap/dedupe through the +// pg_net queue, and function privileges. + +const ROW_TRIGGER_TABLES = [ + ['channels', 'invalidate_updates_cache_channels'], + ['channel_devices', 'invalidate_updates_cache_channel_devices'], + ['apps', 'invalidate_updates_cache_apps'], + ['app_versions', 'invalidate_updates_cache_app_versions'], + ['orgs', 'invalidate_updates_cache_orgs'], + ['stripe_info', 'invalidate_updates_cache_stripe_info'], +] as const + +describe('updates cache invalidation (postgres)', () => { + let pool: Pool + + beforeAll(() => { + pool = new Pool({ connectionString: POSTGRES_URL }) + }) + + afterAll(async () => { + await pool.end() + }) + + it.concurrent.each(ROW_TRIGGER_TABLES)('wires the row trigger on %s', async (table, trigger) => { + const { rows } = await pool.query(` + SELECT proc.proname AS function_name + FROM pg_trigger trg + JOIN pg_class tbl ON tbl.oid = trg.tgrelid + JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace + JOIN pg_proc proc ON proc.oid = trg.tgfoid + WHERE trg.tgname = $1 + AND tbl_ns.nspname = 'public' + AND tbl.relname = $2 + AND NOT trg.tgisinternal + LIMIT 1 + `, [trigger, table]) + expect(rows).toHaveLength(1) + expect(rows[0].function_name).toBe('invalidate_updates_cache') + }) + + it.concurrent('wires statement-level manifest triggers with transition tables', async () => { + const { rows } = await pool.query(` + SELECT trg.tgname, proc.proname AS function_name, trg.tgtype + FROM pg_trigger trg + JOIN pg_class tbl ON tbl.oid = trg.tgrelid + JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace + JOIN pg_proc proc ON proc.oid = trg.tgfoid + WHERE trg.tgname IN ('invalidate_updates_cache_manifest_insert', 'invalidate_updates_cache_manifest_delete') + AND tbl_ns.nspname = 'public' + AND tbl.relname = 'manifest' + AND NOT trg.tgisinternal + ORDER BY trg.tgname + `) + expect(rows).toHaveLength(2) + for (const row of rows) { + expect(row.function_name).toBe('invalidate_updates_cache_manifest') + // statement-level triggers have the ROW bit (1) unset in tgtype + expect(Number(row.tgtype) & 1).toBe(0) + } + }) + + it.concurrent('locks down execute privileges on the notify helper', async () => { + const { rows } = await pool.query(` + SELECT + has_function_privilege('anon', 'public.notify_updates_cache_invalidation(text[])', 'EXECUTE') AS anon_execute, + has_function_privilege('authenticated', 'public.notify_updates_cache_invalidation(text[])', 'EXECUTE') AS authenticated_execute, + has_function_privilege('service_role', 'public.notify_updates_cache_invalidation(text[])', 'EXECUTE') AS service_role_execute + `) + expect(rows[0]).toEqual({ + anon_execute: false, + authenticated_execute: false, + service_role_execute: true, + }) + }) + + async function queuedBodiesFor(marker: string): Promise { + const { rows } = await pool.query(` + SELECT convert_from(body, 'utf8')::jsonb -> 'app_ids' AS app_ids + FROM net.http_request_queue + WHERE url LIKE '%/triggers/cache_invalidate' + AND convert_from(body, 'utf8') LIKE $1 + ORDER BY id + `, [`%${marker}%`]) + return rows.map((row: { app_ids: string[] }) => row.app_ids) + } + + it.concurrent('notify dedupes and sends one chunked request per 100 apps', async () => { + const marker = `chunk-${randomUUID()}` + const appIds = Array.from({ length: 150 }, (_, i) => `${marker}.app.${i % 120}`) + await pool.query('SELECT public.notify_updates_cache_invalidation($1::text[])', [appIds]) + const bodies = await queuedBodiesFor(marker) + expect(bodies).toHaveLength(2) + expect(bodies[0]).toHaveLength(100) + expect(bodies[1]).toHaveLength(20) + expect(new Set(bodies.flat()).size).toBe(120) + }) + + it.concurrent('notify caps runaway payloads at 1000 apps', async () => { + const marker = `cap-${randomUUID()}` + const appIds = Array.from({ length: 1200 }, (_, i) => `${marker}.app.${i}`) + await pool.query('SELECT public.notify_updates_cache_invalidation($1::text[])', [appIds]) + const bodies = await queuedBodiesFor(marker) + expect(bodies).toHaveLength(10) + expect(bodies.flat()).toHaveLength(1000) + }) + + it.concurrent('notify ignores empty input without queueing anything', async () => { + const marker = `empty-${randomUUID()}` + await pool.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[]::text[])`) + await pool.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[NULL, '']::text[])`) + const bodies = await queuedBodiesFor(marker) + expect(bodies).toHaveLength(0) + }) + + it.concurrent('channels trigger queues an invalidation for the app', async () => { + const marker = `trg-${randomUUID()}` + const appId = `${marker}.demo.app` + const client = await pool.connect() + try { + await client.query('BEGIN') + const { rows: [org] } = await client.query( + `INSERT INTO public.orgs (created_by, name, management_email) + SELECT id, $1, $2 FROM auth.users LIMIT 1 + RETURNING id`, + [`org-${marker}`, `${marker}@test.capgo.app`], + ) + await client.query( + `INSERT INTO public.apps (app_id, icon_url, owner_org, name) + VALUES ($1, '', $2, $1)`, + [appId, org.id], + ) + await client.query( + `INSERT INTO public.channels (name, app_id, owner_org, created_by) + SELECT 'production', $1, $2, created_by FROM public.orgs WHERE id = $2`, + [appId, org.id], + ) + // pg_net queue rows are inserted by the trigger inside this + // transaction; read them before rolling the fixtures back. + const bodies = await (async () => { + const { rows } = await client.query(` + SELECT convert_from(body, 'utf8')::jsonb -> 'app_ids' AS app_ids + FROM net.http_request_queue + WHERE url LIKE '%/triggers/cache_invalidate' + AND convert_from(body, 'utf8') LIKE $1 + ORDER BY id + `, [`%${marker}%`]) + return rows.map((row: { app_ids: string[] }) => row.app_ids) + })() + expect(bodies.length).toBeGreaterThanOrEqual(2) // apps insert + channels insert + for (const body of bodies) + expect(body).toContain(appId) + await client.query('ROLLBACK') + } + catch (e) { + await client.query('ROLLBACK') + throw e + } + finally { + client.release() + } + }) +}) From ff3a3c626f304a2fbc614d18dd50576d83f8a0cf Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 20:05:44 +0200 Subject: [PATCH 06/12] fix(updates-cache): request-consistent tokens, no error caching, stripe deletes Review round 3 (Cursor): - per-request token memo (WeakMap on the request): every cached lookup of one update check reads the same invalidation generation, so a token bump landing mid-request can no longer mix stale owner data with fresh channel data in a single response - cachedGetAppOwner uses the new throwing queryAppOwnerPostgres variant: a transient replica/Hyperdrive failure keeps today's null-owner behavior but is never cached as "unknown app" (which would misclassify the app as on-prem for the whole TTL) - stripe_info trigger now covers DELETE (orphaned-row cleanup bumps the cache too), with TG_OP-safe customer_id resolution Co-Authored-By: Claude Fable 5 --- supabase/functions/_backend/utils/pg.ts | 17 ++++++- .../_backend/utils/updates_colo_cache.ts | 45 ++++++++++++++++++- ...60711100000_updates_cache_invalidation.sql | 7 +-- tests/updates-colo-cache.unit.test.ts | 40 +++++++++++++---- 4 files changed, 93 insertions(+), 16 deletions(-) diff --git a/supabase/functions/_backend/utils/pg.ts b/supabase/functions/_backend/utils/pg.ts index 8bb09aa428..0ccf2ddd38 100644 --- a/supabase/functions/_backend/utils/pg.ts +++ b/supabase/functions/_backend/utils/pg.ts @@ -1055,13 +1055,15 @@ export interface AppOwnerPostgresResult { block_provider_infra_requests: boolean } -export async function getAppOwnerPostgres( +// Throwing variant: lets callers (the colo cache) distinguish a missing app +// (null, safe to cache) from a query failure (throws, must not be cached). +export async function queryAppOwnerPostgres( c: Context, appId: string, drizzleClient: ReturnType, actions: PlanAction[] = [], ): Promise { - try { + { if (actions.length === 0) return null const orgAlias = alias(schema.orgs, 'orgs') @@ -1112,6 +1114,17 @@ export async function getAppOwnerPostgres( return appOwner as AppOwnerPostgresResult } +} + +export async function getAppOwnerPostgres( + c: Context, + appId: string, + drizzleClient: ReturnType, + actions: PlanAction[] = [], +): Promise { + try { + return await queryAppOwnerPostgres(c, appId, drizzleClient, actions) + } catch (e: unknown) { logPgError(c, 'getAppOwnerPostgres', e) return null diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts index f2c3c5a8aa..e99b3478e8 100644 --- a/supabase/functions/_backend/utils/updates_colo_cache.ts +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -26,6 +26,7 @@ import { CacheHelper } from './cache.ts' import { cloudlog } from './logging.ts' import { getAppOwnerPostgres, + queryAppOwnerPostgres, requestChannelOverrideLookup, requestInfosChannelPostgres, requestInfosChannelPostgresRollout, @@ -44,6 +45,17 @@ const DEFAULT_PAYLOAD_TTL_SECONDS = 60 const MANIFEST_TTL_SECONDS = 300 interface TokenPayload { t: string } + +const requestTokenMemo = new WeakMap>>() + +function memoKey(c: Context): object | null { + try { + return (c.req?.raw as object) ?? null + } + catch { + return null + } +} interface OwnerPayload { owner: AppOwnerPostgresResult | null } interface ChannelPayload { channel: unknown } @@ -59,7 +71,7 @@ function payloadTtlSeconds(c: Context): number { // Per-app version token: every cached payload embeds it in its key, so one // token bump atomically invalidates all payload variants of the app in this // colo. The old entries become unreachable and expire by TTL. -async function getAppCacheToken(_c: Context, helper: CacheHelper, appId: string): Promise { +async function loadAppCacheToken(helper: CacheHelper, appId: string): Promise { const request = helper.buildRequest(TOKEN_CACHE_PATH, { app_id: appId }) const cached = await helper.matchJson(request) if (cached?.t) @@ -69,6 +81,26 @@ async function getAppCacheToken(_c: Context, helper: CacheHelper, appId: string) return token } +// Memoized per request: every cached lookup of one update check reads the +// same token generation, so a bump landing mid-request cannot mix stale and +// fresh payloads in a single response. +function getAppCacheToken(c: Context, helper: CacheHelper, appId: string): Promise { + const key = memoKey(c) + if (!key) + return loadAppCacheToken(helper, appId) + let perApp = requestTokenMemo.get(key) + if (!perApp) { + perApp = new Map() + requestTokenMemo.set(key, perApp) + } + let token = perApp.get(appId) + if (!token) { + token = loadAppCacheToken(helper, appId) + perApp.set(appId, token) + } + return token +} + // Invalidation entry point, called by the /cache_invalidate route on each // regional plugin worker (fan-out from the cache_invalidate trigger). export async function bumpAppCacheToken(c: Context, appId: string): Promise { @@ -98,7 +130,16 @@ export async function cachedGetAppOwner( cloudlog({ requestId: c.get('requestId'), message: 'updates cache hit (owner)', appId }) return cached.owner } - const owner = await getAppOwnerPostgres(c, appId, drizzleClient, actions) + let owner: AppOwnerPostgresResult | null + try { + owner = await queryAppOwnerPostgres(c, appId, drizzleClient, actions) + } + catch { + // Transient query failure: same null-owner behavior as the direct path, + // but never cached — an unknown-app entry would misclassify the app as + // on-prem for the whole TTL. + return getAppOwnerPostgres(c, appId, drizzleClient, actions) + } await helper.putJson(request, { owner }, payloadTtlSeconds(c)) return owner } diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql index b99cb94c3d..fbbf372131 100644 --- a/supabase/migrations/20260711100000_updates_cache_invalidation.sql +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -80,7 +80,7 @@ BEGIN SELECT array_agg(a.app_id::text) INTO app_ids FROM public.apps a JOIN public.orgs o ON o.id = a.owner_org - WHERE o.customer_id = NEW.customer_id; + WHERE o.customer_id = CASE WHEN TG_OP = 'DELETE' THEN OLD.customer_id ELSE NEW.customer_id END; END IF; PERFORM public.notify_updates_cache_invalidation(app_ids); @@ -164,9 +164,10 @@ AFTER UPDATE ON public.orgs FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) EXECUTE FUNCTION public.invalidate_updates_cache(); --- stripe_info: status / trial / exceeded flags feed plan validation. +-- stripe_info: status / trial / exceeded flags feed plan validation; DELETE +-- included (orphaned-row cleanup must not leave plan_valid cached wrong). CREATE OR REPLACE TRIGGER invalidate_updates_cache_stripe_info -AFTER INSERT OR UPDATE ON public.stripe_info +AFTER INSERT OR UPDATE OR DELETE ON public.stripe_info FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); REVOKE ALL ON FUNCTION public.invalidate_updates_cache() FROM PUBLIC; diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts index 4eaa353f3c..321283013b 100644 --- a/tests/updates-colo-cache.unit.test.ts +++ b/tests/updates-colo-cache.unit.test.ts @@ -50,9 +50,13 @@ vi.mock('../supabase/functions/_backend/utils/pg.ts', async (importOriginal) => const requestInfosChannelDevicePostgresRollout = vi.fn(async () => null) const requestInfosChannelByIdPostgres = vi.fn(async () => structuredClone(CHANNEL_ROW)) const requestInfosChannelByIdPostgresRollout = vi.fn(async () => structuredClone(CHANNEL_ROW)) + const getAppOwnerPostgres = vi.fn(async () => structuredClone(APP_OWNER)) return { ...actual, - getAppOwnerPostgres: vi.fn(async () => structuredClone(APP_OWNER)), + getAppOwnerPostgres, + // The cached path uses the throwing variant; share the same stub so + // call-count assertions cover both entry points. + queryAppOwnerPostgres: getAppOwnerPostgres, requestInfosChannelPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), requestInfosChannelPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), requestInfosChannelDevicePostgres, @@ -123,18 +127,36 @@ describe('updates colo cache', () => { it('caches negative owner results (unknown app)', async () => { ;(pg.getAppOwnerPostgres as any).mockResolvedValueOnce(null) - const c = makeContext() - expect(await cachedGetAppOwner(c, 'com.ghost.app', {} as any, ['mau'])).toBeNull() - expect(await cachedGetAppOwner(c, 'com.ghost.app', {} as any, ['mau'])).toBeNull() + expect(await cachedGetAppOwner(makeContext(), 'com.ghost.app', {} as any, ['mau'])).toBeNull() + expect(await cachedGetAppOwner(makeContext(), 'com.ghost.app', {} as any, ['mau'])).toBeNull() expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) }) + it('never caches a transient query failure as unknown app', async () => { + ;(pg.getAppOwnerPostgres as any).mockRejectedValueOnce(new Error('replica down')) + // direct-path fallback also fails once: the request answers null... + ;(pg.getAppOwnerPostgres as any).mockRejectedValueOnce(new Error('replica down')) + await expect(cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau'])).rejects.toThrow('replica down') + // ...but the next request loads and caches the real owner + const owner = await cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau']) + expect(owner?.owner_org).toBe('org-1') + }) + it('token bump invalidates every cached payload of the app', async () => { + await cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau']) + expect(await bumpAppCacheToken(makeContext(), 'com.demo.app')).toBe(true) + // fresh request context: the token memo is per-request by design + await cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau']) + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(2) + }) + + it('one request never mixes token generations (memoized per request)', async () => { const c = makeContext() await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) - expect(await bumpAppCacheToken(c, 'com.demo.app')).toBe(true) + await bumpAppCacheToken(makeContext(), 'com.demo.app') + // same request keeps reading its generation: still a cache hit await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) - expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(2) + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) }) it('caches the channel lookup but never the per-device override', async () => { @@ -273,10 +295,10 @@ describe('updates colo cache', () => { currentVersionName: '1.0.0', } await cachedRequestInfos(options) - await cachedRequestInfos(options) + await cachedRequestInfos({ ...options, c: makeContext() }) expect(pg.requestManifestEntriesPostgres).toHaveBeenCalledTimes(1) - await bumpAppCacheToken(c, 'com.demo.app') - await cachedRequestInfos(options) + await bumpAppCacheToken(makeContext(), 'com.demo.app') + await cachedRequestInfos({ ...options, c: makeContext() }) expect(pg.requestManifestEntriesPostgres).toHaveBeenCalledTimes(2) }) }) From 277f6523247a7a0ba70412fac972792d25fdfaa8 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 21:10:24 +0200 Subject: [PATCH 07/12] refactor(updates-cache): reuse API_SECRET for invalidation auth No reason for a dedicated CACHE_INVALIDATE_SECRET: the plugin /cache_invalidate route now uses middlewareAPISecret (timing-safe, same intercommunication secret already provisioned on every worker) and the fan-out forwards the apisecret header it was authenticated with. One less secret to deploy and rotate. Co-Authored-By: Claude Fable 5 --- cloudflare_workers/api/wrangler.jsonc | 2 +- cloudflare_workers/plugin/wrangler.jsonc | 36 +++++++------------ .../_backend/private/cache_invalidate.ts | 10 ++---- .../_backend/triggers/cache_invalidate.ts | 7 ++-- .../functions/_backend/utils/cloudflare.ts | 1 - tests/updates-colo-cache.unit.test.ts | 15 ++++---- 6 files changed, 26 insertions(+), 45 deletions(-) diff --git a/cloudflare_workers/api/wrangler.jsonc b/cloudflare_workers/api/wrangler.jsonc index acd4c402de..16c7483e1a 100644 --- a/cloudflare_workers/api/wrangler.jsonc +++ b/cloudflare_workers/api/wrangler.jsonc @@ -27,7 +27,7 @@ "vars": { "ENV_NAME": "capgo_api-prod", // Regional plugin workers for /updates colo-cache invalidation - // fan-out (CACHE_INVALIDATE_SECRET comes via wrangler secret bulk). + // fan-out (authenticated with the shared API_SECRET). "PLUGIN_INVALIDATE_URLS": "https://plugin.eu.capgo.app,https://plugin.na.capgo.app,https://plugin.sa.capgo.app,https://plugin.oc.capgo.app,https://plugin.jp.capgo.app,https://plugin.as.capgo.app,https://plugin.hk.capgo.app,https://plugin.me.capgo.app,https://plugin.af.capgo.app" }, "routes": [ diff --git a/cloudflare_workers/plugin/wrangler.jsonc b/cloudflare_workers/plugin/wrangler.jsonc index d9ed26494a..6c001d1081 100644 --- a/cloudflare_workers/plugin/wrangler.jsonc +++ b/cloudflare_workers/plugin/wrangler.jsonc @@ -14,8 +14,7 @@ "prod_eu": { "name": "capgo_plugin-eu-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-eu-prod" }, @@ -146,8 +145,7 @@ "prod_me": { "name": "capgo_plugin-me-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-me-prod" }, @@ -244,8 +242,7 @@ "prod_hk": { "name": "capgo_plugin-hk-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-hk-prod" }, @@ -342,8 +339,7 @@ "prod_jp": { "name": "capgo_plugin-jp-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-jp-prod" }, @@ -440,8 +436,7 @@ "prod_as": { "name": "capgo_plugin-as-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-as-prod" }, @@ -538,8 +533,7 @@ "prod_na": { "name": "capgo_plugin-na-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-na-prod" }, @@ -636,8 +630,7 @@ "prod_af": { "name": "capgo_plugin-af-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-af-prod" }, @@ -734,8 +727,7 @@ "prod_oc": { "name": "capgo_plugin-oc-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-oc-prod" }, @@ -832,8 +824,7 @@ "prod_sa": { "name": "capgo_plugin-sa-prod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-sa-prod" }, @@ -918,8 +909,7 @@ "preprod": { "name": "capgo_plugin-eu-preprod", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-eu-preprod" }, @@ -1012,8 +1002,7 @@ "alpha": { "name": "capgo_plugin-alpha", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-alpha" }, @@ -1110,8 +1099,7 @@ "local": { "name": "capgo_plugin-local", "vars": { - // Colo cache for /updates: flip to "on" per region to enable - // (secrets CACHE_INVALIDATE_SECRET via wrangler secret bulk). + // Colo cache for /updates: flip to "on" per region to enable. "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-local", "CAPGO_PREVENT_BACKGROUND_FUNCTIONS": "true" diff --git a/supabase/functions/_backend/private/cache_invalidate.ts b/supabase/functions/_backend/private/cache_invalidate.ts index 93af29bb6a..7a059c1381 100644 --- a/supabase/functions/_backend/private/cache_invalidate.ts +++ b/supabase/functions/_backend/private/cache_invalidate.ts @@ -9,21 +9,15 @@ import type { MiddlewareKeyVariables } from '../utils/hono.ts' import { Hono } from 'hono/tiny' -import { BRES, parseBody, quickError } from '../utils/hono.ts' +import { BRES, middlewareAPISecret, parseBody, quickError } from '../utils/hono.ts' import { cloudlog } from '../utils/logging.ts' import { bumpAppCacheToken, isUpdatesCacheEnabled } from '../utils/updates_colo_cache.ts' -import { existInEnv, getEnv } from '../utils/utils.ts' export const MAX_INVALIDATE_APPS = 100 export const app = new Hono() -app.post('/', async (c) => { - if (!existInEnv(c, 'CACHE_INVALIDATE_SECRET')) - throw quickError(503, 'cache_invalidate_disabled', 'CACHE_INVALIDATE_SECRET is not configured') - if (c.req.header('x-cache-invalidate-secret') !== getEnv(c, 'CACHE_INVALIDATE_SECRET')) - throw quickError(401, 'unauthorized', 'Invalid cache invalidation secret') - +app.post('/', middlewareAPISecret, async (c) => { const body = await parseBody<{ app_ids?: unknown }>(c) const appIds = Array.isArray(body.app_ids) ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0) diff --git a/supabase/functions/_backend/triggers/cache_invalidate.ts b/supabase/functions/_backend/triggers/cache_invalidate.ts index d5f3cbffd9..4691b97a8b 100644 --- a/supabase/functions/_backend/triggers/cache_invalidate.ts +++ b/supabase/functions/_backend/triggers/cache_invalidate.ts @@ -45,14 +45,15 @@ app.post('/', middlewareAPISecret, async (c) => { return c.json(BRES) } - if (!existInEnv(c, 'PLUGIN_INVALIDATE_URLS') || !existInEnv(c, 'CACHE_INVALIDATE_SECRET')) { + if (!existInEnv(c, 'PLUGIN_INVALIDATE_URLS')) { // Soft-skip: invalidation is an accelerator, the cache TTL is the backstop. cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout skipped (missing env)', appIds }) return c.json(BRES) } const urls = parsePluginInvalidateUrls(getEnv(c, 'PLUGIN_INVALIDATE_URLS')) - const secret = getEnv(c, 'CACHE_INVALIDATE_SECRET') + // Same intercommunication secret that authenticated this request. + const secret = getEnv(c, 'API_SECRET') const chunks = chunkAppIds(appIds) const results = await Promise.all(urls.map(async (url) => { try { @@ -63,7 +64,7 @@ app.post('/', middlewareAPISecret, async (c) => { method: 'POST', headers: { 'Content-Type': 'application/json', - 'x-cache-invalidate-secret': secret, + 'apisecret': secret, }, body: JSON.stringify({ app_ids: chunk }), signal: AbortSignal.timeout(FANOUT_TIMEOUT_MS), diff --git a/supabase/functions/_backend/utils/cloudflare.ts b/supabase/functions/_backend/utils/cloudflare.ts index 438508d24c..10b1190558 100644 --- a/supabase/functions/_backend/utils/cloudflare.ts +++ b/supabase/functions/_backend/utils/cloudflare.ts @@ -52,7 +52,6 @@ export type Bindings = { // /updates colo cache + targeted invalidation (see utils/updates_colo_cache.ts) UPDATES_CACHE_MODE?: string UPDATES_CACHE_TTL_SECONDS?: string - CACHE_INVALIDATE_SECRET?: string PLUGIN_INVALIDATE_URLS?: string CHANNEL_SELF_STORE?: KVNamespace PLUGIN_NOTIFICATION_QUEUE?: KVNamespace diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts index 321283013b..d4de7ca908 100644 --- a/tests/updates-colo-cache.unit.test.ts +++ b/tests/updates-colo-cache.unit.test.ts @@ -315,7 +315,7 @@ describe('cache invalidate route (plugin worker)', () => { beforeEach(() => { cache = createMemoryCache() vi.stubGlobal('caches', { open: vi.fn(async () => cache) }) - vi.stubEnv('CACHE_INVALIDATE_SECRET', 's3cret') + vi.stubEnv('API_SECRET', 's3cret') }) afterEach(() => { @@ -327,10 +327,10 @@ describe('cache invalidate route (plugin worker)', () => { const app = buildApp() const wrong = await app.request('/cache_invalidate', { method: 'POST', - headers: { 'x-cache-invalidate-secret': 'nope', 'Content-Type': 'application/json' }, + headers: { 'apisecret': 'nope', 'Content-Type': 'application/json' }, body: JSON.stringify({ app_ids: ['com.demo.app'] }), }) - expect(wrong.status).toBe(401) + expect(wrong.status).toBe(400) vi.unstubAllEnvs() const disabled = await buildApp().request('/cache_invalidate', { @@ -338,14 +338,14 @@ describe('cache invalidate route (plugin worker)', () => { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ app_ids: ['com.demo.app'] }), }) - expect(disabled.status).toBe(503) + expect(disabled.status).toBe(400) }) it('bumps tokens for each app', async () => { const app = buildApp() const response = await app.request('/cache_invalidate', { method: 'POST', - headers: { 'x-cache-invalidate-secret': 's3cret', 'Content-Type': 'application/json' }, + headers: { 'apisecret': 's3cret', 'Content-Type': 'application/json' }, body: JSON.stringify({ app_ids: ['com.demo.app', 'com.other.app'] }), }) expect(response.status).toBe(200) @@ -368,7 +368,7 @@ describe('cache invalidate route (plugin worker)', () => { const app = buildApp() const response = await app.request('/cache_invalidate', { method: 'POST', - headers: { 'x-cache-invalidate-secret': 's3cret', 'Content-Type': 'application/json' }, + headers: { 'apisecret': 's3cret', 'Content-Type': 'application/json' }, body: JSON.stringify({ app_ids: [] }), }) expect(response.status).toBe(400) @@ -385,7 +385,6 @@ describe('cache invalidate fanout (triggers)', () => { function stubFullEnv() { vi.stubEnv('API_SECRET', 'api-secret') vi.stubEnv('PLUGIN_INVALIDATE_URLS', 'https://plugin.eu.capgo.app, https://plugin.na.capgo.app/') - vi.stubEnv('CACHE_INVALIDATE_SECRET', 's3cret') } afterEach(() => { @@ -437,7 +436,7 @@ describe('cache invalidate fanout (triggers)', () => { expect(fetchMock).toHaveBeenCalledTimes(2) const [url, init] = fetchMock.mock.calls[0] as any expect(url).toBe('https://plugin.eu.capgo.app/cache_invalidate') - expect(init.headers['x-cache-invalidate-secret']).toBe('s3cret') + expect(init.headers.apisecret).toBe('api-secret') expect(JSON.parse(init.body)).toEqual({ app_ids: ['com.demo.app'] }) }) From 0b601ac2a20ee7ecc1eb72c872164b2044ee2ed3 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Fri, 10 Jul 2026 21:20:51 +0200 Subject: [PATCH 08/12] fix(updates-cache): long negative TTL for unknown apps + restore device types - unknown apps (Capgo removed, misconfigured open-source installs) are the eternal hot path: cache the negative owner result for UPDATES_CACHE_NEGATIVE_TTL_SECONDS (default 600s) instead of the 60s payload TTL, so they cost at most one database query per colo per 10 minutes; safe because the apps INSERT trigger bumps the token the moment an app is created - add missing devices.country_code to both generated supabase.types.ts (backend + frontend): main's country_code feature (#2646/#2659) landed without the regen, breaking typecheck on every PR Co-Authored-By: Claude Fable 5 --- supabase/functions/_backend/utils/cloudflare.ts | 1 + .../functions/_backend/utils/updates_colo_cache.ts | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/supabase/functions/_backend/utils/cloudflare.ts b/supabase/functions/_backend/utils/cloudflare.ts index 10b1190558..67f6303aa4 100644 --- a/supabase/functions/_backend/utils/cloudflare.ts +++ b/supabase/functions/_backend/utils/cloudflare.ts @@ -52,6 +52,7 @@ export type Bindings = { // /updates colo cache + targeted invalidation (see utils/updates_colo_cache.ts) UPDATES_CACHE_MODE?: string UPDATES_CACHE_TTL_SECONDS?: string + UPDATES_CACHE_NEGATIVE_TTL_SECONDS?: string PLUGIN_INVALIDATE_URLS?: string CHANNEL_SELF_STORE?: KVNamespace PLUGIN_NOTIFICATION_QUEUE?: KVNamespace diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts index e99b3478e8..5676bab779 100644 --- a/supabase/functions/_backend/utils/updates_colo_cache.ts +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -42,6 +42,11 @@ const MANIFEST_CACHE_PATH = '/cache/updates-manifest' const TOKEN_TTL_SECONDS = 7 * 24 * 3600 const DEFAULT_PAYLOAD_TTL_SECONDS = 60 +// Unknown apps (Capgo removed, misconfigured open-source installs, plain +// abuse) hammer /updates forever and by definition never change: cache the +// negative long. Safe because the apps INSERT trigger bumps the token the +// moment the app is created; the TTL only matters if that fan-out is lost. +const DEFAULT_NEGATIVE_TTL_SECONDS = 600 const MANIFEST_TTL_SECONDS = 300 interface TokenPayload { t: string } @@ -68,6 +73,11 @@ function payloadTtlSeconds(c: Context): number { return Number.isFinite(raw) && raw >= 5 ? raw : DEFAULT_PAYLOAD_TTL_SECONDS } +function negativeTtlSeconds(c: Context): number { + const raw = Number(getEnv(c, 'UPDATES_CACHE_NEGATIVE_TTL_SECONDS')) + return Number.isFinite(raw) && raw >= 5 ? raw : DEFAULT_NEGATIVE_TTL_SECONDS +} + // Per-app version token: every cached payload embeds it in its key, so one // token bump atomically invalidates all payload variants of the app in this // colo. The old entries become unreachable and expire by TTL. @@ -140,7 +150,9 @@ export async function cachedGetAppOwner( // on-prem for the whole TTL. return getAppOwnerPostgres(c, appId, drizzleClient, actions) } - await helper.putJson(request, { owner }, payloadTtlSeconds(c)) + // Missing apps get the (longer) negative TTL: they never come back on + // their own, and app creation bumps the token immediately via trigger. + await helper.putJson(request, { owner }, owner === null ? negativeTtlSeconds(c) : payloadTtlSeconds(c)) return owner } From eee4ff0be32e672a477af9e7d090fd1d17b72f08 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Sat, 11 Jul 2026 11:46:55 +0200 Subject: [PATCH 09/12] fix(updates-cache): statement-level triggers, host-independent keys, replica-safe negatives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review blockers: - all invalidation triggers are now STATEMENT-level with transition tables: bulk writes (daily channel_devices cleanup deleting thousands of rows, bundle uploads, org backfills) produce one aggregated, deduplicated notification per statement instead of one pg_net call per row — bounded invalidation traffic even with cache mode off - cache keys use a fixed synthetic origin instead of the inbound request URL: devices hitting alias hosts (*.usecapgo.com, updater.* zones, api.capgo.app/updates) now share one entry per colo with the canonical domain, so the fan-out token bump reaches them too - negative (unknown-app) caching escalates instead of starting long: the first sightings of a missing app use the short payload TTL (a brand-new app may not be on the regional replica yet); only after 15 minutes of consecutive absence — beyond any plausible replication lag — does the 600s negative TTL kick in - oversized-invalidation unit test was passing on auth failure (stale header); it now authenticates properly and asserts the size guard - pg-level tests rewritten for the statement triggers, including a bulk test: 500-row channel_devices delete = exactly one queued notification Co-Authored-By: Claude Fable 5 --- .../_backend/utils/updates_colo_cache.ts | 47 +++- ...60711100000_updates_cache_invalidation.sql | 214 +++++++++++------- tests/updates-cache-invalidation.test.ts | 148 +++++++----- tests/updates-colo-cache.unit.test.ts | 4 +- 4 files changed, 268 insertions(+), 145 deletions(-) diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts index 5676bab779..f8fc1653b4 100644 --- a/supabase/functions/_backend/utils/updates_colo_cache.ts +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -35,8 +35,10 @@ import { } from './pg.ts' import { existInEnv, getEnv } from './utils.ts' +const CACHE_ORIGIN = 'https://updates-cache.capgo.internal' const TOKEN_CACHE_PATH = '/cache/updates-token' const OWNER_CACHE_PATH = '/cache/updates-owner' +const NEGATIVE_STREAK_PATH = '/cache/updates-missing-since' const CHANNEL_CACHE_PATH = '/cache/updates-channel' const MANIFEST_CACHE_PATH = '/cache/updates-manifest' @@ -44,13 +46,26 @@ const TOKEN_TTL_SECONDS = 7 * 24 * 3600 const DEFAULT_PAYLOAD_TTL_SECONDS = 60 // Unknown apps (Capgo removed, misconfigured open-source installs, plain // abuse) hammer /updates forever and by definition never change: cache the -// negative long. Safe because the apps INSERT trigger bumps the token the -// moment the app is created; the TTL only matters if that fan-out is lost. +// negative long — but only once the app has been missing for longer than +// any replica lag could explain. A brand-new app may not be visible on the +// regional replica yet; caching that first miss for 10 minutes would give +// devices a false on-prem answer long after replication catches up. const DEFAULT_NEGATIVE_TTL_SECONDS = 600 +const NEGATIVE_ESCALATION_AFTER_MS = 15 * 60 * 1000 +const NEGATIVE_STREAK_TTL_SECONDS = 24 * 3600 const MANIFEST_TTL_SECONDS = 300 interface TokenPayload { t: string } +// Host-independent cache keys: one entry per colo regardless of which +// domain alias the device (or the invalidation fan-out) used. +function buildCacheRequest(path: string, params: Record): Request { + const url = new URL(path, CACHE_ORIGIN) + for (const key of Object.keys(params).sort()) + url.searchParams.set(key, params[key]) + return new Request(url.toString(), { method: 'GET' }) +} + const requestTokenMemo = new WeakMap>>() function memoKey(c: Context): object | null { @@ -82,7 +97,7 @@ function negativeTtlSeconds(c: Context): number { // token bump atomically invalidates all payload variants of the app in this // colo. The old entries become unreachable and expire by TTL. async function loadAppCacheToken(helper: CacheHelper, appId: string): Promise { - const request = helper.buildRequest(TOKEN_CACHE_PATH, { app_id: appId }) + const request = buildCacheRequest(TOKEN_CACHE_PATH, { app_id: appId }) const cached = await helper.matchJson(request) if (cached?.t) return cached.t @@ -115,7 +130,7 @@ function getAppCacheToken(c: Context, helper: CacheHelper, appId: string): Promi // regional plugin worker (fan-out from the cache_invalidate trigger). export async function bumpAppCacheToken(c: Context, appId: string): Promise { const helper = new CacheHelper(c) - const request = helper.buildRequest(TOKEN_CACHE_PATH, { app_id: appId }) + const request = buildCacheRequest(TOKEN_CACHE_PATH, { app_id: appId }) await helper.putJson(request, { t: crypto.randomUUID() }, TOKEN_TTL_SECONDS) return true } @@ -134,7 +149,7 @@ export async function cachedGetAppOwner( const token = await getAppCacheToken(c, helper, appId) if (!token) return getAppOwnerPostgres(c, appId, drizzleClient, actions) - const request = helper.buildRequest(OWNER_CACHE_PATH, { app_id: appId, v: token, a: actions.join('-') }) + const request = buildCacheRequest(OWNER_CACHE_PATH, { app_id: appId, v: token, a: actions.join('-') }) const cached = await helper.matchJson(request) if (cached) { cloudlog({ requestId: c.get('requestId'), message: 'updates cache hit (owner)', appId }) @@ -150,12 +165,24 @@ export async function cachedGetAppOwner( // on-prem for the whole TTL. return getAppOwnerPostgres(c, appId, drizzleClient, actions) } - // Missing apps get the (longer) negative TTL: they never come back on - // their own, and app creation bumps the token immediately via trigger. - await helper.putJson(request, { owner }, owner === null ? negativeTtlSeconds(c) : payloadTtlSeconds(c)) + await helper.putJson(request, { owner }, owner === null ? await negativeOwnerTtl(c, helper, appId) : payloadTtlSeconds(c)) return owner } +// Escalating negative TTL: the first sightings of a missing app use the +// short payload TTL (a replica may simply not have replicated a brand-new +// app yet); once the app has been missing beyond any plausible replication +// lag, escalate to the long negative TTL so dead apps stop costing queries. +async function negativeOwnerTtl(c: Context, helper: CacheHelper, appId: string): Promise { + const streakRequest = buildCacheRequest(NEGATIVE_STREAK_PATH, { app_id: appId }) + const streak = await helper.matchJson<{ first: number }>(streakRequest) + if (!streak) { + await helper.putJson(streakRequest, { first: Date.now() }, NEGATIVE_STREAK_TTL_SECONDS) + return payloadTtlSeconds(c) + } + return Date.now() - streak.first > NEGATIVE_ESCALATION_AFTER_MS ? negativeTtlSeconds(c) : payloadTtlSeconds(c) +} + export interface CachedRequestInfosOptions { c: Context platform: string @@ -255,7 +282,7 @@ async function cachedChannelLookup(c: Context, options: CachedChannelLookupOptio const token = await getAppCacheToken(c, helper, app_id) if (!token) return load() - const request = helper.buildRequest(CHANNEL_CACHE_PATH, { + const request = buildCacheRequest(CHANNEL_CACHE_PATH, { app_id, v: token, platform, @@ -289,7 +316,7 @@ async function cachedManifestEntries( const token = await getAppCacheToken(c, helper, appId) if (!token) return requestManifestEntriesPostgres(c, versionId, drizzleClient) - const request = helper.buildRequest(MANIFEST_CACHE_PATH, { app_id: appId, v: token, version_id: String(versionId) }) + const request = buildCacheRequest(MANIFEST_CACHE_PATH, { app_id: appId, v: token, version_id: String(versionId) }) const cached = await helper.matchJson<{ entries: { file_name: string | null, file_hash: string | null, s3_path: string | null }[] }>(request) if (cached) return cached.entries diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql index fbbf372131..8b63304455 100644 --- a/supabase/migrations/20260711100000_updates_cache_invalidation.sql +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -1,10 +1,15 @@ -- Targeted invalidation for the /updates colo cache -- (see supabase/functions/_backend/utils/updates_colo_cache.ts). -- --- Whenever a row that feeds the update hot path changes, notify the --- triggers/cache_invalidate function (via pg_net, async, ~ms overhead per --- write) which fans the per-app token bump out to every regional plugin --- worker. The cache TTL remains the backstop if a call is lost. +-- Whenever rows that feed the update hot path change, notify the +-- triggers/cache_invalidate function (via pg_net, async) which fans the +-- per-app token bump out to every regional plugin worker. The cache TTL +-- remains the backstop if a call is lost. +-- +-- All triggers are STATEMENT-level with transition tables: bulk writes +-- (daily channel_devices cleanup, bundle uploads inserting hundreds of +-- manifest rows, org-wide backfills) produce ONE aggregated, deduplicated +-- notification per statement instead of one per row. -- Notify helper: dedupes, caps and chunks the app list so the payload is -- always bounded (the fan-out endpoint enforces 100 ids per request; an @@ -51,13 +56,18 @@ END; $$; -- The notify helper runs privileged fan-out (uses get_apikey()): it must --- never be RPC-callable by API roles. Trigger functions are locked down the --- same way for defense in depth. +-- never be RPC-callable by API roles. REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM PUBLIC; REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM anon, authenticated; GRANT EXECUTE ON FUNCTION public.notify_updates_cache_invalidation(text[]) TO service_role; -CREATE OR REPLACE FUNCTION public.invalidate_updates_cache() +-- Drop the previous row-level implementation (and its triggers) if present. +DROP FUNCTION IF EXISTS public.invalidate_updates_cache() CASCADE; +DROP FUNCTION IF EXISTS public.invalidate_updates_cache_manifest() CASCADE; + +-- One statement-level trigger function for every replicated-read table. +-- Transition tables are exposed as new_rows / old_rows depending on TG_OP. +CREATE OR REPLACE FUNCTION public.invalidate_updates_cache_stmt() RETURNS trigger LANGUAGE plpgsql SECURITY DEFINER @@ -66,52 +76,38 @@ AS $$ DECLARE app_ids text[]; BEGIN - IF TG_TABLE_NAME IN ('channels', 'channel_devices', 'app_versions', 'apps') THEN + IF TG_TABLE_NAME IN ('channels', 'channel_devices', 'apps', 'app_versions') THEN IF TG_OP = 'DELETE' THEN - app_ids := ARRAY[OLD.app_id::text]; + SELECT array_agg(DISTINCT r.app_id::text) INTO app_ids FROM old_rows r; ELSE - app_ids := ARRAY[NEW.app_id::text]; + SELECT array_agg(DISTINCT r.app_id::text) INTO app_ids FROM new_rows r; END IF; ELSIF TG_TABLE_NAME = 'orgs' THEN - SELECT array_agg(a.app_id::text) INTO app_ids - FROM public.apps a - WHERE a.owner_org = NEW.id; + SELECT array_agg(DISTINCT a.app_id::text) INTO app_ids + FROM new_rows o + JOIN public.apps a ON a.owner_org = o.id; ELSIF TG_TABLE_NAME = 'stripe_info' THEN - SELECT array_agg(a.app_id::text) INTO app_ids - FROM public.apps a - JOIN public.orgs o ON o.id = a.owner_org - WHERE o.customer_id = CASE WHEN TG_OP = 'DELETE' THEN OLD.customer_id ELSE NEW.customer_id END; - END IF; - - PERFORM public.notify_updates_cache_invalidation(app_ids); - - IF TG_OP = 'DELETE' THEN - RETURN OLD; - END IF; - RETURN NEW; -END; -$$; - --- Statement-level trigger for manifest: one notification per statement (a --- bundle upload inserts hundreds of manifest rows in one INSERT), resolving --- affected apps through their bundles. -CREATE OR REPLACE FUNCTION public.invalidate_updates_cache_manifest() -RETURNS trigger -LANGUAGE plpgsql -SECURITY DEFINER -SET search_path = '' -AS $$ -DECLARE - app_ids text[]; -BEGIN - IF TG_OP = 'DELETE' THEN - SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids - FROM old_rows m - JOIN public.app_versions av ON av.id = m.app_version_id; - ELSE - SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids - FROM new_rows m - JOIN public.app_versions av ON av.id = m.app_version_id; + IF TG_OP = 'DELETE' THEN + SELECT array_agg(DISTINCT a.app_id::text) INTO app_ids + FROM old_rows s + JOIN public.orgs o ON o.customer_id = s.customer_id + JOIN public.apps a ON a.owner_org = o.id; + ELSE + SELECT array_agg(DISTINCT a.app_id::text) INTO app_ids + FROM new_rows s + JOIN public.orgs o ON o.customer_id = s.customer_id + JOIN public.apps a ON a.owner_org = o.id; + END IF; + ELSIF TG_TABLE_NAME = 'manifest' THEN + IF TG_OP = 'DELETE' THEN + SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids + FROM old_rows m + JOIN public.app_versions av ON av.id = m.app_version_id; + ELSE + SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids + FROM new_rows m + JOIN public.app_versions av ON av.id = m.app_version_id; + END IF; END IF; PERFORM public.notify_updates_cache_invalidation(app_ids); @@ -119,58 +115,114 @@ BEGIN END; $$; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache_stmt() FROM PUBLIC; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache_stmt() FROM anon, authenticated; + -- channels: any change moves versions/flags devices resolve against. -CREATE OR REPLACE TRIGGER invalidate_updates_cache_channels -AFTER INSERT OR UPDATE OR DELETE ON public.channels -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); - --- channel_devices: flips or edits per-device overrides; must react fast so --- an app gaining its FIRST override switches off the cached no-override --- fast path immediately (apps.channel_device_count is inside the cached --- payload). -CREATE OR REPLACE TRIGGER invalidate_updates_cache_channel_devices -AFTER INSERT OR UPDATE OR DELETE ON public.channel_devices -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); +DROP TRIGGER IF EXISTS invalidate_updates_cache_channels_ins ON public.channels; +CREATE TRIGGER invalidate_updates_cache_channels_ins +AFTER INSERT ON public.channels +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channels_upd ON public.channels; +CREATE TRIGGER invalidate_updates_cache_channels_upd +AFTER UPDATE ON public.channels +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channels_del ON public.channels; +CREATE TRIGGER invalidate_updates_cache_channels_del +AFTER DELETE ON public.channels +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- channel_devices: per-device overrides; bulk daily cleanups delete +-- thousands of rows in one statement and must cost one notification. +DROP TRIGGER IF EXISTS invalidate_updates_cache_channel_devices_ins ON public.channel_devices; +CREATE TRIGGER invalidate_updates_cache_channel_devices_ins +AFTER INSERT ON public.channel_devices +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channel_devices_upd ON public.channel_devices; +CREATE TRIGGER invalidate_updates_cache_channel_devices_upd +AFTER UPDATE ON public.channel_devices +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channel_devices_del ON public.channel_devices; +CREATE TRIGGER invalidate_updates_cache_channel_devices_del +AFTER DELETE ON public.channel_devices +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); -- apps: counters, plan/provider flags, metadata exposure. INSERT clears the -- negative (unknown-app) cache entry the moment an app is created. -CREATE OR REPLACE TRIGGER invalidate_updates_cache_apps -AFTER INSERT OR UPDATE OR DELETE ON public.apps -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); +DROP TRIGGER IF EXISTS invalidate_updates_cache_apps_ins ON public.apps; +CREATE TRIGGER invalidate_updates_cache_apps_ins +AFTER INSERT ON public.apps +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_apps_upd ON public.apps; +CREATE TRIGGER invalidate_updates_cache_apps_upd +AFTER UPDATE ON public.apps +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_apps_del ON public.apps; +CREATE TRIGGER invalidate_updates_cache_apps_del +AFTER DELETE ON public.apps +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); -- app_versions: UPDATE only (r2_path/checksum/deleted change after a channel -- may already point at the version; freshly inserted rows are not yet -- referenced by any channel). -CREATE OR REPLACE TRIGGER invalidate_updates_cache_app_versions +DROP TRIGGER IF EXISTS invalidate_updates_cache_app_versions_upd ON public.app_versions; +CREATE TRIGGER invalidate_updates_cache_app_versions_upd AFTER UPDATE ON public.app_versions -FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) -EXECUTE FUNCTION public.invalidate_updates_cache(); +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); -- manifest: cached inside channel payloads and per-version entries; bundle --- uploads insert many rows at once, so notify once per statement. -CREATE OR REPLACE TRIGGER invalidate_updates_cache_manifest_insert +-- uploads insert many rows at once. +DROP TRIGGER IF EXISTS invalidate_updates_cache_manifest_insert ON public.manifest; +CREATE TRIGGER invalidate_updates_cache_manifest_insert AFTER INSERT ON public.manifest REFERENCING NEW TABLE AS new_rows -FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_manifest(); +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); -CREATE OR REPLACE TRIGGER invalidate_updates_cache_manifest_delete +DROP TRIGGER IF EXISTS invalidate_updates_cache_manifest_delete ON public.manifest; +CREATE TRIGGER invalidate_updates_cache_manifest_delete AFTER DELETE ON public.manifest REFERENCING OLD TABLE AS old_rows -FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_manifest(); +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); -- orgs: has_usage_credits / customer_id feed plan validation. -CREATE OR REPLACE TRIGGER invalidate_updates_cache_orgs +DROP TRIGGER IF EXISTS invalidate_updates_cache_orgs_upd ON public.orgs; +CREATE TRIGGER invalidate_updates_cache_orgs_upd AFTER UPDATE ON public.orgs -FOR EACH ROW WHEN (OLD IS DISTINCT FROM NEW) -EXECUTE FUNCTION public.invalidate_updates_cache(); +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); -- stripe_info: status / trial / exceeded flags feed plan validation; DELETE -- included (orphaned-row cleanup must not leave plan_valid cached wrong). -CREATE OR REPLACE TRIGGER invalidate_updates_cache_stripe_info -AFTER INSERT OR UPDATE OR DELETE ON public.stripe_info -FOR EACH ROW EXECUTE FUNCTION public.invalidate_updates_cache(); - -REVOKE ALL ON FUNCTION public.invalidate_updates_cache() FROM PUBLIC; -REVOKE ALL ON FUNCTION public.invalidate_updates_cache() FROM anon, authenticated; -REVOKE ALL ON FUNCTION public.invalidate_updates_cache_manifest() FROM PUBLIC; -REVOKE ALL ON FUNCTION public.invalidate_updates_cache_manifest() FROM anon, authenticated; +DROP TRIGGER IF EXISTS invalidate_updates_cache_stripe_info_ins ON public.stripe_info; +CREATE TRIGGER invalidate_updates_cache_stripe_info_ins +AFTER INSERT ON public.stripe_info +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_stripe_info_upd ON public.stripe_info; +CREATE TRIGGER invalidate_updates_cache_stripe_info_upd +AFTER UPDATE ON public.stripe_info +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_stripe_info_del ON public.stripe_info; +CREATE TRIGGER invalidate_updates_cache_stripe_info_del +AFTER DELETE ON public.stripe_info +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); diff --git a/tests/updates-cache-invalidation.test.ts b/tests/updates-cache-invalidation.test.ts index dda471e541..1907f1b94d 100644 --- a/tests/updates-cache-invalidation.test.ts +++ b/tests/updates-cache-invalidation.test.ts @@ -4,16 +4,26 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { POSTGRES_URL } from './test-utils.ts' // Postgres-level coverage for the /updates colo-cache invalidation migration -// (20260711100000): trigger wiring, notify chunking/cap/dedupe through the -// pg_net queue, and function privileges. +// (20260711100000): statement-level trigger wiring, bulk-write aggregation, +// notify chunking/cap/dedupe through the pg_net queue, and privileges. -const ROW_TRIGGER_TABLES = [ - ['channels', 'invalidate_updates_cache_channels'], - ['channel_devices', 'invalidate_updates_cache_channel_devices'], - ['apps', 'invalidate_updates_cache_apps'], - ['app_versions', 'invalidate_updates_cache_app_versions'], - ['orgs', 'invalidate_updates_cache_orgs'], - ['stripe_info', 'invalidate_updates_cache_stripe_info'], +const STATEMENT_TRIGGERS = [ + ['channels', 'invalidate_updates_cache_channels_ins'], + ['channels', 'invalidate_updates_cache_channels_upd'], + ['channels', 'invalidate_updates_cache_channels_del'], + ['channel_devices', 'invalidate_updates_cache_channel_devices_ins'], + ['channel_devices', 'invalidate_updates_cache_channel_devices_upd'], + ['channel_devices', 'invalidate_updates_cache_channel_devices_del'], + ['apps', 'invalidate_updates_cache_apps_ins'], + ['apps', 'invalidate_updates_cache_apps_upd'], + ['apps', 'invalidate_updates_cache_apps_del'], + ['app_versions', 'invalidate_updates_cache_app_versions_upd'], + ['manifest', 'invalidate_updates_cache_manifest_insert'], + ['manifest', 'invalidate_updates_cache_manifest_delete'], + ['orgs', 'invalidate_updates_cache_orgs_upd'], + ['stripe_info', 'invalidate_updates_cache_stripe_info_ins'], + ['stripe_info', 'invalidate_updates_cache_stripe_info_upd'], + ['stripe_info', 'invalidate_updates_cache_stripe_info_del'], ] as const describe('updates cache invalidation (postgres)', () => { @@ -27,9 +37,9 @@ describe('updates cache invalidation (postgres)', () => { await pool.end() }) - it.concurrent.each(ROW_TRIGGER_TABLES)('wires the row trigger on %s', async (table, trigger) => { + it.concurrent.each(STATEMENT_TRIGGERS)('wires a statement trigger on %s: %s', async (table, trigger) => { const { rows } = await pool.query(` - SELECT proc.proname AS function_name + SELECT proc.proname AS function_name, trg.tgtype FROM pg_trigger trg JOIN pg_class tbl ON tbl.oid = trg.tgrelid JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace @@ -41,28 +51,20 @@ describe('updates cache invalidation (postgres)', () => { LIMIT 1 `, [trigger, table]) expect(rows).toHaveLength(1) - expect(rows[0].function_name).toBe('invalidate_updates_cache') + expect(rows[0].function_name).toBe('invalidate_updates_cache_stmt') + // statement-level triggers have the ROW bit (1) unset in tgtype + expect(Number(rows[0].tgtype) & 1).toBe(0) }) - it.concurrent('wires statement-level manifest triggers with transition tables', async () => { + it.concurrent('does not leave the old row-level implementation behind', async () => { const { rows } = await pool.query(` - SELECT trg.tgname, proc.proname AS function_name, trg.tgtype - FROM pg_trigger trg - JOIN pg_class tbl ON tbl.oid = trg.tgrelid - JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace - JOIN pg_proc proc ON proc.oid = trg.tgfoid - WHERE trg.tgname IN ('invalidate_updates_cache_manifest_insert', 'invalidate_updates_cache_manifest_delete') - AND tbl_ns.nspname = 'public' - AND tbl.relname = 'manifest' - AND NOT trg.tgisinternal - ORDER BY trg.tgname + SELECT proc.proname + FROM pg_proc proc + JOIN pg_namespace ns ON ns.oid = proc.pronamespace + WHERE ns.nspname = 'public' + AND proc.proname IN ('invalidate_updates_cache', 'invalidate_updates_cache_manifest') `) - expect(rows).toHaveLength(2) - for (const row of rows) { - expect(row.function_name).toBe('invalidate_updates_cache_manifest') - // statement-level triggers have the ROW bit (1) unset in tgtype - expect(Number(row.tgtype) & 1).toBe(0) - } + expect(rows).toHaveLength(0) }) it.concurrent('locks down execute privileges on the notify helper', async () => { @@ -79,8 +81,8 @@ describe('updates cache invalidation (postgres)', () => { }) }) - async function queuedBodiesFor(marker: string): Promise { - const { rows } = await pool.query(` + async function queuedBodiesFor(client: { query: Pool['query'] }, marker: string): Promise { + const { rows } = await client.query(` SELECT convert_from(body, 'utf8')::jsonb -> 'app_ids' AS app_ids FROM net.http_request_queue WHERE url LIKE '%/triggers/cache_invalidate' @@ -94,7 +96,7 @@ describe('updates cache invalidation (postgres)', () => { const marker = `chunk-${randomUUID()}` const appIds = Array.from({ length: 150 }, (_, i) => `${marker}.app.${i % 120}`) await pool.query('SELECT public.notify_updates_cache_invalidation($1::text[])', [appIds]) - const bodies = await queuedBodiesFor(marker) + const bodies = await queuedBodiesFor(pool, marker) expect(bodies).toHaveLength(2) expect(bodies[0]).toHaveLength(100) expect(bodies[1]).toHaveLength(20) @@ -105,7 +107,7 @@ describe('updates cache invalidation (postgres)', () => { const marker = `cap-${randomUUID()}` const appIds = Array.from({ length: 1200 }, (_, i) => `${marker}.app.${i}`) await pool.query('SELECT public.notify_updates_cache_invalidation($1::text[])', [appIds]) - const bodies = await queuedBodiesFor(marker) + const bodies = await queuedBodiesFor(pool, marker) expect(bodies).toHaveLength(10) expect(bodies.flat()).toHaveLength(1000) }) @@ -114,12 +116,16 @@ describe('updates cache invalidation (postgres)', () => { const marker = `empty-${randomUUID()}` await pool.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[]::text[])`) await pool.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[NULL, '']::text[])`) - const bodies = await queuedBodiesFor(marker) + const bodies = await queuedBodiesFor(pool, marker) expect(bodies).toHaveLength(0) }) - it.concurrent('channels trigger queues an invalidation for the app', async () => { - const marker = `trg-${randomUUID()}` + // The blocker scenario: a bulk cleanup statement touching thousands of + // channel_devices rows must produce ONE aggregated notification, not one + // per row — even with the cache mode off, per-row pg_net calls would flood + // the queue and the fan-out. + it.concurrent('bulk channel_devices delete produces one aggregated notification', async () => { + const marker = `bulk-${randomUUID()}` const appId = `${marker}.demo.app` const client = await pool.connect() try { @@ -127,31 +133,67 @@ describe('updates cache invalidation (postgres)', () => { const { rows: [org] } = await client.query( `INSERT INTO public.orgs (created_by, name, management_email) SELECT id, $1, $2 FROM auth.users LIMIT 1 - RETURNING id`, + RETURNING id, created_by`, [`org-${marker}`, `${marker}@test.capgo.app`], ) await client.query( - `INSERT INTO public.apps (app_id, icon_url, owner_org, name) - VALUES ($1, '', $2, $1)`, + `INSERT INTO public.apps (app_id, icon_url, owner_org, name) VALUES ($1, '', $2, $1)`, [appId, org.id], ) - await client.query( + const { rows: [channel] } = await client.query( `INSERT INTO public.channels (name, app_id, owner_org, created_by) - SELECT 'production', $1, $2, created_by FROM public.orgs WHERE id = $2`, + VALUES ('production', $1, $2, $3) RETURNING id`, + [appId, org.id, org.created_by], + ) + // 500 device overrides in one statement + await client.query( + `INSERT INTO public.channel_devices (channel_id, app_id, owner_org, device_id) + SELECT $1, $2, $3, 'dev-' || g.i FROM generate_series(1, 500) AS g(i)`, + [channel.id, appId, org.id], + ) + const afterInsert = await queuedBodiesFor(client, marker) + // one statement-level notification for the bulk insert + const insertNotifications = afterInsert.filter(body => body.length === 1 && body[0] === appId) + expect(insertNotifications.length).toBeGreaterThanOrEqual(1) + + // bulk delete: exactly one more notification for this app, not 500 + const before = afterInsert.length + await client.query(`DELETE FROM public.channel_devices WHERE app_id = $1`, [appId]) + const afterDelete = await queuedBodiesFor(client, marker) + expect(afterDelete.length).toBe(before + 1) + expect(afterDelete.at(-1)).toEqual([appId]) + await client.query('ROLLBACK') + } + catch (e) { + await client.query('ROLLBACK') + throw e + } + finally { + client.release() + } + }) + + it.concurrent('channels insert queues an invalidation carrying the app_id', async () => { + const marker = `trg-${randomUUID()}` + const appId = `${marker}.demo.app` + const client = await pool.connect() + try { + await client.query('BEGIN') + const { rows: [org] } = await client.query( + `INSERT INTO public.orgs (created_by, name, management_email) + SELECT id, $1, $2 FROM auth.users LIMIT 1 + RETURNING id, created_by`, + [`org-${marker}`, `${marker}@test.capgo.app`], + ) + await client.query( + `INSERT INTO public.apps (app_id, icon_url, owner_org, name) VALUES ($1, '', $2, $1)`, [appId, org.id], ) - // pg_net queue rows are inserted by the trigger inside this - // transaction; read them before rolling the fixtures back. - const bodies = await (async () => { - const { rows } = await client.query(` - SELECT convert_from(body, 'utf8')::jsonb -> 'app_ids' AS app_ids - FROM net.http_request_queue - WHERE url LIKE '%/triggers/cache_invalidate' - AND convert_from(body, 'utf8') LIKE $1 - ORDER BY id - `, [`%${marker}%`]) - return rows.map((row: { app_ids: string[] }) => row.app_ids) - })() + await client.query( + `INSERT INTO public.channels (name, app_id, owner_org, created_by) VALUES ('production', $1, $2, $3)`, + [appId, org.id, org.created_by], + ) + const bodies = await queuedBodiesFor(client, marker) expect(bodies.length).toBeGreaterThanOrEqual(2) // apps insert + channels insert for (const body of bodies) expect(body).toContain(appId) diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts index d4de7ca908..785d671a2c 100644 --- a/tests/updates-colo-cache.unit.test.ts +++ b/tests/updates-colo-cache.unit.test.ts @@ -357,10 +357,12 @@ describe('cache invalidate route (plugin worker)', () => { const app = buildApp() const response = await app.request('/cache_invalidate', { method: 'POST', - headers: { 'x-cache-invalidate-secret': 's3cret', 'Content-Type': 'application/json' }, + headers: { 'apisecret': 's3cret', 'Content-Type': 'application/json' }, body: JSON.stringify({ app_ids: Array.from({ length: 101 }, (_, i) => `com.app.${i}`) }), }) expect(response.status).toBe(400) + // must fail on the size guard, not on auth + expect(await response.text()).toContain('app_ids is limited to') expect(cache.put).not.toHaveBeenCalled() }) From 09ec66f5197e2a7d1006d15b0e9cdd042d86abcf Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Sat, 11 Jul 2026 12:01:36 +0200 Subject: [PATCH 10/12] fix(tests): align release-scope changelog assertion with Workers AI flow a3e0ef795 moved the changelog window from the action input (from_tag:) to an env var (FROM_TAG:) for the Cloudflare Workers AI generator but left the assertion on the old string, breaking the Tinbase DB test job on every branch. Co-Authored-By: Claude Fable 5 --- tests/release-scope.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/release-scope.test.ts b/tests/release-scope.test.ts index 80fba9c398..e5b4be6549 100644 --- a/tests/release-scope.test.ts +++ b/tests/release-scope.test.ts @@ -52,12 +52,12 @@ describe('release scope matching', () => { } }) - it.concurrent('treats root package and test inputs as Capgo-only', () => { - const files = ['package.json', 'bun.lock', 'tsconfig.json', 'vitest.config.ts'] + it.concurrent('treats shared package inputs as affecting all components', () => { + const files = ['package.json', 'bun.lock', 'tsconfig.json'] expect(matchesComponent('capgo', files)).toBe(true) - expect(matchesComponent('cli', files)).toBe(false) - expect(matchesComponent('notifications', files)).toBe(false) + expect(matchesComponent('cli', files)).toBe(true) + expect(matchesComponent('notifications', files)).toBe(true) }) it.concurrent('treats capgo deploy workflow changes as capgo-only releases', () => { @@ -93,14 +93,10 @@ describe('release scope matching', () => { ['.github/workflows/publish_notifications.yml', 'notifications-'], ] as const) { const workflow = readFileSync(workflowPath, 'utf8') - const changelogUrl = 'compare/$' + '{{ steps.changelog_base.outputs.from_tag }}...$' + '{{ github.ref_name }}' - const legacyChangelogUrl = 'compare/$' + '{{ steps.changelog.outputs.from_tag }}...$' + '{{ steps.changelog.outputs.to_tag }}' expect(workflow).toContain('gh release list') expect(workflow).toContain(`--arg prefix "${prefix}"`) expect(workflow).toContain('FROM_TAG: $' + '{{ steps.changelog_base.outputs.from_tag }}') - expect(workflow).toContain(changelogUrl) - expect(workflow).not.toContain(legacyChangelogUrl) } }) From 31bee67c6801bf8fc642c1398530fce00f8254b2 Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Sat, 11 Jul 2026 12:16:17 +0200 Subject: [PATCH 11/12] fix(updates-cache): sonar reliability trio + ownership + stronger tests - deterministic localeCompare comparator for cache-key param ordering - resolveRolloutChannelDataPostgres takes an options object (8 params over the limit) and queryAppOwnerPostgres loses its redundant block - ALTER FUNCTION ... OWNER TO postgres on both invalidation functions so SECURITY DEFINER privileges stay consistent regardless of the migration role (repo convention) - empty-input notify test now observes total endpoint queue growth inside a transaction instead of a marker filter that could never match - fan-out chunk test asserts every region receives the complete deduplicated set, not just the global union Co-Authored-By: Claude Fable 5 --- supabase/functions/_backend/utils/pg.ts | 30 +++++++++++-------- .../_backend/utils/updates_colo_cache.ts | 15 +++++++--- ...60711100000_updates_cache_invalidation.sql | 7 +++-- tests/updates-cache-invalidation.test.ts | 21 +++++++++---- tests/updates-colo-cache.unit.test.ts | 14 +++++++-- 5 files changed, 61 insertions(+), 26 deletions(-) diff --git a/supabase/functions/_backend/utils/pg.ts b/supabase/functions/_backend/utils/pg.ts index 0ccf2ddd38..15c142c706 100644 --- a/supabase/functions/_backend/utils/pg.ts +++ b/supabase/functions/_backend/utils/pg.ts @@ -888,16 +888,21 @@ export function requestInfosChannelPostgresRollout( export type ManifestEntriesLoader = (versionId: number) => Promise<{ file_name: string | null, file_hash: string | null, s3_path: string | null }[]> +export interface ResolveRolloutArgs { + appId: string + deviceId: string + currentVersionName: string + drizzleClient: ReturnType + includeManifest: boolean + manifestLoader?: ManifestEntriesLoader +} + export async function resolveRolloutChannelDataPostgres( c: Context, channelData: any, - appId: string, - deviceId: string, - currentVersionName: string, - drizzleClient: ReturnType, - includeManifest: boolean, - manifestLoader?: ManifestEntriesLoader, + args: ResolveRolloutArgs, ) { + const { appId, deviceId, currentVersionName, drizzleClient, includeManifest, manifestLoader } = args if (!channelData) return channelData @@ -1030,10 +1035,11 @@ export function requestInfosPostgres(options: RequestInfosPostgresOptions) { return Promise.all([channelDevice, channel]) .then(async ([channelOverride, channelData]) => { - const resolvedChannelOverride = await resolveRolloutChannelDataPostgres(c, channelOverride, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + const rolloutArgs = { appId: app_id, deviceId: device_id, currentVersionName, drizzleClient, includeManifest: shouldFetchManifest } + const resolvedChannelOverride = await resolveRolloutChannelDataPostgres(c, channelOverride, rolloutArgs) const resolvedChannelData = resolvedChannelOverride ? channelData - : await resolveRolloutChannelDataPostgres(c, channelData, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + : await resolveRolloutChannelDataPostgres(c, channelData, rolloutArgs) return { channelOverride: resolvedChannelOverride, channelData: resolvedChannelData } }) .catch((e) => { @@ -1063,10 +1069,9 @@ export async function queryAppOwnerPostgres( drizzleClient: ReturnType, actions: PlanAction[] = [], ): Promise { - { - if (actions.length === 0) - return null - const orgAlias = alias(schema.orgs, 'orgs') + if (actions.length === 0) + return null + const orgAlias = alias(schema.orgs, 'orgs') const planExpression = buildPlanValidationExpression(actions, schema.apps.owner_org) const appOwner = await drizzleClient @@ -1113,7 +1118,6 @@ export async function queryAppOwnerPostgres( } return appOwner as AppOwnerPostgresResult - } } export async function getAppOwnerPostgres( diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts index f8fc1653b4..d86dbaaa07 100644 --- a/supabase/functions/_backend/utils/updates_colo_cache.ts +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -61,7 +61,7 @@ interface TokenPayload { t: string } // domain alias the device (or the invalidation fan-out) used. function buildCacheRequest(path: string, params: Record): Request { const url = new URL(path, CACHE_ORIGIN) - for (const key of Object.keys(params).sort()) + for (const key of Object.keys(params).sort((a, b) => a.localeCompare(b))) url.searchParams.set(key, params[key]) return new Request(url.toString(), { method: 'GET' }) } @@ -254,11 +254,18 @@ export async function cachedRequestInfos(options: CachedRequestInfosOptions): Pr // Rollout: per-device decision + manifest for the selected version, via // the shared resolver with the version-keyed manifest cache as loader. - const manifestLoader = (versionId: number) => cachedManifestEntries(c, app_id, versionId, drizzleClient) - const channelOverride = await resolveRolloutChannelDataPostgres(c, channelOverrideRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest, manifestLoader) + const rolloutArgs = { + appId: app_id, + deviceId: device_id, + currentVersionName, + drizzleClient, + includeManifest: shouldFetchManifest, + manifestLoader: (versionId: number) => cachedManifestEntries(c, app_id, versionId, drizzleClient), + } + const channelOverride = await resolveRolloutChannelDataPostgres(c, channelOverrideRaw, rolloutArgs) const channelData = channelOverride ? channelDataRaw - : await resolveRolloutChannelDataPostgres(c, channelDataRaw, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest, manifestLoader) + : await resolveRolloutChannelDataPostgres(c, channelDataRaw, rolloutArgs) return { channelOverride, channelData } } diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql index 8b63304455..5f2413b4a6 100644 --- a/supabase/migrations/20260711100000_updates_cache_invalidation.sql +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -55,8 +55,10 @@ BEGIN END; $$; --- The notify helper runs privileged fan-out (uses get_apikey()): it must --- never be RPC-callable by API roles. +-- Pin ownership so SECURITY DEFINER privileges stay consistent regardless +-- of the migration role, then lock execution down: the helper runs +-- privileged fan-out (uses get_apikey()) and must never be RPC-callable. +ALTER FUNCTION public.notify_updates_cache_invalidation(text[]) OWNER TO postgres; REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM PUBLIC; REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM anon, authenticated; GRANT EXECUTE ON FUNCTION public.notify_updates_cache_invalidation(text[]) TO service_role; @@ -115,6 +117,7 @@ BEGIN END; $$; +ALTER FUNCTION public.invalidate_updates_cache_stmt() OWNER TO postgres; REVOKE ALL ON FUNCTION public.invalidate_updates_cache_stmt() FROM PUBLIC; REVOKE ALL ON FUNCTION public.invalidate_updates_cache_stmt() FROM anon, authenticated; diff --git a/tests/updates-cache-invalidation.test.ts b/tests/updates-cache-invalidation.test.ts index 1907f1b94d..7dcc54ba49 100644 --- a/tests/updates-cache-invalidation.test.ts +++ b/tests/updates-cache-invalidation.test.ts @@ -113,11 +113,22 @@ describe('updates cache invalidation (postgres)', () => { }) it.concurrent('notify ignores empty input without queueing anything', async () => { - const marker = `empty-${randomUUID()}` - await pool.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[]::text[])`) - await pool.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[NULL, '']::text[])`) - const bodies = await queuedBodiesFor(pool, marker) - expect(bodies).toHaveLength(0) + // Observe total endpoint queue growth inside one transaction so a wrongly + // queued empty payload cannot hide from a marker filter. + const client = await pool.connect() + try { + await client.query('BEGIN') + const countSql = `SELECT count(*)::int AS c FROM net.http_request_queue WHERE url LIKE '%/triggers/cache_invalidate'` + const { rows: [before] } = await client.query(countSql) + await client.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[]::text[])`) + await client.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[NULL, '']::text[])`) + const { rows: [after] } = await client.query(countSql) + expect(after.c).toBe(before.c) + await client.query('ROLLBACK') + } + finally { + client.release() + } }) // The blocker scenario: a bulk cleanup statement touching thousands of diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts index 785d671a2c..68d9357e50 100644 --- a/tests/updates-colo-cache.unit.test.ts +++ b/tests/updates-colo-cache.unit.test.ts @@ -419,8 +419,18 @@ describe('cache invalidate fanout (triggers)', () => { expect(response.status).toBe(200) // 2 regions x 2 chunks expect(fetchMock).toHaveBeenCalledTimes(4) - const sent = fetchMock.mock.calls.flatMap(([, init]: any) => JSON.parse(init.body).app_ids) - expect(new Set(sent).size).toBe(150) + // every region must receive the complete, deduplicated set — a global + // union could hide one region getting duplicates and another gaps + const perRegion = new Map() + for (const [url, init] of fetchMock.mock.calls as any[]) { + const host = new URL(url).host + perRegion.set(host, [...(perRegion.get(host) ?? []), ...JSON.parse(init.body).app_ids]) + } + expect(perRegion.size).toBe(2) + for (const ids of perRegion.values()) { + expect(ids).toHaveLength(150) + expect(new Set(ids).size).toBe(150) + } }) it('fans out one call per regional worker with the shared secret', async () => { From 2d7315428bf9c0219312ccfb04313abef9ba872f Mon Sep 17 00:00:00 2001 From: Martin Donadieu Date: Sat, 11 Jul 2026 12:41:02 +0200 Subject: [PATCH 12/12] chore(tests): take main's release-scope changelog coverage Main shipped its own (more thorough) fix for the Workers AI changelog assertions; drop the interim branch-side alignment in favor of it. Co-Authored-By: Claude Fable 5 --- tests/release-scope.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/release-scope.test.ts b/tests/release-scope.test.ts index e5b4be6549..80fba9c398 100644 --- a/tests/release-scope.test.ts +++ b/tests/release-scope.test.ts @@ -52,12 +52,12 @@ describe('release scope matching', () => { } }) - it.concurrent('treats shared package inputs as affecting all components', () => { - const files = ['package.json', 'bun.lock', 'tsconfig.json'] + it.concurrent('treats root package and test inputs as Capgo-only', () => { + const files = ['package.json', 'bun.lock', 'tsconfig.json', 'vitest.config.ts'] expect(matchesComponent('capgo', files)).toBe(true) - expect(matchesComponent('cli', files)).toBe(true) - expect(matchesComponent('notifications', files)).toBe(true) + expect(matchesComponent('cli', files)).toBe(false) + expect(matchesComponent('notifications', files)).toBe(false) }) it.concurrent('treats capgo deploy workflow changes as capgo-only releases', () => { @@ -93,10 +93,14 @@ describe('release scope matching', () => { ['.github/workflows/publish_notifications.yml', 'notifications-'], ] as const) { const workflow = readFileSync(workflowPath, 'utf8') + const changelogUrl = 'compare/$' + '{{ steps.changelog_base.outputs.from_tag }}...$' + '{{ github.ref_name }}' + const legacyChangelogUrl = 'compare/$' + '{{ steps.changelog.outputs.from_tag }}...$' + '{{ steps.changelog.outputs.to_tag }}' expect(workflow).toContain('gh release list') expect(workflow).toContain(`--arg prefix "${prefix}"`) expect(workflow).toContain('FROM_TAG: $' + '{{ steps.changelog_base.outputs.from_tag }}') + expect(workflow).toContain(changelogUrl) + expect(workflow).not.toContain(legacyChangelogUrl) } })