diff --git a/cloudflare_workers/api/index.ts b/cloudflare_workers/api/index.ts index 723f1556d1..cb5bb386c5 100644 --- a/cloudflare_workers/api/index.ts +++ b/cloudflare_workers/api/index.ts @@ -51,6 +51,7 @@ import { app as replication } from '../../supabase/functions/_backend/public/rep import { app as statistics } from '../../supabase/functions/_backend/public/statistics/index.ts' import { app as translation } from '../../supabase/functions/_backend/public/translation.ts' import { app as webhooks } from '../../supabase/functions/_backend/public/webhooks/index.ts' +import { app as cache_invalidate } from '../../supabase/functions/_backend/triggers/cache_invalidate.ts' import { app as credit_usage_alerts } from '../../supabase/functions/_backend/triggers/credit_usage_alerts.ts' import { app as cron_clean_orphan_images } from '../../supabase/functions/_backend/triggers/cron_clean_orphan_images.ts' import { app as cron_clear_versions } from '../../supabase/functions/_backend/triggers/cron_clear_versions.ts' @@ -151,6 +152,7 @@ appPrivate.route('/sso/verify-dns', sso_verify_dns) const functionNameTriggers = 'triggers' const appTriggers = createHono(functionNameTriggers, version) appTriggers.route('/ok', ok) +appTriggers.route('/cache_invalidate', cache_invalidate) appTriggers.route('/cron_email', cron_email) appTriggers.route('/cron_clear_versions', cron_clear_versions) appTriggers.route('/cron_clean_orphan_images', cron_clean_orphan_images) diff --git a/cloudflare_workers/api/wrangler.jsonc b/cloudflare_workers/api/wrangler.jsonc index 5254bf2d1c..16c7483e1a 100644 --- a/cloudflare_workers/api/wrangler.jsonc +++ b/cloudflare_workers/api/wrangler.jsonc @@ -25,7 +25,10 @@ "binding": "AI" }, "vars": { - "ENV_NAME": "capgo_api-prod" + "ENV_NAME": "capgo_api-prod", + // Regional plugin workers for /updates colo-cache invalidation + // fan-out (authenticated with the shared API_SECRET). + "PLUGIN_INVALIDATE_URLS": "https://plugin.eu.capgo.app,https://plugin.na.capgo.app,https://plugin.sa.capgo.app,https://plugin.oc.capgo.app,https://plugin.jp.capgo.app,https://plugin.as.capgo.app,https://plugin.hk.capgo.app,https://plugin.me.capgo.app,https://plugin.af.capgo.app" }, "routes": [ { diff --git a/cloudflare_workers/plugin/index.ts b/cloudflare_workers/plugin/index.ts index 41d87e8954..0fe69d5992 100644 --- a/cloudflare_workers/plugin/index.ts +++ b/cloudflare_workers/plugin/index.ts @@ -1,6 +1,7 @@ import { app as channel_self } from '../../supabase/functions/_backend/plugins/channel_self.ts' import { app as stats } from '../../supabase/functions/_backend/plugins/stats.ts' import { app as updates } from '../../supabase/functions/_backend/plugins/updates.ts' +import { app as cache_invalidate } from '../../supabase/functions/_backend/private/cache_invalidate.ts' import { app as latency } from '../../supabase/functions/_backend/private/latency.ts' import { app as ok } from '../../supabase/functions/_backend/public/ok.ts' import { createAllCatch, createHono, useCors } from '../../supabase/functions/_backend/utils/hono.ts' @@ -32,6 +33,7 @@ app.route('/updates', updates) app.route('/stats', stats) app.route('/ok', ok) app.route('/latency', latency) +app.route('/cache_invalidate', cache_invalidate) createAllCatch(app, functionName) diff --git a/cloudflare_workers/plugin/wrangler.jsonc b/cloudflare_workers/plugin/wrangler.jsonc index f6e7bbb775..6c001d1081 100644 --- a/cloudflare_workers/plugin/wrangler.jsonc +++ b/cloudflare_workers/plugin/wrangler.jsonc @@ -14,6 +14,8 @@ "prod_eu": { "name": "capgo_plugin-eu-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-eu-prod" }, "observability": { @@ -143,6 +145,8 @@ "prod_me": { "name": "capgo_plugin-me-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-me-prod" }, "observability": { @@ -238,6 +242,8 @@ "prod_hk": { "name": "capgo_plugin-hk-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-hk-prod" }, "observability": { @@ -333,6 +339,8 @@ "prod_jp": { "name": "capgo_plugin-jp-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-jp-prod" }, "observability": { @@ -428,6 +436,8 @@ "prod_as": { "name": "capgo_plugin-as-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-as-prod" }, "observability": { @@ -523,6 +533,8 @@ "prod_na": { "name": "capgo_plugin-na-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-na-prod" }, "observability": { @@ -618,6 +630,8 @@ "prod_af": { "name": "capgo_plugin-af-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-af-prod" }, "placement": { @@ -713,6 +727,8 @@ "prod_oc": { "name": "capgo_plugin-oc-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-oc-prod" }, "observability": { @@ -808,6 +824,8 @@ "prod_sa": { "name": "capgo_plugin-sa-prod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-sa-prod" }, "observability": { @@ -891,6 +909,8 @@ "preprod": { "name": "capgo_plugin-eu-preprod", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-eu-preprod" }, "observability": { @@ -982,6 +1002,8 @@ "alpha": { "name": "capgo_plugin-alpha", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-alpha" }, "observability": { @@ -1077,6 +1099,8 @@ "local": { "name": "capgo_plugin-local", "vars": { + // Colo cache for /updates: flip to "on" per region to enable. + "UPDATES_CACHE_MODE": "off", "ENV_NAME": "capgo_plugin-local", "CAPGO_PREVENT_BACKGROUND_FUNCTIONS": "true" }, diff --git a/supabase/functions/_backend/private/cache_invalidate.ts b/supabase/functions/_backend/private/cache_invalidate.ts new file mode 100644 index 0000000000..7a059c1381 --- /dev/null +++ b/supabase/functions/_backend/private/cache_invalidate.ts @@ -0,0 +1,38 @@ +// Colo cache invalidation endpoint, mounted on every regional plugin worker. +// +// The plugin workers are placement-pinned, so a request to each regional +// domain lands in the colo whose cache serves that region's devices. The +// cache_invalidate trigger handler fans a token bump out to all regions; +// each bump makes every cached /updates payload of the app unreachable +// (see updates_colo_cache.ts). Auth is a dedicated shared secret so this +// stays independent from the API-secret used by Supabase triggers. + +import type { MiddlewareKeyVariables } from '../utils/hono.ts' +import { Hono } from 'hono/tiny' +import { BRES, middlewareAPISecret, parseBody, quickError } from '../utils/hono.ts' +import { cloudlog } from '../utils/logging.ts' +import { bumpAppCacheToken, isUpdatesCacheEnabled } from '../utils/updates_colo_cache.ts' + +export const MAX_INVALIDATE_APPS = 100 + +export const app = new Hono() + +app.post('/', middlewareAPISecret, async (c) => { + const body = await parseBody<{ app_ids?: unknown }>(c) + const appIds = Array.isArray(body.app_ids) + ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0) + : [] + if (appIds.length === 0) + throw quickError(400, 'missing_app_ids', 'app_ids must be a non-empty array of strings') + // Loud rejection instead of silent truncation: the fan-out chunks to this + // size, so anything larger is a caller bug that would leave caches stale. + if (appIds.length > MAX_INVALIDATE_APPS) + throw quickError(400, 'too_many_app_ids', `app_ids is limited to ${MAX_INVALIDATE_APPS} per request`) + + // Bump even when the cache mode is off so entries from a prior "on" + // window can never be served stale after a toggle. + const results = await Promise.all(appIds.map(appId => bumpAppCacheToken(c, appId))) + const bumped = results.filter(Boolean).length + cloudlog({ requestId: c.get('requestId'), message: 'updates cache invalidated', appIds, bumped, enabled: isUpdatesCacheEnabled(c) }) + return c.json({ ...BRES, bumped }) +}) diff --git a/supabase/functions/_backend/triggers/cache_invalidate.ts b/supabase/functions/_backend/triggers/cache_invalidate.ts new file mode 100644 index 0000000000..4691b97a8b --- /dev/null +++ b/supabase/functions/_backend/triggers/cache_invalidate.ts @@ -0,0 +1,87 @@ +// Fan-out of /updates colo-cache invalidations to every regional plugin +// worker. +// +// Called by the invalidate_updates_cache() database trigger (via pg_net) +// whenever a row that feeds the update hot path changes (channels, +// channel_devices, apps, app_versions, orgs, stripe_info). The plugin +// workers are placement-pinned, so one POST per regional domain reaches +// every colo that caches this app — end-to-end reaction is ~1s from the +// database commit, with the cache TTL as backstop when a call is lost. + +import type { MiddlewareKeyVariables } from '../utils/hono.ts' +import { Hono } from 'hono/tiny' +import { MAX_INVALIDATE_APPS } from '../private/cache_invalidate.ts' +import { BRES, middlewareAPISecret, parseBody } from '../utils/hono.ts' +import { cloudlog, cloudlogErr, serializeError } from '../utils/logging.ts' +import { existInEnv, getEnv } from '../utils/utils.ts' + +const FANOUT_TIMEOUT_MS = 5000 +const FANOUT_CHUNK_SIZE = MAX_INVALIDATE_APPS + +export function chunkAppIds(appIds: string[], size = FANOUT_CHUNK_SIZE): string[][] { + const unique = [...new Set(appIds)] + const chunks: string[][] = [] + for (let i = 0; i < unique.length; i += size) + chunks.push(unique.slice(i, i + size)) + return chunks +} + +export function parsePluginInvalidateUrls(raw: string): string[] { + return raw + .split(',') + .map(url => url.trim().replace(/\/$/, '')) + .filter(Boolean) +} + +export const app = new Hono() + +app.post('/', middlewareAPISecret, async (c) => { + const body = await parseBody<{ app_ids?: unknown }>(c) + const appIds = Array.isArray(body.app_ids) + ? body.app_ids.filter((appId): appId is string => typeof appId === 'string' && appId.length > 0) + : [] + if (appIds.length === 0) { + cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout skipped (no app_ids)' }) + return c.json(BRES) + } + + if (!existInEnv(c, 'PLUGIN_INVALIDATE_URLS')) { + // Soft-skip: invalidation is an accelerator, the cache TTL is the backstop. + cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout skipped (missing env)', appIds }) + return c.json(BRES) + } + + const urls = parsePluginInvalidateUrls(getEnv(c, 'PLUGIN_INVALIDATE_URLS')) + // Same intercommunication secret that authenticated this request. + const secret = getEnv(c, 'API_SECRET') + const chunks = chunkAppIds(appIds) + const results = await Promise.all(urls.map(async (url) => { + try { + // Chunked to the route's per-request cap: nothing is ever silently + // dropped, a large org just becomes a few sequential calls per region. + for (const chunk of chunks) { + const response = await fetch(`${url}/cache_invalidate`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'apisecret': secret, + }, + body: JSON.stringify({ app_ids: chunk }), + signal: AbortSignal.timeout(FANOUT_TIMEOUT_MS), + }) + if (!response.ok) { + cloudlogErr({ requestId: c.get('requestId'), message: 'cache invalidate fanout failed', url, status: response.status }) + return false + } + } + return true + } + catch (e) { + cloudlogErr({ requestId: c.get('requestId'), message: 'cache invalidate fanout error', url, error: serializeError(e) }) + return false + } + })) + const succeeded = results.filter(Boolean).length + cloudlog({ requestId: c.get('requestId'), message: 'cache invalidate fanout done', appIds, regions: urls.length, succeeded }) + return c.json({ ...BRES, regions: urls.length, succeeded }) +}) diff --git a/supabase/functions/_backend/utils/cloudflare.ts b/supabase/functions/_backend/utils/cloudflare.ts index aba94df294..67f6303aa4 100644 --- a/supabase/functions/_backend/utils/cloudflare.ts +++ b/supabase/functions/_backend/utils/cloudflare.ts @@ -49,6 +49,11 @@ export type Bindings = { NOTIFICATION_EVENTS?: AnalyticsEngineDataset NOTIFICATION_QUEUE?: Queue DB_STOREAPPS: D1Database + // /updates colo cache + targeted invalidation (see utils/updates_colo_cache.ts) + UPDATES_CACHE_MODE?: string + UPDATES_CACHE_TTL_SECONDS?: string + UPDATES_CACHE_NEGATIVE_TTL_SECONDS?: string + PLUGIN_INVALIDATE_URLS?: string CHANNEL_SELF_STORE?: KVNamespace PLUGIN_NOTIFICATION_QUEUE?: KVNamespace LOCAL_READ_REPLICA_SUPABASE_DB_URL?: string diff --git a/supabase/functions/_backend/utils/pg.ts b/supabase/functions/_backend/utils/pg.ts index 0a90f5e65b..15c142c706 100644 --- a/supabase/functions/_backend/utils/pg.ts +++ b/supabase/functions/_backend/utils/pg.ts @@ -24,7 +24,7 @@ const REPLICATION_LAG_CACHE_TTL_MS = REPLICATION_LAG_CACHE_TTL_SECONDS * 1000 type ReplicationStatus = 'ok' | 'lagging' | 'unknown' interface ChannelLookupResult { id: number, name: string, allow_device_self_set: boolean, public: boolean, owner_org: string } -type PlanAction = 'mau' | 'storage' | 'bandwidth' +export type PlanAction = 'mau' | 'storage' | 'bandwidth' type ReadReplicaHyperdriveBinding = | 'HYPERDRIVE_CAPGO_READ_AS_JAPAN' | 'HYPERDRIVE_CAPGO_READ_AS_INDIA' @@ -886,15 +886,23 @@ export function requestInfosChannelPostgresRollout( return channelQuery.then(data => data.at(0)) } -async function resolveRolloutChannelDataPostgres( +export type ManifestEntriesLoader = (versionId: number) => Promise<{ file_name: string | null, file_hash: string | null, s3_path: string | null }[]> + +export interface ResolveRolloutArgs { + appId: string + deviceId: string + currentVersionName: string + drizzleClient: ReturnType + includeManifest: boolean + manifestLoader?: ManifestEntriesLoader +} + +export async function resolveRolloutChannelDataPostgres( c: Context, channelData: any, - appId: string, - deviceId: string, - currentVersionName: string, - drizzleClient: ReturnType, - includeManifest: boolean, + args: ResolveRolloutArgs, ) { + const { appId, deviceId, currentVersionName, drizzleClient, includeManifest, manifestLoader } = args if (!channelData) return channelData @@ -923,8 +931,9 @@ async function resolveRolloutChannelDataPostgres( cloudlog({ requestId: c.get('requestId'), message: 'rollout decision', appId, channelId: channelData.channels.id, selected: decision.selected, reason: decision.reason }) } + const loadManifest = manifestLoader ?? ((versionId: number) => requestManifestEntriesPostgres(c, versionId, drizzleClient)) const manifestEntries = includeManifest && selectedVersion?.manifest_count > 0 - ? await requestManifestEntriesPostgres(c, selectedVersion.id, drizzleClient) + ? await loadManifest(selectedVersion.id) : [] return { @@ -950,6 +959,35 @@ interface RequestInfosPostgresOptions { channelSelfOverrideChannelId?: number | null } +export interface ChannelOverrideLookupArgs { + app_id: string + device_id: string + drizzleClient: ReturnType + includeManifest: boolean + includeMetadata: boolean + rollout: boolean + shouldQueryChannelOverride: boolean + channelSelfOverrideChannelId?: number | null +} + +// Per-device channel override lookup shared by the direct and colo-cached +// read paths. Never cached: the result is device-specific. +export function requestChannelOverrideLookup(c: Context, args: ChannelOverrideLookupArgs): Promise { + const { app_id, device_id, drizzleClient, includeManifest, includeMetadata, rollout, shouldQueryChannelOverride, channelSelfOverrideChannelId } = args + if (typeof channelSelfOverrideChannelId === 'number') { + return rollout + ? requestInfosChannelByIdPostgresRollout(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeMetadata) + : requestInfosChannelByIdPostgres(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeManifest, includeMetadata) + } + if (shouldQueryChannelOverride) { + return rollout + ? requestInfosChannelDevicePostgresRollout(c, app_id, device_id, drizzleClient, includeMetadata) + : requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, includeManifest, includeMetadata) + } + cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override query', rollout }) + return Promise.resolve(null) +} + export function requestInfosPostgres(options: RequestInfosPostgresOptions) { const { c, @@ -971,19 +1009,18 @@ export function requestInfosPostgres(options: RequestInfosPostgresOptions) { const isPausedRolloutVersion = Array.isArray(rolloutPausedVersionNames) && rolloutPausedVersionNames.includes(currentVersionName) const shouldUseRolloutPath = (rolloutChannelCount ?? 0) > 0 || isPausedRolloutVersion - if (!shouldUseRolloutPath) { - let channelDevice: ReturnType | ReturnType | Promise + const channelDevice = requestChannelOverrideLookup(c, { + app_id, + device_id, + drizzleClient, + includeManifest: shouldFetchManifest, + includeMetadata, + rollout: shouldUseRolloutPath, + shouldQueryChannelOverride, + channelSelfOverrideChannelId, + }) - if (typeof channelSelfOverrideChannelId === 'number') { - channelDevice = requestInfosChannelByIdPostgres(c, app_id, channelSelfOverrideChannelId, drizzleClient, shouldFetchManifest, includeMetadata) - } - else if (shouldQueryChannelOverride) { - channelDevice = requestInfosChannelDevicePostgres(c, app_id, device_id, drizzleClient, shouldFetchManifest, includeMetadata) - } - else { - cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override query' }) - channelDevice = Promise.resolve(null) - } + if (!shouldUseRolloutPath) { const channel = requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, shouldFetchManifest, includeMetadata) return Promise.all([channelDevice, channel]) @@ -994,25 +1031,15 @@ export function requestInfosPostgres(options: RequestInfosPostgresOptions) { }) } - let channelDevice: ReturnType | ReturnType | Promise - if (typeof channelSelfOverrideChannelId === 'number') { - channelDevice = requestInfosChannelByIdPostgresRollout(c, app_id, channelSelfOverrideChannelId, drizzleClient, includeMetadata) - } - else if (shouldQueryChannelOverride) { - channelDevice = requestInfosChannelDevicePostgresRollout(c, app_id, device_id, drizzleClient, includeMetadata) - } - else { - cloudlog({ requestId: c.get('requestId'), message: 'Skipping channel device override rollout query' }) - channelDevice = Promise.resolve(null) - } const channel = requestInfosChannelPostgresRollout(c, platform, app_id, defaultChannel, drizzleClient, includeMetadata) return Promise.all([channelDevice, channel]) .then(async ([channelOverride, channelData]) => { - const resolvedChannelOverride = await resolveRolloutChannelDataPostgres(c, channelOverride, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + const rolloutArgs = { appId: app_id, deviceId: device_id, currentVersionName, drizzleClient, includeManifest: shouldFetchManifest } + const resolvedChannelOverride = await resolveRolloutChannelDataPostgres(c, channelOverride, rolloutArgs) const resolvedChannelData = resolvedChannelOverride ? channelData - : await resolveRolloutChannelDataPostgres(c, channelData, app_id, device_id, currentVersionName, drizzleClient, shouldFetchManifest) + : await resolveRolloutChannelDataPostgres(c, channelData, rolloutArgs) return { channelOverride: resolvedChannelOverride, channelData: resolvedChannelData } }) .catch((e) => { @@ -1034,16 +1061,17 @@ export interface AppOwnerPostgresResult { block_provider_infra_requests: boolean } -export async function getAppOwnerPostgres( +// Throwing variant: lets callers (the colo cache) distinguish a missing app +// (null, safe to cache) from a query failure (throws, must not be cached). +export async function queryAppOwnerPostgres( c: Context, appId: string, drizzleClient: ReturnType, actions: PlanAction[] = [], ): Promise { - try { - if (actions.length === 0) - return null - const orgAlias = alias(schema.orgs, 'orgs') + if (actions.length === 0) + return null + const orgAlias = alias(schema.orgs, 'orgs') const planExpression = buildPlanValidationExpression(actions, schema.apps.owner_org) const appOwner = await drizzleClient @@ -1090,6 +1118,16 @@ export async function getAppOwnerPostgres( } return appOwner as AppOwnerPostgresResult +} + +export async function getAppOwnerPostgres( + c: Context, + appId: string, + drizzleClient: ReturnType, + actions: PlanAction[] = [], +): Promise { + try { + return await queryAppOwnerPostgres(c, appId, drizzleClient, actions) } catch (e: unknown) { logPgError(c, 'getAppOwnerPostgres', e) diff --git a/supabase/functions/_backend/utils/update.ts b/supabase/functions/_backend/utils/update.ts index ec7fd65d68..20bfa78d37 100644 --- a/supabase/functions/_backend/utils/update.ts +++ b/supabase/functions/_backend/utils/update.ts @@ -19,6 +19,7 @@ import { cloudlog } from './logging.ts' import { getClientIP } from './rate_limit.ts' import { sendNotifOrgCached } from './notifications.ts' import { closeClient, getAppBlockProviderInfraRequestsPostgres, getAppOwnerPostgres, getDrizzleClient, getPgClient, requestInfosPostgres, setReplicationLagHeader } from './pg.ts' +import { cachedGetAppOwner, cachedRequestInfos, isUpdatesCacheEnabled } from './updates_colo_cache.ts' import { makeDevice } from './plugin_parser.ts' import { s3 } from './s3.ts' import { createStatsBandwidth, createStatsMau, createStatsVersion, onPremStats, sendStatsAndDevice } from './stats.ts' @@ -238,7 +239,10 @@ export async function updateWithPG( return providerBlockedResponse } - const appOwner = await getAppOwnerPostgres(c, app_id, drizzleClient, PLAN_LIMIT) + const useColoCache = isUpdatesCacheEnabled(c) + const appOwner = useColoCache + ? await cachedGetAppOwner(c, app_id, drizzleClient, PLAN_LIMIT) + : await getAppOwnerPostgres(c, app_id, drizzleClient, PLAN_LIMIT) // if version_build is not semver, then make it semver const device = makeDevice(body, appOwner?.allow_device_custom_id) if (!appOwner) { @@ -340,7 +344,7 @@ export async function updateWithPG( // Only query link/comment if plugin supports it (v5.35.0+, v6.35.0+, v7.35.0+, v8.35.0+) AND app has expose_metadata enabled const needsMetadata = appOwner.expose_metadata && !isDeprecatedPluginVersion(pluginVersion, '5.35.0', '6.35.0', '7.35.0', '8.35.0') - const requestedInto = await requestInfosPostgres({ + const requestInfosOptions = { c, platform, app_id, @@ -354,7 +358,10 @@ export async function updateWithPG( currentVersionName: version_name, includeMetadata: needsMetadata, channelSelfOverrideChannelId: channelSelfOverride?.channel_id.id, - }) + } + const requestedInto = useColoCache + ? await cachedRequestInfos(requestInfosOptions) + : await requestInfosPostgres(requestInfosOptions) const { channelOverride } = requestedInto let { channelData } = requestedInto cloudlog({ requestId: c.get('requestId'), message: `channelData exists ? ${channelData !== undefined}, channelOverride exists ? ${channelOverride !== undefined}` }) diff --git a/supabase/functions/_backend/utils/updates_colo_cache.ts b/supabase/functions/_backend/utils/updates_colo_cache.ts new file mode 100644 index 0000000000..d86dbaaa07 --- /dev/null +++ b/supabase/functions/_backend/utils/updates_colo_cache.ts @@ -0,0 +1,334 @@ +// Colo-local cache for the /updates hot path, with targeted invalidation. +// +// Most apps have one public channel and no per-device overrides: their +// channel resolution is identical for every device on the same +// (platform, defaultChannel) and can be served from the colo cache with +// zero database queries. Per-device work (semver compares, rollout +// decisions, override lookups for apps that have them) stays per-request. +// +// Freshness has two layers: +// - a short TTL backstop (UPDATES_CACHE_TTL_SECONDS, default 60s — the same +// class as the Hyperdrive query cache serving this traffic today), and +// - targeted invalidation: every cached entry is keyed under a per-app +// version token; bumping the token (bumpAppCacheToken) makes every entry +// for that app unreachable at once. Database triggers fan the bump out to +// each regional plugin worker (placement-pinned, so each call lands in +// the colo whose cache needs clearing) within ~1s of the commit. +// +// The apps.channel_device_count counter (already denormalized and cached in +// the payload) is the switch: apps with overrides keep their per-device +// lookup on every request, so adding a FIRST override reacts within one +// token bump (~1s) or one TTL at worst. + +import type { Context } from 'hono' +import type { AppOwnerPostgresResult, getDrizzleClient, PlanAction } from './pg.ts' +import { CacheHelper } from './cache.ts' +import { cloudlog } from './logging.ts' +import { + getAppOwnerPostgres, + queryAppOwnerPostgres, + requestChannelOverrideLookup, + requestInfosChannelPostgres, + requestInfosChannelPostgresRollout, + requestManifestEntriesPostgres, + resolveRolloutChannelDataPostgres, +} from './pg.ts' +import { existInEnv, getEnv } from './utils.ts' + +const CACHE_ORIGIN = 'https://updates-cache.capgo.internal' +const TOKEN_CACHE_PATH = '/cache/updates-token' +const OWNER_CACHE_PATH = '/cache/updates-owner' +const NEGATIVE_STREAK_PATH = '/cache/updates-missing-since' +const CHANNEL_CACHE_PATH = '/cache/updates-channel' +const MANIFEST_CACHE_PATH = '/cache/updates-manifest' + +const TOKEN_TTL_SECONDS = 7 * 24 * 3600 +const DEFAULT_PAYLOAD_TTL_SECONDS = 60 +// Unknown apps (Capgo removed, misconfigured open-source installs, plain +// abuse) hammer /updates forever and by definition never change: cache the +// negative long — but only once the app has been missing for longer than +// any replica lag could explain. A brand-new app may not be visible on the +// regional replica yet; caching that first miss for 10 minutes would give +// devices a false on-prem answer long after replication catches up. +const DEFAULT_NEGATIVE_TTL_SECONDS = 600 +const NEGATIVE_ESCALATION_AFTER_MS = 15 * 60 * 1000 +const NEGATIVE_STREAK_TTL_SECONDS = 24 * 3600 +const MANIFEST_TTL_SECONDS = 300 + +interface TokenPayload { t: string } + +// Host-independent cache keys: one entry per colo regardless of which +// domain alias the device (or the invalidation fan-out) used. +function buildCacheRequest(path: string, params: Record): Request { + const url = new URL(path, CACHE_ORIGIN) + for (const key of Object.keys(params).sort((a, b) => a.localeCompare(b))) + url.searchParams.set(key, params[key]) + return new Request(url.toString(), { method: 'GET' }) +} + +const requestTokenMemo = new WeakMap>>() + +function memoKey(c: Context): object | null { + try { + return (c.req?.raw as object) ?? null + } + catch { + return null + } +} +interface OwnerPayload { owner: AppOwnerPostgresResult | null } +interface ChannelPayload { channel: unknown } + +export function isUpdatesCacheEnabled(c: Context): boolean { + return existInEnv(c, 'UPDATES_CACHE_MODE') && getEnv(c, 'UPDATES_CACHE_MODE') === 'on' +} + +function payloadTtlSeconds(c: Context): number { + const raw = Number(getEnv(c, 'UPDATES_CACHE_TTL_SECONDS')) + return Number.isFinite(raw) && raw >= 5 ? raw : DEFAULT_PAYLOAD_TTL_SECONDS +} + +function negativeTtlSeconds(c: Context): number { + const raw = Number(getEnv(c, 'UPDATES_CACHE_NEGATIVE_TTL_SECONDS')) + return Number.isFinite(raw) && raw >= 5 ? raw : DEFAULT_NEGATIVE_TTL_SECONDS +} + +// Per-app version token: every cached payload embeds it in its key, so one +// token bump atomically invalidates all payload variants of the app in this +// colo. The old entries become unreachable and expire by TTL. +async function loadAppCacheToken(helper: CacheHelper, appId: string): Promise { + const request = buildCacheRequest(TOKEN_CACHE_PATH, { app_id: appId }) + const cached = await helper.matchJson(request) + if (cached?.t) + return cached.t + const token = crypto.randomUUID() + await helper.putJson(request, { t: token }, TOKEN_TTL_SECONDS) + return token +} + +// Memoized per request: every cached lookup of one update check reads the +// same token generation, so a bump landing mid-request cannot mix stale and +// fresh payloads in a single response. +function getAppCacheToken(c: Context, helper: CacheHelper, appId: string): Promise { + const key = memoKey(c) + if (!key) + return loadAppCacheToken(helper, appId) + let perApp = requestTokenMemo.get(key) + if (!perApp) { + perApp = new Map() + requestTokenMemo.set(key, perApp) + } + let token = perApp.get(appId) + if (!token) { + token = loadAppCacheToken(helper, appId) + perApp.set(appId, token) + } + return token +} + +// Invalidation entry point, called by the /cache_invalidate route on each +// regional plugin worker (fan-out from the cache_invalidate trigger). +export async function bumpAppCacheToken(c: Context, appId: string): Promise { + const helper = new CacheHelper(c) + const request = buildCacheRequest(TOKEN_CACHE_PATH, { app_id: appId }) + await helper.putJson(request, { t: crypto.randomUUID() }, TOKEN_TTL_SECONDS) + return true +} + +// Drop-in replacement for getAppOwnerPostgres, cached per app. Negative +// results (unknown app) are cached too: unknown-app traffic is the +// enumeration hot path and the apps INSERT trigger bumps the token the +// moment the app is created. +export async function cachedGetAppOwner( + c: Context, + appId: string, + drizzleClient: ReturnType, + actions: PlanAction[], +): Promise { + const helper = new CacheHelper(c) + const token = await getAppCacheToken(c, helper, appId) + if (!token) + return getAppOwnerPostgres(c, appId, drizzleClient, actions) + const request = buildCacheRequest(OWNER_CACHE_PATH, { app_id: appId, v: token, a: actions.join('-') }) + const cached = await helper.matchJson(request) + if (cached) { + cloudlog({ requestId: c.get('requestId'), message: 'updates cache hit (owner)', appId }) + return cached.owner + } + let owner: AppOwnerPostgresResult | null + try { + owner = await queryAppOwnerPostgres(c, appId, drizzleClient, actions) + } + catch { + // Transient query failure: same null-owner behavior as the direct path, + // but never cached — an unknown-app entry would misclassify the app as + // on-prem for the whole TTL. + return getAppOwnerPostgres(c, appId, drizzleClient, actions) + } + await helper.putJson(request, { owner }, owner === null ? await negativeOwnerTtl(c, helper, appId) : payloadTtlSeconds(c)) + return owner +} + +// Escalating negative TTL: the first sightings of a missing app use the +// short payload TTL (a replica may simply not have replicated a brand-new +// app yet); once the app has been missing beyond any plausible replication +// lag, escalate to the long negative TTL so dead apps stop costing queries. +async function negativeOwnerTtl(c: Context, helper: CacheHelper, appId: string): Promise { + const streakRequest = buildCacheRequest(NEGATIVE_STREAK_PATH, { app_id: appId }) + const streak = await helper.matchJson<{ first: number }>(streakRequest) + if (!streak) { + await helper.putJson(streakRequest, { first: Date.now() }, NEGATIVE_STREAK_TTL_SECONDS) + return payloadTtlSeconds(c) + } + return Date.now() - streak.first > NEGATIVE_ESCALATION_AFTER_MS ? negativeTtlSeconds(c) : payloadTtlSeconds(c) +} + +export interface CachedRequestInfosOptions { + c: Context + platform: string + app_id: string + device_id: string + defaultChannel: string + drizzleClient: ReturnType + channelDeviceCount?: number | null + manifestBundleCount?: number | null + rolloutChannelCount?: number | null + rolloutPausedVersionNames?: string[] | null + currentVersionName: string + includeMetadata?: boolean + channelSelfOverrideChannelId?: number | null +} + +// Cached mirror of requestInfosPostgres (pg.ts): the app-level channel +// lookup is cached per (app, platform, defaultChannel, flags); the +// device-level parts (override lookup, rollout decision, rollout manifest) +// run per request exactly like the uncached path. +export async function cachedRequestInfos(options: CachedRequestInfosOptions): Promise<{ channelData: any, channelOverride: any }> { + const { + c, + platform, + app_id, + device_id, + defaultChannel, + drizzleClient, + channelDeviceCount, + manifestBundleCount, + rolloutChannelCount, + rolloutPausedVersionNames, + currentVersionName, + includeMetadata = false, + channelSelfOverrideChannelId, + } = options + const shouldQueryChannelOverride = channelDeviceCount === undefined || channelDeviceCount === null ? true : channelDeviceCount > 0 + const shouldFetchManifest = manifestBundleCount === undefined || manifestBundleCount === null ? true : manifestBundleCount > 0 + const isPausedRolloutVersion = Array.isArray(rolloutPausedVersionNames) && rolloutPausedVersionNames.includes(currentVersionName) + const rollout = (rolloutChannelCount ?? 0) > 0 || isPausedRolloutVersion + + // Device-level override lookup: never cached (per-device result). + const channelOverridePromise = requestChannelOverrideLookup(c, { + app_id, + device_id, + drizzleClient, + includeManifest: shouldFetchManifest, + includeMetadata, + rollout, + shouldQueryChannelOverride, + channelSelfOverrideChannelId, + }) + + // App-level channel resolution: cached. + const channelPromise = cachedChannelLookup(c, { + platform, + app_id, + defaultChannel, + drizzleClient, + includeMetadata, + includeManifest: shouldFetchManifest, + rollout, + }) + + const [channelOverrideRaw, channelDataRaw] = await Promise.all([channelOverridePromise, channelPromise]) + + if (!rollout) + return { channelData: channelDataRaw, channelOverride: channelOverrideRaw } + + // Rollout: per-device decision + manifest for the selected version, via + // the shared resolver with the version-keyed manifest cache as loader. + const rolloutArgs = { + appId: app_id, + deviceId: device_id, + currentVersionName, + drizzleClient, + includeManifest: shouldFetchManifest, + manifestLoader: (versionId: number) => cachedManifestEntries(c, app_id, versionId, drizzleClient), + } + const channelOverride = await resolveRolloutChannelDataPostgres(c, channelOverrideRaw, rolloutArgs) + const channelData = channelOverride + ? channelDataRaw + : await resolveRolloutChannelDataPostgres(c, channelDataRaw, rolloutArgs) + return { channelOverride, channelData } +} + +interface CachedChannelLookupOptions { + platform: string + app_id: string + defaultChannel: string + drizzleClient: ReturnType + includeMetadata: boolean + includeManifest: boolean + rollout: boolean +} + +async function cachedChannelLookup(c: Context, options: CachedChannelLookupOptions): Promise { + const { platform, app_id, defaultChannel, drizzleClient, includeMetadata, includeManifest, rollout } = options + const load = () => rollout + ? requestInfosChannelPostgresRollout(c, platform, app_id, defaultChannel, drizzleClient, includeMetadata) + : requestInfosChannelPostgres(c, platform, app_id, defaultChannel, drizzleClient, includeManifest, includeMetadata) + + const helper = new CacheHelper(c) + const token = await getAppCacheToken(c, helper, app_id) + if (!token) + return load() + const request = buildCacheRequest(CHANNEL_CACHE_PATH, { + app_id, + v: token, + platform, + channel: defaultChannel || '-', + meta: includeMetadata ? '1' : '0', + manifest: includeManifest ? '1' : '0', + rollout: rollout ? '1' : '0', + }) + const cached = await helper.matchJson(request) + if (cached) { + cloudlog({ requestId: c.get('requestId'), message: 'updates cache hit (channel)', appId: app_id, platform }) + return cached.channel ?? undefined + } + const channel = await load() + // `undefined` is not JSON-representable; store null and map back on read. + await helper.putJson(request, { channel: channel ?? null }, payloadTtlSeconds(c)) + return channel +} + +// Manifest entries for a rollout-selected version, keyed under the app's +// version token so a token bump (e.g. the manifest trigger) invalidates them +// together with the channel payloads. Empty results are never cached (the +// bundle may still be uploading). +async function cachedManifestEntries( + c: Context, + appId: string, + versionId: number, + drizzleClient: ReturnType, +): Promise<{ file_name: string | null, file_hash: string | null, s3_path: string | null }[]> { + const helper = new CacheHelper(c) + const token = await getAppCacheToken(c, helper, appId) + if (!token) + return requestManifestEntriesPostgres(c, versionId, drizzleClient) + const request = buildCacheRequest(MANIFEST_CACHE_PATH, { app_id: appId, v: token, version_id: String(versionId) }) + const cached = await helper.matchJson<{ entries: { file_name: string | null, file_hash: string | null, s3_path: string | null }[] }>(request) + if (cached) + return cached.entries + const entries = await requestManifestEntriesPostgres(c, versionId, drizzleClient) + if (entries.length > 0) + await helper.putJson(request, { entries }, MANIFEST_TTL_SECONDS) + return entries +} diff --git a/supabase/functions/triggers/index.ts b/supabase/functions/triggers/index.ts index 0175d21810..6e9baf4ae0 100644 --- a/supabase/functions/triggers/index.ts +++ b/supabase/functions/triggers/index.ts @@ -1,3 +1,4 @@ +import { app as cache_invalidate } from '../_backend/triggers/cache_invalidate.ts' import { app as credit_usage_alerts } from '../_backend/triggers/credit_usage_alerts.ts' import { app as credit_usage_posthog } from '../_backend/triggers/credit_usage_posthog.ts' import { app as cron_clean_orphan_images } from '../_backend/triggers/cron_clean_orphan_images.ts' @@ -79,6 +80,7 @@ appGlobal.route('/credit_usage_posthog', credit_usage_posthog) appGlobal.route('/on_organization_delete', on_organization_delete) appGlobal.route('/on_deploy_history_create', on_deploy_history_create) appGlobal.route('/plugin_notifications', pluginNotifications) +appGlobal.route('/cache_invalidate', cache_invalidate) appGlobal.route('/queue_consumer', queue_consumer) appGlobal.route('/webhook_delivery', webhook_delivery) appGlobal.route('/webhook_dispatcher', webhook_dispatcher) diff --git a/supabase/migrations/20260711100000_updates_cache_invalidation.sql b/supabase/migrations/20260711100000_updates_cache_invalidation.sql new file mode 100644 index 0000000000..5f2413b4a6 --- /dev/null +++ b/supabase/migrations/20260711100000_updates_cache_invalidation.sql @@ -0,0 +1,231 @@ +-- Targeted invalidation for the /updates colo cache +-- (see supabase/functions/_backend/utils/updates_colo_cache.ts). +-- +-- Whenever rows that feed the update hot path change, notify the +-- triggers/cache_invalidate function (via pg_net, async) which fans the +-- per-app token bump out to every regional plugin worker. The cache TTL +-- remains the backstop if a call is lost. +-- +-- All triggers are STATEMENT-level with transition tables: bulk writes +-- (daily channel_devices cleanup, bundle uploads inserting hundreds of +-- manifest rows, org-wide backfills) produce ONE aggregated, deduplicated +-- notification per statement instead of one per row. + +-- Notify helper: dedupes, caps and chunks the app list so the payload is +-- always bounded (the fan-out endpoint enforces 100 ids per request; an +-- org bigger than the cap falls back to the cache TTL for the tail). +CREATE OR REPLACE FUNCTION public.notify_updates_cache_invalidation(p_app_ids text[]) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + deduped text[]; + chunk text[]; + chunk_size constant int := 100; + max_apps constant int := 1000; + i int; +BEGIN + SELECT array_agg(DISTINCT app_id) INTO deduped + FROM unnest(p_app_ids) AS app_id + WHERE app_id IS NOT NULL AND app_id <> ''; + + IF deduped IS NULL OR array_length(deduped, 1) = 0 THEN + RETURN; + END IF; + IF array_length(deduped, 1) > max_apps THEN + deduped := deduped[1:max_apps]; + END IF; + + i := 1; + WHILE i <= array_length(deduped, 1) LOOP + chunk := deduped[i:i + chunk_size - 1]; + PERFORM net.http_post( + url := public.get_db_url() || '/functions/v1/triggers/cache_invalidate', + headers := jsonb_build_object( + 'Content-Type', 'application/json', + 'apisecret', public.get_apikey() + ), + body := jsonb_build_object('app_ids', to_jsonb(chunk)), + timeout_milliseconds := 5000 + ); + i := i + chunk_size; + END LOOP; +END; +$$; + +-- Pin ownership so SECURITY DEFINER privileges stay consistent regardless +-- of the migration role, then lock execution down: the helper runs +-- privileged fan-out (uses get_apikey()) and must never be RPC-callable. +ALTER FUNCTION public.notify_updates_cache_invalidation(text[]) OWNER TO postgres; +REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM PUBLIC; +REVOKE ALL ON FUNCTION public.notify_updates_cache_invalidation(text[]) FROM anon, authenticated; +GRANT EXECUTE ON FUNCTION public.notify_updates_cache_invalidation(text[]) TO service_role; + +-- Drop the previous row-level implementation (and its triggers) if present. +DROP FUNCTION IF EXISTS public.invalidate_updates_cache() CASCADE; +DROP FUNCTION IF EXISTS public.invalidate_updates_cache_manifest() CASCADE; + +-- One statement-level trigger function for every replicated-read table. +-- Transition tables are exposed as new_rows / old_rows depending on TG_OP. +CREATE OR REPLACE FUNCTION public.invalidate_updates_cache_stmt() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = '' +AS $$ +DECLARE + app_ids text[]; +BEGIN + IF TG_TABLE_NAME IN ('channels', 'channel_devices', 'apps', 'app_versions') THEN + IF TG_OP = 'DELETE' THEN + SELECT array_agg(DISTINCT r.app_id::text) INTO app_ids FROM old_rows r; + ELSE + SELECT array_agg(DISTINCT r.app_id::text) INTO app_ids FROM new_rows r; + END IF; + ELSIF TG_TABLE_NAME = 'orgs' THEN + SELECT array_agg(DISTINCT a.app_id::text) INTO app_ids + FROM new_rows o + JOIN public.apps a ON a.owner_org = o.id; + ELSIF TG_TABLE_NAME = 'stripe_info' THEN + IF TG_OP = 'DELETE' THEN + SELECT array_agg(DISTINCT a.app_id::text) INTO app_ids + FROM old_rows s + JOIN public.orgs o ON o.customer_id = s.customer_id + JOIN public.apps a ON a.owner_org = o.id; + ELSE + SELECT array_agg(DISTINCT a.app_id::text) INTO app_ids + FROM new_rows s + JOIN public.orgs o ON o.customer_id = s.customer_id + JOIN public.apps a ON a.owner_org = o.id; + END IF; + ELSIF TG_TABLE_NAME = 'manifest' THEN + IF TG_OP = 'DELETE' THEN + SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids + FROM old_rows m + JOIN public.app_versions av ON av.id = m.app_version_id; + ELSE + SELECT array_agg(DISTINCT av.app_id::text) INTO app_ids + FROM new_rows m + JOIN public.app_versions av ON av.id = m.app_version_id; + END IF; + END IF; + + PERFORM public.notify_updates_cache_invalidation(app_ids); + RETURN NULL; +END; +$$; + +ALTER FUNCTION public.invalidate_updates_cache_stmt() OWNER TO postgres; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache_stmt() FROM PUBLIC; +REVOKE ALL ON FUNCTION public.invalidate_updates_cache_stmt() FROM anon, authenticated; + +-- channels: any change moves versions/flags devices resolve against. +DROP TRIGGER IF EXISTS invalidate_updates_cache_channels_ins ON public.channels; +CREATE TRIGGER invalidate_updates_cache_channels_ins +AFTER INSERT ON public.channels +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channels_upd ON public.channels; +CREATE TRIGGER invalidate_updates_cache_channels_upd +AFTER UPDATE ON public.channels +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channels_del ON public.channels; +CREATE TRIGGER invalidate_updates_cache_channels_del +AFTER DELETE ON public.channels +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- channel_devices: per-device overrides; bulk daily cleanups delete +-- thousands of rows in one statement and must cost one notification. +DROP TRIGGER IF EXISTS invalidate_updates_cache_channel_devices_ins ON public.channel_devices; +CREATE TRIGGER invalidate_updates_cache_channel_devices_ins +AFTER INSERT ON public.channel_devices +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channel_devices_upd ON public.channel_devices; +CREATE TRIGGER invalidate_updates_cache_channel_devices_upd +AFTER UPDATE ON public.channel_devices +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_channel_devices_del ON public.channel_devices; +CREATE TRIGGER invalidate_updates_cache_channel_devices_del +AFTER DELETE ON public.channel_devices +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- apps: counters, plan/provider flags, metadata exposure. INSERT clears the +-- negative (unknown-app) cache entry the moment an app is created. +DROP TRIGGER IF EXISTS invalidate_updates_cache_apps_ins ON public.apps; +CREATE TRIGGER invalidate_updates_cache_apps_ins +AFTER INSERT ON public.apps +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_apps_upd ON public.apps; +CREATE TRIGGER invalidate_updates_cache_apps_upd +AFTER UPDATE ON public.apps +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_apps_del ON public.apps; +CREATE TRIGGER invalidate_updates_cache_apps_del +AFTER DELETE ON public.apps +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- app_versions: UPDATE only (r2_path/checksum/deleted change after a channel +-- may already point at the version; freshly inserted rows are not yet +-- referenced by any channel). +DROP TRIGGER IF EXISTS invalidate_updates_cache_app_versions_upd ON public.app_versions; +CREATE TRIGGER invalidate_updates_cache_app_versions_upd +AFTER UPDATE ON public.app_versions +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- manifest: cached inside channel payloads and per-version entries; bundle +-- uploads insert many rows at once. +DROP TRIGGER IF EXISTS invalidate_updates_cache_manifest_insert ON public.manifest; +CREATE TRIGGER invalidate_updates_cache_manifest_insert +AFTER INSERT ON public.manifest +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_manifest_delete ON public.manifest; +CREATE TRIGGER invalidate_updates_cache_manifest_delete +AFTER DELETE ON public.manifest +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- orgs: has_usage_credits / customer_id feed plan validation. +DROP TRIGGER IF EXISTS invalidate_updates_cache_orgs_upd ON public.orgs; +CREATE TRIGGER invalidate_updates_cache_orgs_upd +AFTER UPDATE ON public.orgs +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +-- stripe_info: status / trial / exceeded flags feed plan validation; DELETE +-- included (orphaned-row cleanup must not leave plan_valid cached wrong). +DROP TRIGGER IF EXISTS invalidate_updates_cache_stripe_info_ins ON public.stripe_info; +CREATE TRIGGER invalidate_updates_cache_stripe_info_ins +AFTER INSERT ON public.stripe_info +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_stripe_info_upd ON public.stripe_info; +CREATE TRIGGER invalidate_updates_cache_stripe_info_upd +AFTER UPDATE ON public.stripe_info +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); + +DROP TRIGGER IF EXISTS invalidate_updates_cache_stripe_info_del ON public.stripe_info; +CREATE TRIGGER invalidate_updates_cache_stripe_info_del +AFTER DELETE ON public.stripe_info +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.invalidate_updates_cache_stmt(); diff --git a/tests/updates-cache-invalidation.test.ts b/tests/updates-cache-invalidation.test.ts new file mode 100644 index 0000000000..7dcc54ba49 --- /dev/null +++ b/tests/updates-cache-invalidation.test.ts @@ -0,0 +1,221 @@ +import { randomUUID } from 'node:crypto' +import { Pool } from 'pg' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { POSTGRES_URL } from './test-utils.ts' + +// Postgres-level coverage for the /updates colo-cache invalidation migration +// (20260711100000): statement-level trigger wiring, bulk-write aggregation, +// notify chunking/cap/dedupe through the pg_net queue, and privileges. + +const STATEMENT_TRIGGERS = [ + ['channels', 'invalidate_updates_cache_channels_ins'], + ['channels', 'invalidate_updates_cache_channels_upd'], + ['channels', 'invalidate_updates_cache_channels_del'], + ['channel_devices', 'invalidate_updates_cache_channel_devices_ins'], + ['channel_devices', 'invalidate_updates_cache_channel_devices_upd'], + ['channel_devices', 'invalidate_updates_cache_channel_devices_del'], + ['apps', 'invalidate_updates_cache_apps_ins'], + ['apps', 'invalidate_updates_cache_apps_upd'], + ['apps', 'invalidate_updates_cache_apps_del'], + ['app_versions', 'invalidate_updates_cache_app_versions_upd'], + ['manifest', 'invalidate_updates_cache_manifest_insert'], + ['manifest', 'invalidate_updates_cache_manifest_delete'], + ['orgs', 'invalidate_updates_cache_orgs_upd'], + ['stripe_info', 'invalidate_updates_cache_stripe_info_ins'], + ['stripe_info', 'invalidate_updates_cache_stripe_info_upd'], + ['stripe_info', 'invalidate_updates_cache_stripe_info_del'], +] as const + +describe('updates cache invalidation (postgres)', () => { + let pool: Pool + + beforeAll(() => { + pool = new Pool({ connectionString: POSTGRES_URL }) + }) + + afterAll(async () => { + await pool.end() + }) + + it.concurrent.each(STATEMENT_TRIGGERS)('wires a statement trigger on %s: %s', async (table, trigger) => { + const { rows } = await pool.query(` + SELECT proc.proname AS function_name, trg.tgtype + FROM pg_trigger trg + JOIN pg_class tbl ON tbl.oid = trg.tgrelid + JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace + JOIN pg_proc proc ON proc.oid = trg.tgfoid + WHERE trg.tgname = $1 + AND tbl_ns.nspname = 'public' + AND tbl.relname = $2 + AND NOT trg.tgisinternal + LIMIT 1 + `, [trigger, table]) + expect(rows).toHaveLength(1) + expect(rows[0].function_name).toBe('invalidate_updates_cache_stmt') + // statement-level triggers have the ROW bit (1) unset in tgtype + expect(Number(rows[0].tgtype) & 1).toBe(0) + }) + + it.concurrent('does not leave the old row-level implementation behind', async () => { + const { rows } = await pool.query(` + SELECT proc.proname + FROM pg_proc proc + JOIN pg_namespace ns ON ns.oid = proc.pronamespace + WHERE ns.nspname = 'public' + AND proc.proname IN ('invalidate_updates_cache', 'invalidate_updates_cache_manifest') + `) + expect(rows).toHaveLength(0) + }) + + it.concurrent('locks down execute privileges on the notify helper', async () => { + const { rows } = await pool.query(` + SELECT + has_function_privilege('anon', 'public.notify_updates_cache_invalidation(text[])', 'EXECUTE') AS anon_execute, + has_function_privilege('authenticated', 'public.notify_updates_cache_invalidation(text[])', 'EXECUTE') AS authenticated_execute, + has_function_privilege('service_role', 'public.notify_updates_cache_invalidation(text[])', 'EXECUTE') AS service_role_execute + `) + expect(rows[0]).toEqual({ + anon_execute: false, + authenticated_execute: false, + service_role_execute: true, + }) + }) + + async function queuedBodiesFor(client: { query: Pool['query'] }, marker: string): Promise { + const { rows } = await client.query(` + SELECT convert_from(body, 'utf8')::jsonb -> 'app_ids' AS app_ids + FROM net.http_request_queue + WHERE url LIKE '%/triggers/cache_invalidate' + AND convert_from(body, 'utf8') LIKE $1 + ORDER BY id + `, [`%${marker}%`]) + return rows.map((row: { app_ids: string[] }) => row.app_ids) + } + + it.concurrent('notify dedupes and sends one chunked request per 100 apps', async () => { + const marker = `chunk-${randomUUID()}` + const appIds = Array.from({ length: 150 }, (_, i) => `${marker}.app.${i % 120}`) + await pool.query('SELECT public.notify_updates_cache_invalidation($1::text[])', [appIds]) + const bodies = await queuedBodiesFor(pool, marker) + expect(bodies).toHaveLength(2) + expect(bodies[0]).toHaveLength(100) + expect(bodies[1]).toHaveLength(20) + expect(new Set(bodies.flat()).size).toBe(120) + }) + + it.concurrent('notify caps runaway payloads at 1000 apps', async () => { + const marker = `cap-${randomUUID()}` + const appIds = Array.from({ length: 1200 }, (_, i) => `${marker}.app.${i}`) + await pool.query('SELECT public.notify_updates_cache_invalidation($1::text[])', [appIds]) + const bodies = await queuedBodiesFor(pool, marker) + expect(bodies).toHaveLength(10) + expect(bodies.flat()).toHaveLength(1000) + }) + + it.concurrent('notify ignores empty input without queueing anything', async () => { + // Observe total endpoint queue growth inside one transaction so a wrongly + // queued empty payload cannot hide from a marker filter. + const client = await pool.connect() + try { + await client.query('BEGIN') + const countSql = `SELECT count(*)::int AS c FROM net.http_request_queue WHERE url LIKE '%/triggers/cache_invalidate'` + const { rows: [before] } = await client.query(countSql) + await client.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[]::text[])`) + await client.query(`SELECT public.notify_updates_cache_invalidation(ARRAY[NULL, '']::text[])`) + const { rows: [after] } = await client.query(countSql) + expect(after.c).toBe(before.c) + await client.query('ROLLBACK') + } + finally { + client.release() + } + }) + + // The blocker scenario: a bulk cleanup statement touching thousands of + // channel_devices rows must produce ONE aggregated notification, not one + // per row — even with the cache mode off, per-row pg_net calls would flood + // the queue and the fan-out. + it.concurrent('bulk channel_devices delete produces one aggregated notification', async () => { + const marker = `bulk-${randomUUID()}` + const appId = `${marker}.demo.app` + const client = await pool.connect() + try { + await client.query('BEGIN') + const { rows: [org] } = await client.query( + `INSERT INTO public.orgs (created_by, name, management_email) + SELECT id, $1, $2 FROM auth.users LIMIT 1 + RETURNING id, created_by`, + [`org-${marker}`, `${marker}@test.capgo.app`], + ) + await client.query( + `INSERT INTO public.apps (app_id, icon_url, owner_org, name) VALUES ($1, '', $2, $1)`, + [appId, org.id], + ) + const { rows: [channel] } = await client.query( + `INSERT INTO public.channels (name, app_id, owner_org, created_by) + VALUES ('production', $1, $2, $3) RETURNING id`, + [appId, org.id, org.created_by], + ) + // 500 device overrides in one statement + await client.query( + `INSERT INTO public.channel_devices (channel_id, app_id, owner_org, device_id) + SELECT $1, $2, $3, 'dev-' || g.i FROM generate_series(1, 500) AS g(i)`, + [channel.id, appId, org.id], + ) + const afterInsert = await queuedBodiesFor(client, marker) + // one statement-level notification for the bulk insert + const insertNotifications = afterInsert.filter(body => body.length === 1 && body[0] === appId) + expect(insertNotifications.length).toBeGreaterThanOrEqual(1) + + // bulk delete: exactly one more notification for this app, not 500 + const before = afterInsert.length + await client.query(`DELETE FROM public.channel_devices WHERE app_id = $1`, [appId]) + const afterDelete = await queuedBodiesFor(client, marker) + expect(afterDelete.length).toBe(before + 1) + expect(afterDelete.at(-1)).toEqual([appId]) + await client.query('ROLLBACK') + } + catch (e) { + await client.query('ROLLBACK') + throw e + } + finally { + client.release() + } + }) + + it.concurrent('channels insert queues an invalidation carrying the app_id', async () => { + const marker = `trg-${randomUUID()}` + const appId = `${marker}.demo.app` + const client = await pool.connect() + try { + await client.query('BEGIN') + const { rows: [org] } = await client.query( + `INSERT INTO public.orgs (created_by, name, management_email) + SELECT id, $1, $2 FROM auth.users LIMIT 1 + RETURNING id, created_by`, + [`org-${marker}`, `${marker}@test.capgo.app`], + ) + await client.query( + `INSERT INTO public.apps (app_id, icon_url, owner_org, name) VALUES ($1, '', $2, $1)`, + [appId, org.id], + ) + await client.query( + `INSERT INTO public.channels (name, app_id, owner_org, created_by) VALUES ('production', $1, $2, $3)`, + [appId, org.id, org.created_by], + ) + const bodies = await queuedBodiesFor(client, marker) + expect(bodies.length).toBeGreaterThanOrEqual(2) // apps insert + channels insert + for (const body of bodies) + expect(body).toContain(appId) + await client.query('ROLLBACK') + } + catch (e) { + await client.query('ROLLBACK') + throw e + } + finally { + client.release() + } + }) +}) diff --git a/tests/updates-colo-cache.unit.test.ts b/tests/updates-colo-cache.unit.test.ts new file mode 100644 index 0000000000..68d9357e50 --- /dev/null +++ b/tests/updates-colo-cache.unit.test.ts @@ -0,0 +1,479 @@ +import { Hono } from 'hono/tiny' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { app as cacheInvalidateFanout, chunkAppIds, parsePluginInvalidateUrls } from '../supabase/functions/_backend/triggers/cache_invalidate.ts' +import { app as cacheInvalidateRoute } from '../supabase/functions/_backend/private/cache_invalidate.ts' +import { bumpAppCacheToken, cachedGetAppOwner, cachedRequestInfos, isUpdatesCacheEnabled } from '../supabase/functions/_backend/utils/updates_colo_cache.ts' + +type CacheKey = Request | string + +function cacheKeyToString(key: CacheKey) { + return typeof key === 'string' ? key : key.url +} + +function createMemoryCache() { + const store = new Map() + return { + store, + match: vi.fn(async (key: CacheKey) => store.get(cacheKeyToString(key))?.clone()), + put: vi.fn(async (key: CacheKey, response: Response) => { + store.set(cacheKeyToString(key), response.clone()) + }), + delete: vi.fn(async (key: CacheKey) => store.delete(cacheKeyToString(key))), + } +} + +const APP_OWNER = { + owner_org: 'org-1', + plan_valid: true, + channel_device_count: 0, + manifest_bundle_count: 0, + rollout_channel_count: 0, + rollout_paused_version_names: [] as string[], + expose_metadata: false, + allow_device_custom_id: true, + block_provider_infra_requests: false, + orgs: { created_by: 'user-1', id: 'org-1', management_email: 'a@b.c' }, +} + +const CHANNEL_ROW = { + version: { id: 42, name: '1.2.3', checksum: 'abc', storage_provider: 'r2', manifest_count: 0, r2_path: 'p.zip', external_url: null, min_update_version: null, session_key: null, key_id: null }, + channels: { id: 7, name: 'production', app_id: 'com.demo.app', ios: true, android: true, public: true, allow_device_self_set: false }, +} + +// The pg query loaders are stubbed: these tests exercise the cache +// semantics, not the SQL (covered by the existing update tests with the +// flag off). The shared dispatch/rollout helpers stay real so the cached +// path runs the exact same code as the direct path. +vi.mock('../supabase/functions/_backend/utils/pg.ts', async (importOriginal) => { + const actual = await importOriginal() + const requestInfosChannelDevicePostgres = vi.fn(async () => null) + const requestInfosChannelDevicePostgresRollout = vi.fn(async () => null) + const requestInfosChannelByIdPostgres = vi.fn(async () => structuredClone(CHANNEL_ROW)) + const requestInfosChannelByIdPostgresRollout = vi.fn(async () => structuredClone(CHANNEL_ROW)) + const getAppOwnerPostgres = vi.fn(async () => structuredClone(APP_OWNER)) + return { + ...actual, + getAppOwnerPostgres, + // The cached path uses the throwing variant; share the same stub so + // call-count assertions cover both entry points. + queryAppOwnerPostgres: getAppOwnerPostgres, + requestInfosChannelPostgres: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelPostgresRollout: vi.fn(async () => structuredClone(CHANNEL_ROW)), + requestInfosChannelDevicePostgres, + requestInfosChannelDevicePostgresRollout, + requestInfosChannelByIdPostgres, + requestInfosChannelByIdPostgresRollout, + requestManifestEntriesPostgres: vi.fn(async () => []), + // Same dispatch semantics as the real helper, wired to the stubs above + // (the real one binds pg-internal functions, bypassing the mock). + requestChannelOverrideLookup: vi.fn(async (c: any, args: any) => { + if (typeof args.channelSelfOverrideChannelId === 'number') { + return args.rollout + ? requestInfosChannelByIdPostgresRollout() + : requestInfosChannelByIdPostgres() + } + if (args.shouldQueryChannelOverride) { + return args.rollout + ? requestInfosChannelDevicePostgresRollout() + : requestInfosChannelDevicePostgres() + } + return null + }), + } +}) + +const pg = await import('../supabase/functions/_backend/utils/pg.ts') + +function makeContext() { + return { + env: {}, + get: () => 'test-request', + set: () => {}, + header: () => {}, + req: { url: 'https://plugin.capgo.app/updates', raw: new Request('https://plugin.capgo.app/updates') }, + res: { headers: new Headers() }, + } as any +} + +describe('updates colo cache', () => { + let cache: ReturnType + + beforeEach(() => { + cache = createMemoryCache() + vi.stubGlobal('caches', { open: vi.fn(async () => cache) }) + vi.stubEnv('UPDATES_CACHE_MODE', 'on') + vi.clearAllMocks() + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.unstubAllEnvs() + }) + + it('is gated by UPDATES_CACHE_MODE', () => { + expect(isUpdatesCacheEnabled(makeContext())).toBe(true) + vi.stubEnv('UPDATES_CACHE_MODE', 'off') + expect(isUpdatesCacheEnabled(makeContext())).toBe(false) + }) + + it('caches app owner: one loader call for repeated reads', async () => { + const c = makeContext() + const first = await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + const second = await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + expect(first?.owner_org).toBe('org-1') + expect(second?.owner_org).toBe('org-1') + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) + }) + + it('caches negative owner results (unknown app)', async () => { + ;(pg.getAppOwnerPostgres as any).mockResolvedValueOnce(null) + expect(await cachedGetAppOwner(makeContext(), 'com.ghost.app', {} as any, ['mau'])).toBeNull() + expect(await cachedGetAppOwner(makeContext(), 'com.ghost.app', {} as any, ['mau'])).toBeNull() + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) + }) + + it('never caches a transient query failure as unknown app', async () => { + ;(pg.getAppOwnerPostgres as any).mockRejectedValueOnce(new Error('replica down')) + // direct-path fallback also fails once: the request answers null... + ;(pg.getAppOwnerPostgres as any).mockRejectedValueOnce(new Error('replica down')) + await expect(cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau'])).rejects.toThrow('replica down') + // ...but the next request loads and caches the real owner + const owner = await cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau']) + expect(owner?.owner_org).toBe('org-1') + }) + + it('token bump invalidates every cached payload of the app', async () => { + await cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau']) + expect(await bumpAppCacheToken(makeContext(), 'com.demo.app')).toBe(true) + // fresh request context: the token memo is per-request by design + await cachedGetAppOwner(makeContext(), 'com.demo.app', {} as any, ['mau']) + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(2) + }) + + it('one request never mixes token generations (memoized per request)', async () => { + const c = makeContext() + await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + await bumpAppCacheToken(makeContext(), 'com.demo.app') + // same request keeps reading its generation: still a cache hit + await cachedGetAppOwner(c, 'com.demo.app', {} as any, ['mau']) + expect(pg.getAppOwnerPostgres).toHaveBeenCalledTimes(1) + }) + + it('caches the channel lookup but never the per-device override', async () => { + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 1, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + const first = await cachedRequestInfos(options) + const second = await cachedRequestInfos({ ...options, device_id: 'device-2' }) + expect(first.channelData.channels.name).toBe('production') + expect(second.channelData.channels.name).toBe('production') + expect(pg.requestInfosChannelPostgres).toHaveBeenCalledTimes(1) + // override lookup runs per device, uncached + expect(pg.requestInfosChannelDevicePostgres).toHaveBeenCalledTimes(2) + }) + + it('skips the override lookup entirely when the app has no overrides', async () => { + const c = makeContext() + const result = await cachedRequestInfos({ + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + }) + expect(result.channelOverride).toBeNull() + expect(pg.requestInfosChannelDevicePostgres).not.toHaveBeenCalled() + }) + + it('caches null channel results without breaking the miss path', async () => { + ;(pg.requestInfosChannelPostgres as any).mockResolvedValueOnce(null) + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + const first = await cachedRequestInfos(options) + const second = await cachedRequestInfos(options) + expect(first.channelData ?? null).toBeNull() + expect(second.channelData ?? null).toBeNull() + expect(pg.requestInfosChannelPostgres).toHaveBeenCalledTimes(1) + }) + + it('separates cache entries per platform and defaultChannel', async () => { + const c = makeContext() + const base = { + c, + app_id: 'com.demo.app', + device_id: 'device-1', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 0, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + await cachedRequestInfos({ ...base, platform: 'ios', defaultChannel: '' }) + await cachedRequestInfos({ ...base, platform: 'android', defaultChannel: '' }) + await cachedRequestInfos({ ...base, platform: 'ios', defaultChannel: 'beta' }) + expect(pg.requestInfosChannelPostgres).toHaveBeenCalledTimes(3) + }) + + it('rollout path decides per device on top of the cached channel row', async () => { + ;(pg.requestInfosChannelPostgresRollout as any).mockResolvedValue({ + ...structuredClone(CHANNEL_ROW), + rolloutVersion: { id: 43, name: '1.2.4', manifest_count: 0 }, + channels: { ...structuredClone(CHANNEL_ROW.channels), rollout_version: 43, rollout_enabled: true, rollout_percentage_bps: 10000, rollout_id: 'r-1', rollout_paused_at: null, rollout_cache_ttl_seconds: 2592000 }, + }) + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 0, + rolloutChannelCount: 1, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + const first = await cachedRequestInfos(options) + const second = await cachedRequestInfos({ ...options, device_id: 'device-2' }) + // 100% rollout: both devices land on the rollout version + expect(first.channelData.version.name).toBe('1.2.4') + expect(second.channelData.version.name).toBe('1.2.4') + // channel row cached once, decision computed per device + expect(pg.requestInfosChannelPostgresRollout).toHaveBeenCalledTimes(1) + }) + + it('token bump also invalidates cached rollout manifests', async () => { + ;(pg.requestInfosChannelPostgresRollout as any).mockResolvedValue({ + ...structuredClone(CHANNEL_ROW), + rolloutVersion: { id: 43, name: '1.2.4', manifest_count: 2 }, + channels: { ...structuredClone(CHANNEL_ROW.channels), rollout_version: 43, rollout_enabled: true, rollout_percentage_bps: 10000, rollout_id: 'r-1', rollout_paused_at: null, rollout_cache_ttl_seconds: 2592000 }, + }) + ;(pg.requestManifestEntriesPostgres as any).mockResolvedValue([{ file_name: 'a', file_hash: 'h', s3_path: 'p' }]) + const c = makeContext() + const options = { + c, + platform: 'ios', + app_id: 'com.demo.app', + device_id: 'device-1', + defaultChannel: '', + drizzleClient: {} as any, + channelDeviceCount: 0, + manifestBundleCount: 1, + rolloutChannelCount: 1, + rolloutPausedVersionNames: [], + currentVersionName: '1.0.0', + } + await cachedRequestInfos(options) + await cachedRequestInfos({ ...options, c: makeContext() }) + expect(pg.requestManifestEntriesPostgres).toHaveBeenCalledTimes(1) + await bumpAppCacheToken(makeContext(), 'com.demo.app') + await cachedRequestInfos({ ...options, c: makeContext() }) + expect(pg.requestManifestEntriesPostgres).toHaveBeenCalledTimes(2) + }) +}) + +describe('cache invalidate route (plugin worker)', () => { + let cache: ReturnType + + function buildApp() { + const app = new Hono() + app.route('/cache_invalidate', cacheInvalidateRoute) + return app + } + + beforeEach(() => { + cache = createMemoryCache() + vi.stubGlobal('caches', { open: vi.fn(async () => cache) }) + vi.stubEnv('API_SECRET', 's3cret') + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.unstubAllEnvs() + }) + + it('rejects when the secret is missing or wrong', async () => { + const app = buildApp() + const wrong = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'nope', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(wrong.status).toBe(400) + + vi.unstubAllEnvs() + const disabled = await buildApp().request('/cache_invalidate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(disabled.status).toBe(400) + }) + + it('bumps tokens for each app', async () => { + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 's3cret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app', 'com.other.app'] }), + }) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ bumped: 2 }) + expect(cache.put).toHaveBeenCalledTimes(2) + }) + + it('rejects oversized app_ids loudly instead of truncating', async () => { + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 's3cret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: Array.from({ length: 101 }, (_, i) => `com.app.${i}`) }), + }) + expect(response.status).toBe(400) + // must fail on the size guard, not on auth + expect(await response.text()).toContain('app_ids is limited to') + expect(cache.put).not.toHaveBeenCalled() + }) + + it('rejects empty app_ids', async () => { + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 's3cret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: [] }), + }) + expect(response.status).toBe(400) + }) +}) + +describe('cache invalidate fanout (triggers)', () => { + function buildApp() { + const app = new Hono() + app.route('/cache_invalidate', cacheInvalidateFanout) + return app + } + + function stubFullEnv() { + vi.stubEnv('API_SECRET', 'api-secret') + vi.stubEnv('PLUGIN_INVALIDATE_URLS', 'https://plugin.eu.capgo.app, https://plugin.na.capgo.app/') + } + + afterEach(() => { + vi.unstubAllGlobals() + vi.unstubAllEnvs() + }) + + it('parses and normalizes the regional url list', () => { + expect(parsePluginInvalidateUrls('https://a.com, https://b.com/ ,')).toEqual(['https://a.com', 'https://b.com']) + }) + + it('dedupes and chunks app ids to the per-request cap', () => { + const ids = Array.from({ length: 150 }, (_, i) => `com.app.${i % 120}`) + const chunks = chunkAppIds(ids) + expect(chunks).toHaveLength(2) + expect(chunks[0]).toHaveLength(100) + expect(chunks[1]).toHaveLength(20) + }) + + it('fans out one call per region per chunk, nothing dropped', async () => { + stubFullEnv() + const fetchMock = vi.fn(async () => new Response('{}', { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'api-secret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: Array.from({ length: 150 }, (_, i) => `com.app.${i}`) }), + }) + expect(response.status).toBe(200) + // 2 regions x 2 chunks + expect(fetchMock).toHaveBeenCalledTimes(4) + // every region must receive the complete, deduplicated set — a global + // union could hide one region getting duplicates and another gaps + const perRegion = new Map() + for (const [url, init] of fetchMock.mock.calls as any[]) { + const host = new URL(url).host + perRegion.set(host, [...(perRegion.get(host) ?? []), ...JSON.parse(init.body).app_ids]) + } + expect(perRegion.size).toBe(2) + for (const ids of perRegion.values()) { + expect(ids).toHaveLength(150) + expect(new Set(ids).size).toBe(150) + } + }) + + it('fans out one call per regional worker with the shared secret', async () => { + stubFullEnv() + const fetchMock = vi.fn(async () => new Response('{}', { status: 200 })) + vi.stubGlobal('fetch', fetchMock) + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'api-secret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(response.status).toBe(200) + expect(await response.json()).toMatchObject({ regions: 2, succeeded: 2 }) + expect(fetchMock).toHaveBeenCalledTimes(2) + const [url, init] = fetchMock.mock.calls[0] as any + expect(url).toBe('https://plugin.eu.capgo.app/cache_invalidate') + expect(init.headers.apisecret).toBe('api-secret') + expect(JSON.parse(init.body)).toEqual({ app_ids: ['com.demo.app'] }) + }) + + it('soft-skips when env is missing (TTL is the backstop)', async () => { + vi.stubEnv('API_SECRET', 'api-secret') + const fetchMock = vi.fn() + vi.stubGlobal('fetch', fetchMock) + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'apisecret': 'api-secret', 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(response.status).toBe(200) + expect(fetchMock).not.toHaveBeenCalled() + }) + + it('rejects without the api secret', async () => { + stubFullEnv() + const app = buildApp() + const response = await app.request('/cache_invalidate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ app_ids: ['com.demo.app'] }), + }) + expect(response.status).toBe(400) + }) +})