From 84fd3f0613f9f8b72fc1be4ab10d312355e073ca Mon Sep 17 00:00:00 2001 From: Filip Ilic Date: Tue, 7 Apr 2026 18:13:42 +0200 Subject: [PATCH] fix(core): chunk IN-clause batch queries to stay under D1's 100-param limit SeoRepository.getMany, BylineRepository.getContentBylinesMany, and BylineRepository.findByUserIds each built a single SELECT with one bound parameter per id. Called from handleContentList with up to 100 ids, these exceeded D1's 100-bound-parameter per-statement limit and threw "D1_ERROR: too many SQL variables", breaking the admin content list page on any collection with enough items. Chunk each batch query into groups of 90 ids. With one slot consumed by the collection/slug filter the remaining 99 ids would just barely fit, so 90 leaves headroom. Known similar instances exist in comment.ts, plugin-storage.ts, and taxonomies/index.ts but are not on the content-list hot path and are left out of scope. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Filip Ilic --- .changeset/old-badgers-see.md | 5 + .../core/src/database/repositories/byline.ts | 105 ++++++++++-------- .../core/src/database/repositories/seo.ts | 45 +++++--- 3 files changed, 93 insertions(+), 62 deletions(-) create mode 100644 .changeset/old-badgers-see.md diff --git a/.changeset/old-badgers-see.md b/.changeset/old-badgers-see.md new file mode 100644 index 000000000..bca552271 --- /dev/null +++ b/.changeset/old-badgers-see.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Fixes `D1_ERROR: too many SQL variables` on the admin content list when a collection has ~100+ items. SEO and byline batch-hydration queries now chunk their `IN` clauses to stay under D1's 100-bound-parameter statement limit. diff --git a/packages/core/src/database/repositories/byline.ts b/packages/core/src/database/repositories/byline.ts index b78f54235..29ee2fe76 100644 --- a/packages/core/src/database/repositories/byline.ts +++ b/packages/core/src/database/repositories/byline.ts @@ -57,6 +57,33 @@ function rowToByline(row: BylineRow): BylineSummary { export class BylineRepository { constructor(private db: Kysely) {} + /** Single batched SELECT for getContentBylinesMany. Caller must keep + * `contentIds` under D1's 100-parameter statement limit. */ + private async fetchBylineCreditsChunk(collectionSlug: string, contentIds: string[]) { + return this.db + .selectFrom("_emdash_content_bylines as cb") + .innerJoin("_emdash_bylines as b", "b.id", "cb.byline_id") + .select([ + "cb.content_id as content_id", + "cb.sort_order as sort_order", + "cb.role_label as role_label", + "b.id as id", + "b.slug as slug", + "b.display_name as display_name", + "b.bio as bio", + "b.avatar_media_id as avatar_media_id", + "b.website_url as website_url", + "b.user_id as user_id", + "b.is_guest as is_guest", + "b.created_at as created_at", + "b.updated_at as updated_at", + ]) + .where("cb.collection_slug", "=", collectionSlug) + .where("cb.content_id", "in", contentIds) + .orderBy("cb.sort_order", "asc") + .execute(); + } + async findById(id: string): Promise { const row = await this.db .selectFrom("_emdash_bylines") @@ -249,8 +276,11 @@ export class BylineRepository { } /** - * Batch-fetch byline credits for multiple content items in a single query. + * Batch-fetch byline credits for multiple content items. * Returns a Map keyed by contentId. + * + * Chunked to stay under D1's 100-parameter statement limit (one slot is + * consumed by `collectionSlug`, leaving room for 99 ids; chunk 90 for headroom). */ async getContentBylinesMany( collectionSlug: string, @@ -259,41 +289,22 @@ export class BylineRepository { const result = new Map(); if (contentIds.length === 0) return result; - const rows = await this.db - .selectFrom("_emdash_content_bylines as cb") - .innerJoin("_emdash_bylines as b", "b.id", "cb.byline_id") - .select([ - "cb.content_id as content_id", - "cb.sort_order as sort_order", - "cb.role_label as role_label", - "b.id as id", - "b.slug as slug", - "b.display_name as display_name", - "b.bio as bio", - "b.avatar_media_id as avatar_media_id", - "b.website_url as website_url", - "b.user_id as user_id", - "b.is_guest as is_guest", - "b.created_at as created_at", - "b.updated_at as updated_at", - ]) - .where("cb.collection_slug", "=", collectionSlug) - .where("cb.content_id", "in", contentIds) - .orderBy("cb.sort_order", "asc") - .execute(); - - for (const row of rows) { - const contentId = row.content_id; - const credit: ContentBylineCredit = { - byline: rowToByline(row), - sortOrder: row.sort_order, - roleLabel: row.role_label, - }; - const existing = result.get(contentId); - if (existing) { - existing.push(credit); - } else { - result.set(contentId, [credit]); + const CHUNK_SIZE = 90; + for (let i = 0; i < contentIds.length; i += CHUNK_SIZE) { + const chunk = contentIds.slice(i, i + CHUNK_SIZE); + const rows = await this.fetchBylineCreditsChunk(collectionSlug, chunk); + for (const row of rows) { + const credit: ContentBylineCredit = { + byline: rowToByline(row), + sortOrder: row.sort_order, + roleLabel: row.role_label, + }; + const existing = result.get(row.content_id); + if (existing) { + existing.push(credit); + } else { + result.set(row.content_id, [credit]); + } } } @@ -308,15 +319,19 @@ export class BylineRepository { const result = new Map(); if (userIds.length === 0) return result; - const rows = await this.db - .selectFrom("_emdash_bylines") - .selectAll() - .where("user_id", "in", userIds) - .execute(); - - for (const row of rows) { - if (row.user_id) { - result.set(row.user_id, rowToByline(row)); + // Chunked to stay under D1's 100-parameter statement limit. + const CHUNK_SIZE = 90; + for (let i = 0; i < userIds.length; i += CHUNK_SIZE) { + const chunk = userIds.slice(i, i + CHUNK_SIZE); + const rows = await this.db + .selectFrom("_emdash_bylines") + .selectAll() + .where("user_id", "in", chunk) + .execute(); + for (const row of rows) { + if (row.user_id) { + result.set(row.user_id, rowToByline(row)); + } } } return result; diff --git a/packages/core/src/database/repositories/seo.ts b/packages/core/src/database/repositories/seo.ts index 1af31b4f8..3874d4348 100644 --- a/packages/core/src/database/repositories/seo.ts +++ b/packages/core/src/database/repositories/seo.ts @@ -36,6 +36,17 @@ function hasAnyField(input: ContentSeoInput): boolean { export class SeoRepository { constructor(private db: Kysely) {} + /** Single batched SELECT for an IN-clause chunk. Caller is responsible for + * keeping the chunk under D1's 100-parameter statement limit. */ + private async fetchSeoChunk(collection: string, contentIds: string[]) { + return this.db + .selectFrom("_emdash_seo") + .selectAll() + .where("collection", "=", collection) + .where("content_id", "in", contentIds) + .execute(); + } + /** * Get SEO data for a content item. Returns null defaults if no row exists. */ @@ -61,36 +72,36 @@ export class SeoRepository { } /** - * Get SEO data for multiple content items in a single query. + * Get SEO data for multiple content items. * Returns a Map keyed by content_id. Items without SEO rows get defaults. + * + * The IN clause is chunked because D1 (and SQLite by default) caps a single + * statement at 100 bound parameters. With one slot consumed by `collection`, + * we can fit at most 99 ids per batch — chunk size 90 leaves headroom. */ async getMany(collection: string, contentIds: string[]): Promise> { const result = new Map(); if (contentIds.length === 0) return result; - // Batch query — single SELECT with IN clause - const rows = await this.db - .selectFrom("_emdash_seo") - .selectAll() - .where("collection", "=", collection) - .where("content_id", "in", contentIds) - .execute(); - - // Index fetched rows by content_id - const rowMap = new Map(rows.map((r) => [r.content_id, r])); - - for (const id of contentIds) { - const row = rowMap.get(id); - if (row) { - result.set(id, { + const CHUNK_SIZE = 90; + for (let i = 0; i < contentIds.length; i += CHUNK_SIZE) { + const chunk = contentIds.slice(i, i + CHUNK_SIZE); + const rows = await this.fetchSeoChunk(collection, chunk); + for (const row of rows) { + result.set(row.content_id, { title: row.seo_title ?? null, description: row.seo_description ?? null, image: row.seo_image ?? null, canonical: row.seo_canonical ?? null, noIndex: row.seo_no_index === 1, }); - } else { + } + } + + // Fill in defaults for ids that had no SEO row. + for (const id of contentIds) { + if (!result.has(id)) { result.set(id, { ...SEO_DEFAULTS }); } }