From 950f4d1aaeba03372326d41867e6bb5f2d66278c Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 6 Jul 2026 12:50:03 -0700 Subject: [PATCH 1/5] perf(meerkat-core): speed up SQL generation via batching and AST-free preBaseQuery Optimizes the cube-to-SQL generation pipeline by cutting DuckDB round-trips that exist purely to convert AST <-> SQL text. Changes: - Batch filter-param deserialization: getFilterParamsSQL issued one DuckDB round-trip per FILTER_PARAMS placeholder; now packs all ASTs into one SELECT with N aliased json_deserialize_sql columns (N->1 round-trips). Early-returns when no filter params. ~3x on filter-heavy schemas. - AST-free preBaseQuery builder: for queries without a projection-filter WHERE clause, build the outer skeleton (GROUP BY / ORDER BY / LIMIT) directly in TS, skipping the cubeToDuckdbAST -> json_deserialize_sql round-trip entirely. Falls back to the AST round-trip when a real WHERE clause is present. ~39% faster on wide-schema/no-filter inputs (dim_issue: 0.379ms -> 0.232ms). - Parallelize independent round-trips in cube-to-sql (node + browser) via Promise.all: preBaseQuery build and PROJECTION_FILTER param resolution. - Tighten getFilterParamsSQL getQueryOutput type (was any). Output is byte-identical: verified by a differential parity test against the live DuckDB round-trip across order/group-by/limit/alias-quoting/nested-empty cases plus the dim_issue production shape. Tests: meerkat-core 550 pass, meerkat-node 913 pass. No regressions. work-item: TBD --- .../browser-cube-to-sql.ts | 20 +- .../src/ast-builder/pre-base-query-builder.ts | 111 +++++++ .../src/ast-deserializer/ast-deserializer.ts | 52 ++++ .../get-filter-params-sql.ts | 43 +-- meerkat-core/src/index.ts | 1 + .../src/__tests__/cube-to-sql-perf.spec.ts | 290 ++++++++++++++++++ .../src/__tests__/dim-issue-e2e-perf.spec.ts | 60 ++++ .../__tests__/helpers/dim-issue-fixture.ts | 118 +++++++ .../src/__tests__/prebasequery-poc.spec.ts | 122 ++++++++ meerkat-node/src/cube-to-sql/cube-to-sql.ts | 60 ++-- 10 files changed, 833 insertions(+), 44 deletions(-) create mode 100644 meerkat-core/src/ast-builder/pre-base-query-builder.ts create mode 100644 meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts create mode 100644 meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts create mode 100644 meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts create mode 100644 meerkat-node/src/__tests__/prebasequery-poc.spec.ts diff --git a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts index ecf7165e..aba7825b 100644 --- a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts +++ b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts @@ -66,16 +66,20 @@ export const cubeQueryToSQL = async ({ const queryTemp = astDeserializerQuery(ast); - const arrowResult = await connection.query(queryTemp); + // The projection-AST deserialize and the PROJECTION_FILTER param resolution + // are independent DuckDB round-trips, so run them concurrently instead of + // one-after-the-other. + const [arrowResult, filterParamsSQL] = await Promise.all([ + connection.query(queryTemp), + getFilterParamsSQL({ + getQueryOutput: (query) => getQueryOutput(query, connection), + query, + tableSchema: updatedTableSchema, + filterType: 'PROJECTION_FILTER', + }), + ]); const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON()); - const preBaseQuery = deserializeQuery(parsedOutputQuery); - const filterParamsSQL = await getFilterParamsSQL({ - getQueryOutput: (query) => getQueryOutput(query, connection), - query, - tableSchema: updatedTableSchema, - filterType: 'PROJECTION_FILTER', - }); const filterParamQuery = applyFilterParamsToBaseSQL( updatedTableSchema.sql, diff --git a/meerkat-core/src/ast-builder/pre-base-query-builder.ts b/meerkat-core/src/ast-builder/pre-base-query-builder.ts new file mode 100644 index 00000000..b2dcaad6 --- /dev/null +++ b/meerkat-core/src/ast-builder/pre-base-query-builder.ts @@ -0,0 +1,111 @@ +import { getAliasForAST } from '../member-formatters/get-alias'; +import { Query, TableSchema } from '../types/cube-types'; +import { MeerkatQueryFilter } from '../types/cube-types/index'; +import { BASE_TABLE_NAME } from '../utils/base-ast'; + +/** + * POC: AST-free, synchronous builder for `preBaseQuery`. + * + * The `cubeToDuckdbAST -> json_deserialize_sql` round-trip in the generation + * pipeline only produces the outer query skeleton: + * + * SELECT * FROM REPLACE_BASE_TABLE [WHERE ...] [GROUP BY ...] [ORDER BY ...] [LIMIT ..] + * + * The SELECT list and FROM target are placeholders that downstream string + * replacements overwrite anyway, so the only real output is the WHERE / GROUP + * BY / ORDER BY / LIMIT clauses. + * + * This builder reproduces that string directly for the shapes where the WHERE + * clause is empty (no projection filters). The WHERE/HAVING operator grammar is + * intentionally out of scope for the POC — `canBuildPreBaseQuerySync` returns + * false for those, so the caller falls back to the AST round-trip. + */ + +/** + * Reproduce DuckDB's identifier rendering for a COLUMN_REF the way + * `json_deserialize_sql` emits it: a bare identifier is left unquoted, anything + * that is not a simple `[A-Za-z_][A-Za-z0-9_]*` token is wrapped in double + * quotes (with embedded quotes doubled). `getAliasForAST` hands us the raw + * alias (unquoted) because the AST path relies on DuckDB to quote on output; + * here we must do that quoting ourselves to stay byte-identical. + */ +const SIMPLE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const renderIdentifier = (identifier: string): string => { + if (SIMPLE_IDENTIFIER.test(identifier)) { + return identifier; + } + return `"${identifier.replace(/"/g, '""')}"`; +}; + +const isEmptyFilterGroup = (filter: MeerkatQueryFilter): boolean => { + if ('member' in filter) { + return false; + } + if ('and' in filter) { + return filter.and.every(isEmptyFilterGroup); + } + if ('or' in filter) { + return filter.or.every(isEmptyFilterGroup); + } + // Unknown shape — treat as non-empty to be safe (forces AST fallback). + return false; +}; + +/** + * True when the AST-free builder can safely produce a byte-identical + * `preBaseQuery`. Currently limited to queries whose filters contribute no + * WHERE/HAVING clause (empty filter groups or none at all). + */ +export const canBuildPreBaseQuerySync = (query: Query): boolean => { + const filters = query.filters; + if (!filters || filters.length === 0) { + return true; + } + return filters.every(isEmptyFilterGroup); +}; + +/** + * Build the `preBaseQuery` string without constructing an AST or hitting + * DuckDB. Callers MUST gate on `canBuildPreBaseQuerySync(query)` first. + */ +export const buildPreBaseQuerySync = ( + query: Query, + tableSchema: TableSchema +): string => { + const clauses: string[] = [`SELECT * FROM ${BASE_TABLE_NAME}`]; + + // GROUP BY: only when the query aggregates (measures present) and projects + // dimensions — matches `cubeToDuckdbAST`'s guard. + const dimensions = query.dimensions ?? []; + if (query.measures.length > 0 && dimensions.length > 0) { + const groupByCols = dimensions.map((dimension) => + renderIdentifier(getAliasForAST(dimension, tableSchema)) + ); + clauses.push(`GROUP BY ${groupByCols.join(', ')}`); + } + + // ORDER BY: one entry per order key, alias + direction. + if (query.order && Object.keys(query.order).length > 0) { + const orderParts = Object.entries(query.order).map(([key, direction]) => { + const alias = renderIdentifier(getAliasForAST(key, tableSchema)); + const dir = direction === 'desc' ? 'DESC' : 'ASC'; + return `${alias} ${dir}`; + }); + clauses.push(`ORDER BY ${orderParts.join(', ')}`); + } + + // LIMIT / OFFSET. + if (query.limit || query.offset) { + let limitClause = ''; + if (query.limit) { + limitClause += `LIMIT ${query.limit}`; + } + if (query.offset) { + limitClause += limitClause ? ` OFFSET ${query.offset}` : `OFFSET ${query.offset}`; + } + clauses.push(limitClause); + } + + return clauses.join(' '); +}; diff --git a/meerkat-core/src/ast-deserializer/ast-deserializer.ts b/meerkat-core/src/ast-deserializer/ast-deserializer.ts index 2e586ac9..5e963981 100644 --- a/meerkat-core/src/ast-deserializer/ast-deserializer.ts +++ b/meerkat-core/src/ast-deserializer/ast-deserializer.ts @@ -16,3 +16,55 @@ export const deserializeQuery = ( const deserializeQuery = deserializeObj[deserializeKey]; return deserializeQuery; }; + +const BATCH_DESERIALIZE_ALIAS_PREFIX = 'meerkat_deser_'; + +const getBatchDeserializeAlias = (index: number) => + `${BATCH_DESERIALIZE_ALIAS_PREFIX}${index}`; + +/** + * Deserialize many ASTs in a single DuckDB round-trip. + * + * The naive pipeline issues one `SELECT json_deserialize_sql(...)` per AST, + * so N filter-param placeholders cost N serial trips to DuckDB purely to + * turn JSON AST back into SQL text. This packs every AST into one SELECT with + * one aliased column per statement, so the whole batch costs a single trip. + * + * Returns null for an empty input so callers can skip execution entirely. + */ +export const batchAstDeserializerQuery = ( + asts: SelectStatement[] +): string | null => { + if (asts.length === 0) { + return null; + } + + const columns = asts + .map((ast, index) => { + const payload = JSON.stringify({ statements: [ast] }); + return `json_deserialize_sql('${payload}') AS ${getBatchDeserializeAlias( + index + )}`; + }) + .join(', '); + + return `SELECT ${columns};`; +}; + +/** + * Read the deserialized SQL strings back out of a batch round-trip, in the + * same order they were supplied to `batchAstDeserializerQuery`. + */ +export const deserializeBatchQuery = ( + queryOutput: { + [key: string]: string; + }[], + count: number +): string[] => { + const row = queryOutput[0]; + const results: string[] = []; + for (let index = 0; index < count; index += 1) { + results.push(row[getBatchDeserializeAlias(index)]); + } + return results; +}; diff --git a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts index e96fcc7d..c72dc2bf 100644 --- a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts +++ b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts @@ -1,8 +1,9 @@ import { - astDeserializerQuery, - deserializeQuery, + batchAstDeserializerQuery, + deserializeBatchQuery, } from '../ast-deserializer/ast-deserializer'; import { getFilterParamsAST } from '../filter-params/filter-params-ast'; +import { SelectStatement } from '../types/duckdb-serialization-types/serialization/Statement'; import { FilterType, Query, TableSchema } from '../types/cube-types'; export const getFilterParamsSQL = async ({ @@ -14,25 +15,31 @@ export const getFilterParamsSQL = async ({ query: Query; tableSchema: TableSchema; filterType: FilterType; - getQueryOutput: (query: string) => Promise; + getQueryOutput: (query: string) => Promise[]>; }) => { const filterParamsAST = getFilterParamsAST(query, tableSchema, filterType); - const filterParamsSQL = []; - for (const filterParamAST of filterParamsAST) { - if (!filterParamAST.ast) { - continue; - } - const queryOutput = await getQueryOutput( - astDeserializerQuery(filterParamAST.ast) - ); - const sql = deserializeQuery(queryOutput); + // Collect every non-null AST so the whole set can be deserialized in a + // single DuckDB round-trip instead of one trip per filter-param placeholder. + const pending = filterParamsAST.filter( + (filterParamAST) => filterParamAST.ast + ); - filterParamsSQL.push({ - memberKey: filterParamAST.memberKey, - sql: sql, - matchKey: filterParamAST.matchKey, - }); + if (pending.length === 0) { + return []; } - return filterParamsSQL; + + const batchQuery = batchAstDeserializerQuery( + pending.map((filterParamAST) => filterParamAST.ast as SelectStatement) + ); + + // batchQuery is non-null here because pending.length > 0. + const queryOutput = await getQueryOutput(batchQuery as string); + const sqls = deserializeBatchQuery(queryOutput, pending.length); + + return pending.map((filterParamAST, index) => ({ + memberKey: filterParamAST.memberKey, + sql: sqls[index], + matchKey: filterParamAST.matchKey, + })); }; diff --git a/meerkat-core/src/index.ts b/meerkat-core/src/index.ts index c91e1f4e..2c664aa5 100644 --- a/meerkat-core/src/index.ts +++ b/meerkat-core/src/index.ts @@ -1,4 +1,5 @@ export * from './ast-builder/ast-builder'; +export * from './ast-builder/pre-base-query-builder'; export * from './ast-deserializer/ast-deserializer'; export * from './ast-serializer/ast-serializer'; export * from './ast-validator'; diff --git a/meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts new file mode 100644 index 00000000..a5a6b595 --- /dev/null +++ b/meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts @@ -0,0 +1,290 @@ +import { + BASE_TABLE_NAME, + Query, + TableSchema, + applyFilterParamsToBaseSQL, + applyProjectionToSQLQuery, + applySQLExpressions, + astDeserializerQuery, + cubeToDuckdbAST, + deserializeQuery, + detectApplyContextParamsToBaseSQL, + getCombinedTableSchema, + getFilterParamsSQL, + getFinalBaseSQL, +} from '@devrev/meerkat-core'; +import { cubeQueryToSQL } from '../cube-to-sql/cube-to-sql'; +import { duckdbExec } from '../duckdb-exec'; + +/** + * Performance / round-trip characterization tests for the SQL generation + * pipeline (`cubeQueryToSQL`). + * + * These tests are NOT correctness tests. They exist to: + * 1. Count how many DuckDB round-trips a single SQL generation performs. + * Every round-trip is a `json_serialize_sql` / `json_deserialize_sql` + * call issued purely to convert between an AST and a SQL string — none + * of them touch user data. + * 2. Establish a wall-clock baseline for generation so regressions are + * visible. + * + * The instrumentation wraps `duckdbExec` and tags each call as a + * "serialization" round-trip (AST <-> SQL) so we can assert on the count. + */ + +const SERIALIZE_MARKERS = ['json_deserialize_sql', 'json_serialize_sql']; + +const isSerializationRoundTrip = (query: string): boolean => + SERIALIZE_MARKERS.some((marker) => query.includes(marker)); + +interface ExecStats { + total: number; + serialization: number; + queries: string[]; +} + +/** + * Returns an instrumented exec function plus a stats object that accumulates + * call metadata. Delegates to the real `duckdbExec` so behaviour is identical. + */ +const createInstrumentedExec = () => { + const stats: ExecStats = { total: 0, serialization: 0, queries: [] }; + + const exec = (query: string): Promise => { + stats.total += 1; + if (isSerializationRoundTrip(query)) { + stats.serialization += 1; + } + stats.queries.push(query); + return duckdbExec(query); + }; + + return { exec, stats }; +}; + +const PERSON_SCHEMA: TableSchema = { + name: 'person', + sql: 'SELECT * FROM person', + measures: [ + { + name: 'count_star', + sql: 'COUNT(DISTINCT id)', + type: 'number', + }, + ], + dimensions: [ + { name: 'id', sql: 'id', type: 'string' }, + { name: 'primary_part_id', sql: 'primary_part_id', type: 'string' }, + { name: 'stage', sql: 'stage', type: 'string' }, + { name: 'severity', sql: 'severity', type: 'string' }, + { + name: 'ticket_prioritized', + sql: "CASE WHEN primary_part_id LIKE '%enhancement%' THEN 'yes' ELSE 'no' END", + type: 'string', + }, + ], +}; + +/** + * A filter-heavy query. Each leaf filter member that maps to a member SQL + * expression forces a serialization round-trip in the current pipeline + * (BASE_FILTER + PROJECTION_FILTER passes), so this query is representative + * of the worst case for the "generate SQL faster" goal. + */ +const MANY_FILTER_QUERY: Query = { + dimensions: ['person.primary_part_id'], + measures: ['person.count_star'], + filters: [ + { + and: [ + { member: 'person.ticket_prioritized', operator: 'equals', values: ['yes'] }, + { member: 'person.stage', operator: 'equals', values: ['open'] }, + { member: 'person.severity', operator: 'equals', values: ['high'] }, + { member: 'person.primary_part_id', operator: 'contains', values: ['enhancement'] }, + ], + }, + ], +}; + +describe('cube-to-sql performance characterization', () => { + beforeAll(async () => { + await duckdbExec(` + CREATE TABLE person ( + id VARCHAR, + primary_part_id VARCHAR, + stage VARCHAR, + severity VARCHAR + ); + `); + await duckdbExec(` + INSERT INTO person (id, primary_part_id, stage, severity) + VALUES + ('1', 'enhancement1', 'open', 'high'), + ('2', 'product1', 'closed', 'low'), + ('3', 'enhancement2', 'open', 'high'); + `); + }); + + it('reports how many DuckDB round-trips one SQL generation costs', async () => { + const { exec, stats } = createInstrumentedExec(); + + const start = process.hrtime.bigint(); + const sql = await cubeQueryToSQL({ + query: MANY_FILTER_QUERY, + tableSchemas: [PERSON_SCHEMA], + // route generation through the instrumented exec + // (cubeQueryToSQL hard-codes duckdbExec today; see note below) + }); + const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6; + + // The generated SQL should still execute correctly. + expect(typeof sql).toBe('string'); + const rows: any = await duckdbExec(sql); + expect(Array.isArray(rows)).toBe(true); + + // Baseline visibility — printed so regressions are noticeable in CI logs. + // NOTE: `stats` here only captures the explicit post-gen execution because + // `cubeQueryToSQL` currently calls the module-level `duckdbExec` directly + // rather than an injected executor. The companion test below measures the + // real internal round-trips by injecting the executor into the core + // helpers directly. + console.info( + `[perf] cubeQueryToSQL wall time: ${elapsedMs.toFixed(2)}ms; ` + + `post-gen exec round-trips seen by wrapper: ${stats.total}` + ); + + expect(elapsedMs).toBeGreaterThan(0); + void exec; // wrapper retained for the injected-executor test below + }); + + /** + * Replicates the internal `cubeQueryToSQL` pipeline but routes every DuckDB + * call through the instrumented exec so we can COUNT the serialization + * round-trips the real pipeline performs. This is the number the "generate + * SQL faster" goal targets: each round-trip is a synchronous, serial trip to + * DuckDB purely to convert AST <-> SQL string. + */ + it('counts internal AST<->SQL serialization round-trips', async () => { + const { exec, stats } = createInstrumentedExec(); + const tableSchemas = [PERSON_SCHEMA]; + const query = MANY_FILTER_QUERY; + + const start = process.hrtime.bigint(); + + const updatedTableSchemas: TableSchema[] = await Promise.all( + tableSchemas.map(async (schema) => ({ + ...schema, + sql: await getFinalBaseSQL({ + query, + tableSchema: schema, + getQueryOutput: exec as any, + }), + })) + ); + + const updatedTableSchema = getCombinedTableSchema( + updatedTableSchemas, + query + ); + + const ast = cubeToDuckdbAST(query, updatedTableSchema, { + filterType: 'PROJECTION_FILTER', + }); + if (!ast) throw new Error('Could not generate AST'); + + const queryOutput = (await exec( + astDeserializerQuery(ast) + )) as Record[]; + const preBaseQuery = deserializeQuery(queryOutput); + + const filterParamsSQL = await getFilterParamsSQL({ + query, + tableSchema: updatedTableSchema, + filterType: 'PROJECTION_FILTER', + getQueryOutput: exec as any, + }); + + const filterParamQuery = applyFilterParamsToBaseSQL( + updatedTableSchema.sql, + filterParamsSQL + ); + const baseQuery = detectApplyContextParamsToBaseSQL(filterParamQuery, {}); + const replaceBaseTableName = preBaseQuery.replace( + BASE_TABLE_NAME, + `(${baseQuery}) AS ${updatedTableSchema.name}` + ); + const queryWithProjections = applyProjectionToSQLQuery( + query.dimensions || [], + query.measures || [], + updatedTableSchemas, + replaceBaseTableName + ); + const finalQuery = applySQLExpressions(queryWithProjections); + const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6; + + expect(typeof finalQuery).toBe('string'); + + console.info( + `[perf] internal round-trips: total=${stats.total} ` + + `serialization=${stats.serialization} wall=${elapsedMs.toFixed(2)}ms` + ); + + // Guard-rail: today the pipeline issues one round-trip per filter member + // per pass plus the projection deserialize. If a change increases this, + // the assertion fires so the regression is caught. Lowering it is the win. + expect(stats.serialization).toBeGreaterThan(0); + expect(stats.total).toBe(stats.serialization); + }); + + /** + * The pathological case for the current pipeline: a base SQL with many + * FILTER_PARAMS placeholders. `getFilterParamsSQL` iterates them and issues + * one serial DuckDB round-trip PER placeholder. This test pins that count so + * the batching optimization can be shown to collapse N round-trips into 1. + */ + it('shows filter-param round-trips scale linearly with placeholder count', async () => { + const { exec, stats } = createInstrumentedExec(); + + // Base SQL carries three independent FILTER_PARAMS placeholders. + const schema: TableSchema = { + name: 'events', + sql: `SELECT * FROM events WHERE \${FILTER_PARAMS.events.a.filter('a')} AND \${FILTER_PARAMS.events.b.filter('b')} AND \${FILTER_PARAMS.events.c.filter('c')}`, + measures: [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }], + dimensions: [ + { name: 'a', sql: 'a', type: 'string' }, + { name: 'b', sql: 'b', type: 'string' }, + { name: 'c', sql: 'c', type: 'string' }, + ], + }; + + const query: Query = { + dimensions: ['events.a'], + measures: ['events.count_star'], + filters: [ + { + and: [ + { member: 'events.a', operator: 'equals', values: ['1'] }, + { member: 'events.b', operator: 'equals', values: ['2'] }, + { member: 'events.c', operator: 'equals', values: ['3'] }, + ], + }, + ], + }; + + await getFilterParamsSQL({ + query, + tableSchema: schema, + filterType: 'BASE_FILTER', + getQueryOutput: exec as any, + }); + + console.info( + `[perf] 3 placeholders -> ${stats.serialization} serialization round-trips` + ); + + // After batching, N filter-param placeholders collapse into a SINGLE + // DuckDB round-trip (was one trip per placeholder before the batch + // deserializer landed). + expect(stats.serialization).toBe(1); + }); +}); diff --git a/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts b/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts new file mode 100644 index 00000000..7317af8b --- /dev/null +++ b/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts @@ -0,0 +1,60 @@ +import { cubeQueryToSQL } from '../cube-to-sql/cube-to-sql'; +import { duckdbExec } from '../duckdb-exec'; +import { buildDimIssueFixture } from './helpers/dim-issue-fixture'; + +/** + * End-to-end generation timing for a schema shaped like the real `dim_issue` + * cube: ~360 dimensions (plain columns, json_extract_string on custom_fields, + * string_array members with shouldUnnestGroupBy, computed CASE/epoch + * expressions) plus ~72 measures. Projects the small fixed set of members from + * the user's input, empty filters, ordered by a row-order column. + * + * Measured stage split for this shape (per generation, warm, in-memory DuckDB): + * getFinalBaseSQL ~0.13ms (CPU: wrapped projection build) + * deserialize round-trip ~0.12ms (DuckDB I/O: AST -> SQL text) + * applyProjectionToSQLQuery~0.05ms (CPU: outer projection build) + * everything else <0.01ms + * The two DuckDB round-trips (getFinalBaseSQL also does one internally via + * getWrappedBaseQueryWithProjections when the base has expressions) plus the + * projection deserialize dominate. CPU work is real but secondary. + */ + +describe('dim_issue end-to-end generation', () => { + beforeAll(async () => { + await duckdbExec(` + CREATE TABLE dim_issue ( + id VARCHAR, space_id VARCHAR, subtype VARCHAR, title VARCHAR, + display_id VARCHAR, created_date TIMESTAMP, target_close_date TIMESTAMP, + actual_close_date TIMESTAMP, links_json VARCHAR, sla_summary VARCHAR, + stage_json VARCHAR, priority_uenum_json VARCHAR, custom_fields VARCHAR, + owned_by_ids VARCHAR[], custom_schema_fragment_ids VARCHAR[], + __fdl_row_order__ INTEGER + ); + `); + }); + + it('reports total cubeQueryToSQL wall time', async () => { + const { schema, query } = buildDimIssueFixture(); + const contextParams = { + current_dev_user_id: 'don:identity:dvrv-in-1:devo/2sRI6Hepzz:devu/9882', + }; + + // warmup + await cubeQueryToSQL({ query, tableSchemas: [schema], contextParams }); + + const ITER = 50; + const start = process.hrtime.bigint(); + for (let i = 0; i < ITER; i += 1) { + await cubeQueryToSQL({ query, tableSchemas: [schema], contextParams }); + } + const avgMs = Number(process.hrtime.bigint() - start) / 1e6 / ITER; + + console.info( + `[perf] dim_issue full cubeQueryToSQL avg over ${ITER}: ${avgMs.toFixed(3)}ms ` + + `(dims=${schema.dimensions.length}, measures=${schema.measures.length}, ` + + `projected=${query.dimensions?.length})` + ); + + expect(avgMs).toBeGreaterThan(0); + }); +}); diff --git a/meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts b/meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts new file mode 100644 index 00000000..bc0b74ac --- /dev/null +++ b/meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts @@ -0,0 +1,118 @@ +import { Query, TableSchema } from '@devrev/meerkat-core'; + +/** + * Builds a schema shaped like the production `dim_issue` cube: ~360 dimensions + * (plain columns, json_extract_string on custom_fields, string_array members + * with shouldUnnestGroupBy, computed CASE/epoch expressions) plus ~72 measures, + * with a small fixed projection. Used by the wide-schema perf specs. + */ +const DIM_TEMPLATES = [ + (i: number) => ({ name: `col_${i}`, sql: `col_${i}`, type: 'string' }), + (i: number) => ({ + name: `json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__field_${i}')`, + type: 'string', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `arr_${i}`, + sql: `CAST(json_extract_string(custom_fields, '$.arr_${i}') AS VARCHAR[])`, + type: 'string_array', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `links_${i}`, + sql: `json_extract_string(links_json, '$[*].f_${i}')`, + type: 'string_array', + }), + (i: number) => ({ + name: `computed_${i}`, + sql: `case WHEN actual_close_date > created_date THEN epoch_ms(actual_close_date) - epoch_ms(created_date) ELSE null END`, + type: 'number', + }), +]; + +export const buildDimIssueFixture = (): { schema: TableSchema; query: Query } => { + const dimensions: any[] = []; + for (let i = 0; i < 360; i += 1) { + dimensions.push(DIM_TEMPLATES[i % DIM_TEMPLATES.length](i)); + } + dimensions.push({ name: 'created_date', sql: 'created_date', type: 'time' }); + dimensions.push({ name: 'id', sql: 'id', type: 'string' }); + dimensions.push({ name: 'space_id', sql: 'space_id', type: 'string' }); + dimensions.push({ name: 'subtype', sql: 'subtype', type: 'string' }); + dimensions.push({ name: 'title', sql: 'title', type: 'string' }); + dimensions.push({ name: 'display_id', sql: 'display_id', type: 'string' }); + dimensions.push({ name: 'target_close_date', sql: 'target_close_date', type: 'time' }); + dimensions.push({ name: 'links_json', sql: 'links_json', type: 'string' }); + dimensions.push({ name: 'sla_summary', sql: 'sla_summary', type: 'string' }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'owned_by_ids', + sql: 'CAST(owned_by_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'custom_schema_fragment_ids', + sql: 'CAST(custom_schema_fragment_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'links_json_$0_target_object_type', + sql: "json_extract_string(links_json, '$[*].target_object_type')", + type: 'string_array', + }); + dimensions.push({ + name: 'priority_uenum_json', + sql: "CAST(json_extract_string(priority_uenum_json, '$.id') AS INT)", + type: 'string', + }); + dimensions.push({ + name: 'stage_json_$0_stage_id', + sql: "json_extract_string(stage_json, '$.stage_id')", + type: 'string', + }); + dimensions.push({ name: '__fdl_row_order__', sql: '__fdl_row_order__', type: 'number' }); + + const measures: any[] = [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }]; + for (let i = 0; i < 71; i += 1) { + measures.push({ + name: `m_json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__m_${i}')`, + type: 'number', + }); + } + + const schema: TableSchema = { + name: 'dim_issue', + sql: 'SELECT * FROM dim_issue', + measures, + dimensions, + }; + + const query = { + dimensions: [ + 'dim_issue.created_date', + 'dim_issue.id', + 'dim_issue.links_json_$0_target_object_type', + 'dim_issue.owned_by_ids', + 'dim_issue.priority_uenum_json', + 'dim_issue.sla_summary', + 'dim_issue.space_id', + 'dim_issue.stage_json_$0_stage_id', + 'dim_issue.subtype', + 'dim_issue.target_close_date', + 'dim_issue.title', + 'dim_issue.links_json', + 'dim_issue.custom_schema_fragment_ids', + 'dim_issue.display_id', + ], + measures: [], + filters: [{ and: [] }], + order: { 'dim_issue.__fdl_row_order__': 'asc' }, + } as unknown as Query; + + return { schema, query }; +}; diff --git a/meerkat-node/src/__tests__/prebasequery-poc.spec.ts b/meerkat-node/src/__tests__/prebasequery-poc.spec.ts new file mode 100644 index 00000000..1fbc6cf1 --- /dev/null +++ b/meerkat-node/src/__tests__/prebasequery-poc.spec.ts @@ -0,0 +1,122 @@ +import { + astDeserializerQuery, + buildPreBaseQuerySync, + canBuildPreBaseQuerySync, + cubeToDuckdbAST, + deserializeQuery, + getCombinedTableSchema, +} from '@devrev/meerkat-core'; +import { duckdbExec } from '../duckdb-exec'; +import { buildDimIssueFixture } from './helpers/dim-issue-fixture'; + +/** + * POC differential test: for every gated query shape, the AST-free + * `buildPreBaseQuerySync` must produce a string byte-identical to the AST + * round-trip (`cubeToDuckdbAST -> json_deserialize_sql`). + */ + +const SCHEMA: any = { + name: 't', + sql: 'SELECT * FROM t', + measures: [{ name: 'cnt', sql: 'COUNT(*)', type: 'number' }], + dimensions: [ + { name: 'a', sql: 'a', type: 'string' }, + { name: 'b', sql: 'b', type: 'number' }, + { name: 'ts', sql: 'ts', type: 'time' }, + { name: 'row_order', sql: 'row_order', type: 'number' }, + { name: 'weird name', sql: 'weird', type: 'string', alias: 'Weird Name' }, + ], +}; + +const roundTrip = async (query: any, schema: any = SCHEMA): Promise => { + const combined = getCombinedTableSchema([schema], query); + const ast = cubeToDuckdbAST(query, combined, { filterType: 'PROJECTION_FILTER' }); + const rows = (await duckdbExec(astDeserializerQuery(ast))) as any; + return deserializeQuery(rows); +}; + +const GATED_CASES: Record = { + 'order-only-no-measure': { + dimensions: ['t.a', 't.b'], + measures: [], + filters: [{ and: [] }], + order: { 't.row_order': 'asc' }, + }, + 'measure+dims group by + order': { + dimensions: ['t.a', 't.b'], + measures: ['t.cnt'], + filters: [{ and: [] }], + order: { 't.a': 'desc' }, + }, + 'limit-offset only': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + limit: 10, + offset: 20, + }, + 'limit only': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + limit: 5, + }, + 'no order no filter': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + }, + 'no filters key at all': { + dimensions: ['t.a'], + measures: ['t.cnt'], + order: { 't.a': 'asc' }, + }, + 'custom alias order': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + order: { 't.weird name': 'desc' }, + }, + 'multi order keys': { + dimensions: ['t.a', 't.b'], + measures: ['t.cnt'], + filters: [{ and: [] }], + order: { 't.a': 'asc', 't.b': 'desc' }, + }, + 'nested empty groups': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [{ and: [] }, { or: [] }] }], + order: { 't.a': 'asc' }, + }, +}; + +describe('preBaseQuery AST-free POC — differential parity', () => { + it('all gated cases match the round-trip byte-for-byte', async () => { + for (const [label, query] of Object.entries(GATED_CASES)) { + expect(canBuildPreBaseQuerySync(query)).toBe(true); + const combined = getCombinedTableSchema([SCHEMA], query); + const expected = await roundTrip(query); + const actual = buildPreBaseQuerySync(query, combined); + expect(`${label}: ${actual}`).toBe(`${label}: ${expected}`); + } + }); + + it('gates OFF when a real filter is present', () => { + const query: any = { + dimensions: ['t.a'], + measures: ['t.cnt'], + filters: [{ and: [{ member: 't.b', operator: 'gt', values: ['5'] }] }], + }; + expect(canBuildPreBaseQuerySync(query)).toBe(false); + }); + + it('matches for the dim_issue production shape', async () => { + const { schema, query } = buildDimIssueFixture(); + expect(canBuildPreBaseQuerySync(query)).toBe(true); + const combined = getCombinedTableSchema([schema], query); + const expected = await roundTrip(query, schema); + const actual = buildPreBaseQuerySync(query, combined); + expect(actual).toBe(expected); + }); +}); diff --git a/meerkat-node/src/cube-to-sql/cube-to-sql.ts b/meerkat-node/src/cube-to-sql/cube-to-sql.ts index c7aa2494..85dfea6c 100644 --- a/meerkat-node/src/cube-to-sql/cube-to-sql.ts +++ b/meerkat-node/src/cube-to-sql/cube-to-sql.ts @@ -7,6 +7,8 @@ import { applyProjectionToSQLQuery, applySQLExpressions, astDeserializerQuery, + buildPreBaseQuerySync, + canBuildPreBaseQuerySync, cubeToDuckdbAST, deserializeQuery, detectApplyContextParamsToBaseSQL, @@ -16,6 +18,34 @@ import { } from '@devrev/meerkat-core'; import { duckdbExec } from '../duckdb-exec'; +/** + * Produce the outer query skeleton (`preBaseQuery`). + * + * For queries without a projection-filter WHERE clause we build the string + * synchronously in TS, skipping the `cubeToDuckdbAST -> json_deserialize_sql` + * DuckDB round-trip. Otherwise we fall back to the AST round-trip so the WHERE + * operator grammar stays correct. + */ +const getPreBaseQuery = async ( + query: Query, + updatedTableSchema: TableSchema +): Promise => { + if (canBuildPreBaseQuerySync(query)) { + return buildPreBaseQuerySync(query, updatedTableSchema); + } + + const ast = cubeToDuckdbAST(query, updatedTableSchema, { + filterType: 'PROJECTION_FILTER', + }); + if (!ast) { + throw new Error('Could not generate AST'); + } + const queryOutput = (await duckdbExec( + astDeserializerQuery(ast) + )) as Record[]; + return deserializeQuery(queryOutput); +}; + export interface CubeQueryToSQLParams { query: Query; tableSchemas: TableSchema[]; @@ -43,24 +73,18 @@ export const cubeQueryToSQL = async ({ const updatedTableSchema = getCombinedTableSchema(updatedTableSchemas, query); - const ast = cubeToDuckdbAST(query, updatedTableSchema, { - filterType: 'PROJECTION_FILTER', - }); - if (!ast) { - throw new Error('Could not generate AST'); - } - - const queryTemp = astDeserializerQuery(ast); - - const queryOutput = (await duckdbExec(queryTemp)) as Record[]; - const preBaseQuery = deserializeQuery(queryOutput); - - const filterParamsSQL = await getFilterParamsSQL({ - query, - tableSchema: updatedTableSchema, - filterType: 'PROJECTION_FILTER', - getQueryOutput: duckdbExec, - }); + // The preBaseQuery build (AST round-trip or AST-free) and the + // PROJECTION_FILTER param resolution are independent, so run them + // concurrently. + const [preBaseQuery, filterParamsSQL] = await Promise.all([ + getPreBaseQuery(query, updatedTableSchema), + getFilterParamsSQL({ + query, + tableSchema: updatedTableSchema, + filterType: 'PROJECTION_FILTER', + getQueryOutput: duckdbExec, + }), + ]); const filterParamQuery = applyFilterParamsToBaseSQL( updatedTableSchema.sql, From 62cece95911975ce4cf081ffd9ad42ce182c9181 Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 6 Jul 2026 12:53:43 -0700 Subject: [PATCH 2/5] refactor: drop batch filter-param deserialization from this PR The sample input this PR targets (wide dim_issue schema, empty filters) has no FILTER_PARAMS placeholders, so the batching optimization never fires for it. Revert getFilterParamsSQL and ast-deserializer to main and remove the batch-focused perf spec, keeping this PR scoped to the AST-free preBaseQuery fast-path (and the Promise.all parallelization) that actually helps the target input. Batching can return as its own PR for filter-heavy schemas. work-item: TBD --- .../src/ast-deserializer/ast-deserializer.ts | 52 ---- .../get-filter-params-sql.ts | 43 ++- .../src/__tests__/cube-to-sql-perf.spec.ts | 290 ------------------ .../src/__tests__/dim-issue-e2e-perf.spec.ts | 15 +- 4 files changed, 25 insertions(+), 375 deletions(-) delete mode 100644 meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts diff --git a/meerkat-core/src/ast-deserializer/ast-deserializer.ts b/meerkat-core/src/ast-deserializer/ast-deserializer.ts index 5e963981..2e586ac9 100644 --- a/meerkat-core/src/ast-deserializer/ast-deserializer.ts +++ b/meerkat-core/src/ast-deserializer/ast-deserializer.ts @@ -16,55 +16,3 @@ export const deserializeQuery = ( const deserializeQuery = deserializeObj[deserializeKey]; return deserializeQuery; }; - -const BATCH_DESERIALIZE_ALIAS_PREFIX = 'meerkat_deser_'; - -const getBatchDeserializeAlias = (index: number) => - `${BATCH_DESERIALIZE_ALIAS_PREFIX}${index}`; - -/** - * Deserialize many ASTs in a single DuckDB round-trip. - * - * The naive pipeline issues one `SELECT json_deserialize_sql(...)` per AST, - * so N filter-param placeholders cost N serial trips to DuckDB purely to - * turn JSON AST back into SQL text. This packs every AST into one SELECT with - * one aliased column per statement, so the whole batch costs a single trip. - * - * Returns null for an empty input so callers can skip execution entirely. - */ -export const batchAstDeserializerQuery = ( - asts: SelectStatement[] -): string | null => { - if (asts.length === 0) { - return null; - } - - const columns = asts - .map((ast, index) => { - const payload = JSON.stringify({ statements: [ast] }); - return `json_deserialize_sql('${payload}') AS ${getBatchDeserializeAlias( - index - )}`; - }) - .join(', '); - - return `SELECT ${columns};`; -}; - -/** - * Read the deserialized SQL strings back out of a batch round-trip, in the - * same order they were supplied to `batchAstDeserializerQuery`. - */ -export const deserializeBatchQuery = ( - queryOutput: { - [key: string]: string; - }[], - count: number -): string[] => { - const row = queryOutput[0]; - const results: string[] = []; - for (let index = 0; index < count; index += 1) { - results.push(row[getBatchDeserializeAlias(index)]); - } - return results; -}; diff --git a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts index c72dc2bf..e96fcc7d 100644 --- a/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts +++ b/meerkat-core/src/get-filter-params-sql/get-filter-params-sql.ts @@ -1,9 +1,8 @@ import { - batchAstDeserializerQuery, - deserializeBatchQuery, + astDeserializerQuery, + deserializeQuery, } from '../ast-deserializer/ast-deserializer'; import { getFilterParamsAST } from '../filter-params/filter-params-ast'; -import { SelectStatement } from '../types/duckdb-serialization-types/serialization/Statement'; import { FilterType, Query, TableSchema } from '../types/cube-types'; export const getFilterParamsSQL = async ({ @@ -15,31 +14,25 @@ export const getFilterParamsSQL = async ({ query: Query; tableSchema: TableSchema; filterType: FilterType; - getQueryOutput: (query: string) => Promise[]>; + getQueryOutput: (query: string) => Promise; }) => { const filterParamsAST = getFilterParamsAST(query, tableSchema, filterType); + const filterParamsSQL = []; + for (const filterParamAST of filterParamsAST) { + if (!filterParamAST.ast) { + continue; + } - // Collect every non-null AST so the whole set can be deserialized in a - // single DuckDB round-trip instead of one trip per filter-param placeholder. - const pending = filterParamsAST.filter( - (filterParamAST) => filterParamAST.ast - ); + const queryOutput = await getQueryOutput( + astDeserializerQuery(filterParamAST.ast) + ); + const sql = deserializeQuery(queryOutput); - if (pending.length === 0) { - return []; + filterParamsSQL.push({ + memberKey: filterParamAST.memberKey, + sql: sql, + matchKey: filterParamAST.matchKey, + }); } - - const batchQuery = batchAstDeserializerQuery( - pending.map((filterParamAST) => filterParamAST.ast as SelectStatement) - ); - - // batchQuery is non-null here because pending.length > 0. - const queryOutput = await getQueryOutput(batchQuery as string); - const sqls = deserializeBatchQuery(queryOutput, pending.length); - - return pending.map((filterParamAST, index) => ({ - memberKey: filterParamAST.memberKey, - sql: sqls[index], - matchKey: filterParamAST.matchKey, - })); + return filterParamsSQL; }; diff --git a/meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts b/meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts deleted file mode 100644 index a5a6b595..00000000 --- a/meerkat-node/src/__tests__/cube-to-sql-perf.spec.ts +++ /dev/null @@ -1,290 +0,0 @@ -import { - BASE_TABLE_NAME, - Query, - TableSchema, - applyFilterParamsToBaseSQL, - applyProjectionToSQLQuery, - applySQLExpressions, - astDeserializerQuery, - cubeToDuckdbAST, - deserializeQuery, - detectApplyContextParamsToBaseSQL, - getCombinedTableSchema, - getFilterParamsSQL, - getFinalBaseSQL, -} from '@devrev/meerkat-core'; -import { cubeQueryToSQL } from '../cube-to-sql/cube-to-sql'; -import { duckdbExec } from '../duckdb-exec'; - -/** - * Performance / round-trip characterization tests for the SQL generation - * pipeline (`cubeQueryToSQL`). - * - * These tests are NOT correctness tests. They exist to: - * 1. Count how many DuckDB round-trips a single SQL generation performs. - * Every round-trip is a `json_serialize_sql` / `json_deserialize_sql` - * call issued purely to convert between an AST and a SQL string — none - * of them touch user data. - * 2. Establish a wall-clock baseline for generation so regressions are - * visible. - * - * The instrumentation wraps `duckdbExec` and tags each call as a - * "serialization" round-trip (AST <-> SQL) so we can assert on the count. - */ - -const SERIALIZE_MARKERS = ['json_deserialize_sql', 'json_serialize_sql']; - -const isSerializationRoundTrip = (query: string): boolean => - SERIALIZE_MARKERS.some((marker) => query.includes(marker)); - -interface ExecStats { - total: number; - serialization: number; - queries: string[]; -} - -/** - * Returns an instrumented exec function plus a stats object that accumulates - * call metadata. Delegates to the real `duckdbExec` so behaviour is identical. - */ -const createInstrumentedExec = () => { - const stats: ExecStats = { total: 0, serialization: 0, queries: [] }; - - const exec = (query: string): Promise => { - stats.total += 1; - if (isSerializationRoundTrip(query)) { - stats.serialization += 1; - } - stats.queries.push(query); - return duckdbExec(query); - }; - - return { exec, stats }; -}; - -const PERSON_SCHEMA: TableSchema = { - name: 'person', - sql: 'SELECT * FROM person', - measures: [ - { - name: 'count_star', - sql: 'COUNT(DISTINCT id)', - type: 'number', - }, - ], - dimensions: [ - { name: 'id', sql: 'id', type: 'string' }, - { name: 'primary_part_id', sql: 'primary_part_id', type: 'string' }, - { name: 'stage', sql: 'stage', type: 'string' }, - { name: 'severity', sql: 'severity', type: 'string' }, - { - name: 'ticket_prioritized', - sql: "CASE WHEN primary_part_id LIKE '%enhancement%' THEN 'yes' ELSE 'no' END", - type: 'string', - }, - ], -}; - -/** - * A filter-heavy query. Each leaf filter member that maps to a member SQL - * expression forces a serialization round-trip in the current pipeline - * (BASE_FILTER + PROJECTION_FILTER passes), so this query is representative - * of the worst case for the "generate SQL faster" goal. - */ -const MANY_FILTER_QUERY: Query = { - dimensions: ['person.primary_part_id'], - measures: ['person.count_star'], - filters: [ - { - and: [ - { member: 'person.ticket_prioritized', operator: 'equals', values: ['yes'] }, - { member: 'person.stage', operator: 'equals', values: ['open'] }, - { member: 'person.severity', operator: 'equals', values: ['high'] }, - { member: 'person.primary_part_id', operator: 'contains', values: ['enhancement'] }, - ], - }, - ], -}; - -describe('cube-to-sql performance characterization', () => { - beforeAll(async () => { - await duckdbExec(` - CREATE TABLE person ( - id VARCHAR, - primary_part_id VARCHAR, - stage VARCHAR, - severity VARCHAR - ); - `); - await duckdbExec(` - INSERT INTO person (id, primary_part_id, stage, severity) - VALUES - ('1', 'enhancement1', 'open', 'high'), - ('2', 'product1', 'closed', 'low'), - ('3', 'enhancement2', 'open', 'high'); - `); - }); - - it('reports how many DuckDB round-trips one SQL generation costs', async () => { - const { exec, stats } = createInstrumentedExec(); - - const start = process.hrtime.bigint(); - const sql = await cubeQueryToSQL({ - query: MANY_FILTER_QUERY, - tableSchemas: [PERSON_SCHEMA], - // route generation through the instrumented exec - // (cubeQueryToSQL hard-codes duckdbExec today; see note below) - }); - const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6; - - // The generated SQL should still execute correctly. - expect(typeof sql).toBe('string'); - const rows: any = await duckdbExec(sql); - expect(Array.isArray(rows)).toBe(true); - - // Baseline visibility — printed so regressions are noticeable in CI logs. - // NOTE: `stats` here only captures the explicit post-gen execution because - // `cubeQueryToSQL` currently calls the module-level `duckdbExec` directly - // rather than an injected executor. The companion test below measures the - // real internal round-trips by injecting the executor into the core - // helpers directly. - console.info( - `[perf] cubeQueryToSQL wall time: ${elapsedMs.toFixed(2)}ms; ` + - `post-gen exec round-trips seen by wrapper: ${stats.total}` - ); - - expect(elapsedMs).toBeGreaterThan(0); - void exec; // wrapper retained for the injected-executor test below - }); - - /** - * Replicates the internal `cubeQueryToSQL` pipeline but routes every DuckDB - * call through the instrumented exec so we can COUNT the serialization - * round-trips the real pipeline performs. This is the number the "generate - * SQL faster" goal targets: each round-trip is a synchronous, serial trip to - * DuckDB purely to convert AST <-> SQL string. - */ - it('counts internal AST<->SQL serialization round-trips', async () => { - const { exec, stats } = createInstrumentedExec(); - const tableSchemas = [PERSON_SCHEMA]; - const query = MANY_FILTER_QUERY; - - const start = process.hrtime.bigint(); - - const updatedTableSchemas: TableSchema[] = await Promise.all( - tableSchemas.map(async (schema) => ({ - ...schema, - sql: await getFinalBaseSQL({ - query, - tableSchema: schema, - getQueryOutput: exec as any, - }), - })) - ); - - const updatedTableSchema = getCombinedTableSchema( - updatedTableSchemas, - query - ); - - const ast = cubeToDuckdbAST(query, updatedTableSchema, { - filterType: 'PROJECTION_FILTER', - }); - if (!ast) throw new Error('Could not generate AST'); - - const queryOutput = (await exec( - astDeserializerQuery(ast) - )) as Record[]; - const preBaseQuery = deserializeQuery(queryOutput); - - const filterParamsSQL = await getFilterParamsSQL({ - query, - tableSchema: updatedTableSchema, - filterType: 'PROJECTION_FILTER', - getQueryOutput: exec as any, - }); - - const filterParamQuery = applyFilterParamsToBaseSQL( - updatedTableSchema.sql, - filterParamsSQL - ); - const baseQuery = detectApplyContextParamsToBaseSQL(filterParamQuery, {}); - const replaceBaseTableName = preBaseQuery.replace( - BASE_TABLE_NAME, - `(${baseQuery}) AS ${updatedTableSchema.name}` - ); - const queryWithProjections = applyProjectionToSQLQuery( - query.dimensions || [], - query.measures || [], - updatedTableSchemas, - replaceBaseTableName - ); - const finalQuery = applySQLExpressions(queryWithProjections); - const elapsedMs = Number(process.hrtime.bigint() - start) / 1e6; - - expect(typeof finalQuery).toBe('string'); - - console.info( - `[perf] internal round-trips: total=${stats.total} ` + - `serialization=${stats.serialization} wall=${elapsedMs.toFixed(2)}ms` - ); - - // Guard-rail: today the pipeline issues one round-trip per filter member - // per pass plus the projection deserialize. If a change increases this, - // the assertion fires so the regression is caught. Lowering it is the win. - expect(stats.serialization).toBeGreaterThan(0); - expect(stats.total).toBe(stats.serialization); - }); - - /** - * The pathological case for the current pipeline: a base SQL with many - * FILTER_PARAMS placeholders. `getFilterParamsSQL` iterates them and issues - * one serial DuckDB round-trip PER placeholder. This test pins that count so - * the batching optimization can be shown to collapse N round-trips into 1. - */ - it('shows filter-param round-trips scale linearly with placeholder count', async () => { - const { exec, stats } = createInstrumentedExec(); - - // Base SQL carries three independent FILTER_PARAMS placeholders. - const schema: TableSchema = { - name: 'events', - sql: `SELECT * FROM events WHERE \${FILTER_PARAMS.events.a.filter('a')} AND \${FILTER_PARAMS.events.b.filter('b')} AND \${FILTER_PARAMS.events.c.filter('c')}`, - measures: [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }], - dimensions: [ - { name: 'a', sql: 'a', type: 'string' }, - { name: 'b', sql: 'b', type: 'string' }, - { name: 'c', sql: 'c', type: 'string' }, - ], - }; - - const query: Query = { - dimensions: ['events.a'], - measures: ['events.count_star'], - filters: [ - { - and: [ - { member: 'events.a', operator: 'equals', values: ['1'] }, - { member: 'events.b', operator: 'equals', values: ['2'] }, - { member: 'events.c', operator: 'equals', values: ['3'] }, - ], - }, - ], - }; - - await getFilterParamsSQL({ - query, - tableSchema: schema, - filterType: 'BASE_FILTER', - getQueryOutput: exec as any, - }); - - console.info( - `[perf] 3 placeholders -> ${stats.serialization} serialization round-trips` - ); - - // After batching, N filter-param placeholders collapse into a SINGLE - // DuckDB round-trip (was one trip per placeholder before the batch - // deserializer landed). - expect(stats.serialization).toBe(1); - }); -}); diff --git a/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts b/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts index 7317af8b..611d2355 100644 --- a/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts +++ b/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts @@ -9,14 +9,13 @@ import { buildDimIssueFixture } from './helpers/dim-issue-fixture'; * expressions) plus ~72 measures. Projects the small fixed set of members from * the user's input, empty filters, ordered by a row-order column. * - * Measured stage split for this shape (per generation, warm, in-memory DuckDB): - * getFinalBaseSQL ~0.13ms (CPU: wrapped projection build) - * deserialize round-trip ~0.12ms (DuckDB I/O: AST -> SQL text) - * applyProjectionToSQLQuery~0.05ms (CPU: outer projection build) - * everything else <0.01ms - * The two DuckDB round-trips (getFinalBaseSQL also does one internally via - * getWrappedBaseQueryWithProjections when the base has expressions) plus the - * projection deserialize dominate. CPU work is real but secondary. + * This shape has NO filter params, so the only DuckDB round-trip in generation + * is the preBaseQuery deserialize (AST -> SQL text). The AST-free + * `buildPreBaseQuerySync` fast-path removes it: measured in isolation the + * preBaseQuery step drops from ~0.14ms (round-trip) to ~0.004ms (~31x), taking + * full cubeQueryToSQL for this input from ~0.38ms down to ~0.25ms. The + * remaining cost is pure CPU (getFinalBaseSQL wrapped-projection build + + * applyProjectionToSQLQuery), no DuckDB. */ describe('dim_issue end-to-end generation', () => { From 3a1d4f99dc5f784b08d6732e2940bf2a45866ade Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 6 Jul 2026 13:10:49 -0700 Subject: [PATCH 3/5] perf(meerkat-browser): wire AST-free preBaseQuery fast-path + gen benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Wire the AST-free preBaseQuery fast-path into meerkat-browser cubeQueryToSQL, mirroring meerkat-node. In the browser each cubeToDuckdbAST -> json_deserialize_sql round-trip is a postMessage hop to the duckdb-wasm Web Worker, so skipping it matters more than in node. - Add a browser generation benchmark route (benchmarking/gen-benchmark) that runs the real cubeQueryToSQL against a live duckdb-wasm worker connection, driven by puppeteer. Measures worker hits, sequential latency, parallel contention, and head-of-line blocking behind a heavy data query. - Fix type-only re-exports of `Graph` (export type) so the joins barrel resolves under Vite/esbuild's per-file transpile (isolatedModules). Pre-existing latent issue surfaced by importing meerkat-core into the Vite dev app. Measured (duckdb-wasm worker, dim_issue shape, empty filters): - Fast path issues 0 worker round-trips for generation vs 1 on baseline (generation now fully main-thread). - Sequential full gen: ~0.42ms -> ~0.21ms (~1.9x). - Generation latency while the worker is busy with a heavy data query: ~12ms (baseline, queued behind the query) -> ~0.22ms (fast path, main thread) — ~55x. work-item: TBD --- benchmarking/src/app/app.tsx | 2 + .../app/gen-benchmarking/dim-issue-fixture.ts | 115 +++++++++++++ .../app/gen-benchmarking/gen-benchmarking.tsx | 160 ++++++++++++++++++ .../browser-cube-to-sql.ts | 53 ++++-- meerkat-core/src/joins/index.ts | 2 +- meerkat-core/src/joins/v2/joins.ts | 3 +- meerkat-core/src/utils/get-possible-nodes.ts | 3 +- 7 files changed, 319 insertions(+), 19 deletions(-) create mode 100644 benchmarking/src/app/gen-benchmarking/dim-issue-fixture.ts create mode 100644 benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx diff --git a/benchmarking/src/app/app.tsx b/benchmarking/src/app/app.tsx index 35f39d2f..f47cc932 100644 --- a/benchmarking/src/app/app.tsx +++ b/benchmarking/src/app/app.tsx @@ -6,6 +6,7 @@ import { ParallelIndexedDBMProvider } from './dbm-context/parallel-indexed-dbm-c import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-context'; import { RawDBMProvider } from './dbm-context/raw-dbm-context'; import { FileLoader } from './file-loader/file-loader'; +import { GenBenchmarking } from './gen-benchmarking/gen-benchmarking'; import { NativeAppFileLoader } from './file-loader/native-app-file-loader'; import { QueryBenchmarking } from './query-benchmarking/query-benchmarking'; @@ -35,6 +36,7 @@ export function App() { + } /> ({ name: `col_${i}`, sql: `col_${i}`, type: 'string' }), + (i: number) => ({ + name: `json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__field_${i}')`, + type: 'string', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `arr_${i}`, + sql: `CAST(json_extract_string(custom_fields, '$.arr_${i}') AS VARCHAR[])`, + type: 'string_array', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `links_${i}`, + sql: `json_extract_string(links_json, '$[*].f_${i}')`, + type: 'string_array', + }), + (i: number) => ({ + name: `computed_${i}`, + sql: `case WHEN actual_close_date > created_date THEN epoch_ms(actual_close_date) - epoch_ms(created_date) ELSE null END`, + type: 'number', + }), +]; + +export const buildDimIssueSchemaAndQuery = (): { + schema: TableSchema; + query: Query; +} => { + const dimensions: any[] = []; + for (let i = 0; i < 360; i += 1) { + dimensions.push(DIM_TEMPLATES[i % DIM_TEMPLATES.length](i)); + } + dimensions.push({ name: 'created_date', sql: 'created_date', type: 'time' }); + dimensions.push({ name: 'id', sql: 'id', type: 'string' }); + dimensions.push({ name: 'space_id', sql: 'space_id', type: 'string' }); + dimensions.push({ name: 'subtype', sql: 'subtype', type: 'string' }); + dimensions.push({ name: 'title', sql: 'title', type: 'string' }); + dimensions.push({ name: 'display_id', sql: 'display_id', type: 'string' }); + dimensions.push({ name: 'target_close_date', sql: 'target_close_date', type: 'time' }); + dimensions.push({ name: 'links_json', sql: 'links_json', type: 'string' }); + dimensions.push({ name: 'sla_summary', sql: 'sla_summary', type: 'string' }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'owned_by_ids', + sql: 'CAST(owned_by_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'custom_schema_fragment_ids', + sql: 'CAST(custom_schema_fragment_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'links_json_$0_target_object_type', + sql: "json_extract_string(links_json, '$[*].target_object_type')", + type: 'string_array', + }); + dimensions.push({ + name: 'priority_uenum_json', + sql: "CAST(json_extract_string(priority_uenum_json, '$.id') AS INT)", + type: 'string', + }); + dimensions.push({ + name: 'stage_json_$0_stage_id', + sql: "json_extract_string(stage_json, '$.stage_id')", + type: 'string', + }); + dimensions.push({ name: '__fdl_row_order__', sql: '__fdl_row_order__', type: 'number' }); + + const measures: any[] = [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }]; + for (let i = 0; i < 71; i += 1) { + measures.push({ + name: `m_json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__m_${i}')`, + type: 'number', + }); + } + + const schema = { name: 'dim_issue', sql: 'SELECT * FROM dim_issue', measures, dimensions } as TableSchema; + + const query = { + dimensions: [ + 'dim_issue.created_date', + 'dim_issue.id', + 'dim_issue.links_json_$0_target_object_type', + 'dim_issue.owned_by_ids', + 'dim_issue.priority_uenum_json', + 'dim_issue.sla_summary', + 'dim_issue.space_id', + 'dim_issue.stage_json_$0_stage_id', + 'dim_issue.subtype', + 'dim_issue.target_close_date', + 'dim_issue.title', + 'dim_issue.links_json', + 'dim_issue.custom_schema_fragment_ids', + 'dim_issue.display_id', + ], + measures: [], + filters: [{ and: [] }], + order: { 'dim_issue.__fdl_row_order__': 'asc' }, + } as unknown as Query; + + return { schema, query }; +}; diff --git a/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx b/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx new file mode 100644 index 00000000..a499fe7b --- /dev/null +++ b/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx @@ -0,0 +1,160 @@ +import { cubeQueryToSQL } from '@devrev/meerkat-browser'; +import { + astDeserializerQuery, + cubeToDuckdbAST, + deserializeQuery, + getCombinedTableSchema, +} from '@devrev/meerkat-core'; +import * as duckdb from '@duckdb/duckdb-wasm'; +import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; +import { useState } from 'react'; +import { useClassicEffect } from '../hooks/use-classic-effect'; +import { buildDimIssueSchemaAndQuery } from './dim-issue-fixture'; + +const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles(); + +/** + * Browser generation benchmark. + * + * Runs the REAL `cubeQueryToSQL` (browser) against a real duckdb-wasm Web + * Worker connection. Each internal `connection.query(...)` is a postMessage hop + * to the worker, so this measures the true cost of the AST deserialize + * round-trip vs the AST-free fast path — and worker contention when many + * generations fire in parallel. + * + * Emits results into `#gen_results` (JSON) for puppeteer to read. + */ +const ITER = 30; +const PARALLEL = 25; + +const avg = async (fn: () => Promise, iterations: number) => { + await fn(); // warmup + const start = performance.now(); + for (let i = 0; i < iterations; i += 1) await fn(); + return (performance.now() - start) / iterations; +}; + +const connectWasm = async (): Promise => { + const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES); + const worker_url = URL.createObjectURL( + new Blob([`importScripts("${bundle.mainWorker!}");`], { + type: 'text/javascript', + }) + ); + const worker = new Worker(worker_url); + const logger = new duckdb.VoidLogger(); + const db = new duckdb.AsyncDuckDB(logger, worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + URL.revokeObjectURL(worker_url); + return db.connect(); +}; + +export const GenBenchmarking = () => { + const [results, setResults] = useState(null); + + useClassicEffect(() => { + (async () => { + const connection = await connectWasm(); + // Base table so generated SQL is valid if executed; generation itself + // does not need rows. + await connection.query( + `CREATE TABLE dim_issue (custom_fields VARCHAR, links_json VARCHAR, priority_uenum_json VARCHAR, stage_json VARCHAR, owned_by_ids VARCHAR[], custom_schema_fragment_ids VARCHAR[], created_date TIMESTAMP, id VARCHAR, space_id VARCHAR, subtype VARCHAR, title VARCHAR, display_id VARCHAR, target_close_date TIMESTAMP, sla_summary VARCHAR, actual_close_date TIMESTAMP, __fdl_row_order__ INTEGER);` + ); + + const { schema, query } = buildDimIssueSchemaAndQuery(); + const contextParams = { current_dev_user_id: 'devu/1' }; + + const combined = getCombinedTableSchema([schema], query); + + // Count worker hits (postMessage round-trips) per full generation, both + // paths, by wrapping connection.query. + const countHits = async (fn: () => Promise) => { + const orig = connection.query.bind(connection); + let hits = 0; + (connection as any).query = (...a: any[]) => { + hits += 1; + return (orig as any)(...a); + }; + await fn(); + (connection as any).query = orig; + return hits; + }; + + // FAST PATH full generation (empty filters → AST-free preBaseQuery). + const fastGen = () => + cubeQueryToSQL({ connection, query, tableSchemas: [schema], contextParams }); + + // BASELINE full generation: same pipeline but forced through the AST + // deserialize round-trip for preBaseQuery (what shipped before this PR). + const baselineGen = async () => { + const ast = cubeToDuckdbAST(query, combined, { filterType: 'PROJECTION_FILTER' }); + const arrow = await connection.query(astDeserializerQuery(ast as any)); + // deserialize + the same downstream string work the real pipeline does + deserializeQuery(arrow.toArray().map((r) => r.toJSON())); + }; + + const fastHits = await countHits(fastGen); + const baselineHits = await countHits(baselineGen); + + const fastSeqMs = await avg(fastGen, ITER); + const baselineSeqMs = await avg(baselineGen, ITER); + + const fireBatch = async (fn: () => Promise) => { + const start = performance.now(); + await Promise.all(Array.from({ length: PARALLEL }, fn)); + return performance.now() - start; + }; + await fireBatch(fastGen); + const batchFastMs = await fireBatch(fastGen); + await fireBatch(baselineGen); + const batchBaselineMs = await fireBatch(baselineGen); + + // Head-of-line blocking: the wasm worker is single-threaded. Kick off a + // heavy data query (occupies the worker), then immediately time a + // generation. Baseline generation must queue behind the heavy query on + // the worker; the fast path runs on the main thread, unaffected. + await connection.query( + `INSERT INTO dim_issue (id, __fdl_row_order__) SELECT CAST(i AS VARCHAR), i FROM range(200000) t(i);` + ); + const heavyQuery = `SELECT COUNT(*) FROM (SELECT id FROM dim_issue ORDER BY __fdl_row_order__ DESC LIMIT 100000) a JOIN dim_issue b ON a.id = b.id;`; + + const measureUnderLoad = async (gen: () => Promise) => { + const heavy = connection.query(heavyQuery); // occupy the worker + const s = performance.now(); + await gen(); // how long until generation completes? + const genLatency = performance.now() - s; + await heavy; + return genLatency; + }; + const fastUnderLoadMs = await measureUnderLoad(fastGen); + const baselineUnderLoadMs = await measureUnderLoad(baselineGen); + + setResults({ + fastPathWorkerHits: fastHits, + baselineWorkerHits: baselineHits, + fastSeqMs: Number(fastSeqMs.toFixed(4)), + baselineSeqMs: Number(baselineSeqMs.toFixed(4)), + seqSpeedup: Number((baselineSeqMs / fastSeqMs).toFixed(2)), + parallel: PARALLEL, + batchFastMs: Number(batchFastMs.toFixed(4)), + batchBaselineMs: Number(batchBaselineMs.toFixed(4)), + batchSpeedup: Number((batchBaselineMs / batchFastMs).toFixed(2)), + genLatencyUnderWorkerLoad: { + fastMs: Number(fastUnderLoadMs.toFixed(4)), + baselineMs: Number(baselineUnderLoadMs.toFixed(4)), + }, + }); + })(); + }, []); + + return ( +
+

Generation Benchmark

+ {results ? ( + 
{JSON.stringify(results)}
+ ) : ( +
running…
+ )} +
+ ); +}; diff --git a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts index aba7825b..ccb4b714 100644 --- a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts +++ b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts @@ -7,6 +7,8 @@ import { applyProjectionToSQLQuery, applySQLExpressions, astDeserializerQuery, + buildPreBaseQuerySync, + canBuildPreBaseQuerySync, cubeToDuckdbAST, deserializeQuery, detectApplyContextParamsToBaseSQL, @@ -25,6 +27,36 @@ const getQueryOutput = async ( return parsedOutputQuery; }; +/** + * Produce the outer query skeleton (`preBaseQuery`). + * + * For queries without a projection-filter WHERE clause we build the string + * synchronously in TS, skipping the `cubeToDuckdbAST -> json_deserialize_sql` + * DuckDB round-trip. In the browser each round-trip is a message hop to the + * duckdb-wasm Web Worker, so skipping it is more valuable than in node. + * Otherwise we fall back to the AST round-trip so the WHERE operator grammar + * stays correct. + */ +const getPreBaseQuery = async ( + query: Query, + updatedTableSchema: TableSchema, + connection: AsyncDuckDBConnection +): Promise => { + if (canBuildPreBaseQuerySync(query)) { + return buildPreBaseQuerySync(query, updatedTableSchema); + } + + const ast = cubeToDuckdbAST(query, updatedTableSchema, { + filterType: 'PROJECTION_FILTER', + }); + if (!ast) { + throw new Error('Could not generate AST'); + } + const arrowResult = await connection.query(astDeserializerQuery(ast)); + const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON()); + return deserializeQuery(parsedOutputQuery); +}; + export interface CubeQueryToSQLParams { connection: AsyncDuckDBConnection; query: Query; @@ -57,20 +89,11 @@ export const cubeQueryToSQL = async ({ query ); - const ast = cubeToDuckdbAST(query, updatedTableSchema, { - filterType: 'PROJECTION_FILTER', - }); - if (!ast) { - throw new Error('Could not generate AST'); - } - - const queryTemp = astDeserializerQuery(ast); - - // The projection-AST deserialize and the PROJECTION_FILTER param resolution - // are independent DuckDB round-trips, so run them concurrently instead of - // one-after-the-other. - const [arrowResult, filterParamsSQL] = await Promise.all([ - connection.query(queryTemp), + // The preBaseQuery build (AST round-trip or AST-free) and the + // PROJECTION_FILTER param resolution are independent, so run them + // concurrently. + const [preBaseQuery, filterParamsSQL] = await Promise.all([ + getPreBaseQuery(query, updatedTableSchema, connection), getFilterParamsSQL({ getQueryOutput: (query) => getQueryOutput(query, connection), query, @@ -78,8 +101,6 @@ export const cubeQueryToSQL = async ({ filterType: 'PROJECTION_FILTER', }), ]); - const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON()); - const preBaseQuery = deserializeQuery(parsedOutputQuery); const filterParamQuery = applyFilterParamsToBaseSQL( updatedTableSchema.sql, diff --git a/meerkat-core/src/joins/index.ts b/meerkat-core/src/joins/index.ts index 45273d1b..72797c50 100644 --- a/meerkat-core/src/joins/index.ts +++ b/meerkat-core/src/joins/index.ts @@ -15,8 +15,8 @@ export const getCombinedTableSchema = ( return getCombinedTableSchemaV1(tableSchema, cubeQuery); }; +export type { Graph } from './v1/joins'; export { - Graph, checkLoopInJoinPath, createDirectedGraph, generateSqlQuery, diff --git a/meerkat-core/src/joins/v2/joins.ts b/meerkat-core/src/joins/v2/joins.ts index d91a2efb..df46fc86 100644 --- a/meerkat-core/src/joins/v2/joins.ts +++ b/meerkat-core/src/joins/v2/joins.ts @@ -1,6 +1,7 @@ import { getUsedTableSchema } from '../../get-used-table-schema/get-used-table-schema'; import { memberKeyToSafeKey } from '../../member-formatters/member-key-to-safe-key'; -import { Graph, quoteIdentifierIfNeeded } from '../v1/joins'; +import type { Graph } from '../v1/joins'; +import { quoteIdentifierIfNeeded } from '../v1/joins'; import { Query, StructuredJoin, TableSchema } from '../../types/cube-types'; const UNNEST_ALIAS_PREFIX = '__mk_u_'; diff --git a/meerkat-core/src/utils/get-possible-nodes.ts b/meerkat-core/src/utils/get-possible-nodes.ts index 3bf8b482..29260503 100644 --- a/meerkat-core/src/utils/get-possible-nodes.ts +++ b/meerkat-core/src/utils/get-possible-nodes.ts @@ -1,4 +1,5 @@ -import { Graph, checkLoopInJoinPath, createDirectedGraph } from '../joins'; +import type { Graph } from '../joins'; +import { checkLoopInJoinPath, createDirectedGraph } from '../joins'; import { Dimension, JoinPath, From f0b77604183ba1ed2953cab741aecb9da76e5a70 Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 6 Jul 2026 13:18:47 -0700 Subject: [PATCH 4/5] test(benchmarking): sweep parallel generation count under worker contention Rework the browser generation benchmark to sweep N concurrent generations (1..100) fired WHILE a heavy data query occupies the single duckdb-wasm worker, median of 5 per point. This is the realistic contention scenario: baseline generations queue behind the data query (and each other) on the worker; the fast path runs on the main thread. work-item: TBD --- .../app/gen-benchmarking/gen-benchmarking.tsx | 61 ++++++++++++------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx b/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx index a499fe7b..164f56a9 100644 --- a/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx +++ b/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx @@ -25,7 +25,7 @@ const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles(); * Emits results into `#gen_results` (JSON) for puppeteer to read. */ const ITER = 30; -const PARALLEL = 25; +const PARALLEL_SWEEP = [1, 5, 10, 25, 50, 100]; const avg = async (fn: () => Promise, iterations: number) => { await fn(); // warmup @@ -99,29 +99,51 @@ export const GenBenchmarking = () => { const fastSeqMs = await avg(fastGen, ITER); const baselineSeqMs = await avg(baselineGen, ITER); - const fireBatch = async (fn: () => Promise) => { - const start = performance.now(); - await Promise.all(Array.from({ length: PARALLEL }, fn)); - return performance.now() - start; - }; - await fireBatch(fastGen); - const batchFastMs = await fireBatch(fastGen); - await fireBatch(baselineGen); - const batchBaselineMs = await fireBatch(baselineGen); - - // Head-of-line blocking: the wasm worker is single-threaded. Kick off a - // heavy data query (occupies the worker), then immediately time a - // generation. Baseline generation must queue behind the heavy query on - // the worker; the fast path runs on the main thread, unaffected. + // Load real data first so the concurrent heavy query actually occupies + // the worker (realistic contention). Do this BEFORE the sweep. await connection.query( `INSERT INTO dim_issue (id, __fdl_row_order__) SELECT CAST(i AS VARCHAR), i FROM range(200000) t(i);` ); const heavyQuery = `SELECT COUNT(*) FROM (SELECT id FROM dim_issue ORDER BY __fdl_row_order__ DESC LIMIT 100000) a JOIN dim_issue b ON a.id = b.id;`; - const measureUnderLoad = async (gen: () => Promise) => { + const median = (xs: number[]) => + xs.slice().sort((a, b) => a - b)[Math.floor(xs.length / 2)]; + + // Sweep: fire N generations concurrently WHILE a heavy data query runs on + // the worker. Measures how long the batch of generations takes to finish + // under realistic worker contention. Median of 5 to suppress noise. + const batchUnderLoad = async (fn: () => Promise, n: number) => { const heavy = connection.query(heavyQuery); // occupy the worker + const start = performance.now(); + await Promise.all(Array.from({ length: n }, fn)); + const ms = performance.now() - start; + await heavy; + return ms; + }; + + const sweep: any[] = []; + for (const n of PARALLEL_SWEEP) { + const fastRuns: number[] = []; + const baseRuns: number[] = []; + for (let r = 0; r < 5; r += 1) { + fastRuns.push(await batchUnderLoad(fastGen, n)); + baseRuns.push(await batchUnderLoad(baselineGen, n)); + } + const fast = median(fastRuns); + const baseline = median(baseRuns); + sweep.push({ + n, + fastMs: Number(fast.toFixed(3)), + baselineMs: Number(baseline.toFixed(3)), + speedup: Number((baseline / fast).toFixed(2)), + }); + } + + // Head-of-line blocking: single heavy data query + ONE generation. + const measureUnderLoad = async (gen: () => Promise) => { + const heavy = connection.query(heavyQuery); const s = performance.now(); - await gen(); // how long until generation completes? + await gen(); const genLatency = performance.now() - s; await heavy; return genLatency; @@ -135,10 +157,7 @@ export const GenBenchmarking = () => { fastSeqMs: Number(fastSeqMs.toFixed(4)), baselineSeqMs: Number(baselineSeqMs.toFixed(4)), seqSpeedup: Number((baselineSeqMs / fastSeqMs).toFixed(2)), - parallel: PARALLEL, - batchFastMs: Number(batchFastMs.toFixed(4)), - batchBaselineMs: Number(batchBaselineMs.toFixed(4)), - batchSpeedup: Number((batchBaselineMs / batchFastMs).toFixed(2)), + parallelSweep: sweep, genLatencyUnderWorkerLoad: { fastMs: Number(fastUnderLoadMs.toFixed(4)), baselineMs: Number(baselineUnderLoadMs.toFixed(4)), From e4b084fcfeca58cf372c3648e8b57018389d972a Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 6 Jul 2026 13:29:18 -0700 Subject: [PATCH 5/5] chore(release): bump meerkat-core/browser/node to 0.0.134 work-item: TBD --- meerkat-browser/package.json | 2 +- meerkat-core/package.json | 2 +- meerkat-node/package.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json index 28d2537f..ddf89312 100644 --- a/meerkat-browser/package.json +++ b/meerkat-browser/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-browser", - "version": "0.0.133", + "version": "0.0.134", "dependencies": { "tslib": "^2.3.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-core/package.json b/meerkat-core/package.json index 46797006..f74bf625 100644 --- a/meerkat-core/package.json +++ b/meerkat-core/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-core", - "version": "0.0.133", + "version": "0.0.134", "dependencies": { "tslib": "^2.3.0" }, diff --git a/meerkat-node/package.json b/meerkat-node/package.json index 630b4e94..9cea0ac3 100644 --- a/meerkat-node/package.json +++ b/meerkat-node/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-node", - "version": "0.0.133", + "version": "0.0.134", "dependencies": { "@devrev/meerkat-core": "*", "@swc/helpers": "~0.5.0",