diff --git a/benchmarking/src/app/app.tsx b/benchmarking/src/app/app.tsx index 35f39d2f..f47cc932 100644 --- a/benchmarking/src/app/app.tsx +++ b/benchmarking/src/app/app.tsx @@ -6,6 +6,7 @@ import { ParallelIndexedDBMProvider } from './dbm-context/parallel-indexed-dbm-c import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-context'; import { RawDBMProvider } from './dbm-context/raw-dbm-context'; import { FileLoader } from './file-loader/file-loader'; +import { GenBenchmarking } from './gen-benchmarking/gen-benchmarking'; import { NativeAppFileLoader } from './file-loader/native-app-file-loader'; import { QueryBenchmarking } from './query-benchmarking/query-benchmarking'; @@ -35,6 +36,7 @@ export function App() { + } /> ({ name: `col_${i}`, sql: `col_${i}`, type: 'string' }), + (i: number) => ({ + name: `json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__field_${i}')`, + type: 'string', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `arr_${i}`, + sql: `CAST(json_extract_string(custom_fields, '$.arr_${i}') AS VARCHAR[])`, + type: 'string_array', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `links_${i}`, + sql: `json_extract_string(links_json, '$[*].f_${i}')`, + type: 'string_array', + }), + (i: number) => ({ + name: `computed_${i}`, + sql: `case WHEN actual_close_date > created_date THEN epoch_ms(actual_close_date) - epoch_ms(created_date) ELSE null END`, + type: 'number', + }), +]; + +export const buildDimIssueSchemaAndQuery = (): { + schema: TableSchema; + query: Query; +} => { + const dimensions: any[] = []; + for (let i = 0; i < 360; i += 1) { + dimensions.push(DIM_TEMPLATES[i % DIM_TEMPLATES.length](i)); + } + dimensions.push({ name: 'created_date', sql: 'created_date', type: 'time' }); + dimensions.push({ name: 'id', sql: 'id', type: 'string' }); + dimensions.push({ name: 'space_id', sql: 'space_id', type: 'string' }); + dimensions.push({ name: 'subtype', sql: 'subtype', type: 'string' }); + dimensions.push({ name: 'title', sql: 'title', type: 'string' }); + dimensions.push({ name: 'display_id', sql: 'display_id', type: 'string' }); + dimensions.push({ name: 'target_close_date', sql: 'target_close_date', type: 'time' }); + dimensions.push({ name: 'links_json', sql: 'links_json', type: 'string' }); + dimensions.push({ name: 'sla_summary', sql: 'sla_summary', type: 'string' }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'owned_by_ids', + sql: 'CAST(owned_by_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'custom_schema_fragment_ids', + sql: 'CAST(custom_schema_fragment_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'links_json_$0_target_object_type', + sql: "json_extract_string(links_json, '$[*].target_object_type')", + type: 'string_array', + }); + dimensions.push({ + name: 'priority_uenum_json', + sql: "CAST(json_extract_string(priority_uenum_json, '$.id') AS INT)", + type: 'string', + }); + dimensions.push({ + name: 'stage_json_$0_stage_id', + sql: "json_extract_string(stage_json, '$.stage_id')", + type: 'string', + }); + dimensions.push({ name: '__fdl_row_order__', sql: '__fdl_row_order__', type: 'number' }); + + const measures: any[] = [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }]; + for (let i = 0; i < 71; i += 1) { + measures.push({ + name: `m_json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__m_${i}')`, + type: 'number', + }); + } + + const schema = { name: 'dim_issue', sql: 'SELECT * FROM dim_issue', measures, dimensions } as TableSchema; + + const query = { + dimensions: [ + 'dim_issue.created_date', + 'dim_issue.id', + 'dim_issue.links_json_$0_target_object_type', + 'dim_issue.owned_by_ids', + 'dim_issue.priority_uenum_json', + 'dim_issue.sla_summary', + 'dim_issue.space_id', + 'dim_issue.stage_json_$0_stage_id', + 'dim_issue.subtype', + 'dim_issue.target_close_date', + 'dim_issue.title', + 'dim_issue.links_json', + 'dim_issue.custom_schema_fragment_ids', + 'dim_issue.display_id', + ], + measures: [], + filters: [{ and: [] }], + order: { 'dim_issue.__fdl_row_order__': 'asc' }, + } as unknown as Query; + + return { schema, query }; +}; diff --git a/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx b/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx new file mode 100644 index 00000000..164f56a9 --- /dev/null +++ b/benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx @@ -0,0 +1,179 @@ +import { cubeQueryToSQL } from '@devrev/meerkat-browser'; +import { + astDeserializerQuery, + cubeToDuckdbAST, + deserializeQuery, + getCombinedTableSchema, +} from '@devrev/meerkat-core'; +import * as duckdb from '@duckdb/duckdb-wasm'; +import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm'; +import { useState } from 'react'; +import { useClassicEffect } from '../hooks/use-classic-effect'; +import { buildDimIssueSchemaAndQuery } from './dim-issue-fixture'; + +const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles(); + +/** + * Browser generation benchmark. + * + * Runs the REAL `cubeQueryToSQL` (browser) against a real duckdb-wasm Web + * Worker connection. Each internal `connection.query(...)` is a postMessage hop + * to the worker, so this measures the true cost of the AST deserialize + * round-trip vs the AST-free fast path — and worker contention when many + * generations fire in parallel. + * + * Emits results into `#gen_results` (JSON) for puppeteer to read. + */ +const ITER = 30; +const PARALLEL_SWEEP = [1, 5, 10, 25, 50, 100]; + +const avg = async (fn: () => Promise, iterations: number) => { + await fn(); // warmup + const start = performance.now(); + for (let i = 0; i < iterations; i += 1) await fn(); + return (performance.now() - start) / iterations; +}; + +const connectWasm = async (): Promise => { + const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES); + const worker_url = URL.createObjectURL( + new Blob([`importScripts("${bundle.mainWorker!}");`], { + type: 'text/javascript', + }) + ); + const worker = new Worker(worker_url); + const logger = new duckdb.VoidLogger(); + const db = new duckdb.AsyncDuckDB(logger, worker); + await db.instantiate(bundle.mainModule, bundle.pthreadWorker); + URL.revokeObjectURL(worker_url); + return db.connect(); +}; + +export const GenBenchmarking = () => { + const [results, setResults] = useState(null); + + useClassicEffect(() => { + (async () => { + const connection = await connectWasm(); + // Base table so generated SQL is valid if executed; generation itself + // does not need rows. + await connection.query( + `CREATE TABLE dim_issue (custom_fields VARCHAR, links_json VARCHAR, priority_uenum_json VARCHAR, stage_json VARCHAR, owned_by_ids VARCHAR[], custom_schema_fragment_ids VARCHAR[], created_date TIMESTAMP, id VARCHAR, space_id VARCHAR, subtype VARCHAR, title VARCHAR, display_id VARCHAR, target_close_date TIMESTAMP, sla_summary VARCHAR, actual_close_date TIMESTAMP, __fdl_row_order__ INTEGER);` + ); + + const { schema, query } = buildDimIssueSchemaAndQuery(); + const contextParams = { current_dev_user_id: 'devu/1' }; + + const combined = getCombinedTableSchema([schema], query); + + // Count worker hits (postMessage round-trips) per full generation, both + // paths, by wrapping connection.query. + const countHits = async (fn: () => Promise) => { + const orig = connection.query.bind(connection); + let hits = 0; + (connection as any).query = (...a: any[]) => { + hits += 1; + return (orig as any)(...a); + }; + await fn(); + (connection as any).query = orig; + return hits; + }; + + // FAST PATH full generation (empty filters → AST-free preBaseQuery). + const fastGen = () => + cubeQueryToSQL({ connection, query, tableSchemas: [schema], contextParams }); + + // BASELINE full generation: same pipeline but forced through the AST + // deserialize round-trip for preBaseQuery (what shipped before this PR). + const baselineGen = async () => { + const ast = cubeToDuckdbAST(query, combined, { filterType: 'PROJECTION_FILTER' }); + const arrow = await connection.query(astDeserializerQuery(ast as any)); + // deserialize + the same downstream string work the real pipeline does + deserializeQuery(arrow.toArray().map((r) => r.toJSON())); + }; + + const fastHits = await countHits(fastGen); + const baselineHits = await countHits(baselineGen); + + const fastSeqMs = await avg(fastGen, ITER); + const baselineSeqMs = await avg(baselineGen, ITER); + + // Load real data first so the concurrent heavy query actually occupies + // the worker (realistic contention). Do this BEFORE the sweep. + await connection.query( + `INSERT INTO dim_issue (id, __fdl_row_order__) SELECT CAST(i AS VARCHAR), i FROM range(200000) t(i);` + ); + const heavyQuery = `SELECT COUNT(*) FROM (SELECT id FROM dim_issue ORDER BY __fdl_row_order__ DESC LIMIT 100000) a JOIN dim_issue b ON a.id = b.id;`; + + const median = (xs: number[]) => + xs.slice().sort((a, b) => a - b)[Math.floor(xs.length / 2)]; + + // Sweep: fire N generations concurrently WHILE a heavy data query runs on + // the worker. Measures how long the batch of generations takes to finish + // under realistic worker contention. Median of 5 to suppress noise. + const batchUnderLoad = async (fn: () => Promise, n: number) => { + const heavy = connection.query(heavyQuery); // occupy the worker + const start = performance.now(); + await Promise.all(Array.from({ length: n }, fn)); + const ms = performance.now() - start; + await heavy; + return ms; + }; + + const sweep: any[] = []; + for (const n of PARALLEL_SWEEP) { + const fastRuns: number[] = []; + const baseRuns: number[] = []; + for (let r = 0; r < 5; r += 1) { + fastRuns.push(await batchUnderLoad(fastGen, n)); + baseRuns.push(await batchUnderLoad(baselineGen, n)); + } + const fast = median(fastRuns); + const baseline = median(baseRuns); + sweep.push({ + n, + fastMs: Number(fast.toFixed(3)), + baselineMs: Number(baseline.toFixed(3)), + speedup: Number((baseline / fast).toFixed(2)), + }); + } + + // Head-of-line blocking: single heavy data query + ONE generation. + const measureUnderLoad = async (gen: () => Promise) => { + const heavy = connection.query(heavyQuery); + const s = performance.now(); + await gen(); + const genLatency = performance.now() - s; + await heavy; + return genLatency; + }; + const fastUnderLoadMs = await measureUnderLoad(fastGen); + const baselineUnderLoadMs = await measureUnderLoad(baselineGen); + + setResults({ + fastPathWorkerHits: fastHits, + baselineWorkerHits: baselineHits, + fastSeqMs: Number(fastSeqMs.toFixed(4)), + baselineSeqMs: Number(baselineSeqMs.toFixed(4)), + seqSpeedup: Number((baselineSeqMs / fastSeqMs).toFixed(2)), + parallelSweep: sweep, + genLatencyUnderWorkerLoad: { + fastMs: Number(fastUnderLoadMs.toFixed(4)), + baselineMs: Number(baselineUnderLoadMs.toFixed(4)), + }, + }); + })(); + }, []); + + return ( +
+

Generation Benchmark

+ {results ? ( + 
{JSON.stringify(results)}
+ ) : ( +
running…
+ )} +
+ ); +}; diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json index 28d2537f..ddf89312 100644 --- a/meerkat-browser/package.json +++ b/meerkat-browser/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-browser", - "version": "0.0.133", + "version": "0.0.134", "dependencies": { "tslib": "^2.3.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts index ecf7165e..ccb4b714 100644 --- a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts +++ b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts @@ -7,6 +7,8 @@ import { applyProjectionToSQLQuery, applySQLExpressions, astDeserializerQuery, + buildPreBaseQuerySync, + canBuildPreBaseQuerySync, cubeToDuckdbAST, deserializeQuery, detectApplyContextParamsToBaseSQL, @@ -25,6 +27,36 @@ const getQueryOutput = async ( return parsedOutputQuery; }; +/** + * Produce the outer query skeleton (`preBaseQuery`). + * + * For queries without a projection-filter WHERE clause we build the string + * synchronously in TS, skipping the `cubeToDuckdbAST -> json_deserialize_sql` + * DuckDB round-trip. In the browser each round-trip is a message hop to the + * duckdb-wasm Web Worker, so skipping it is more valuable than in node. + * Otherwise we fall back to the AST round-trip so the WHERE operator grammar + * stays correct. + */ +const getPreBaseQuery = async ( + query: Query, + updatedTableSchema: TableSchema, + connection: AsyncDuckDBConnection +): Promise => { + if (canBuildPreBaseQuerySync(query)) { + return buildPreBaseQuerySync(query, updatedTableSchema); + } + + const ast = cubeToDuckdbAST(query, updatedTableSchema, { + filterType: 'PROJECTION_FILTER', + }); + if (!ast) { + throw new Error('Could not generate AST'); + } + const arrowResult = await connection.query(astDeserializerQuery(ast)); + const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON()); + return deserializeQuery(parsedOutputQuery); +}; + export interface CubeQueryToSQLParams { connection: AsyncDuckDBConnection; query: Query; @@ -57,25 +89,18 @@ export const cubeQueryToSQL = async ({ query ); - const ast = cubeToDuckdbAST(query, updatedTableSchema, { - filterType: 'PROJECTION_FILTER', - }); - if (!ast) { - throw new Error('Could not generate AST'); - } - - const queryTemp = astDeserializerQuery(ast); - - const arrowResult = await connection.query(queryTemp); - const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON()); - - const preBaseQuery = deserializeQuery(parsedOutputQuery); - const filterParamsSQL = await getFilterParamsSQL({ - getQueryOutput: (query) => getQueryOutput(query, connection), - query, - tableSchema: updatedTableSchema, - filterType: 'PROJECTION_FILTER', - }); + // The preBaseQuery build (AST round-trip or AST-free) and the + // PROJECTION_FILTER param resolution are independent, so run them + // concurrently. + const [preBaseQuery, filterParamsSQL] = await Promise.all([ + getPreBaseQuery(query, updatedTableSchema, connection), + getFilterParamsSQL({ + getQueryOutput: (query) => getQueryOutput(query, connection), + query, + tableSchema: updatedTableSchema, + filterType: 'PROJECTION_FILTER', + }), + ]); const filterParamQuery = applyFilterParamsToBaseSQL( updatedTableSchema.sql, diff --git a/meerkat-core/package.json b/meerkat-core/package.json index 46797006..f74bf625 100644 --- a/meerkat-core/package.json +++ b/meerkat-core/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-core", - "version": "0.0.133", + "version": "0.0.134", "dependencies": { "tslib": "^2.3.0" }, diff --git a/meerkat-core/src/ast-builder/pre-base-query-builder.ts b/meerkat-core/src/ast-builder/pre-base-query-builder.ts new file mode 100644 index 00000000..b2dcaad6 --- /dev/null +++ b/meerkat-core/src/ast-builder/pre-base-query-builder.ts @@ -0,0 +1,111 @@ +import { getAliasForAST } from '../member-formatters/get-alias'; +import { Query, TableSchema } from '../types/cube-types'; +import { MeerkatQueryFilter } from '../types/cube-types/index'; +import { BASE_TABLE_NAME } from '../utils/base-ast'; + +/** + * POC: AST-free, synchronous builder for `preBaseQuery`. + * + * The `cubeToDuckdbAST -> json_deserialize_sql` round-trip in the generation + * pipeline only produces the outer query skeleton: + * + * SELECT * FROM REPLACE_BASE_TABLE [WHERE ...] [GROUP BY ...] [ORDER BY ...] [LIMIT ..] + * + * The SELECT list and FROM target are placeholders that downstream string + * replacements overwrite anyway, so the only real output is the WHERE / GROUP + * BY / ORDER BY / LIMIT clauses. + * + * This builder reproduces that string directly for the shapes where the WHERE + * clause is empty (no projection filters). The WHERE/HAVING operator grammar is + * intentionally out of scope for the POC — `canBuildPreBaseQuerySync` returns + * false for those, so the caller falls back to the AST round-trip. + */ + +/** + * Reproduce DuckDB's identifier rendering for a COLUMN_REF the way + * `json_deserialize_sql` emits it: a bare identifier is left unquoted, anything + * that is not a simple `[A-Za-z_][A-Za-z0-9_]*` token is wrapped in double + * quotes (with embedded quotes doubled). `getAliasForAST` hands us the raw + * alias (unquoted) because the AST path relies on DuckDB to quote on output; + * here we must do that quoting ourselves to stay byte-identical. + */ +const SIMPLE_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/; + +const renderIdentifier = (identifier: string): string => { + if (SIMPLE_IDENTIFIER.test(identifier)) { + return identifier; + } + return `"${identifier.replace(/"/g, '""')}"`; +}; + +const isEmptyFilterGroup = (filter: MeerkatQueryFilter): boolean => { + if ('member' in filter) { + return false; + } + if ('and' in filter) { + return filter.and.every(isEmptyFilterGroup); + } + if ('or' in filter) { + return filter.or.every(isEmptyFilterGroup); + } + // Unknown shape — treat as non-empty to be safe (forces AST fallback). + return false; +}; + +/** + * True when the AST-free builder can safely produce a byte-identical + * `preBaseQuery`. Currently limited to queries whose filters contribute no + * WHERE/HAVING clause (empty filter groups or none at all). + */ +export const canBuildPreBaseQuerySync = (query: Query): boolean => { + const filters = query.filters; + if (!filters || filters.length === 0) { + return true; + } + return filters.every(isEmptyFilterGroup); +}; + +/** + * Build the `preBaseQuery` string without constructing an AST or hitting + * DuckDB. Callers MUST gate on `canBuildPreBaseQuerySync(query)` first. + */ +export const buildPreBaseQuerySync = ( + query: Query, + tableSchema: TableSchema +): string => { + const clauses: string[] = [`SELECT * FROM ${BASE_TABLE_NAME}`]; + + // GROUP BY: only when the query aggregates (measures present) and projects + // dimensions — matches `cubeToDuckdbAST`'s guard. + const dimensions = query.dimensions ?? []; + if (query.measures.length > 0 && dimensions.length > 0) { + const groupByCols = dimensions.map((dimension) => + renderIdentifier(getAliasForAST(dimension, tableSchema)) + ); + clauses.push(`GROUP BY ${groupByCols.join(', ')}`); + } + + // ORDER BY: one entry per order key, alias + direction. + if (query.order && Object.keys(query.order).length > 0) { + const orderParts = Object.entries(query.order).map(([key, direction]) => { + const alias = renderIdentifier(getAliasForAST(key, tableSchema)); + const dir = direction === 'desc' ? 'DESC' : 'ASC'; + return `${alias} ${dir}`; + }); + clauses.push(`ORDER BY ${orderParts.join(', ')}`); + } + + // LIMIT / OFFSET. + if (query.limit || query.offset) { + let limitClause = ''; + if (query.limit) { + limitClause += `LIMIT ${query.limit}`; + } + if (query.offset) { + limitClause += limitClause ? ` OFFSET ${query.offset}` : `OFFSET ${query.offset}`; + } + clauses.push(limitClause); + } + + return clauses.join(' '); +}; diff --git a/meerkat-core/src/index.ts b/meerkat-core/src/index.ts index c91e1f4e..2c664aa5 100644 --- a/meerkat-core/src/index.ts +++ b/meerkat-core/src/index.ts @@ -1,4 +1,5 @@ export * from './ast-builder/ast-builder'; +export * from './ast-builder/pre-base-query-builder'; export * from './ast-deserializer/ast-deserializer'; export * from './ast-serializer/ast-serializer'; export * from './ast-validator'; diff --git a/meerkat-core/src/joins/index.ts b/meerkat-core/src/joins/index.ts index 45273d1b..72797c50 100644 --- a/meerkat-core/src/joins/index.ts +++ b/meerkat-core/src/joins/index.ts @@ -15,8 +15,8 @@ export const getCombinedTableSchema = ( return getCombinedTableSchemaV1(tableSchema, cubeQuery); }; +export type { Graph } from './v1/joins'; export { - Graph, checkLoopInJoinPath, createDirectedGraph, generateSqlQuery, diff --git a/meerkat-core/src/joins/v2/joins.ts b/meerkat-core/src/joins/v2/joins.ts index d91a2efb..df46fc86 100644 --- a/meerkat-core/src/joins/v2/joins.ts +++ b/meerkat-core/src/joins/v2/joins.ts @@ -1,6 +1,7 @@ import { getUsedTableSchema } from '../../get-used-table-schema/get-used-table-schema'; import { memberKeyToSafeKey } from '../../member-formatters/member-key-to-safe-key'; -import { Graph, quoteIdentifierIfNeeded } from '../v1/joins'; +import type { Graph } from '../v1/joins'; +import { quoteIdentifierIfNeeded } from '../v1/joins'; import { Query, StructuredJoin, TableSchema } from '../../types/cube-types'; const UNNEST_ALIAS_PREFIX = '__mk_u_'; diff --git a/meerkat-core/src/utils/get-possible-nodes.ts b/meerkat-core/src/utils/get-possible-nodes.ts index 3bf8b482..29260503 100644 --- a/meerkat-core/src/utils/get-possible-nodes.ts +++ b/meerkat-core/src/utils/get-possible-nodes.ts @@ -1,4 +1,5 @@ -import { Graph, checkLoopInJoinPath, createDirectedGraph } from '../joins'; +import type { Graph } from '../joins'; +import { checkLoopInJoinPath, createDirectedGraph } from '../joins'; import { Dimension, JoinPath, diff --git a/meerkat-node/package.json b/meerkat-node/package.json index 630b4e94..9cea0ac3 100644 --- a/meerkat-node/package.json +++ b/meerkat-node/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-node", - "version": "0.0.133", + "version": "0.0.134", "dependencies": { "@devrev/meerkat-core": "*", "@swc/helpers": "~0.5.0", diff --git a/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts b/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts new file mode 100644 index 00000000..611d2355 --- /dev/null +++ b/meerkat-node/src/__tests__/dim-issue-e2e-perf.spec.ts @@ -0,0 +1,59 @@ +import { cubeQueryToSQL } from '../cube-to-sql/cube-to-sql'; +import { duckdbExec } from '../duckdb-exec'; +import { buildDimIssueFixture } from './helpers/dim-issue-fixture'; + +/** + * End-to-end generation timing for a schema shaped like the real `dim_issue` + * cube: ~360 dimensions (plain columns, json_extract_string on custom_fields, + * string_array members with shouldUnnestGroupBy, computed CASE/epoch + * expressions) plus ~72 measures. Projects the small fixed set of members from + * the user's input, empty filters, ordered by a row-order column. + * + * This shape has NO filter params, so the only DuckDB round-trip in generation + * is the preBaseQuery deserialize (AST -> SQL text). The AST-free + * `buildPreBaseQuerySync` fast-path removes it: measured in isolation the + * preBaseQuery step drops from ~0.14ms (round-trip) to ~0.004ms (~31x), taking + * full cubeQueryToSQL for this input from ~0.38ms down to ~0.25ms. The + * remaining cost is pure CPU (getFinalBaseSQL wrapped-projection build + + * applyProjectionToSQLQuery), no DuckDB. + */ + +describe('dim_issue end-to-end generation', () => { + beforeAll(async () => { + await duckdbExec(` + CREATE TABLE dim_issue ( + id VARCHAR, space_id VARCHAR, subtype VARCHAR, title VARCHAR, + display_id VARCHAR, created_date TIMESTAMP, target_close_date TIMESTAMP, + actual_close_date TIMESTAMP, links_json VARCHAR, sla_summary VARCHAR, + stage_json VARCHAR, priority_uenum_json VARCHAR, custom_fields VARCHAR, + owned_by_ids VARCHAR[], custom_schema_fragment_ids VARCHAR[], + __fdl_row_order__ INTEGER + ); + `); + }); + + it('reports total cubeQueryToSQL wall time', async () => { + const { schema, query } = buildDimIssueFixture(); + const contextParams = { + current_dev_user_id: 'don:identity:dvrv-in-1:devo/2sRI6Hepzz:devu/9882', + }; + + // warmup + await cubeQueryToSQL({ query, tableSchemas: [schema], contextParams }); + + const ITER = 50; + const start = process.hrtime.bigint(); + for (let i = 0; i < ITER; i += 1) { + await cubeQueryToSQL({ query, tableSchemas: [schema], contextParams }); + } + const avgMs = Number(process.hrtime.bigint() - start) / 1e6 / ITER; + + console.info( + `[perf] dim_issue full cubeQueryToSQL avg over ${ITER}: ${avgMs.toFixed(3)}ms ` + + `(dims=${schema.dimensions.length}, measures=${schema.measures.length}, ` + + `projected=${query.dimensions?.length})` + ); + + expect(avgMs).toBeGreaterThan(0); + }); +}); diff --git a/meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts b/meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts new file mode 100644 index 00000000..bc0b74ac --- /dev/null +++ b/meerkat-node/src/__tests__/helpers/dim-issue-fixture.ts @@ -0,0 +1,118 @@ +import { Query, TableSchema } from '@devrev/meerkat-core'; + +/** + * Builds a schema shaped like the production `dim_issue` cube: ~360 dimensions + * (plain columns, json_extract_string on custom_fields, string_array members + * with shouldUnnestGroupBy, computed CASE/epoch expressions) plus ~72 measures, + * with a small fixed projection. Used by the wide-schema perf specs. + */ +const DIM_TEMPLATES = [ + (i: number) => ({ name: `col_${i}`, sql: `col_${i}`, type: 'string' }), + (i: number) => ({ + name: `json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__field_${i}')`, + type: 'string', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `arr_${i}`, + sql: `CAST(json_extract_string(custom_fields, '$.arr_${i}') AS VARCHAR[])`, + type: 'string_array', + }), + (i: number) => ({ + modifier: { shouldUnnestGroupBy: false }, + name: `links_${i}`, + sql: `json_extract_string(links_json, '$[*].f_${i}')`, + type: 'string_array', + }), + (i: number) => ({ + name: `computed_${i}`, + sql: `case WHEN actual_close_date > created_date THEN epoch_ms(actual_close_date) - epoch_ms(created_date) ELSE null END`, + type: 'number', + }), +]; + +export const buildDimIssueFixture = (): { schema: TableSchema; query: Query } => { + const dimensions: any[] = []; + for (let i = 0; i < 360; i += 1) { + dimensions.push(DIM_TEMPLATES[i % DIM_TEMPLATES.length](i)); + } + dimensions.push({ name: 'created_date', sql: 'created_date', type: 'time' }); + dimensions.push({ name: 'id', sql: 'id', type: 'string' }); + dimensions.push({ name: 'space_id', sql: 'space_id', type: 'string' }); + dimensions.push({ name: 'subtype', sql: 'subtype', type: 'string' }); + dimensions.push({ name: 'title', sql: 'title', type: 'string' }); + dimensions.push({ name: 'display_id', sql: 'display_id', type: 'string' }); + dimensions.push({ name: 'target_close_date', sql: 'target_close_date', type: 'time' }); + dimensions.push({ name: 'links_json', sql: 'links_json', type: 'string' }); + dimensions.push({ name: 'sla_summary', sql: 'sla_summary', type: 'string' }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'owned_by_ids', + sql: 'CAST(owned_by_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'custom_schema_fragment_ids', + sql: 'CAST(custom_schema_fragment_ids AS VARCHAR[])', + type: 'string_array', + }); + dimensions.push({ + modifier: { shouldUnnestGroupBy: false }, + name: 'links_json_$0_target_object_type', + sql: "json_extract_string(links_json, '$[*].target_object_type')", + type: 'string_array', + }); + dimensions.push({ + name: 'priority_uenum_json', + sql: "CAST(json_extract_string(priority_uenum_json, '$.id') AS INT)", + type: 'string', + }); + dimensions.push({ + name: 'stage_json_$0_stage_id', + sql: "json_extract_string(stage_json, '$.stage_id')", + type: 'string', + }); + dimensions.push({ name: '__fdl_row_order__', sql: '__fdl_row_order__', type: 'number' }); + + const measures: any[] = [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }]; + for (let i = 0; i < 71; i += 1) { + measures.push({ + name: `m_json_${i}`, + sql: `json_extract_string(custom_fields, '$.ctype__m_${i}')`, + type: 'number', + }); + } + + const schema: TableSchema = { + name: 'dim_issue', + sql: 'SELECT * FROM dim_issue', + measures, + dimensions, + }; + + const query = { + dimensions: [ + 'dim_issue.created_date', + 'dim_issue.id', + 'dim_issue.links_json_$0_target_object_type', + 'dim_issue.owned_by_ids', + 'dim_issue.priority_uenum_json', + 'dim_issue.sla_summary', + 'dim_issue.space_id', + 'dim_issue.stage_json_$0_stage_id', + 'dim_issue.subtype', + 'dim_issue.target_close_date', + 'dim_issue.title', + 'dim_issue.links_json', + 'dim_issue.custom_schema_fragment_ids', + 'dim_issue.display_id', + ], + measures: [], + filters: [{ and: [] }], + order: { 'dim_issue.__fdl_row_order__': 'asc' }, + } as unknown as Query; + + return { schema, query }; +}; diff --git a/meerkat-node/src/__tests__/prebasequery-poc.spec.ts b/meerkat-node/src/__tests__/prebasequery-poc.spec.ts new file mode 100644 index 00000000..1fbc6cf1 --- /dev/null +++ b/meerkat-node/src/__tests__/prebasequery-poc.spec.ts @@ -0,0 +1,122 @@ +import { + astDeserializerQuery, + buildPreBaseQuerySync, + canBuildPreBaseQuerySync, + cubeToDuckdbAST, + deserializeQuery, + getCombinedTableSchema, +} from '@devrev/meerkat-core'; +import { duckdbExec } from '../duckdb-exec'; +import { buildDimIssueFixture } from './helpers/dim-issue-fixture'; + +/** + * POC differential test: for every gated query shape, the AST-free + * `buildPreBaseQuerySync` must produce a string byte-identical to the AST + * round-trip (`cubeToDuckdbAST -> json_deserialize_sql`). + */ + +const SCHEMA: any = { + name: 't', + sql: 'SELECT * FROM t', + measures: [{ name: 'cnt', sql: 'COUNT(*)', type: 'number' }], + dimensions: [ + { name: 'a', sql: 'a', type: 'string' }, + { name: 'b', sql: 'b', type: 'number' }, + { name: 'ts', sql: 'ts', type: 'time' }, + { name: 'row_order', sql: 'row_order', type: 'number' }, + { name: 'weird name', sql: 'weird', type: 'string', alias: 'Weird Name' }, + ], +}; + +const roundTrip = async (query: any, schema: any = SCHEMA): Promise => { + const combined = getCombinedTableSchema([schema], query); + const ast = cubeToDuckdbAST(query, combined, { filterType: 'PROJECTION_FILTER' }); + const rows = (await duckdbExec(astDeserializerQuery(ast))) as any; + return deserializeQuery(rows); +}; + +const GATED_CASES: Record = { + 'order-only-no-measure': { + dimensions: ['t.a', 't.b'], + measures: [], + filters: [{ and: [] }], + order: { 't.row_order': 'asc' }, + }, + 'measure+dims group by + order': { + dimensions: ['t.a', 't.b'], + measures: ['t.cnt'], + filters: [{ and: [] }], + order: { 't.a': 'desc' }, + }, + 'limit-offset only': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + limit: 10, + offset: 20, + }, + 'limit only': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + limit: 5, + }, + 'no order no filter': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + }, + 'no filters key at all': { + dimensions: ['t.a'], + measures: ['t.cnt'], + order: { 't.a': 'asc' }, + }, + 'custom alias order': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [] }], + order: { 't.weird name': 'desc' }, + }, + 'multi order keys': { + dimensions: ['t.a', 't.b'], + measures: ['t.cnt'], + filters: [{ and: [] }], + order: { 't.a': 'asc', 't.b': 'desc' }, + }, + 'nested empty groups': { + dimensions: ['t.a'], + measures: [], + filters: [{ and: [{ and: [] }, { or: [] }] }], + order: { 't.a': 'asc' }, + }, +}; + +describe('preBaseQuery AST-free POC — differential parity', () => { + it('all gated cases match the round-trip byte-for-byte', async () => { + for (const [label, query] of Object.entries(GATED_CASES)) { + expect(canBuildPreBaseQuerySync(query)).toBe(true); + const combined = getCombinedTableSchema([SCHEMA], query); + const expected = await roundTrip(query); + const actual = buildPreBaseQuerySync(query, combined); + expect(`${label}: ${actual}`).toBe(`${label}: ${expected}`); + } + }); + + it('gates OFF when a real filter is present', () => { + const query: any = { + dimensions: ['t.a'], + measures: ['t.cnt'], + filters: [{ and: [{ member: 't.b', operator: 'gt', values: ['5'] }] }], + }; + expect(canBuildPreBaseQuerySync(query)).toBe(false); + }); + + it('matches for the dim_issue production shape', async () => { + const { schema, query } = buildDimIssueFixture(); + expect(canBuildPreBaseQuerySync(query)).toBe(true); + const combined = getCombinedTableSchema([schema], query); + const expected = await roundTrip(query, schema); + const actual = buildPreBaseQuerySync(query, combined); + expect(actual).toBe(expected); + }); +}); diff --git a/meerkat-node/src/cube-to-sql/cube-to-sql.ts b/meerkat-node/src/cube-to-sql/cube-to-sql.ts index c7aa2494..85dfea6c 100644 --- a/meerkat-node/src/cube-to-sql/cube-to-sql.ts +++ b/meerkat-node/src/cube-to-sql/cube-to-sql.ts @@ -7,6 +7,8 @@ import { applyProjectionToSQLQuery, applySQLExpressions, astDeserializerQuery, + buildPreBaseQuerySync, + canBuildPreBaseQuerySync, cubeToDuckdbAST, deserializeQuery, detectApplyContextParamsToBaseSQL, @@ -16,6 +18,34 @@ import { } from '@devrev/meerkat-core'; import { duckdbExec } from '../duckdb-exec'; +/** + * Produce the outer query skeleton (`preBaseQuery`). + * + * For queries without a projection-filter WHERE clause we build the string + * synchronously in TS, skipping the `cubeToDuckdbAST -> json_deserialize_sql` + * DuckDB round-trip. Otherwise we fall back to the AST round-trip so the WHERE + * operator grammar stays correct. + */ +const getPreBaseQuery = async ( + query: Query, + updatedTableSchema: TableSchema +): Promise => { + if (canBuildPreBaseQuerySync(query)) { + return buildPreBaseQuerySync(query, updatedTableSchema); + } + + const ast = cubeToDuckdbAST(query, updatedTableSchema, { + filterType: 'PROJECTION_FILTER', + }); + if (!ast) { + throw new Error('Could not generate AST'); + } + const queryOutput = (await duckdbExec( + astDeserializerQuery(ast) + )) as Record[]; + return deserializeQuery(queryOutput); +}; + export interface CubeQueryToSQLParams { query: Query; tableSchemas: TableSchema[]; @@ -43,24 +73,18 @@ export const cubeQueryToSQL = async ({ const updatedTableSchema = getCombinedTableSchema(updatedTableSchemas, query); - const ast = cubeToDuckdbAST(query, updatedTableSchema, { - filterType: 'PROJECTION_FILTER', - }); - if (!ast) { - throw new Error('Could not generate AST'); - } - - const queryTemp = astDeserializerQuery(ast); - - const queryOutput = (await duckdbExec(queryTemp)) as Record[]; - const preBaseQuery = deserializeQuery(queryOutput); - - const filterParamsSQL = await getFilterParamsSQL({ - query, - tableSchema: updatedTableSchema, - filterType: 'PROJECTION_FILTER', - getQueryOutput: duckdbExec, - }); + // The preBaseQuery build (AST round-trip or AST-free) and the + // PROJECTION_FILTER param resolution are independent, so run them + // concurrently. + const [preBaseQuery, filterParamsSQL] = await Promise.all([ + getPreBaseQuery(query, updatedTableSchema), + getFilterParamsSQL({ + query, + tableSchema: updatedTableSchema, + filterType: 'PROJECTION_FILTER', + getQueryOutput: duckdbExec, + }), + ]); const filterParamQuery = applyFilterParamsToBaseSQL( updatedTableSchema.sql,