perf: AST-free preBaseQuery generation (node + browser, skip DuckDB round-trip)

benchmarking/src/app/app.tsx

@@ -6,6 +6,7 @@ import { ParallelIndexedDBMProvider } from './dbm-context/parallel-indexed-dbm-c
 import { ParallelMemoryDBMProvider } from './dbm-context/parallel-memory-dbm-context';
 import { RawDBMProvider } from './dbm-context/raw-dbm-context';
 import { FileLoader } from './file-loader/file-loader';
+import { GenBenchmarking } from './gen-benchmarking/gen-benchmarking';
 import { NativeAppFileLoader } from './file-loader/native-app-file-loader';
 import { QueryBenchmarking } from './query-benchmarking/query-benchmarking';
 
@@ -35,6 +36,7 @@ export function App() {
         </ul>
       </nav>
       <Routes>
+        <Route path="/gen-benchmark" element={<GenBenchmarking />} />
         <Route
           path="/raw-dbm"
           element={

benchmarking/src/app/gen-benchmarking/dim-issue-fixture.ts

@@ -0,0 +1,115 @@
+import { Query, TableSchema } from '@devrev/meerkat-core';
+
+/**
+ * Wide schema shaped like the production `dim_issue` cube (~360 dims + ~72
+ * measures), with the same small 14-member projection, empty filters, and a
+ * row-order ORDER BY. Mirrors the node fixture used in meerkat-node tests.
+ */
+const DIM_TEMPLATES = [
+  (i: number) => ({ name: `col_${i}`, sql: `col_${i}`, type: 'string' }),
+  (i: number) => ({
+    name: `json_${i}`,
+    sql: `json_extract_string(custom_fields, '$.ctype__field_${i}')`,
+    type: 'string',
+  }),
+  (i: number) => ({
+    modifier: { shouldUnnestGroupBy: false },
+    name: `arr_${i}`,
+    sql: `CAST(json_extract_string(custom_fields, '$.arr_${i}') AS VARCHAR[])`,
+    type: 'string_array',
+  }),
+  (i: number) => ({
+    modifier: { shouldUnnestGroupBy: false },
+    name: `links_${i}`,
+    sql: `json_extract_string(links_json, '$[*].f_${i}')`,
+    type: 'string_array',
+  }),
+  (i: number) => ({
+    name: `computed_${i}`,
+    sql: `case WHEN actual_close_date > created_date THEN epoch_ms(actual_close_date) - epoch_ms(created_date) ELSE null END`,
+    type: 'number',
+  }),
+];
+
+export const buildDimIssueSchemaAndQuery = (): {
+  schema: TableSchema;
+  query: Query;
+} => {
+  const dimensions: any[] = [];
+  for (let i = 0; i < 360; i += 1) {
+    dimensions.push(DIM_TEMPLATES[i % DIM_TEMPLATES.length](i));
+  }
+  dimensions.push({ name: 'created_date', sql: 'created_date', type: 'time' });
+  dimensions.push({ name: 'id', sql: 'id', type: 'string' });
+  dimensions.push({ name: 'space_id', sql: 'space_id', type: 'string' });
+  dimensions.push({ name: 'subtype', sql: 'subtype', type: 'string' });
+  dimensions.push({ name: 'title', sql: 'title', type: 'string' });
+  dimensions.push({ name: 'display_id', sql: 'display_id', type: 'string' });
+  dimensions.push({ name: 'target_close_date', sql: 'target_close_date', type: 'time' });
+  dimensions.push({ name: 'links_json', sql: 'links_json', type: 'string' });
+  dimensions.push({ name: 'sla_summary', sql: 'sla_summary', type: 'string' });
+  dimensions.push({
+    modifier: { shouldUnnestGroupBy: false },
+    name: 'owned_by_ids',
+    sql: 'CAST(owned_by_ids AS VARCHAR[])',
+    type: 'string_array',
+  });
+  dimensions.push({
+    modifier: { shouldUnnestGroupBy: false },
+    name: 'custom_schema_fragment_ids',
+    sql: 'CAST(custom_schema_fragment_ids AS VARCHAR[])',
+    type: 'string_array',
+  });
+  dimensions.push({
+    modifier: { shouldUnnestGroupBy: false },
+    name: 'links_json_$0_target_object_type',
+    sql: "json_extract_string(links_json, '$[*].target_object_type')",
+    type: 'string_array',
+  });
+  dimensions.push({
+    name: 'priority_uenum_json',
+    sql: "CAST(json_extract_string(priority_uenum_json, '$.id') AS INT)",
+    type: 'string',
+  });
+  dimensions.push({
+    name: 'stage_json_$0_stage_id',
+    sql: "json_extract_string(stage_json, '$.stage_id')",
+    type: 'string',
+  });
+  dimensions.push({ name: '__fdl_row_order__', sql: '__fdl_row_order__', type: 'number' });
+
+  const measures: any[] = [{ name: 'count_star', sql: 'COUNT(*)', type: 'number' }];
+  for (let i = 0; i < 71; i += 1) {
+    measures.push({
+      name: `m_json_${i}`,
+      sql: `json_extract_string(custom_fields, '$.ctype__m_${i}')`,
+      type: 'number',
+    });
+  }
+
+  const schema = { name: 'dim_issue', sql: 'SELECT * FROM dim_issue', measures, dimensions } as TableSchema;
+
+  const query = {
+    dimensions: [
+      'dim_issue.created_date',
+      'dim_issue.id',
+      'dim_issue.links_json_$0_target_object_type',
+      'dim_issue.owned_by_ids',
+      'dim_issue.priority_uenum_json',
+      'dim_issue.sla_summary',
+      'dim_issue.space_id',
+      'dim_issue.stage_json_$0_stage_id',
+      'dim_issue.subtype',
+      'dim_issue.target_close_date',
+      'dim_issue.title',
+      'dim_issue.links_json',
+      'dim_issue.custom_schema_fragment_ids',
+      'dim_issue.display_id',
+    ],
+    measures: [],
+    filters: [{ and: [] }],
+    order: { 'dim_issue.__fdl_row_order__': 'asc' },
+  } as unknown as Query;
+
+  return { schema, query };
+};

benchmarking/src/app/gen-benchmarking/gen-benchmarking.tsx

@@ -0,0 +1,179 @@
+import { cubeQueryToSQL } from '@devrev/meerkat-browser';
+import {
+  astDeserializerQuery,
+  cubeToDuckdbAST,
+  deserializeQuery,
+  getCombinedTableSchema,
+} from '@devrev/meerkat-core';
+import * as duckdb from '@duckdb/duckdb-wasm';
+import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
+import { useState } from 'react';
+import { useClassicEffect } from '../hooks/use-classic-effect';
+import { buildDimIssueSchemaAndQuery } from './dim-issue-fixture';
+
+const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
+
+/**
+ * Browser generation benchmark.
+ *
+ * Runs the REAL `cubeQueryToSQL` (browser) against a real duckdb-wasm Web
+ * Worker connection. Each internal `connection.query(...)` is a postMessage hop
+ * to the worker, so this measures the true cost of the AST deserialize
+ * round-trip vs the AST-free fast path — and worker contention when many
+ * generations fire in parallel.
+ *
+ * Emits results into `#gen_results` (JSON) for puppeteer to read.
+ */
+const ITER = 30;
+const PARALLEL_SWEEP = [1, 5, 10, 25, 50, 100];
+
+const avg = async (fn: () => Promise<void>, iterations: number) => {
+  await fn(); // warmup
+  const start = performance.now();
+  for (let i = 0; i < iterations; i += 1) await fn();
+  return (performance.now() - start) / iterations;
+};
+
+const connectWasm = async (): Promise<AsyncDuckDBConnection> => {
+  const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+  const worker_url = URL.createObjectURL(
+    new Blob([`importScripts("${bundle.mainWorker!}");`], {
+      type: 'text/javascript',
+    })
+  );
+  const worker = new Worker(worker_url);
+  const logger = new duckdb.VoidLogger();
+  const db = new duckdb.AsyncDuckDB(logger, worker);
+  await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+  URL.revokeObjectURL(worker_url);
+  return db.connect();
+};
+
+export const GenBenchmarking = () => {
+  const [results, setResults] = useState<any>(null);
+
+  useClassicEffect(() => {
+    (async () => {
+      const connection = await connectWasm();
+      // Base table so generated SQL is valid if executed; generation itself
+      // does not need rows.
+      await connection.query(
+        `CREATE TABLE dim_issue (custom_fields VARCHAR, links_json VARCHAR, priority_uenum_json VARCHAR, stage_json VARCHAR, owned_by_ids VARCHAR[], custom_schema_fragment_ids VARCHAR[], created_date TIMESTAMP, id VARCHAR, space_id VARCHAR, subtype VARCHAR, title VARCHAR, display_id VARCHAR, target_close_date TIMESTAMP, sla_summary VARCHAR, actual_close_date TIMESTAMP, __fdl_row_order__ INTEGER);`
+      );
+
+      const { schema, query } = buildDimIssueSchemaAndQuery();
+      const contextParams = { current_dev_user_id: 'devu/1' };
+
+      const combined = getCombinedTableSchema([schema], query);
+
+      // Count worker hits (postMessage round-trips) per full generation, both
+      // paths, by wrapping connection.query.
+      const countHits = async (fn: () => Promise<unknown>) => {
+        const orig = connection.query.bind(connection);
+        let hits = 0;
+        (connection as any).query = (...a: any[]) => {
+          hits += 1;
+          return (orig as any)(...a);
+        };
+        await fn();
+        (connection as any).query = orig;
+        return hits;
+      };
+
+      // FAST PATH full generation (empty filters → AST-free preBaseQuery).
+      const fastGen = () =>
+        cubeQueryToSQL({ connection, query, tableSchemas: [schema], contextParams });
+
+      // BASELINE full generation: same pipeline but forced through the AST
+      // deserialize round-trip for preBaseQuery (what shipped before this PR).
+      const baselineGen = async () => {
+        const ast = cubeToDuckdbAST(query, combined, { filterType: 'PROJECTION_FILTER' });
+        const arrow = await connection.query(astDeserializerQuery(ast as any));
+        // deserialize + the same downstream string work the real pipeline does
+        deserializeQuery(arrow.toArray().map((r) => r.toJSON()));
+      };
+
+      const fastHits = await countHits(fastGen);
+      const baselineHits = await countHits(baselineGen);
+
+      const fastSeqMs = await avg(fastGen, ITER);
+      const baselineSeqMs = await avg(baselineGen, ITER);
+
+      // Load real data first so the concurrent heavy query actually occupies
+      // the worker (realistic contention). Do this BEFORE the sweep.
+      await connection.query(
+        `INSERT INTO dim_issue (id, __fdl_row_order__) SELECT CAST(i AS VARCHAR), i FROM range(200000) t(i);`
+      );
+      const heavyQuery = `SELECT COUNT(*) FROM (SELECT id FROM dim_issue ORDER BY __fdl_row_order__ DESC LIMIT 100000) a JOIN dim_issue b ON a.id = b.id;`;
+
+      const median = (xs: number[]) =>
+        xs.slice().sort((a, b) => a - b)[Math.floor(xs.length / 2)];
+
+      // Sweep: fire N generations concurrently WHILE a heavy data query runs on
+      // the worker. Measures how long the batch of generations takes to finish
+      // under realistic worker contention. Median of 5 to suppress noise.
+      const batchUnderLoad = async (fn: () => Promise<unknown>, n: number) => {
+        const heavy = connection.query(heavyQuery); // occupy the worker
+        const start = performance.now();
+        await Promise.all(Array.from({ length: n }, fn));
+        const ms = performance.now() - start;
+        await heavy;
+        return ms;
+      };
+
+      const sweep: any[] = [];
+      for (const n of PARALLEL_SWEEP) {
+        const fastRuns: number[] = [];
+        const baseRuns: number[] = [];
+        for (let r = 0; r < 5; r += 1) {
+          fastRuns.push(await batchUnderLoad(fastGen, n));
+          baseRuns.push(await batchUnderLoad(baselineGen, n));
+        }
+        const fast = median(fastRuns);
+        const baseline = median(baseRuns);
+        sweep.push({
+          n,
+          fastMs: Number(fast.toFixed(3)),
+          baselineMs: Number(baseline.toFixed(3)),
+          speedup: Number((baseline / fast).toFixed(2)),
+        });
+      }
+
+      // Head-of-line blocking: single heavy data query + ONE generation.
+      const measureUnderLoad = async (gen: () => Promise<unknown>) => {
+        const heavy = connection.query(heavyQuery);
+        const s = performance.now();
+        await gen();
+        const genLatency = performance.now() - s;
+        await heavy;
+        return genLatency;
+      };
+      const fastUnderLoadMs = await measureUnderLoad(fastGen);
+      const baselineUnderLoadMs = await measureUnderLoad(baselineGen);
+
+      setResults({
+        fastPathWorkerHits: fastHits,
+        baselineWorkerHits: baselineHits,
+        fastSeqMs: Number(fastSeqMs.toFixed(4)),
+        baselineSeqMs: Number(baselineSeqMs.toFixed(4)),
+        seqSpeedup: Number((baselineSeqMs / fastSeqMs).toFixed(2)),
+        parallelSweep: sweep,
+        genLatencyUnderWorkerLoad: {
+          fastMs: Number(fastUnderLoadMs.toFixed(4)),
+          baselineMs: Number(baselineUnderLoadMs.toFixed(4)),
+        },
+      });
+    })();
+  }, []);
+
+  return (
+    <div>
+      <h1>Generation Benchmark</h1>
+      {results ? (
+        <pre id="gen_results">{JSON.stringify(results)}</pre>
+      ) : (
+        <div>running…</div>
+      )}
+    </div>
+  );
+};

meerkat-browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-browser",
-  "version": "0.0.133",
+  "version": "0.0.134",
   "dependencies": {
     "tslib": "^2.3.0",
     "@devrev/meerkat-core": "*",