feat(dbm): add per-query onEvent to QueryOptions

meerkat-browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-browser",
-  "version": "0.0.133",
+  "version": "0.0.134",
   "dependencies": {
     "tslib": "^2.3.0",
     "@devrev/meerkat-core": "*",

meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts

@@ -2,6 +2,7 @@ import {
   BASE_TABLE_NAME,
   ContextParams,
   Query,
+  SqlGenerationOnEvent,
   TableSchema,
   applyFilterParamsToBaseSQL,
   applyProjectionToSQLQuery,
@@ -13,6 +14,7 @@ import {
   getCombinedTableSchema,
   getFilterParamsSQL,
   getFinalBaseSQL,
+  timePhase,
 } from '@devrev/meerkat-core';
 import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
 
@@ -30,90 +32,132 @@ export interface CubeQueryToSQLParams {
   query: Query;
   tableSchemas: TableSchema[];
   contextParams?: ContextParams;
+  /**
+   * Optional per-call callback for SQL-generation phase timings. Invoked once
+   * per phase plus once for `total`. No-op when omitted.
+   */
+  onEvent?: SqlGenerationOnEvent;
 }
 
 export const cubeQueryToSQL = async ({
   connection,
   query,
   tableSchemas,
   contextParams,
+  onEvent,
 }: CubeQueryToSQLParams) => {
-  const updatedTableSchemas: TableSchema[] = await Promise.all(
-    tableSchemas.map(async (schema: TableSchema) => {
-      const baseFilterParamsSQL = await getFinalBaseSQL({
-        query,
-        tableSchema: schema,
-        getQueryOutput: (query) => getQueryOutput(query, connection),
-      });
-      return {
-        ...schema,
-        sql: baseFilterParamsSQL,
-      };
-    })
+  const totalStart = performance.now();
+
+  const updatedTableSchemas = await timePhase(
+    'base_sql',
+    () =>
+      Promise.all(
+        tableSchemas.map(async (schema: TableSchema) => {
+          const baseFilterParamsSQL = await getFinalBaseSQL({
+            query,
+            tableSchema: schema,
+            getQueryOutput: (query) => getQueryOutput(query, connection),
+          });
+          return {
+            ...schema,
+            sql: baseFilterParamsSQL,
+          };
+        })
+      ),
+    onEvent
   );
 
   const updatedTableSchema = await getCombinedTableSchema(
     updatedTableSchemas,
     query
   );
 
-  const ast = cubeToDuckdbAST(query, updatedTableSchema, {
-    filterType: 'PROJECTION_FILTER',
-  });
+  const ast = await timePhase(
+    'ast_build',
+    async () =>
+      cubeToDuckdbAST(query, updatedTableSchema, {
+        filterType: 'PROJECTION_FILTER',
+      }),
+    onEvent
+  );
   if (!ast) {
     throw new Error('Could not generate AST');
   }
 
-  const queryTemp = astDeserializerQuery(ast);
+  const preBaseQuery = await timePhase(
+    'ast_deserialize_roundtrip',
+    async () => {
+      const queryTemp = astDeserializerQuery(ast);
+      const arrowResult = await connection.query(queryTemp);
+      const parsedOutputQuery = arrowResult
+        .toArray()
+        .map((row) => row.toJSON());
+      return deserializeQuery(parsedOutputQuery);
+    },
+    onEvent
+  );
 
-  const arrowResult = await connection.query(queryTemp);
-  const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON());
+  const filterParamsSQL = await timePhase(
+    'filter_params',
+    () =>
+      getFilterParamsSQL({
+        getQueryOutput: (query) => getQueryOutput(query, connection),
+        query,
+        tableSchema: updatedTableSchema,
+        filterType: 'PROJECTION_FILTER',
+      }),
+    onEvent
+  );
 
-  const preBaseQuery = deserializeQuery(parsedOutputQuery);
-  const filterParamsSQL = await getFilterParamsSQL({
-    getQueryOutput: (query) => getQueryOutput(query, connection),
-    query,
-    tableSchema: updatedTableSchema,
-    filterType: 'PROJECTION_FILTER',
-  });
+  const finalQuery = await timePhase(
+    'projections',
+    async () => {
+      const filterParamQuery = applyFilterParamsToBaseSQL(
+        updatedTableSchema.sql,
+        filterParamsSQL
+      );
 
-  const filterParamQuery = applyFilterParamsToBaseSQL(
-    updatedTableSchema.sql,
-    filterParamsSQL
-  );
+      /**
+       * Replace CONTEXT_PARAMS with context params
+       */
+      const baseQuery = detectApplyContextParamsToBaseSQL(
+        filterParamQuery,
+        contextParams || {}
+      );
 
-  /**
-   * Replace CONTEXT_PARAMS with context params
-   */
-  const baseQuery = detectApplyContextParamsToBaseSQL(
-    filterParamQuery,
-    contextParams || {}
-  );
+      /**
+       * Replace BASE_TABLE_NAME with cube query
+       */
+      const replaceBaseTableName = preBaseQuery.replace(
+        BASE_TABLE_NAME,
+        `(${baseQuery}) AS ${updatedTableSchema.name}`
+      );
 
-  /**
-   * Replace BASE_TABLE_NAME with cube query
-   */
-  const replaceBaseTableName = preBaseQuery.replace(
-    BASE_TABLE_NAME,
-    `(${baseQuery}) AS ${updatedTableSchema.name}`
-  );
+      /**
+       * Add measures to the query
+       */
+      const measures = query.measures;
+      const dimensions = query.dimensions || [];
+      const queryWithProjections = applyProjectionToSQLQuery(
+        dimensions,
+        measures,
+        updatedTableSchemas,
+        replaceBaseTableName
+      );
 
-  /**
-   * Add measures to the query
-   */
-  const measures = query.measures;
-  const dimensions = query.dimensions || [];
-  const queryWithProjections = applyProjectionToSQLQuery(
-    dimensions,
-    measures,
-    updatedTableSchemas,
-    replaceBaseTableName
+      /**
+       * Replace SQL expression placeholders with actual SQL
+       */
+      return applySQLExpressions(queryWithProjections);
+    },
+    onEvent
   );
 
-  /**
-   * Replace SQL expression placeholders with actual SQL
-   */
-  const finalQuery = applySQLExpressions(queryWithProjections);
+  onEvent?.({
+    event_name: 'sql_generation_duration',
+    phase: 'total',
+    duration: performance.now() - totalStart,
+  });
 
   return finalQuery;
 };

meerkat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-core",
-  "version": "0.0.133",
+  "version": "0.0.134",
   "dependencies": {
     "tslib": "^2.3.0"
   },

meerkat-core/src/index.ts

@@ -47,3 +47,4 @@ export type {
   EnsureColumnAliasBatchResult,
 } from './utils/ensure-sql-expression-column-alias';
 export * from './utils/ensure-table-schema-alias-sql';
+export * from './instrumentation/sql-generation-event';

meerkat-core/src/instrumentation/sql-generation-event.spec.ts

@@ -0,0 +1,60 @@
+import { SqlGenerationEvent, timePhase } from './sql-generation-event';
+
+describe('timePhase', () => {
+  it('returns the wrapped result unchanged', async () => {
+    const result = await timePhase('base_sql', async () => 'sql-string');
+    expect(result).toBe('sql-string');
+  });
+
+  it('does not invoke onEvent when omitted', async () => {
+    // Simply must not throw when no callback is supplied.
+    await expect(
+      timePhase('ast_build', async () => 42)
+    ).resolves.toBe(42);
+  });
+
+  it('emits a sql_generation_duration event for the phase when onEvent is supplied', async () => {
+    const events: SqlGenerationEvent[] = [];
+
+    await timePhase('filter_params', async () => 'x', (e) => events.push(e));
+
+    expect(events).toHaveLength(1);
+    expect(events[0]).toEqual(
+      expect.objectContaining({
+        event_name: 'sql_generation_duration',
+        phase: 'filter_params',
+      })
+    );
+    expect(typeof events[0].duration).toBe('number');
+    expect(events[0].duration).toBeGreaterThanOrEqual(0);
+  });
+
+  it('forwards metadata onto the event', async () => {
+    const events: SqlGenerationEvent[] = [];
+
+    await timePhase(
+      'projections',
+      async () => null,
+      (e) => events.push(e),
+      { queryId: 'q-1' }
+    );
+
+    expect(events[0].metadata).toEqual({ queryId: 'q-1' });
+  });
+
+  it('does not emit an event when the phase throws', async () => {
+    const events: SqlGenerationEvent[] = [];
+
+    await expect(
+      timePhase(
+        'ast_deserialize_roundtrip',
+        async () => {
+          throw new Error('boom');
+        },
+        (e) => events.push(e)
+      )
+    ).rejects.toThrow('boom');
+
+    expect(events).toHaveLength(0);
+  });
+});

meerkat-core/src/instrumentation/sql-generation-event.ts

@@ -0,0 +1,59 @@
+/**
+ * Instrumentation for the cube-query -> SQL generation path. This lives in
+ * meerkat-core (which meerkat-node/meerkat-browser already depend on) so the
+ * SQL builders can emit timings without taking a dependency on meerkat-dbm's
+ * DBMEvent. It is intentionally a separate event shape from DBMEvent: the two
+ * cover different phases (generation vs. execution) and different packages.
+ */
+
+/**
+ * Distinct phases of `cubeQueryToSQL`. `total` wraps the whole call; the rest
+ * are the individually timed steps that add up to (roughly) it.
+ */
+export type SqlGenerationPhase =
+  | 'base_sql'
+  | 'ast_build'
+  | 'ast_deserialize_roundtrip'
+  | 'filter_params'
+  | 'projections'
+  | 'total';
+
+export interface SqlGenerationEvent {
+  event_name: 'sql_generation_duration';
+  phase: SqlGenerationPhase;
+  /** Duration in milliseconds. */
+  duration: number;
+  metadata?: object;
+}
+
+/**
+ * Optional per-call callback for SQL-generation timings. Supplied by the caller
+ * of `cubeQueryToSQL`; invoked once per phase (plus once for `total`).
+ */
+export type SqlGenerationOnEvent = (event: SqlGenerationEvent) => void;
+
+/**
+ * Time an async phase and, when an `onEvent` is supplied, emit its duration.
+ * Returns the phase's result untouched so call sites read as a thin wrapper.
+ * The event fires even if `fn` throws is NOT desired here — a failed phase has
+ * no meaningful duration to report, so timing is emitted only on success.
+ */
+export const timePhase = async <T>(
+  phase: SqlGenerationPhase,
+  fn: () => Promise<T>,
+  onEvent?: SqlGenerationOnEvent,
+  metadata?: object
+): Promise<T> => {
+  if (!onEvent) {
+    return fn();
+  }
+  const start = performance.now();
+  const result = await fn();
+  onEvent({
+    event_name: 'sql_generation_duration',
+    phase,
+    duration: performance.now() - start,
+    metadata,
+  });
+  return result;
+};

meerkat-dbm/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-dbm",
-  "version": "0.1.47",
+  "version": "0.1.48",
   "dependencies": {
     "@duckdb/duckdb-wasm": "1.32.0",
     "apache-arrow": "^17.0.0",