diff --git a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts index ecf7165e..942984e2 100644 --- a/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts +++ b/meerkat-browser/src/browser-cube-to-sql/browser-cube-to-sql.ts @@ -54,7 +54,8 @@ export const cubeQueryToSQL = async ({ const updatedTableSchema = await getCombinedTableSchema( updatedTableSchemas, - query + query, + (q) => getQueryOutput(q, connection) ); const ast = cubeToDuckdbAST(query, updatedTableSchema, { diff --git a/meerkat-core/src/joins/index.spec.ts b/meerkat-core/src/joins/index.spec.ts index 6e352fdb..0b6d3237 100644 --- a/meerkat-core/src/joins/index.spec.ts +++ b/meerkat-core/src/joins/index.spec.ts @@ -41,7 +41,7 @@ const withArrayCols = ( }); describe('joins router', () => { - it('routes to v2 when joinPathsV2 is set', () => { + it('routes to v2 when joinPathsV2 is set', async () => { const schemas = [ withArrayCols('issues', ['id'], ['owned_by_ids']), scalar('users'), @@ -58,12 +58,12 @@ describe('joins router', () => { ], ], }; - const result = getCombinedTableSchema(schemas, cubeQuery); + const result = await getCombinedTableSchema(schemas, cubeQuery); expect(result.sql).toContain('UNNEST(owned_by_ids)'); expect(result.sql).not.toMatch(/CONTAINS/i); }); - it('routes to v1 when joinPathsV2 is absent', () => { + it('routes to v1 when joinPathsV2 is absent', async () => { const schemas = [ { ...scalar('orders', ['id', 'customer_id']), @@ -76,7 +76,7 @@ describe('joins router', () => { dimensions: ['customers.id'], joinPaths: [[{ left: 'orders', right: 'customers', on: 'customer_id' }]], }; - const result = getCombinedTableSchema(schemas, cubeQuery); + const result = await getCombinedTableSchema(schemas, cubeQuery); expect(result.sql).toContain('orders.customer_id = customers.id'); expect(result.sql).not.toMatch(/UNNEST/); }); diff --git a/meerkat-core/src/joins/index.ts b/meerkat-core/src/joins/index.ts index 45273d1b..e937e65c 100644 --- a/meerkat-core/src/joins/index.ts +++ b/meerkat-core/src/joins/index.ts @@ -1,16 +1,18 @@ import type { Query } from '../types/cube-types'; +import type { GetQueryOutput } from '../utils/duckdb-ast-parse-serialize'; import { hasJoinPathsV2 } from './accessors'; import { getCombinedTableSchemaV2 } from './v2/joins'; import { getCombinedTableSchema as getCombinedTableSchemaV1 } from './v1/joins'; export { hasAnyJoinPaths, hasJoinPathsV2 } from './accessors'; -export const getCombinedTableSchema = ( +export const getCombinedTableSchema = async ( tableSchema: Parameters[0], - cubeQuery: Query + cubeQuery: Query, + getQueryOutput?: GetQueryOutput ) => { if (hasJoinPathsV2(cubeQuery)) { - return getCombinedTableSchemaV2(tableSchema, cubeQuery); + return getCombinedTableSchemaV2(tableSchema, cubeQuery, getQueryOutput); } return getCombinedTableSchemaV1(tableSchema, cubeQuery); }; diff --git a/meerkat-core/src/joins/v2/joins.spec.ts b/meerkat-core/src/joins/v2/joins.spec.ts index 349215bb..04be0d9d 100644 --- a/meerkat-core/src/joins/v2/joins.spec.ts +++ b/meerkat-core/src/joins/v2/joins.spec.ts @@ -1,4 +1,5 @@ -import { StructuredJoin, TableSchema } from '../../types/cube-types'; +import { MeerkatQueryFilter, StructuredJoin, TableSchema } from '../../types/cube-types'; +import { GetQueryOutput } from '../../utils/duckdb-ast-parse-serialize'; import { createDirectedGraphV2, generateSqlQueryV2 } from './joins'; const scalar = (name: string, cols: string[] = ['id']): TableSchema => ({ @@ -43,7 +44,7 @@ const sqlMapOf = (schemas: TableSchema[]): { [k: string]: string } => ); describe('joins-v2', () => { - it('emits a plain equi-join when from is scalar', () => { + it('emits a plain equi-join when from is scalar', async () => { const schemas = [scalar('orders', ['id', 'customer_id']), scalar('customers')]; const sqlMap = sqlMapOf(schemas); const paths: StructuredJoin[][] = [ @@ -55,13 +56,13 @@ describe('joins-v2', () => { ], ]; const graph = createDirectedGraphV2(schemas, sqlMap, paths); - const sql = generateSqlQueryV2(paths, sqlMap, graph, schemas); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); expect(sql).not.toMatch(/UNNEST/); expect(sql).toContain('orders.customer_id = customers.id'); }); - it('wraps the base with UNNEST when from.column is array-typed', () => { + it('wraps the base with UNNEST when from.column is array-typed', async () => { const schemas = [ withArrayCols('issues', ['id'], ['owned_by_ids']), scalar('users'), @@ -76,14 +77,14 @@ describe('joins-v2', () => { ], ]; const graph = createDirectedGraphV2(schemas, sqlMap, paths); - const sql = generateSqlQueryV2(paths, sqlMap, graph, schemas); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); expect(sql).toContain('UNNEST(owned_by_ids) AS __mk_u_owned_by_ids'); expect(sql).toContain('issues.__mk_u_owned_by_ids = users.id'); expect(sql).not.toMatch(/CONTAINS/i); }); - it('shares one UNNEST projection across edges on the same base array column', () => { + it('shares one UNNEST projection across edges on the same base array column', async () => { const schemas = [ withArrayCols('issues', ['id'], ['owned_by_ids']), scalar('users'), @@ -103,14 +104,14 @@ describe('joins-v2', () => { ], ]; const graph = createDirectedGraphV2(schemas, sqlMap, paths); - const sql = generateSqlQueryV2(paths, sqlMap, graph, schemas); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); expect(sql.match(/UNNEST\(owned_by_ids\)/g)).toHaveLength(1); expect(sql).toContain('issues.__mk_u_owned_by_ids = users.id'); expect(sql).toContain('issues.__mk_u_owned_by_ids = admins.id'); }); - it('inlines dim.sql for composite-child synthetic array columns whose name is not a real column', () => { + it('inlines dim.sql for composite-child synthetic array columns whose name is not a real column', async () => { // The synthetic dim `tags_$0_tag_id` is not a real column on the // base table — only the parent `tags` struct array is. UNNEST must // reference the dim's `sql` expression, not the synthetic name. @@ -141,7 +142,7 @@ describe('joins-v2', () => { ], ]; const graph = createDirectedGraphV2(schemas, sqlMap, paths); - const sql = generateSqlQueryV2(paths, sqlMap, graph, schemas); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); expect(sql).toContain( "UNNEST(json_extract_string(tags, '$[*].tag_id')) AS __mk_u_tags_$0_tag_id" @@ -149,7 +150,7 @@ describe('joins-v2', () => { expect(sql).toContain('parts.__mk_u_tags_$0_tag_id = tags.id'); }); - it('wraps a multi-hop intermediate table when its from.column is array-typed', () => { + it('wraps a multi-hop intermediate table when its from.column is array-typed', async () => { const schemas = [ scalar('tickets', ['id', 'part_id']), withArrayCols('parts', ['id'], ['tag_ids']), @@ -169,14 +170,14 @@ describe('joins-v2', () => { ], ]; const graph = createDirectedGraphV2(schemas, sqlMap, paths); - const sql = generateSqlQueryV2(paths, sqlMap, graph, schemas); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); expect(sql).toContain('tickets.part_id = parts.id'); expect(sql).toContain('UNNEST(tag_ids) AS __mk_u_tag_ids'); expect(sql).toContain('parts.__mk_u_tag_ids = tags.id'); }); - it('strips table qualifier from CAST-wrapped array dim sql (ensureTableSchemasAlias output)', () => { + it('strips table qualifier from CAST-wrapped array dim sql (ensureTableSchemasAlias output)', async () => { // After ensureTableSchemasAlias runs, the dim's sql becomes // `CAST(issue.owned_by_ids AS VARCHAR[])` instead of plain `owned_by_ids`. // The UNNEST wrap must strip the `issue.` qualifier since the inner @@ -208,7 +209,7 @@ describe('joins-v2', () => { ], ]; const graph = createDirectedGraphV2(schemas, sqlMap, paths); - const sql = generateSqlQueryV2(paths, sqlMap, graph, schemas); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); // The UNNEST expression must NOT contain `issue.` since it's in an unnamed subquery scope expect(sql).toContain('UNNEST(CAST(owned_by_ids AS VARCHAR[]))'); @@ -235,4 +236,297 @@ describe('joins-v2', () => { ); }); + it('aliases bridge tables when same table appears multiple times in a path', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'target_id', 'link_type_id']), + scalar('part', ['id']), + scalar('user', ['id']), + ]; + const sqlMap = sqlMapOf(schemas); + const bridgeTables = new Set(['link']); + const paths: StructuredJoin[][] = [ + [ + { from: { table: 'issue', column: 'id' }, to: { table: 'link', column: 'source_id' } }, + { from: { table: 'link', column: 'target_id' }, to: { table: 'part', column: 'id' } }, + { from: { table: 'issue', column: 'id' }, to: { table: 'link', column: 'source_id' } }, + { from: { table: 'link', column: 'target_id' }, to: { table: 'user', column: 'id' } }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas, undefined, bridgeTables); + + expect(sql).toContain('issue.id = link.source_id'); + expect(sql).toContain('link.target_id = part.id'); + expect(sql).toContain('issue.id = link__1.source_id'); + expect(sql).toContain('link__1.target_id = user.id'); + }); + + it('aliases bridge tables across separate paths', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'target_id', 'link_type_id']), + scalar('part', ['id']), + scalar('user', ['id']), + ]; + const sqlMap = sqlMapOf(schemas); + const bridgeTables = new Set(['link']); + const paths: StructuredJoin[][] = [ + [ + { from: { table: 'issue', column: 'id' }, to: { table: 'link', column: 'source_id' } }, + { from: { table: 'link', column: 'target_id' }, to: { table: 'part', column: 'id' } }, + ], + [ + { from: { table: 'issue', column: 'id' }, to: { table: 'link', column: 'source_id' } }, + { from: { table: 'link', column: 'target_id' }, to: { table: 'user', column: 'id' } }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas, undefined, bridgeTables); + + expect(sql).toContain('issue.id = link.source_id'); + expect(sql).toContain('link.target_id = part.id'); + expect(sql).toContain('issue.id = link__1.source_id'); + expect(sql).toContain('link__1.target_id = user.id'); + }); + + it('does not alias when table is not a bridge even if it repeats', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'target_id']), + scalar('part', ['id']), + ]; + const sqlMap = sqlMapOf(schemas); + const paths: StructuredJoin[][] = [ + [ + { from: { table: 'issue', column: 'id' }, to: { table: 'link', column: 'source_id' } }, + { from: { table: 'link', column: 'target_id' }, to: { table: 'part', column: 'id' } }, + { from: { table: 'issue', column: 'id' }, to: { table: 'link', column: 'source_id' } }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + const sql = await generateSqlQueryV2(paths, sqlMap, graph, schemas); + + expect(sql).not.toContain('link__1'); + }); + + it('throws when condition is present but getQueryOutput is not provided', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'link_type_id']), + ]; + const sqlMap = sqlMapOf(schemas); + const condition: MeerkatQueryFilter = { + member: 'link.link_type_id', + operator: 'equals', + values: ['1234'], + }; + const paths: StructuredJoin[][] = [ + [ + { + from: { table: 'issue', column: 'id' }, + to: { table: 'link', column: 'source_id' }, + condition, + }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + await expect( + generateSqlQueryV2(paths, sqlMap, graph, schemas) + ).rejects.toThrow(/getQueryOutput is required/); + }); + + it('appends serialized condition to ON clause when getQueryOutput is provided', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'link_type_id']), + ]; + const sqlMap = sqlMapOf(schemas); + const condition: MeerkatQueryFilter = { + member: 'link.link_type_id', + operator: 'equals', + values: ['1234'], + }; + const paths: StructuredJoin[][] = [ + [ + { + from: { table: 'issue', column: 'id' }, + to: { table: 'link', column: 'source_id' }, + condition, + }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + + const mockGetQueryOutput: GetQueryOutput = async () => { + return [ + { + result: "SELECT (link_type_id = '1234') AS __meerkat_batch_expr_0__;", + }, + ]; + }; + + const sql = await generateSqlQueryV2( + paths, + sqlMap, + graph, + schemas, + mockGetQueryOutput + ); + + expect(sql).toContain('issue.id = link.source_id'); + expect(sql).toContain("AND (link_type_id = '1234')"); + }); + + it('serializes multiple conditions across edges in a single batch', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'target_id', 'link_type_id']), + scalar('part', ['id']), + scalar('user', ['id']), + ]; + const sqlMap = sqlMapOf(schemas); + const bridgeTables = new Set(['link']); + const paths: StructuredJoin[][] = [ + [ + { + from: { table: 'issue', column: 'id' }, + to: { table: 'link', column: 'source_id' }, + condition: { + member: 'link.link_type_id', + operator: 'equals', + values: ['type_a'], + }, + }, + { + from: { table: 'link', column: 'target_id' }, + to: { table: 'part', column: 'id' }, + }, + { + from: { table: 'part', column: 'id' }, + to: { table: 'link', column: 'source_id' }, + condition: { + member: 'link.link_type_id', + operator: 'equals', + values: ['type_b'], + }, + }, + { + from: { table: 'link', column: 'target_id' }, + to: { table: 'user', column: 'id' }, + }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + + let callCount = 0; + const mockGetQueryOutput: GetQueryOutput = async () => { + callCount++; + return [ + { + result: + "SELECT (link_type_id = 'type_a') AS __meerkat_batch_expr_0__, (link_type_id = 'type_b') AS __meerkat_batch_expr_1__;", + }, + ]; + }; + + const sql = await generateSqlQueryV2( + paths, + sqlMap, + graph, + schemas, + mockGetQueryOutput, + bridgeTables + ); + + expect(callCount).toBe(1); + expect(sql).toContain("issue.id = link.source_id AND (link_type_id = 'type_a')"); + expect(sql).toContain('link.target_id = part.id'); + expect(sql).toContain("part.id = link__1.source_id AND (link_type_id = 'type_b')"); + expect(sql).toContain('link__1.target_id = user.id'); + }); + + it('handles notSet condition (IS NULL)', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'deleted_at']), + ]; + const sqlMap = sqlMapOf(schemas); + const paths: StructuredJoin[][] = [ + [ + { + from: { table: 'issue', column: 'id' }, + to: { table: 'link', column: 'source_id' }, + condition: { + member: 'link.deleted_at', + operator: 'notSet', + }, + }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + + const mockGetQueryOutput: GetQueryOutput = async () => { + return [ + { + result: 'SELECT (deleted_at IS NULL) AS __meerkat_batch_expr_0__;', + }, + ]; + }; + + const sql = await generateSqlQueryV2( + paths, + sqlMap, + graph, + schemas, + mockGetQueryOutput + ); + + expect(sql).toContain('issue.id = link.source_id AND (deleted_at IS NULL)'); + }); + + it('handles OR expression with multiple conditions', async () => { + const schemas = [ + scalar('issue', ['id']), + scalar('link', ['id', 'source_id', 'link_type_id']), + ]; + const sqlMap = sqlMapOf(schemas); + const condition: MeerkatQueryFilter = { + or: [ + { member: 'link.link_type_id', operator: 'equals', values: ['type_a'] }, + { member: 'link.link_type_id', operator: 'equals', values: ['type_b'] }, + ], + }; + const paths: StructuredJoin[][] = [ + [ + { + from: { table: 'issue', column: 'id' }, + to: { table: 'link', column: 'source_id' }, + condition, + }, + ], + ]; + const graph = createDirectedGraphV2(schemas, sqlMap, paths); + + const mockGetQueryOutput: GetQueryOutput = async () => { + return [ + { + result: + "SELECT ((link_type_id = 'type_a') OR (link_type_id = 'type_b')) AS __meerkat_batch_expr_0__;", + }, + ]; + }; + + const sql = await generateSqlQueryV2( + paths, + sqlMap, + graph, + schemas, + mockGetQueryOutput + ); + + expect(sql).toContain('issue.id = link.source_id AND'); + expect(sql).toContain("(link_type_id = 'type_a') OR (link_type_id = 'type_b')"); + }); + }); diff --git a/meerkat-core/src/joins/v2/joins.ts b/meerkat-core/src/joins/v2/joins.ts index d91a2efb..1699d2b9 100644 --- a/meerkat-core/src/joins/v2/joins.ts +++ b/meerkat-core/src/joins/v2/joins.ts @@ -1,7 +1,21 @@ +import { cubeFilterToDuckdbAST } from '../../cube-filter-transformer/factory'; +import { traverseMeerkatQueryFilter } from '../../filter-params/filter-params-ast'; import { getUsedTableSchema } from '../../get-used-table-schema/get-used-table-schema'; import { memberKeyToSafeKey } from '../../member-formatters/member-key-to-safe-key'; +import { + MeerkatQueryFilter, + Query, + StructuredJoin, + TableSchema, +} from '../../types/cube-types'; +import { ParsedExpression } from '../../types/duckdb-serialization-types/serialization/ParsedExpression'; +import { getBaseAST } from '../../utils/base-ast'; +import { cubeFiltersEnrichment } from '../../utils/cube-filter-enrichment'; +import { + GetQueryOutput, + serializeExpressions, +} from '../../utils/duckdb-ast-parse-serialize'; import { Graph, quoteIdentifierIfNeeded } from '../v1/joins'; -import { Query, StructuredJoin, TableSchema } from '../../types/cube-types'; const UNNEST_ALIAS_PREFIX = '__mk_u_'; const ARRAY_DIMENSION_TYPES = new Set(['string_array', 'number_array']); @@ -88,7 +102,8 @@ const collectArrayJoinSources = ( * the names emitted in the SELECT list stay in lock-step with the names * referenced in the ON clause. */ -const getUnnestAlias = (column: string): string => `${UNNEST_ALIAS_PREFIX}${column}`; +const getUnnestAlias = (column: string): string => + `${UNNEST_ALIAS_PREFIX}${column}`; /** * ` AS ` — re-aliases the unnested @@ -96,9 +111,14 @@ const getUnnestAlias = (column: string): string => `${UNNEST_ALIAS_PREFIX}${colu * via the same `tableName____mk_u_` identifier they use for any other * member. */ -const buildSafeKeyAliasProjection = (tableName: string, column: string): string => { +const buildSafeKeyAliasProjection = ( + tableName: string, + column: string +): string => { const unnestAlias = getUnnestAlias(column); - return `${unnestAlias} AS ${memberKeyToSafeKey(`${tableName}.${unnestAlias}`)}`; + return `${unnestAlias} AS ${memberKeyToSafeKey( + `${tableName}.${unnestAlias}` + )}`; }; const wrapTableSqlForArrayFrom = ( @@ -124,10 +144,97 @@ const wrapTableSqlForArrayFrom = ( )}`; }; -const buildPredicate = (edge: StructuredJoin, fromIsArray: boolean): string => { - const leftColumn = fromIsArray ? getUnnestAlias(edge.from.column) : edge.from.column; +const buildEquiJoinPredicate = ( + edge: StructuredJoin, + fromIsArray: boolean +): string => { + const leftColumn = fromIsArray + ? getUnnestAlias(edge.from.column) + : edge.from.column; return `${edge.from.table}.${leftColumn} = ${edge.to.table}.${edge.to.column}`; }; +const conditionFilterToAST = ( + condition: MeerkatQueryFilter, + tableSchema: TableSchema | undefined +): ParsedExpression | null => { + if (!tableSchema) return null; + const filters = [JSON.parse(JSON.stringify(condition))]; + const enriched = cubeFiltersEnrichment(filters, tableSchema); + if (!enriched) return null; + return ( + cubeFilterToDuckdbAST(enriched, getBaseAST(), { isAlias: false }) ?? null + ); +}; + +const getReferencedTables = (cubeQuery: Query): Set => { + const tables = new Set(); + const extractTable = (member: string) => { + const dotIndex = member.indexOf('.'); + if (dotIndex > 0) tables.add(member.slice(0, dotIndex)); + }; + + cubeQuery.measures.forEach(extractTable); + cubeQuery.dimensions?.forEach(extractTable); + if (cubeQuery.filters) { + traverseMeerkatQueryFilter(cubeQuery.filters, (filter) => { + extractTable(filter.member); + }); + } + return tables; +}; + +const inferBridgeTables = ( + paths: StructuredJoin[][], + cubeQuery: Query +): Set => { + const referenced = getReferencedTables(cubeQuery); + const bridges = new Set(); + for (const path of paths) { + for (const edge of path) { + if (!referenced.has(edge.to.table)) { + bridges.add(edge.to.table); + } + } + } + return bridges; +}; + +const aliasBridgeTables = ( + paths: StructuredJoin[][], + bridgeTables: Set +): StructuredJoin[][] => { + const seen = new Map(); + if (paths[0]?.[0]) seen.set(paths[0][0].from.table, 1); + + return paths.map((path) => { + const result: StructuredJoin[] = []; + let nextFromAlias: string | undefined; + + for (let i = 0; i < path.length; i++) { + let edge = path[i]; + if (nextFromAlias) { + edge = { ...edge, from: { ...edge.from, table: nextFromAlias } }; + nextFromAlias = undefined; + } + + const count = seen.get(edge.to.table) ?? 0; + seen.set(edge.to.table, count + 1); + + if (count === 0 || !bridgeTables.has(edge.to.table)) { + result.push(edge); + continue; + } + + const alias = `${edge.to.table}__${count}`; + result.push({ ...edge, to: { ...edge.to, table: alias } }); + + if (i + 1 < path.length && path[i + 1].from.table === edge.to.table) { + nextFromAlias = alias; + } + } + return result; + }); +}; export const createDirectedGraphV2 = ( tableSchemas: TableSchema[], @@ -141,7 +248,9 @@ export const createDirectedGraphV2 = ( for (const edge of path) { const { from, to } = edge; if (from.table === to.table) { - throw new Error(`Invalid structured join: self-join on "${from.table}"`); + throw new Error( + `Invalid structured join: self-join on "${from.table}"` + ); } if (!tableSchemaSqlMap[from.table] || !tableSchemaSqlMap[to.table]) { continue; @@ -154,11 +263,14 @@ export const createDirectedGraphV2 = ( ); } if (graph[from.table]?.[to.table]?.[from.column]) { - throw new Error('An invalid path was detected.'); + continue; } graph[from.table] ??= {}; graph[from.table][to.table] ??= {}; - graph[from.table][to.table][from.column] = buildPredicate(edge, fromIsArray); + graph[from.table][to.table][from.column] = buildEquiJoinPredicate( + edge, + fromIsArray + ); } } return graph; @@ -171,12 +283,14 @@ export const createDirectedGraphV2 = ( * gets UNNEST projections so the join becomes a hash-joinable equi-join * (`base.__mk_u_col = right.col`) instead of a `CONTAINS(...)` scan. */ -export const generateSqlQueryV2 = ( +export const generateSqlQueryV2 = async ( paths: StructuredJoin[][], tableSchemaSqlMap: { [key: string]: string }, directedGraph: Graph, - tableSchemas: TableSchema[] -): string => { + tableSchemas: TableSchema[], + getQueryOutput?: GetQueryOutput, + bridgeTables?: Set +): Promise => { if (paths.length === 0) { throw new Error( 'Invalid path, multiple data sources are present without a join path.' @@ -200,42 +314,99 @@ export const generateSqlQueryV2 = ( tableSchemas ); + const resolvedPaths = aliasBridgeTables(paths, bridgeTables ?? new Set()); + const visited = new Map(); + const edgeOrder: { edge: StructuredJoin; equiJoin: string }[] = []; + const conditionASTs: ParsedExpression[] = []; + const conditionIndexByEdge: number[] = []; - for (const path of paths) { + for (const path of resolvedPaths) { for (const edge of path) { const prev = visited.get(edge.to.table); - if (prev?.from.table === edge.from.table) continue; if (prev) { + const prevFrom = prev.from.table.replace(/__\d+$/, ''); + const currFrom = edge.from.table.replace(/__\d+$/, ''); + if (prevFrom === currFrom) continue; throw new Error( `Path ambiguity, node ${edge.to.table} visited from different sources` ); } visited.set(edge.to.table, edge); - const onClause = - directedGraph[edge.from.table][edge.to.table][edge.from.column]; - const rightArrayCols = arraySourcesByTable.get(edge.to.table); - const rightSubquery = rightArrayCols?.size - ? wrapTableSqlForArrayFrom( - tableSchemaSqlMap[edge.to.table], - edge.to.table, - rightArrayCols, - tableSchemas - ) - : `(${tableSchemaSqlMap[edge.to.table]}) AS ${quoteIdentifierIfNeeded(edge.to.table)}`; - query += ` LEFT JOIN ${rightSubquery} ON ${onClause}`; + const equiJoin = + directedGraph[edge.from.table]?.[edge.to.table]?.[edge.from.column] ?? + buildEquiJoinPredicate( + edge, + isArrayColumn(tableSchemas, edge.from.table, edge.from.column) + ); + + edgeOrder.push({ edge, equiJoin }); + + if (edge.condition) { + const toTable = edge.to.table; + const toTableOriginal = toTable.replace(/__\d+$/, ''); + const toTableSchema = tableSchemas.find( + (s) => s.name === toTable || s.name === toTableOriginal + ); + const ast = conditionFilterToAST(edge.condition, toTableSchema); + if (ast) { + conditionIndexByEdge.push(conditionASTs.length); + conditionASTs.push(ast); + } else { + conditionIndexByEdge.push(-1); + } + } else { + conditionIndexByEdge.push(-1); + } + } + } + + let conditionSqls: string[] = []; + if (conditionASTs.length > 0) { + if (!getQueryOutput) { + throw new Error( + 'getQueryOutput is required when join edges have filter conditions' + ); } + conditionSqls = await serializeExpressions(conditionASTs, getQueryOutput); + } + + for (let i = 0; i < edgeOrder.length; i++) { + const { edge, equiJoin } = edgeOrder[i]; + const condIdx = conditionIndexByEdge[i]; + const onClause = + condIdx >= 0 ? `${equiJoin} AND ${conditionSqls[condIdx]}` : equiJoin; + + const toTable = edge.to.table; + const toTableOriginal = toTable.replace(/__\d+$/, ''); + const toTableSql = + tableSchemaSqlMap[toTable] ?? tableSchemaSqlMap[toTableOriginal]; + + const rightArrayCols = arraySourcesByTable.get(toTable); + const rightSubquery = rightArrayCols?.size + ? wrapTableSqlForArrayFrom( + toTableSql, + toTable, + rightArrayCols, + tableSchemas + ) + : `(${toTableSql}) AS ${quoteIdentifierIfNeeded(toTable)}`; + query += ` LEFT JOIN ${rightSubquery} ON ${onClause}`; } return query; }; -const hasLoop = (paths: StructuredJoin[][]): boolean => { +const hasLoop = ( + paths: StructuredJoin[][], + bridgeTables: Set +): boolean => { for (const path of paths) { const visited = new Set(); if (path[0]) visited.add(path[0].from.table); for (const edge of path) { + if (bridgeTables.has(edge.to.table)) continue; if (visited.has(edge.to.table)) return true; visited.add(edge.to.table); } @@ -243,10 +414,11 @@ const hasLoop = (paths: StructuredJoin[][]): boolean => { return false; }; -export const getCombinedTableSchemaV2 = ( +export const getCombinedTableSchemaV2 = async ( tableSchema: TableSchema[], - cubeQuery: Query -): TableSchema => { + cubeQuery: Query, + getQueryOutput?: GetQueryOutput +): Promise => { if (tableSchema.length === 1) return tableSchema[0]; // joinPathsV2 callers control which tables participate; everyone else @@ -257,7 +429,8 @@ export const getCombinedTableSchemaV2 = ( if (activeTables.length === 1) return activeTables[0]; const paths = cubeQuery.joinPathsV2 ?? []; - if (hasLoop(paths)) { + const bridgeTables = inferBridgeTables(paths, cubeQuery); + if (hasLoop(paths, bridgeTables)) { throw new Error( `A loop was detected in the joins. ${JSON.stringify(paths)}` ); @@ -267,7 +440,14 @@ export const getCombinedTableSchemaV2 = ( activeTables.map((s) => [s.name, s.sql]) ); const graph = createDirectedGraphV2(activeTables, tableSchemaSqlMap, paths); - const sql = generateSqlQueryV2(paths, tableSchemaSqlMap, graph, activeTables); + const sql = await generateSqlQueryV2( + paths, + tableSchemaSqlMap, + graph, + activeTables, + getQueryOutput, + bridgeTables + ); return { name: 'MEERKAT_GENERATED_TABLE', diff --git a/meerkat-core/src/types/cube-types/query.ts b/meerkat-core/src/types/cube-types/query.ts index 266011e6..9515259b 100644 --- a/meerkat-core/src/types/cube-types/query.ts +++ b/meerkat-core/src/types/cube-types/query.ts @@ -141,6 +141,7 @@ interface JoinNode { export type StructuredJoin = { from: { table: string; column: string }; to: { table: string; column: string }; + condition?: MeerkatQueryFilter; }; /** diff --git a/meerkat-node/src/cube-to-sql/cube-to-sql.ts b/meerkat-node/src/cube-to-sql/cube-to-sql.ts index c7aa2494..afbfd26c 100644 --- a/meerkat-node/src/cube-to-sql/cube-to-sql.ts +++ b/meerkat-node/src/cube-to-sql/cube-to-sql.ts @@ -41,7 +41,7 @@ export const cubeQueryToSQL = async ({ }) ); - const updatedTableSchema = getCombinedTableSchema(updatedTableSchemas, query); + const updatedTableSchema = await getCombinedTableSchema(updatedTableSchemas, query, duckdbExec); const ast = cubeToDuckdbAST(query, updatedTableSchema, { filterType: 'PROJECTION_FILTER',