diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx index 8982ce968f79..238d5ad4de3b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/EntityLineage/LineageNodeLabelV1.tsx @@ -84,9 +84,11 @@ const EntityLabel = ({ node }: Pick) => { childrenCount > 0 ? 'with-footer' : '' )}> -
- {getServiceIcon(node)} -
+ {!node.isTempTable && ( +
+ {getServiceIcon(node)} +
+ )} { return; } + if (node.data?.node?.isTempTable) { + return; + } + if (node.type === EntityLineageNodeType.LOAD_MORE) { selectLoadMoreNode(node); } else { diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.test.ts index e04831a671ce..b4339d0b5df6 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.test.ts @@ -11,6 +11,7 @@ * limitations under the License. */ import { Edge, Node, Viewport } from 'reactflow'; +import { EntityType } from '../enums/entity.enum'; import { BoundingBox, boundsIntersect, @@ -218,11 +219,12 @@ describe('CanvasUtils', () => { const result = getEdgeCoordinates(edge, sourceNode, targetNode); + // Y uses node.height (100) from createMockNode, so midpoint = 50 expect(result).toEqual({ sourceX: 400, - sourceY: 33, + sourceY: 50, targetX: 490, - targetY: 33, + targetY: 50, }); }); @@ -237,9 +239,9 @@ describe('CanvasUtils', () => { expect(result).not.toBeNull(); expect(result?.sourceX).toBe(500); - expect(result?.sourceY).toBe(233); + expect(result?.sourceY).toBe(250); // 200 + 100/2 expect(result?.targetX).toBe(590); - expect(result?.targetY).toBe(333); + expect(result?.targetY).toBe(350); // 300 + 100/2 }); }); @@ -391,6 +393,54 @@ describe('CanvasUtils', () => { expect(result?.targetX).toBe(490); }); }); + + describe('temp lineage nodes', () => { + const createTempNode = (id: string, measuredHeight: number): Node => ({ + id, + position: { x: 0, y: 0 }, + data: { + node: { + id, + name: id, + entityType: EntityType.TABLE, + isTempTable: true, + columns: [], + }, + isRootNode: false, + }, + width: 400, + height: measuredHeight, + }); + + it('uses measured node.height for temp node entity-level edge', () => { + const edge = createMockEdge('edge1', 'temp_staging', 'node2', false); + const tempNode = createTempNode('temp_staging', 80); + const targetNode = createMockNode('node2'); + targetNode.position = { x: 500, y: 0 }; + + const result = getEdgeCoordinates(edge, tempNode, targetNode); + + expect(result).not.toBeNull(); + expect(result?.sourceY).toBe(40); // 0 + 80/2 (measured height), not 33 (getNodeHeight formula) + expect(result?.targetY).toBe(50); // 0 + 100/2 + }); + + it('centers edge at actual midpoint when measured height differs from computed height', () => { + const edge = createMockEdge('edge1', 'node1', 'node2', false); + const sourceNode = createMockNode('node1'); + sourceNode.height = 150; + sourceNode.position = { x: 0, y: 100 }; + const targetNode = createMockNode('node2'); + targetNode.height = 200; + targetNode.position = { x: 500, y: 100 }; + + const result = getEdgeCoordinates(edge, sourceNode, targetNode); + + expect(result).not.toBeNull(); + expect(result?.sourceY).toBe(175); // 100 + 150/2 + expect(result?.targetY).toBe(200); // 100 + 200/2 + }); + }); }); describe('getEdgeBounds', () => { @@ -414,7 +464,8 @@ describe('CanvasUtils', () => { expect(result).not.toBeNull(); expect(result!.minX).toBeLessThan(351); expect(result!.maxX).toBeGreaterThan(500); - expect(result!.minY).toBeLessThan(0); + // sourceY = 50 (node.height=100 / 2), padding=50 → minY = 0 + expect(result!.minY).toBeLessThanOrEqual(0); expect(result!.maxY).toBeGreaterThan(100); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.ts index ffe7060f00e1..232b80cc2ee2 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CanvasUtils.ts @@ -93,11 +93,19 @@ const getBaseNodeHeightFromType = ( return isRootNode ? baseHeight + 10 : baseHeight; }; +const getNodeYPadding = (node: Node): number => { + const { children } = getEntityChildrenAndLabel(node.data.node); + + const sourceYPadding = children.length > 0 ? 48 : 0; + + return sourceYPadding; +}; + export function getNodeHeight( node: Node, - isColumnLineage: boolean, + isColumnLineage?: boolean, columnCount?: number -) { +): number { const isRootNode = node.data?.isRootNode ?? false; const visibleColumnCount = isColumnLineage @@ -124,15 +132,6 @@ export function getNodeHeight( return height; } -const getNodeYPadding = (node: Node): number => { - const { children } = getEntityChildrenAndLabel(node.data.node); - - const sourceYPadding = children.length > 0 ? 48 : 0; - - // Add padding for the node's border - return sourceYPadding; -}; - interface ColumnLineageData { columnIds: string[]; columnIndex: number; @@ -252,10 +251,10 @@ function getColumnLineageCoordinates( function getEntityLineageCoordinates( sourceNode: Node, targetNode: Node, - isColumnLineage: boolean + _isColumnLineage: boolean ): EdgeCoordinates { - const sourceHeight = getNodeHeight(sourceNode, isColumnLineage, 0); - const targetHeight = getNodeHeight(targetNode, isColumnLineage, 0); + const sourceHeight = sourceNode.height ?? 0; + const targetHeight = targetNode.height ?? 0; return { sourceX: sourceNode.position.x + (sourceNode.width ?? 0), diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx index d782162cd683..b73c3450bee4 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.test.tsx @@ -15,6 +15,7 @@ import { Edge, Node } from 'reactflow'; import { EdgeDetails, LineageData, + LineageNodeType, } from '../components/Lineage/Lineage.interface'; import { SourceType } from '../components/SearchedData/SearchedData.interface'; import { EntityType } from '../enums/entity.enum'; @@ -620,7 +621,9 @@ describe('Test EntityLineageUtils utility', () => { describe('getEntityChildrenAndLabel', () => { it('should return empty values for null input', () => { - const result = getEntityChildrenAndLabel(null as any); + const result = getEntityChildrenAndLabel( + null as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -633,7 +636,9 @@ describe('Test EntityLineageUtils utility', () => { const node = { entityType: 'UNKNOWN', }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -652,7 +657,9 @@ describe('Test EntityLineageUtils utility', () => { columns, flattenColumns: columns, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: columns, @@ -666,7 +673,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.TABLE, columns: [], }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -683,7 +692,9 @@ describe('Test EntityLineageUtils utility', () => { columns, flattenChildren, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: flattenChildren, @@ -701,7 +712,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.DASHBOARD, charts, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: charts, @@ -715,7 +728,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.DASHBOARD, charts: [], }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -733,7 +748,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.MLMODEL, mlFeatures, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: mlFeatures, @@ -747,7 +764,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.MLMODEL, mlFeatures: [], }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -766,7 +785,9 @@ describe('Test EntityLineageUtils utility', () => { columns, flattenColumns: columns, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: columns, @@ -786,7 +807,9 @@ describe('Test EntityLineageUtils utility', () => { columns, }, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: columns, @@ -799,7 +822,9 @@ describe('Test EntityLineageUtils utility', () => { const node = { entityType: EntityType.CONTAINER, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -819,7 +844,9 @@ describe('Test EntityLineageUtils utility', () => { schemaFields, }, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: schemaFields, @@ -832,7 +859,9 @@ describe('Test EntityLineageUtils utility', () => { const node = { entityType: EntityType.TOPIC, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -852,7 +881,9 @@ describe('Test EntityLineageUtils utility', () => { schemaFields, }, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: schemaFields, @@ -872,7 +903,9 @@ describe('Test EntityLineageUtils utility', () => { schemaFields, }, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: schemaFields, @@ -893,7 +926,9 @@ describe('Test EntityLineageUtils utility', () => { schemaFields: requestFields, }, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: responseFields, @@ -906,7 +941,9 @@ describe('Test EntityLineageUtils utility', () => { const node = { entityType: EntityType.API_ENDPOINT, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -924,7 +961,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.SEARCH_INDEX, fields, }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: fields, @@ -938,7 +977,9 @@ describe('Test EntityLineageUtils utility', () => { entityType: EntityType.SEARCH_INDEX, fields: [], }; - const result = getEntityChildrenAndLabel(node as any); + const result = getEntityChildrenAndLabel( + node as unknown as LineageNodeType + ); expect(result).toEqual({ children: [], @@ -1370,17 +1411,19 @@ describe('parseLineageData', () => { jest.clearAllMocks(); // Setup default mock implementations - mockUniqWith.mockImplementation((array: any) => array || []); - mockIsEqual.mockImplementation((a: any, b: any) => a === b); - mockGet.mockImplementation((obj: any, path: any, defaultValue?: any) => { + mockUniqWith.mockImplementation((array) => array ?? []); + mockIsEqual.mockImplementation((a, b) => a === b); + mockGet.mockImplementation((obj, path, defaultValue?) => { if (!obj || !path) { return defaultValue; } - const pathStr = Array.isArray(path) ? path.join('.') : String(path); + const pathStr = Array.isArray(path) + ? (path as string[]).join('.') + : String(path); const keys = pathStr.split('.'); - let result = obj; + let result: unknown = obj; for (const key of keys) { - result = result?.[key]; + result = (result as Record)?.[key]; if (result === undefined) { return defaultValue; } @@ -1596,13 +1639,17 @@ describe('parseLineageData', () => { describe('Pagination handling', () => { it('should create load more nodes for entities with pagination', () => { // Mock get function to return pipeline type for filtering - mockGet.mockImplementation((obj: any, path: any) => { + mockGet.mockImplementation((obj, path) => { if (path === 'entityType') { - return obj?.entityType || EntityType.TABLE; + return ( + (obj as Record)?.entityType || EntityType.TABLE + ); } - const pathStr = Array.isArray(path) ? path.join('.') : String(path); + const pathStr = Array.isArray(path) + ? (path as string[]).join('.') + : String(path); - return obj?.[pathStr]; + return (obj as Record)?.[pathStr]; }); const result = parseLineageData( @@ -1636,13 +1683,15 @@ describe('parseLineageData', () => { }, }; - mockGet.mockImplementation((obj: any, path: any) => { + mockGet.mockImplementation((obj, path) => { if (path === 'entityType') { - return obj?.entityType; + return (obj as Record)?.entityType; } - const pathStr = Array.isArray(path) ? path.join('.') : String(path); + const pathStr = Array.isArray(path) + ? (path as string[]).join('.') + : String(path); - return obj?.[pathStr]; + return (obj as Record)?.[pathStr]; }); const dataWithPipeline = { @@ -1742,11 +1791,11 @@ describe('parseLineageData', () => { }; // Mock uniqWith to actually remove duplicates - mockUniqWith.mockImplementation((array: any, compareFn?: any) => { + mockUniqWith.mockImplementation((array, compareFn?) => { if (!array) { return []; } - const unique: any[] = []; + const unique: unknown[] = []; for (const item of array) { if ( !unique.some((existing) => @@ -1757,12 +1806,15 @@ describe('parseLineageData', () => { } } - return unique; + return unique as ReturnType; }); - mockIsEqual.mockImplementation( - (a: any, b: any) => a.fullyQualifiedName === b.fullyQualifiedName - ); + mockIsEqual.mockImplementation((a, b) => { + const aObj = a as { fullyQualifiedName?: string }; + const bObj = b as { fullyQualifiedName?: string }; + + return aObj.fullyQualifiedName === bObj.fullyQualifiedName; + }); parseLineageData(dataWithDuplicates, mockEntityFqn, mockRootFqn); @@ -1780,7 +1832,7 @@ describe('getLineageTableConfig', () => { }); it('should return empty arrays for null CSV data', () => { - const result = getLineageTableConfig(null as any); + const result = getLineageTableConfig(null as unknown as string[][]); expect(result.columns).toEqual([]); expect(result.dataSource).toEqual([]); @@ -1915,3 +1967,192 @@ describe('getLineageTableConfig', () => { }); }); }); + +describe('extractTempLineageNodes (via parseLineageData)', () => { + const actualLodash = jest.requireActual('lodash'); + + beforeEach(() => { + mockUniqWith.mockImplementation(actualLodash.uniqWith); + mockIsEqual.mockImplementation(actualLodash.isEqual); + mockGet.mockImplementation(actualLodash.get); + }); + + const makeNodeData = (id: string, fqn: string) => ({ + entity: { + id, + type: EntityType.TABLE, + fullyQualifiedName: fqn, + name: fqn, + columns: [], + }, + paging: { entityDownstreamCount: 0, entityUpstreamCount: 0 }, + }); + + const makeLineageData = ( + downstreamEdges: Record = {} + ): LineageData => ({ + nodes: { + 'id-a': makeNodeData('id-a', 'db.tableA'), + 'id-b': makeNodeData('id-b', 'db.tableB'), + }, + downstreamEdges, + upstreamEdges: {}, + }); + + it('creates a temp node with correct shape for an unknown FQN', () => { + const data = makeLineageData({ + e1: { + fromEntity: { + id: 'id-a', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableA', + }, + toEntity: { + id: 'id-b', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableB', + }, + tempLineageTables: [ + { fromEntity: 'tmp_staging', toEntity: 'db.tableB' }, + ], + }, + }); + + const { nodes } = parseLineageData(data, 'db.tableA', 'db.tableA'); + const tempNodes = nodes.filter((n) => n.isTempTable); + + expect(tempNodes).toHaveLength(1); + expect(tempNodes[0].id).toBe('temp_tmp_staging'); + expect(tempNodes[0].fullyQualifiedName).toBe('tmp_staging'); + expect(tempNodes[0].entityType).toBe(EntityType.TABLE); + expect(tempNodes[0].isTempTable).toBe(true); + }); + + it('does not create a temp node when the FQN already exists as a real node', () => { + const data = makeLineageData({ + e1: { + fromEntity: { + id: 'id-a', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableA', + }, + toEntity: { + id: 'id-b', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableB', + }, + tempLineageTables: [ + { fromEntity: 'db.tableA', toEntity: 'tmp_staging' }, + ], + }, + }); + + const { nodes } = parseLineageData(data, 'db.tableA', 'db.tableA'); + const tempNodes = nodes.filter((n) => n.isTempTable); + + expect(tempNodes).toHaveLength(1); + expect(tempNodes[0].fullyQualifiedName).toBe('tmp_staging'); + }); + + it('deduplicates temp nodes when the same FQN appears in multiple hops', () => { + const data = makeLineageData({ + e1: { + fromEntity: { + id: 'id-a', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableA', + }, + toEntity: { + id: 'id-b', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableB', + }, + tempLineageTables: [ + { fromEntity: 'tmp_staging', toEntity: 'tmp_mid' }, + { fromEntity: 'tmp_staging', toEntity: 'db.tableB' }, + ], + }, + }); + + const { nodes } = parseLineageData(data, 'db.tableA', 'db.tableA'); + const stagingNodes = nodes.filter( + (n) => n.fullyQualifiedName === 'tmp_staging' + ); + + expect(stagingNodes).toHaveLength(1); + }); + + it('creates one hop edge per tempLineageTables entry', () => { + const data = makeLineageData({ + e1: { + fromEntity: { + id: 'id-a', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableA', + }, + toEntity: { + id: 'id-b', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableB', + }, + tempLineageTables: [ + { fromEntity: 'db.tableA', toEntity: 'tmp_staging' }, + { fromEntity: 'tmp_staging', toEntity: 'db.tableB' }, + ], + }, + }); + + const { edges } = parseLineageData(data, 'db.tableA', 'db.tableA'); + + expect(edges).toHaveLength(2); + expect(edges[0].fromEntity.fullyQualifiedName).toBe('db.tableA'); + expect(edges[0].toEntity.fullyQualifiedName).toBe('tmp_staging'); + expect(edges[1].fromEntity.fullyQualifiedName).toBe('tmp_staging'); + expect(edges[1].toEntity.fullyQualifiedName).toBe('db.tableB'); + }); + + it('removes the original edge when it is expanded into temp hop edges', () => { + const data = makeLineageData({ + e1: { + fromEntity: { + id: 'id-a', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableA', + }, + toEntity: { + id: 'id-b', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableB', + }, + tempLineageTables: [{ fromEntity: 'db.tableA', toEntity: 'db.tableB' }], + }, + }); + + const { edges } = parseLineageData(data, 'db.tableA', 'db.tableA'); + + expect(edges.some((e) => e.tempLineageTables?.length)).toBe(false); + }); + + it('preserves regular edges that have no tempLineageTables', () => { + const data = makeLineageData({ + e1: { + fromEntity: { + id: 'id-a', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableA', + }, + toEntity: { + id: 'id-b', + type: EntityType.TABLE, + fullyQualifiedName: 'db.tableB', + }, + }, + }); + + const { edges, nodes } = parseLineageData(data, 'db.tableA', 'db.tableA'); + + expect(edges).toHaveLength(1); + expect(edges[0].fromEntity.fullyQualifiedName).toBe('db.tableA'); + expect(nodes.filter((n) => n.isTempTable)).toHaveLength(0); + }); +}); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx index d405b497b710..bca9adb9cd99 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityLineageUtils.tsx @@ -100,7 +100,11 @@ import { Pipeline } from '../generated/entity/data/pipeline'; import { SearchIndex } from '../generated/entity/data/searchIndex'; import { Table } from '../generated/entity/data/table'; import { Topic } from '../generated/entity/data/topic'; -import { ColumnLineage, LineageDetails } from '../generated/type/entityLineage'; +import { + ColumnLineage, + LineageDetails, + TempLineageTable, +} from '../generated/type/entityLineage'; import { EntityReference } from '../generated/type/entityReference'; import { TagSource } from '../generated/type/tagLabel'; import { useLineageStore } from '../hooks/useLineageStore'; @@ -117,12 +121,7 @@ import { jsonToCSV } from './StringsUtils'; import { showErrorToast } from './ToastUtils'; interface LayoutedElements { - node: Array< - Node & { - nodeHeight: number; - childrenHeight: number; - } - >; + node: Array; edge: Edge[]; } @@ -207,6 +206,91 @@ export const getLayoutedElements = ( return { node: uNode, edge: edgesRequired }; }; +/** + * This function returns all the columns as children as well flattened children for subfield columns. + * It also returns the label for the children and the total height of the children. + * + * @param {Node} selectedNode - The node for which to retrieve the downstream nodes and edges. + * @param {string[]} columnsHavingLineage - All nodes in the lineage. + * @return {{ nodes: Node[]; edges: Edge[], nodeIds: string[], edgeIds: string[] }} - + * An object containing the downstream nodes and edges. + */ +export function getEntityChildrenAndLabel(node: LineageNodeType) { + if (!node) { + return { + children: [], + childrenHeading: '', + childrenCount: 0, + }; + } + const entityMappings: Record< + string, + { data: EntityChildren; label: string; childrenCount: number } + > = { + [EntityType.TABLE]: { + data: node.flattenChildren ?? node.columns ?? [], + label: t('label.column-plural'), + childrenCount: node.columns?.length ?? 0, + }, + [EntityType.DASHBOARD]: { + data: node.charts ?? [], + label: t('label.chart-plural'), + childrenCount: node.charts?.length ?? 0, + }, + [EntityType.MLMODEL]: { + data: node.mlFeatures ?? [], + label: t('label.feature-plural'), + childrenCount: node.mlFeatures?.length ?? 0, + }, + [EntityType.DASHBOARD_DATA_MODEL]: { + data: node.flattenChildren ?? node.columns ?? [], + label: t('label.column-plural'), + childrenCount: node.columns?.length ?? 0, + }, + [EntityType.CONTAINER]: { + data: node.flattenChildren ?? node.dataModel?.columns ?? [], + label: t('label.column-plural'), + childrenCount: node.dataModel?.columns?.length ?? 0, + }, + [EntityType.TOPIC]: { + data: node.flattenChildren ?? node.messageSchema?.schemaFields ?? [], + label: t('label.field-plural'), + childrenCount: node.messageSchema?.schemaFields?.length ?? 0, + }, + [EntityType.API_ENDPOINT]: { + data: + node.flattenChildren ?? + node?.responseSchema?.schemaFields ?? + node?.requestSchema?.schemaFields ?? + [], + label: t('label.field-plural'), + childrenCount: + node?.responseSchema?.schemaFields?.length ?? + node?.requestSchema?.schemaFields?.length ?? + 0, + }, + [EntityType.SEARCH_INDEX]: { + data: node.flattenChildren ?? node.fields ?? [], + label: t('label.field-plural'), + childrenCount: node.fields?.length ?? 0, + }, + }; + + const { data, label, childrenCount } = entityMappings[ + node.entityType as EntityType + ] || { + data: [], + label: '', + childrenCount: 0, + }; + + return { + children: data, + childrenHeading: label, + childrenCount, + }; +} + export const getELKLayoutedElements = async ( nodes: Node[], edges: Edge[], @@ -571,91 +655,6 @@ export const removeLineageHandler = async (data: EdgeData): Promise => { } }; -/** - * This function returns all the columns as children as well flattened children for subfield columns. - * It also returns the label for the children and the total height of the children. - * - * @param {Node} selectedNode - The node for which to retrieve the downstream nodes and edges. - * @param {string[]} columnsHavingLineage - All nodes in the lineage. - * @return {{ nodes: Node[]; edges: Edge[], nodeIds: string[], edgeIds: string[] }} - - * An object containing the downstream nodes and edges. - */ -export const getEntityChildrenAndLabel = (node: LineageNodeType) => { - if (!node) { - return { - children: [], - childrenHeading: '', - childrenCount: 0, - }; - } - const entityMappings: Record< - string, - { data: EntityChildren; label: string; childrenCount: number } - > = { - [EntityType.TABLE]: { - data: node.flattenChildren ?? node.columns ?? [], - label: t('label.column-plural'), - childrenCount: node.columns?.length ?? 0, - }, - [EntityType.DASHBOARD]: { - data: node.charts ?? [], - label: t('label.chart-plural'), - childrenCount: node.charts?.length ?? 0, - }, - [EntityType.MLMODEL]: { - data: node.mlFeatures ?? [], - label: t('label.feature-plural'), - childrenCount: node.mlFeatures?.length ?? 0, - }, - [EntityType.DASHBOARD_DATA_MODEL]: { - data: node.flattenChildren ?? node.columns ?? [], - label: t('label.column-plural'), - childrenCount: node.columns?.length ?? 0, - }, - [EntityType.CONTAINER]: { - data: node.flattenChildren ?? node.dataModel?.columns ?? [], - label: t('label.column-plural'), - childrenCount: node.dataModel?.columns?.length ?? 0, - }, - [EntityType.TOPIC]: { - data: node.flattenChildren ?? node.messageSchema?.schemaFields ?? [], - label: t('label.field-plural'), - childrenCount: node.messageSchema?.schemaFields?.length ?? 0, - }, - [EntityType.API_ENDPOINT]: { - data: - node.flattenChildren ?? - node?.responseSchema?.schemaFields ?? - node?.requestSchema?.schemaFields ?? - [], - label: t('label.field-plural'), - childrenCount: - node?.responseSchema?.schemaFields?.length ?? - node?.requestSchema?.schemaFields?.length ?? - 0, - }, - [EntityType.SEARCH_INDEX]: { - data: node.flattenChildren ?? node.fields ?? [], - label: t('label.field-plural'), - childrenCount: node.fields?.length ?? 0, - }, - }; - - const { data, label, childrenCount } = entityMappings[ - node.entityType as EntityType - ] || { - data: [], - label: '', - childrenCount: 0, - }; - - return { - children: data, - childrenHeading: label, - childrenCount, - }; -}; - // Nodes Icons export const getEntityNodeIcon = (label: string) => { switch (label) { @@ -696,6 +695,65 @@ export const positionNodesUsingElk = async ( return obj; }; +export function getUpstreamDownstreamNodesEdges( + edges: EdgeDetails[], + nodes: EntityReference[], + currentNode: string +) { + const downstreamEdges: EdgeDetails[] = []; + const upstreamEdges: EdgeDetails[] = []; + const downstreamNodes: EntityReference[] = []; + const upstreamNodes: EntityReference[] = []; + const activeNode = nodes.find( + (node) => node.fullyQualifiedName === currentNode + ); + + if (!activeNode) { + return { downstreamEdges, upstreamEdges, downstreamNodes, upstreamNodes }; + } + + function findDownstream(node: EntityReference) { + const directDownstream = edges.filter( + (edge) => edge.fromEntity.fullyQualifiedName === node.fullyQualifiedName + ); + downstreamEdges.push(...directDownstream); + directDownstream.forEach((edge) => { + const toNode = nodes.find( + (item) => item.fullyQualifiedName === edge.toEntity.fullyQualifiedName + ); + if (!isUndefined(toNode)) { + if (!downstreamNodes.includes(toNode)) { + downstreamNodes.push(toNode); + findDownstream(toNode); + } + } + }); + } + + function findUpstream(node: EntityReference) { + const directUpstream = edges.filter( + (edge) => edge.toEntity.fullyQualifiedName === node.fullyQualifiedName + ); + upstreamEdges.push(...directUpstream); + directUpstream.forEach((edge) => { + const fromNode = nodes.find( + (item) => item.fullyQualifiedName === edge.fromEntity.fullyQualifiedName + ); + if (!isUndefined(fromNode)) { + if (!upstreamNodes.includes(fromNode)) { + upstreamNodes.push(fromNode); + findUpstream(fromNode); + } + } + }); + } + + findDownstream(activeNode); + findUpstream(activeNode); + + return { downstreamEdges, upstreamEdges, downstreamNodes, upstreamNodes }; +} + export const createNodes = ( nodesData: LineageNodeType[], edgesData: EdgeDetails[], @@ -1193,65 +1251,6 @@ export const createNewEdge = (edge: Edge) => { return selectedEdge; }; -export const getUpstreamDownstreamNodesEdges = ( - edges: EdgeDetails[], - nodes: EntityReference[], - currentNode: string -) => { - const downstreamEdges: EdgeDetails[] = []; - const upstreamEdges: EdgeDetails[] = []; - const downstreamNodes: EntityReference[] = []; - const upstreamNodes: EntityReference[] = []; - const activeNode = nodes.find( - (node) => node.fullyQualifiedName === currentNode - ); - - if (!activeNode) { - return { downstreamEdges, upstreamEdges, downstreamNodes, upstreamNodes }; - } - - function findDownstream(node: EntityReference) { - const directDownstream = edges.filter( - (edge) => edge.fromEntity.fullyQualifiedName === node.fullyQualifiedName - ); - downstreamEdges.push(...directDownstream); - directDownstream.forEach((edge) => { - const toNode = nodes.find( - (item) => item.fullyQualifiedName === edge.toEntity.fullyQualifiedName - ); - if (!isUndefined(toNode)) { - if (!downstreamNodes.includes(toNode)) { - downstreamNodes.push(toNode); - findDownstream(toNode); - } - } - }); - } - - function findUpstream(node: EntityReference) { - const directUpstream = edges.filter( - (edge) => edge.toEntity.fullyQualifiedName === node.fullyQualifiedName - ); - upstreamEdges.push(...directUpstream); - directUpstream.forEach((edge) => { - const fromNode = nodes.find( - (item) => item.fullyQualifiedName === edge.fromEntity.fullyQualifiedName - ); - if (!isUndefined(fromNode)) { - if (!upstreamNodes.includes(fromNode)) { - upstreamNodes.push(fromNode); - findUpstream(fromNode); - } - } - }); - } - - findDownstream(activeNode); - findUpstream(activeNode); - - return { downstreamEdges, upstreamEdges, downstreamNodes, upstreamNodes }; -}; - export const getExportEntity = (entity: LineageSourceType) => { const { name, @@ -1520,31 +1519,21 @@ const processNodeArray = ( ); }; -const processPipelineEdge = (edge: EdgeDetails, pipelineNode: Pipeline) => { - const pipelineEntityType = get(pipelineNode, 'entityType'); - - // Create two edges: fromEntity -> pipeline and pipeline -> toEntity - const edgeFromToPipeline = { - fromEntity: edge.fromEntity, - toEntity: { - id: pipelineNode.id, - type: pipelineEntityType, - fullyQualifiedName: pipelineNode.fullyQualifiedName ?? '', - }, - extraInfo: edge, - }; - - const edgePipelineToTo = { - fromEntity: { - id: pipelineNode.id, - type: pipelineEntityType, - fullyQualifiedName: pipelineNode.fullyQualifiedName ?? '', - }, - toEntity: edge.toEntity, - extraInfo: edge, +const processPipelineEdge = ( + edge: EdgeDetails, + pipelineNode: Pipeline +): EdgeDetails[] => { + const pipelineEntityType = String(get(pipelineNode, 'entityType')); + const pipelineRef = { + id: pipelineNode.id, + type: pipelineEntityType, + fullyQualifiedName: pipelineNode.fullyQualifiedName ?? '', }; - return [edgeFromToPipeline, edgePipelineToTo]; + return [ + { fromEntity: edge.fromEntity, toEntity: pipelineRef, extraInfo: edge }, + { fromEntity: pipelineRef, toEntity: edge.toEntity, extraInfo: edge }, + ]; }; const processEdges = ( @@ -1615,6 +1604,83 @@ const processPagination = ( return { newNodes, newEdges }; }; +const getOrCreateTempNode = ( + nameOrFqn: string, + existingByFqn: Map, + newTempNodes: Map +): LineageNodeType => { + const existing = existingByFqn.get(nameOrFqn); + if (existing) { + return existing; + } + if (newTempNodes.has(nameOrFqn)) { + return newTempNodes.get(nameOrFqn) as LineageNodeType; + } + const tempNode: LineageNodeType = { + id: `temp_${nameOrFqn}`, + name: nameOrFqn, + displayName: nameOrFqn, + fullyQualifiedName: nameOrFqn, + type: EntityType.TABLE, + entityType: EntityType.TABLE, + isTempTable: true, + columns: [], + }; + newTempNodes.set(nameOrFqn, tempNode); + + return tempNode; +}; + +const extractTempLineageNodes = ( + edges: EdgeDetails[], + existingNodes: LineageNodeType[] +): { nodes: LineageNodeType[]; edges: EdgeDetails[] } => { + const newTempNodes = new Map(); + const newEdges: EdgeDetails[] = []; + + const existingByFqn = new Map( + existingNodes + .filter((n) => n.fullyQualifiedName) + .map((n) => [n.fullyQualifiedName!, n]) + ); + + edges.forEach((edge) => { + if (!edge.tempLineageTables?.length) { + return; + } + edge.tempLineageTables.forEach((hop: TempLineageTable) => { + const fromNode = getOrCreateTempNode( + hop.fromEntity, + existingByFqn, + newTempNodes + ); + const toNode = getOrCreateTempNode( + hop.toEntity, + existingByFqn, + newTempNodes + ); + newEdges.push({ + fromEntity: { + id: fromNode.id, + type: fromNode.type ?? EntityType.TABLE, + fullyQualifiedName: fromNode.fullyQualifiedName, + }, + toEntity: { + id: toNode.id, + type: toNode.type ?? EntityType.TABLE, + fullyQualifiedName: toNode.fullyQualifiedName, + }, + }); + }); + }); + + const pureNewNodes = Array.from(newTempNodes.values()).filter( + (n) => !existingByFqn.has(n.fullyQualifiedName ?? '') + ); + + return { nodes: pureNewNodes, edges: newEdges }; +}; + export const parseLineageData = ( data: LineageData, entityFqn: string, // This contains fqn of node or entity that is being viewed in lineage page @@ -1656,14 +1722,23 @@ export const parseLineageData = ( ...newEdges, ]; + const { nodes: tempNodes, edges: tempEdges } = extractTempLineageNodes( + finalEdges, + finalNodes + ); + // Find the main entity const entity = nodesArray.find( (node) => node.fullyQualifiedName === entityFqn ) as LineageNodeType; + const baseEdges = tempEdges.length + ? finalEdges.filter((e) => !e.tempLineageTables?.length) + : finalEdges; + return { - nodes: finalNodes, - edges: finalEdges, + nodes: [...finalNodes, ...tempNodes], + edges: [...baseEdges, ...tempEdges], entity, }; };