From eade7977c20ee8fcd76e07e29737cc58de91f5e1 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 19 May 2026 12:19:28 +0000 Subject: [PATCH] feat: implement applyProposedDdl utility for handling DDL statements and tracking schema changes - Added applyProposedDdl function to apply SQL commands and track added/dropped tables, columns, and foreign keys. - Introduced MermaidDiagramHighlight interface to manage highlighting of added elements in ER diagrams. - Enhanced buildMermaidErDiagram function to support highlighting new tables, columns, and foreign keys. - Created comprehensive tests for applyProposedDdl and buildMermaidErDiagram functionalities, covering various DDL operations and their effects on the schema. - Added end-to-end tests for previewing connection diagrams, ensuring correct handling of SQL commands and validation of responses. --- backend/src/common/data-injection.tokens.ts | 1 + .../preview-connection-diagram.ds.ts | 6 + .../connection-diagram-preview-request.dto.ts | 17 + ...connection-diagram-preview-response.dto.ts | 68 +++ .../connection/connection.controller.ts | 36 ++ .../entities/connection/connection.module.ts | 6 + .../preview-connection-diagram.use.case.ts | 121 +++++ .../use-cases/use-cases.interfaces.ts | 9 + .../utils/apply-proposed-ddl.util.ts | 425 ++++++++++++++++++ .../utils/build-mermaid-er-diagram.util.ts | 86 +++- .../connection-diagram-preview.test.ts | 258 +++++++++++ ...aas-connection-diagram-preview-e2e.test.ts | 288 ++++++++++++ 12 files changed, 1317 insertions(+), 4 deletions(-) create mode 100644 backend/src/entities/connection/application/data-structures/preview-connection-diagram.ds.ts create mode 100644 backend/src/entities/connection/application/dto/connection-diagram-preview-request.dto.ts create mode 100644 backend/src/entities/connection/application/dto/connection-diagram-preview-response.dto.ts create mode 100644 backend/src/entities/connection/use-cases/preview-connection-diagram.use.case.ts create mode 100644 backend/src/entities/connection/utils/apply-proposed-ddl.util.ts create mode 100644 backend/test/ava-tests/connection-diagram-preview.test.ts create mode 100644 backend/test/ava-tests/non-saas-tests/non-saas-connection-diagram-preview-e2e.test.ts diff --git a/backend/src/common/data-injection.tokens.ts b/backend/src/common/data-injection.tokens.ts index f55f49a74..a819f1631 100644 --- a/backend/src/common/data-injection.tokens.ts +++ b/backend/src/common/data-injection.tokens.ts @@ -45,6 +45,7 @@ export enum UseCaseType { UNFREEZE_CONNECTION = 'UNFREEZE_CONNECTION', UPDATE_CONNECTION_TITLE = 'UPDATE_CONNECTION_TITLE', GET_CONNECTION_DIAGRAM = 'GET_CONNECTION_DIAGRAM', + PREVIEW_CONNECTION_DIAGRAM = 'PREVIEW_CONNECTION_DIAGRAM', FIND_ALL_USER_GROUPS = 'FIND_ALL_USER_GROUPS', INVITE_USER_IN_GROUP = 'INVITE_USER_IN_GROUP', diff --git a/backend/src/entities/connection/application/data-structures/preview-connection-diagram.ds.ts b/backend/src/entities/connection/application/data-structures/preview-connection-diagram.ds.ts new file mode 100644 index 000000000..1eef25278 --- /dev/null +++ b/backend/src/entities/connection/application/data-structures/preview-connection-diagram.ds.ts @@ -0,0 +1,6 @@ +export class PreviewConnectionDiagramDs { + connectionId: string; + masterPwd: string; + userId: string; + sqlCommands: Array; +} diff --git a/backend/src/entities/connection/application/dto/connection-diagram-preview-request.dto.ts b/backend/src/entities/connection/application/dto/connection-diagram-preview-request.dto.ts new file mode 100644 index 000000000..192716bd5 --- /dev/null +++ b/backend/src/entities/connection/application/dto/connection-diagram-preview-request.dto.ts @@ -0,0 +1,17 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { ArrayMaxSize, ArrayMinSize, IsArray, IsString, MaxLength } from 'class-validator'; + +export class ConnectionDiagramPreviewRequestDTO { + @ApiProperty({ + type: [String], + description: + 'Array of SQL DDL statements (CREATE TABLE, ALTER TABLE, DROP TABLE) to apply to the diagram preview. Statements are parsed and applied to an in-memory copy of the schema; nothing is executed against the real database.', + example: ['ALTER TABLE users ADD COLUMN age INTEGER'], + }) + @IsArray() + @ArrayMinSize(1) + @ArrayMaxSize(100) + @IsString({ each: true }) + @MaxLength(20000, { each: true }) + sqlCommands: Array; +} diff --git a/backend/src/entities/connection/application/dto/connection-diagram-preview-response.dto.ts b/backend/src/entities/connection/application/dto/connection-diagram-preview-response.dto.ts new file mode 100644 index 000000000..f8a9e5f4e --- /dev/null +++ b/backend/src/entities/connection/application/dto/connection-diagram-preview-response.dto.ts @@ -0,0 +1,68 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; + +export class ConnectionDiagramPreviewStatementResultDTO { + @ApiProperty() + sql: string; + + @ApiProperty({ enum: ['applied', 'skipped', 'error'] }) + status: 'applied' | 'skipped' | 'error'; + + @ApiProperty({ required: false, nullable: true }) + message?: string; +} + +export class ConnectionDiagramPreviewDiffDTO { + @ApiProperty({ type: [String] }) + addedTables: Array; + + @ApiProperty({ type: [String] }) + droppedTables: Array; + + @ApiProperty({ + description: 'Map of table name -> array of column names that would be added', + type: 'object', + additionalProperties: { type: 'array', items: { type: 'string' } }, + }) + addedColumns: Record>; + + @ApiProperty({ + description: 'Map of table name -> array of column names that would be removed', + type: 'object', + additionalProperties: { type: 'array', items: { type: 'string' } }, + }) + droppedColumns: Record>; + + @ApiProperty({ + description: 'Map of table name -> array of "column->refTable.refColumn" descriptors for new foreign keys', + type: 'object', + additionalProperties: { type: 'array', items: { type: 'string' } }, + }) + addedForeignKeys: Record>; + + @ApiProperty({ type: [ConnectionDiagramPreviewStatementResultDTO] }) + statementResults: Array; +} + +export class ConnectionDiagramPreviewResponseDTO { + @ApiProperty() + connectionId: string; + + @ApiProperty({ enum: ConnectionTypesEnum }) + databaseType: ConnectionTypesEnum; + + @ApiProperty({ + description: + 'Mermaid erDiagram source string representing the schema AFTER applying the provided SQL statements. Added entities are styled green via a classDef directive; new columns are marked with a "NEW" attribute key; new foreign keys are marked with "[NEW]" in the relationship label.', + }) + diagram: string; + + @ApiProperty({ description: 'Human-readable description of the projected database structure (post-changes).' }) + description: string; + + @ApiProperty({ type: ConnectionDiagramPreviewDiffDTO }) + diff: ConnectionDiagramPreviewDiffDTO; + + @ApiProperty() + generatedAt: string; +} diff --git a/backend/src/entities/connection/connection.controller.ts b/backend/src/entities/connection/connection.controller.ts index 68badb4df..4b06ca9d9 100644 --- a/backend/src/entities/connection/connection.controller.ts +++ b/backend/src/entities/connection/connection.controller.ts @@ -50,11 +50,14 @@ import { FoundPermissionsInConnectionDs } from './application/data-structures/fo import { GetConnectionDiagramDs } from './application/data-structures/get-connection-diagram.ds.js'; import { GetGroupsInConnectionDs } from './application/data-structures/get-groups-in-connection.ds.js'; import { GetPermissionsInConnectionDs } from './application/data-structures/get-permissions-in-connection.ds.js'; +import { PreviewConnectionDiagramDs } from './application/data-structures/preview-connection-diagram.ds.js'; import { RestoredConnectionDs } from './application/data-structures/restored-connection.ds.js'; import { UpdateConnectionDs } from './application/data-structures/update-connection.ds.js'; import { UpdateConnectionTitleDs } from './application/data-structures/update-connection-title.ds.js'; import { UpdateMasterPasswordDs } from './application/data-structures/update-master-password.ds.js'; import { ValidateConnectionMasterPasswordDs } from './application/data-structures/validate-connection-master-password.ds.js'; +import { ConnectionDiagramPreviewRequestDTO } from './application/dto/connection-diagram-preview-request.dto.js'; +import { ConnectionDiagramPreviewResponseDTO } from './application/dto/connection-diagram-preview-response.dto.js'; import { ConnectionDiagramResponseDTO } from './application/dto/connection-diagram-response.dto.js'; import { CreateConnectionDto } from './application/dto/create-connection.dto.js'; import { CreateGroupInConnectionDTO } from './application/dto/create-group-in-connection.dto.js'; @@ -79,6 +82,7 @@ import { IGetConnectionDiagram, IGetPermissionsForGroupInConnection, IGetUserGroupsInConnection, + IPreviewConnectionDiagram, IRefreshConnectionAgentToken, IRestoreConnection, ITestConnection, @@ -140,6 +144,8 @@ export class ConnectionController { private readonly updateConnectionTitleUseCase: IUpdateConnectionTitle, @Inject(UseCaseType.GET_CONNECTION_DIAGRAM) private readonly getConnectionDiagramUseCase: IGetConnectionDiagram, + @Inject(UseCaseType.PREVIEW_CONNECTION_DIAGRAM) + private readonly previewConnectionDiagramUseCase: IPreviewConnectionDiagram, @Inject(BaseType.GLOBAL_DB_CONTEXT) protected _dbContext: IGlobalDatabaseContext, private readonly amplitudeService: AmplitudeService, @@ -754,4 +760,34 @@ export class ConnectionController { }; return await this.getConnectionDiagramUseCase.execute(inputData, InTransactionEnum.OFF); } + + @ApiOperation({ + summary: + 'Preview Mermaid diagram with proposed DDL changes applied (SQL only). Nothing is executed against the real database — statements are parsed and applied to an in-memory copy of the schema.', + }) + @ApiBody({ type: ConnectionDiagramPreviewRequestDTO }) + @ApiResponse({ + status: 200, + type: ConnectionDiagramPreviewResponseDTO, + }) + @UseGuards(ConnectionDiagramGuard) + @Timeout(90000) + @Post('/connection/diagram/:connectionId/preview') + async previewConnectionDiagram( + @SlugUuid('connectionId') connectionId: string, + @MasterPassword() masterPwd: string, + @UserId() userId: string, + @Body() body: ConnectionDiagramPreviewRequestDTO, + ): Promise { + if (!connectionId) { + throw new BadRequestException(Messages.CONNECTION_ID_MISSING); + } + const inputData: PreviewConnectionDiagramDs = { + connectionId, + masterPwd, + userId, + sqlCommands: body.sqlCommands, + }; + return await this.previewConnectionDiagramUseCase.execute(inputData, InTransactionEnum.OFF); + } } diff --git a/backend/src/entities/connection/connection.module.ts b/backend/src/entities/connection/connection.module.ts index b33619ab6..4a2ab6bc5 100644 --- a/backend/src/entities/connection/connection.module.ts +++ b/backend/src/entities/connection/connection.module.ts @@ -28,6 +28,7 @@ import { GetConnectionDiagramUseCase } from './use-cases/get-connection-diagram. import { GetPermissionsForGroupInConnectionUseCase } from './use-cases/get-permissions-for-group-in-connection.use.case.js'; import { GetUserGroupsInConnectionUseCase } from './use-cases/get-user-groups-in-connection.use.case.js'; import { GetUserPermissionsForGroupInConnectionUseCase } from './use-cases/get-user-permissions-for-group-in-connection.use.case.js'; +import { PreviewConnectionDiagramUseCase } from './use-cases/preview-connection-diagram.use.case.js'; import { RefreshConnectionAgentTokenUseCase } from './use-cases/refresh-connection-agent-token.use.case.js'; import { RestoreConnectionUseCase } from './use-cases/restore-connection-use.case.js'; import { TestConnectionUseCase } from './use-cases/test-connection.use.case.js'; @@ -140,6 +141,10 @@ import { ValidateConnectionTokenUseCase } from './use-cases/validate-connection- provide: UseCaseType.GET_CONNECTION_DIAGRAM, useClass: GetConnectionDiagramUseCase, }, + { + provide: UseCaseType.PREVIEW_CONNECTION_DIAGRAM, + useClass: PreviewConnectionDiagramUseCase, + }, ], controllers: [ConnectionController], }) @@ -168,6 +173,7 @@ export class ConnectionModule implements NestModule { { path: '/connection/unfreeze/:connectionId', method: RequestMethod.PUT }, { path: '/connection/title/:connectionId', method: RequestMethod.PUT }, { path: '/connection/diagram/:connectionId', method: RequestMethod.GET }, + { path: '/connection/diagram/:connectionId/preview', method: RequestMethod.POST }, ) .apply(AuthWithApiMiddleware) .forRoutes({ path: 'connections', method: RequestMethod.GET }); diff --git a/backend/src/entities/connection/use-cases/preview-connection-diagram.use.case.ts b/backend/src/entities/connection/use-cases/preview-connection-diagram.use.case.ts new file mode 100644 index 000000000..5f25e5c10 --- /dev/null +++ b/backend/src/entities/connection/use-cases/preview-connection-diagram.use.case.ts @@ -0,0 +1,121 @@ +import { BadRequestException, HttpException, HttpStatus, Inject, Injectable, Scope } from '@nestjs/common'; +import { validateSchemaCache } from '@rocketadmin/shared-code/dist/src/caching/schema-cache-validator.js'; +import { getDataAccessObject } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/create-data-access-object.js'; +import { ForeignKeyDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/foreign-key.ds.js'; +import { PrimaryKeyDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/primary-key.ds.js'; +import { TableStructureDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/table-structure.ds.js'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { ExceptionOperations } from '../../../exceptions/custom-exceptions/exception-operation.js'; +import { UnknownSQLException } from '../../../exceptions/custom-exceptions/unknown-sql-exception.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { isConnectionTypeAgent } from '../../../helpers/is-connection-entity-agent.js'; +import { PreviewConnectionDiagramDs } from '../application/data-structures/preview-connection-diagram.ds.js'; +import { ConnectionDiagramPreviewResponseDTO } from '../application/dto/connection-diagram-preview-response.dto.js'; +import { applyProposedDdl, SchemaDiff } from '../utils/apply-proposed-ddl.util.js'; +import { buildMermaidErDiagram, MermaidTableInput } from '../utils/build-mermaid-er-diagram.util.js'; +import { isSqlConnectionType } from '../utils/is-sql-connection-type.util.js'; +import { IPreviewConnectionDiagram } from './use-cases.interfaces.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class PreviewConnectionDiagramUseCase + extends AbstractUseCase + implements IPreviewConnectionDiagram +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: PreviewConnectionDiagramDs): Promise { + const { connectionId, masterPwd, userId, sqlCommands } = inputData; + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection(connectionId, masterPwd); + if (!connection) { + throw new HttpException({ message: Messages.CONNECTION_NOT_FOUND }, HttpStatus.BAD_REQUEST); + } + if (!isSqlConnectionType(connection.type)) { + throw new BadRequestException(Messages.DIAGRAM_NOT_SUPPORTED_FOR_CONNECTION_TYPE); + } + + const dao = getDataAccessObject(connection); + const userEmail = isConnectionTypeAgent(connection.type) + ? await this._dbContext.userRepository.getUserEmailOrReturnNull(userId) + : undefined; + + await validateSchemaCache(dao, userEmail); + dao.invalidateMetadataCache(); + + let tables: Array<{ tableName: string; isView: boolean }>; + try { + tables = await dao.getTablesFromDB(userEmail); + } catch (e) { + throw new UnknownSQLException(e.message, ExceptionOperations.FAILED_TO_GET_TABLES); + } + + const realTables = tables.filter((t) => !t.isView); + const tableInputs: Array = await Promise.all( + realTables.map((t) => this.collectTableInfo(dao, t.tableName, userEmail)), + ); + + const { mutatedTables, diff } = applyProposedDdl(tableInputs, sqlCommands, connection.type as ConnectionTypesEnum); + + const { diagram, description } = buildMermaidErDiagram(connection.database || null, mutatedTables, { + addedTables: diff.addedTables, + addedColumns: diff.addedColumns, + addedForeignKeys: diff.addedForeignKeys, + }); + + return { + connectionId, + databaseType: connection.type as ConnectionTypesEnum, + diagram, + description, + diff: serializeDiff(diff), + generatedAt: new Date().toISOString(), + }; + } + + private async collectTableInfo( + dao: ReturnType, + tableName: string, + userEmail: string | undefined, + ): Promise { + const [structure, primaryColumns, foreignKeys] = await Promise.all([ + this.safe>(() => dao.getTableStructure(tableName, userEmail), []), + this.safe>(() => dao.getTablePrimaryColumns(tableName, userEmail), []), + this.safe>(() => dao.getTableForeignKeys(tableName, userEmail), []), + ]); + return { tableName, structure, primaryColumns, foreignKeys }; + } + + private async safe(fn: () => Promise, fallback: T): Promise { + try { + return await fn(); + } catch { + return fallback; + } + } +} + +function serializeDiff(diff: SchemaDiff): ConnectionDiagramPreviewResponseDTO['diff'] { + return { + addedTables: Array.from(diff.addedTables), + droppedTables: Array.from(diff.droppedTables), + addedColumns: mapSetToObject(diff.addedColumns), + droppedColumns: mapSetToObject(diff.droppedColumns), + addedForeignKeys: mapSetToObject(diff.addedForeignKeys), + statementResults: diff.statementResults, + }; +} + +function mapSetToObject(map: Map>): Record> { + const out: Record> = {}; + for (const [key, set] of map.entries()) { + out[key] = Array.from(set); + } + return out; +} diff --git a/backend/src/entities/connection/use-cases/use-cases.interfaces.ts b/backend/src/entities/connection/use-cases/use-cases.interfaces.ts index 15332c227..117a0bf00 100644 --- a/backend/src/entities/connection/use-cases/use-cases.interfaces.ts +++ b/backend/src/entities/connection/use-cases/use-cases.interfaces.ts @@ -14,6 +14,7 @@ import { FoundPermissionsInConnectionDs } from '../application/data-structures/f import { GetConnectionDiagramDs } from '../application/data-structures/get-connection-diagram.ds.js'; import { GetGroupsInConnectionDs } from '../application/data-structures/get-groups-in-connection.ds.js'; import { GetPermissionsInConnectionDs } from '../application/data-structures/get-permissions-in-connection.ds.js'; +import { PreviewConnectionDiagramDs } from '../application/data-structures/preview-connection-diagram.ds.js'; import { RestoredConnectionDs } from '../application/data-structures/restored-connection.ds.js'; import { TestConnectionResultDs } from '../application/data-structures/test-connection-result.ds.js'; import { TokenDs } from '../application/data-structures/token.ds.js'; @@ -22,6 +23,7 @@ import { UpdateConnectionDs } from '../application/data-structures/update-connec import { UpdateConnectionTitleDs } from '../application/data-structures/update-connection-title.ds.js'; import { UpdateMasterPasswordDs } from '../application/data-structures/update-master-password.ds.js'; import { ValidateConnectionMasterPasswordDs } from '../application/data-structures/validate-connection-master-password.ds.js'; +import { ConnectionDiagramPreviewResponseDTO } from '../application/dto/connection-diagram-preview-response.dto.js'; import { ConnectionDiagramResponseDTO } from '../application/dto/connection-diagram-response.dto.js'; import { CreatedConnectionDTO } from '../application/dto/created-connection.dto.js'; import { FoundUserGroupsInConnectionDTO } from '../application/dto/found-user-groups-in-connection.dto.js'; @@ -112,3 +114,10 @@ export interface IUpdateConnectionTitle { export interface IGetConnectionDiagram { execute(inputData: GetConnectionDiagramDs, inTransaction: InTransactionEnum): Promise; } + +export interface IPreviewConnectionDiagram { + execute( + inputData: PreviewConnectionDiagramDs, + inTransaction: InTransactionEnum, + ): Promise; +} diff --git a/backend/src/entities/connection/utils/apply-proposed-ddl.util.ts b/backend/src/entities/connection/utils/apply-proposed-ddl.util.ts new file mode 100644 index 000000000..56bc50a67 --- /dev/null +++ b/backend/src/entities/connection/utils/apply-proposed-ddl.util.ts @@ -0,0 +1,425 @@ +import { ForeignKeyDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/foreign-key.ds.js'; +import { PrimaryKeyDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/primary-key.ds.js'; +import { TableStructureDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/table-structure.ds.js'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import sqlParser from 'node-sql-parser'; +import { connectionTypeToParserDialect } from '../../table-schema/utils/assert-dialect-supported.js'; +import { MermaidTableInput } from './build-mermaid-er-diagram.util.js'; + +const { Parser } = sqlParser; + +export interface SchemaDiff { + addedTables: Set; + droppedTables: Set; + addedColumns: Map>; + droppedColumns: Map>; + addedForeignKeys: Map>; + statementResults: Array; +} + +export interface StatementResult { + sql: string; + status: 'applied' | 'skipped' | 'error'; + message?: string; +} + +export interface ApplyDdlResult { + mutatedTables: Array; + diff: SchemaDiff; +} + +interface MutableTable extends MermaidTableInput {} + +export function applyProposedDdl( + tables: Array, + sqlCommands: Array, + connectionType: ConnectionTypesEnum, +): ApplyDdlResult { + const tableMap = new Map(); + for (const table of tables) { + tableMap.set(normalizeIdent(table.tableName), cloneTable(table)); + } + + const diff: SchemaDiff = { + addedTables: new Set(), + droppedTables: new Set(), + addedColumns: new Map>(), + droppedColumns: new Map>(), + addedForeignKeys: new Map>(), + statementResults: [], + }; + + const dialect = connectionTypeToParserDialect(connectionType); + const parser = new Parser(); + + for (const rawSql of sqlCommands) { + const sql = (rawSql ?? '').trim(); + if (!sql) { + diff.statementResults.push({ sql: rawSql, status: 'skipped', message: 'empty statement' }); + continue; + } + let ast: unknown; + try { + ast = parser.astify(sql, { database: dialect }); + } catch (err) { + diff.statementResults.push({ + sql, + status: 'error', + message: `parse error: ${(err as Error).message}`, + }); + continue; + } + const statements = Array.isArray(ast) ? ast : [ast]; + for (const statement of statements) { + try { + const handled = applyStatement(statement, tableMap, diff); + diff.statementResults.push( + handled.applied + ? { sql, status: 'applied' } + : { sql, status: 'skipped', message: handled.reason ?? 'unsupported statement' }, + ); + } catch (err) { + diff.statementResults.push({ + sql, + status: 'error', + message: (err as Error).message, + }); + } + } + } + + const mutatedTables = Array.from(tableMap.values()); + return { mutatedTables, diff }; +} + +function applyStatement( + statement: any, + tableMap: Map, + diff: SchemaDiff, +): { applied: boolean; reason?: string } { + if (!statement || typeof statement !== 'object') { + return { applied: false, reason: 'unrecognized statement' }; + } + const type = String(statement.type ?? '').toLowerCase(); + const keyword = String(statement.keyword ?? '').toLowerCase(); + + if (type === 'create' && keyword === 'table') { + return applyCreateTable(statement, tableMap, diff); + } + if (type === 'drop' && keyword === 'table') { + return applyDropTable(statement, tableMap, diff); + } + if (type === 'alter' && keyword === 'table') { + return applyAlterTable(statement, tableMap, diff); + } + return { applied: false, reason: `unsupported statement type "${type} ${keyword}"` }; +} + +function applyCreateTable( + statement: any, + tableMap: Map, + diff: SchemaDiff, +): { applied: boolean; reason?: string } { + const tableName = extractTableName(statement.table); + if (!tableName) return { applied: false, reason: 'create table: cannot resolve table name' }; + const key = normalizeIdent(tableName); + if (tableMap.has(key)) { + return { applied: false, reason: `create table: "${tableName}" already exists` }; + } + + const newTable: MutableTable = { + tableName, + structure: [], + primaryColumns: [], + foreignKeys: [], + }; + + const defs: Array = Array.isArray(statement.create_definitions) ? statement.create_definitions : []; + for (const def of defs) { + if (!def || typeof def !== 'object') continue; + if (def.resource === 'column') { + const colName = extractColumnName(def.column); + if (!colName) continue; + newTable.structure.push(buildColumnStructure(colName, def)); + if (def.primary_key) { + newTable.primaryColumns.push({ column_name: colName, data_type: extractDataType(def.definition) }); + } + const refFk = buildForeignKeyFromColumnRef(colName, def.reference_definition); + if (refFk) newTable.foreignKeys.push(refFk); + } else if (def.resource === 'constraint') { + handleInlineConstraint(def, newTable); + } + } + + tableMap.set(key, newTable); + diff.addedTables.add(tableName); + for (const col of newTable.structure) { + addToMapSet(diff.addedColumns, tableName, col.column_name); + } + for (const fk of newTable.foreignKeys) { + addToMapSet(diff.addedForeignKeys, tableName, fkKey(fk)); + } + return { applied: true }; +} + +function applyDropTable( + statement: any, + tableMap: Map, + diff: SchemaDiff, +): { applied: boolean; reason?: string } { + const namesNode = Array.isArray(statement.name) + ? statement.name + : Array.isArray(statement.table) + ? statement.table + : []; + let any = false; + for (const node of namesNode) { + const tableName = node?.table; + if (!tableName) continue; + const key = normalizeIdent(tableName); + if (tableMap.has(key)) { + tableMap.delete(key); + diff.droppedTables.add(tableName); + any = true; + } + } + if (!any) return { applied: false, reason: 'drop table: no matching tables' }; + return { applied: true }; +} + +function applyAlterTable( + statement: any, + tableMap: Map, + diff: SchemaDiff, +): { applied: boolean; reason?: string } { + const tableName = extractTableName(statement.table); + if (!tableName) return { applied: false, reason: 'alter table: cannot resolve table name' }; + const key = normalizeIdent(tableName); + const target = tableMap.get(key); + if (!target) return { applied: false, reason: `alter table: "${tableName}" does not exist` }; + + const exprs: Array = Array.isArray(statement.expr) ? statement.expr : []; + let appliedAny = false; + const reasons: Array = []; + + for (const expr of exprs) { + if (!expr || typeof expr !== 'object') continue; + const action = String(expr.action ?? '').toLowerCase(); + const resource = String(expr.resource ?? '').toLowerCase(); + + if (action === 'add' && resource === 'column') { + const colName = extractColumnName(expr.column); + if (!colName) { + reasons.push('add column: missing column name'); + continue; + } + if (target.structure.some((c) => normalizeIdent(c.column_name) === normalizeIdent(colName))) { + reasons.push(`add column: "${colName}" already exists`); + continue; + } + target.structure.push(buildColumnStructure(colName, expr)); + addToMapSet(diff.addedColumns, target.tableName, colName); + const refFk = buildForeignKeyFromColumnRef(colName, expr.reference_definition); + if (refFk) { + target.foreignKeys.push(refFk); + addToMapSet(diff.addedForeignKeys, target.tableName, fkKey(refFk)); + } + appliedAny = true; + } else if (action === 'drop' && resource === 'column') { + const colName = extractColumnName(expr.column); + if (!colName) { + reasons.push('drop column: missing column name'); + continue; + } + const before = target.structure.length; + target.structure = target.structure.filter((c) => normalizeIdent(c.column_name) !== normalizeIdent(colName)); + target.primaryColumns = target.primaryColumns.filter( + (p) => normalizeIdent(p.column_name) !== normalizeIdent(colName), + ); + target.foreignKeys = target.foreignKeys.filter( + (fk) => normalizeIdent(fk.column_name) !== normalizeIdent(colName), + ); + if (target.structure.length === before) { + reasons.push(`drop column: "${colName}" not found`); + continue; + } + addToMapSet(diff.droppedColumns, target.tableName, colName); + appliedAny = true; + } else if (action === 'add' && resource === 'constraint') { + const cd = expr.create_definitions; + if (cd && typeof cd === 'object') { + const handled = handleInlineConstraint(cd, target); + if (handled.added && handled.fk) { + addToMapSet(diff.addedForeignKeys, target.tableName, fkKey(handled.fk)); + appliedAny = true; + } else if (handled.reason) { + reasons.push(handled.reason); + } + } else { + reasons.push('add constraint: malformed definition'); + } + } else if (action === 'drop' && resource === 'constraint') { + const constraintName = expr.constraint; + if (typeof constraintName === 'string') { + const before = target.foreignKeys.length; + target.foreignKeys = target.foreignKeys.filter((fk) => fk.constraint_name !== constraintName); + if (target.foreignKeys.length !== before) appliedAny = true; + else reasons.push(`drop constraint: "${constraintName}" not found`); + } else { + reasons.push('drop constraint: missing constraint name'); + } + } else { + reasons.push(`unsupported alter expression: action="${action}" resource="${resource}"`); + } + } + + if (!appliedAny) { + return { applied: false, reason: reasons.join('; ') || 'no applicable alter expressions' }; + } + return { applied: true }; +} + +function handleInlineConstraint( + def: any, + target: MutableTable, +): { added: boolean; fk?: ForeignKeyDS; reason?: string } { + const constraintType = String(def?.constraint_type ?? '').toLowerCase(); + if (constraintType === 'primary key') { + const cols: Array = (Array.isArray(def.definition) ? def.definition : []) + .map(extractColumnName) + .filter((c: string | null): c is string => Boolean(c)); + for (const colName of cols) { + if (!target.primaryColumns.some((p) => normalizeIdent(p.column_name) === normalizeIdent(colName))) { + const col = target.structure.find((c) => normalizeIdent(c.column_name) === normalizeIdent(colName)); + target.primaryColumns.push({ + column_name: colName, + data_type: col?.data_type ?? 'unknown', + }); + } + } + return { added: cols.length > 0 }; + } + if (constraintType === 'foreign key') { + const sourceCols: Array = (Array.isArray(def.definition) ? def.definition : []) + .map(extractColumnName) + .filter((c: string | null): c is string => Boolean(c)); + const ref = def.reference_definition; + const refTable = extractTableName(ref?.table); + const refCols: Array = (Array.isArray(ref?.definition) ? ref.definition : []) + .map(extractColumnName) + .filter((c: string | null): c is string => Boolean(c)); + if (!refTable || sourceCols.length === 0) { + return { added: false, reason: 'foreign key: missing referenced table or source columns' }; + } + const constraintName = + typeof def.constraint === 'string' ? def.constraint : `fk_${target.tableName}_${sourceCols.join('_')}`; + const fk: ForeignKeyDS = { + column_name: sourceCols[0], + referenced_column_name: refCols[0] ?? 'id', + referenced_table_name: refTable, + constraint_name: constraintName, + }; + target.foreignKeys.push(fk); + return { added: true, fk }; + } + return { added: false, reason: `unsupported inline constraint "${constraintType}"` }; +} + +function buildColumnStructure(colName: string, def: any): TableStructureDS { + const dataType = extractDataType(def?.definition); + const notNull = def?.nullable && String(def.nullable.type).toLowerCase().includes('not null'); + const defaultVal = extractDefault(def?.default_val); + return { + column_name: colName, + data_type: dataType, + data_type_params: '', + udt_name: dataType, + allow_null: !notNull, + character_maximum_length: typeof def?.definition?.length === 'number' ? def.definition.length : null, + column_default: defaultVal, + }; +} + +function extractDataType(definition: any): string { + if (!definition || typeof definition !== 'object') return 'unknown'; + const dt = definition.dataType; + return typeof dt === 'string' && dt.length > 0 ? dt.toLowerCase() : 'unknown'; +} + +function extractDefault(defaultVal: any): string | null { + if (!defaultVal || typeof defaultVal !== 'object') return null; + const v = defaultVal.value; + if (v && typeof v === 'object') { + if ('value' in v) { + const inner = (v as { value: unknown }).value; + return inner === null || inner === undefined ? null : String(inner); + } + } + return null; +} + +function buildForeignKeyFromColumnRef(colName: string, referenceDefinition: any): ForeignKeyDS | null { + if (!referenceDefinition || typeof referenceDefinition !== 'object') return null; + const refTable = extractTableName(referenceDefinition.table); + const refCols: Array = (Array.isArray(referenceDefinition.definition) ? referenceDefinition.definition : []) + .map(extractColumnName) + .filter((c: string | null): c is string => Boolean(c)); + if (!refTable) return null; + return { + column_name: colName, + referenced_table_name: refTable, + referenced_column_name: refCols[0] ?? 'id', + constraint_name: `fk_${colName}_${refTable}`, + }; +} + +function extractTableName(tableNode: any): string | null { + if (!Array.isArray(tableNode) || tableNode.length === 0) return null; + const first = tableNode[0]; + if (first && typeof first === 'object' && typeof first.table === 'string') return first.table; + return null; +} + +function extractColumnName(columnNode: any): string | null { + if (!columnNode || typeof columnNode !== 'object') return null; + if (typeof columnNode === 'string') return columnNode; + const col = (columnNode as Record).column; + if (typeof col === 'string') return col; + if (col && typeof col === 'object') { + const expr = (col as Record).expr; + if (expr && typeof expr === 'object') { + const value = (expr as Record).value; + if (typeof value === 'string') return value; + } + } + return null; +} + +function normalizeIdent(name: string): string { + return name.replace(/^[`"[]|[`"\]]$/g, '').toLowerCase(); +} + +function cloneTable(t: MermaidTableInput): MutableTable { + return { + tableName: t.tableName, + structure: t.structure.map((c) => ({ ...c })), + primaryColumns: t.primaryColumns.map((p) => ({ ...p })), + foreignKeys: t.foreignKeys.map((f) => ({ ...f })), + }; +} + +function addToMapSet(map: Map>, key: string, value: string): void { + let set = map.get(key); + if (!set) { + set = new Set(); + map.set(key, set); + } + set.add(value); +} + +function fkKey(fk: ForeignKeyDS): string { + return `${fk.column_name}->${fk.referenced_table_name}.${fk.referenced_column_name}`; +} + +// Re-export types used in payloads +export type { PrimaryKeyDS, TableStructureDS }; diff --git a/backend/src/entities/connection/utils/build-mermaid-er-diagram.util.ts b/backend/src/entities/connection/utils/build-mermaid-er-diagram.util.ts index 54ef3e224..1e45f1f1b 100644 --- a/backend/src/entities/connection/utils/build-mermaid-er-diagram.util.ts +++ b/backend/src/entities/connection/utils/build-mermaid-er-diagram.util.ts @@ -14,9 +14,21 @@ export interface MermaidDiagramResult { description: string; } +export interface MermaidDiagramHighlight { + addedTables?: Set; + addedColumns?: Map>; + addedForeignKeys?: Map>; +} + +const ADDED_CLASS_NAME = 'addedEntity'; +const ADDED_CLASS_DEF = ` classDef ${ADDED_CLASS_NAME} fill:#d4edda,stroke:#28a745,color:#155724`; +const ADDED_COLUMN_MARKER = 'NEW'; +const ADDED_FK_MARKER = '[NEW]'; + export function buildMermaidErDiagram( databaseName: string | null, tables: Array, + highlight?: MermaidDiagramHighlight, ): MermaidDiagramResult { const aliasByTable = new Map(); const usedAliases = new Set(); @@ -24,12 +36,17 @@ export function buildMermaidErDiagram( aliasByTable.set(t.tableName, makeUniqueAlias(t.tableName, usedAliases)); } + const addedTablesNorm = normalizeSet(highlight?.addedTables); + const addedColumnsNorm = normalizeMap(highlight?.addedColumns); + const addedFksNorm = normalizeMap(highlight?.addedForeignKeys); + const lines: Array = ['erDiagram']; for (const table of tables) { const alias = aliasByTable.get(table.tableName)!; const pkColumnNames = new Set(table.primaryColumns.map((p) => p.column_name)); const fkColumnNames = new Set(table.foreignKeys.map((fk) => fk.column_name)); + const tableAddedCols = addedColumnsNorm.get(normalizeIdent(table.tableName)) ?? new Set(); const aliasDiffersFromOriginal = alias !== table.tableName; const header = aliasDiffersFromOriginal @@ -46,6 +63,7 @@ export function buildMermaidErDiagram( const markers: Array = []; if (pkColumnNames.has(column.column_name)) markers.push('PK'); if (fkColumnNames.has(column.column_name)) markers.push('FK'); + if (tableAddedCols.has(normalizeIdent(column.column_name))) markers.push(ADDED_COLUMN_MARKER); const comment = buildColumnComment(column); const tail = [markers.join(','), comment].filter((p) => p && p.length > 0).join(' '); lines.push(` ${dataType} ${colName}${tail ? ' ' + tail : ''}`); @@ -57,24 +75,50 @@ export function buildMermaidErDiagram( let relationshipCount = 0; for (const table of tables) { const sourceAlias = aliasByTable.get(table.tableName)!; + const tableAddedFks = addedFksNorm.get(normalizeIdent(table.tableName)) ?? new Set(); for (const fk of table.foreignKeys) { const targetAlias = aliasByTable.get(fk.referenced_table_name); if (!targetAlias) continue; - const label = `"${sanitizeQuotedText(fk.column_name)} -> ${sanitizeQuotedText(fk.referenced_column_name)}"`; - lines.push(` ${sourceAlias} }o--|| ${targetAlias} : ${label}`); + const isAdded = tableAddedFks.has(fkKey(fk)); + const labelText = `${sanitizeQuotedText(fk.column_name)} -> ${sanitizeQuotedText(fk.referenced_column_name)}${isAdded ? ' ' + ADDED_FK_MARKER : ''}`; + lines.push(` ${sourceAlias} }o--|| ${targetAlias} : "${labelText}"`); relationshipCount++; } } + const addedAliases: Array = []; + for (const table of tables) { + if (addedTablesNorm.has(normalizeIdent(table.tableName))) { + addedAliases.push(aliasByTable.get(table.tableName)!); + } + } + if (addedAliases.length > 0) { + lines.push(ADDED_CLASS_DEF); + for (const alias of addedAliases) { + lines.push(` class ${alias} ${ADDED_CLASS_NAME}`); + } + } + const diagram = lines.join('\n'); - const description = buildDescription(databaseName, tables, relationshipCount); + const description = buildDescription(databaseName, tables, relationshipCount, { + addedTables: addedTablesNorm, + addedColumns: addedColumnsNorm, + addedForeignKeys: addedFksNorm, + }); return { diagram, description }; } +interface BuildDescriptionHighlight { + addedTables: Set; + addedColumns: Map>; + addedForeignKeys: Map>; +} + function buildDescription( databaseName: string | null, tables: Array, relationshipCount: number, + highlight: BuildDescriptionHighlight, ): string { const dbLabel = databaseName ? `Database "${databaseName}"` : 'Database'; const tablesPart = `${tables.length} ${pluralize(tables.length, 'table', 'tables')}`; @@ -92,7 +136,15 @@ function buildDescription( t.foreignKeys.length > 0 ? `FKs: ${t.foreignKeys.map((fk) => `${fk.column_name}->${fk.referenced_table_name}.${fk.referenced_column_name}`).join(', ')}` : 'no foreign keys'; - return `- ${t.tableName} (${t.structure.length} ${pluralize(t.structure.length, 'column', 'columns')}; ${pkPart}; ${fkPart})`; + const isNewTable = highlight.addedTables.has(normalizeIdent(t.tableName)); + const newCols = highlight.addedColumns.get(normalizeIdent(t.tableName)); + const newFks = highlight.addedForeignKeys.get(normalizeIdent(t.tableName)); + const markers: Array = []; + if (isNewTable) markers.push('NEW TABLE'); + if (newCols && newCols.size > 0 && !isNewTable) markers.push(`new columns: ${Array.from(newCols).join(', ')}`); + if (newFks && newFks.size > 0 && !isNewTable) markers.push(`new FKs: ${Array.from(newFks).join(', ')}`); + const markerSuffix = markers.length > 0 ? ` [${markers.join('; ')}]` : ''; + return `- ${t.tableName} (${t.structure.length} ${pluralize(t.structure.length, 'column', 'columns')}; ${pkPart}; ${fkPart})${markerSuffix}`; }); return [header, 'Tables:', ...tableSummaries].join('\n'); @@ -162,3 +214,29 @@ function sanitizeIdentifier(value: string): string { function sanitizeQuotedText(value: string): string { return value.replace(/"/g, "'").replace(/[\r\n\t]+/g, ' '); } + +function normalizeIdent(name: string): string { + return name.replace(/^[`"[]|[`"\]]$/g, '').toLowerCase(); +} + +function normalizeSet(input?: Set): Set { + const out = new Set(); + if (!input) return out; + for (const v of input) out.add(normalizeIdent(v)); + return out; +} + +function normalizeMap(input?: Map>): Map> { + const out = new Map>(); + if (!input) return out; + for (const [k, set] of input.entries()) { + const normalized = new Set(); + for (const v of set) normalized.add(normalizeIdent(v)); + out.set(normalizeIdent(k), normalized); + } + return out; +} + +function fkKey(fk: ForeignKeyDS): string { + return `${fk.column_name}->${fk.referenced_table_name}.${fk.referenced_column_name}`; +} diff --git a/backend/test/ava-tests/connection-diagram-preview.test.ts b/backend/test/ava-tests/connection-diagram-preview.test.ts new file mode 100644 index 000000000..b1c2f5e80 --- /dev/null +++ b/backend/test/ava-tests/connection-diagram-preview.test.ts @@ -0,0 +1,258 @@ +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import test from 'ava'; +import { applyProposedDdl } from '../../src/entities/connection/utils/apply-proposed-ddl.util.js'; +import { + buildMermaidErDiagram, + MermaidTableInput, +} from '../../src/entities/connection/utils/build-mermaid-er-diagram.util.js'; + +function baseUsersTable(): MermaidTableInput { + return { + tableName: 'users', + structure: [ + { + column_name: 'id', + data_type: 'integer', + data_type_params: '', + udt_name: 'int4', + allow_null: false, + character_maximum_length: null, + column_default: null, + }, + { + column_name: 'email', + data_type: 'varchar', + data_type_params: '', + udt_name: 'varchar', + allow_null: false, + character_maximum_length: 255, + column_default: null, + }, + ], + primaryColumns: [{ column_name: 'id', data_type: 'integer' }], + foreignKeys: [], + }; +} + +test('applyProposedDdl: applies ADD COLUMN and tracks it in addedColumns', (t) => { + const tables = [baseUsersTable()]; + const { mutatedTables, diff } = applyProposedDdl( + tables, + ['ALTER TABLE users ADD COLUMN age INTEGER NOT NULL DEFAULT 0'], + ConnectionTypesEnum.postgres, + ); + t.is(mutatedTables.length, 1); + const users = mutatedTables[0]; + t.true(users.structure.some((c) => c.column_name === 'age')); + t.deepEqual([...(diff.addedColumns.get('users') ?? new Set())], ['age']); + t.is(diff.statementResults.length, 1); + t.is(diff.statementResults[0].status, 'applied'); +}); + +test('applyProposedDdl: CREATE TABLE with inline REFERENCES produces table + FK in diff', (t) => { + const tables = [baseUsersTable()]; + const { mutatedTables, diff } = applyProposedDdl( + tables, + ['CREATE TABLE posts (id SERIAL PRIMARY KEY, user_id INTEGER REFERENCES users(id), body TEXT)'], + ConnectionTypesEnum.postgres, + ); + t.is(mutatedTables.length, 2); + const posts = mutatedTables.find((tt) => tt.tableName === 'posts')!; + t.truthy(posts); + t.is(posts.foreignKeys.length, 1); + t.is(posts.foreignKeys[0].referenced_table_name, 'users'); + t.is(posts.foreignKeys[0].column_name, 'user_id'); + t.deepEqual([...diff.addedTables], ['posts']); + t.deepEqual([...(diff.addedForeignKeys.get('posts') ?? new Set())], ['user_id->users.id']); +}); + +test('applyProposedDdl: DROP COLUMN removes column and lists it in droppedColumns', (t) => { + const tables = [baseUsersTable()]; + const { mutatedTables, diff } = applyProposedDdl( + tables, + ['ALTER TABLE users DROP COLUMN email'], + ConnectionTypesEnum.postgres, + ); + t.is(mutatedTables[0].structure.length, 1); + t.false(mutatedTables[0].structure.some((c) => c.column_name === 'email')); + t.deepEqual([...(diff.droppedColumns.get('users') ?? new Set())], ['email']); +}); + +test('applyProposedDdl: DROP TABLE removes the table and lists it in droppedTables', (t) => { + const tables = [baseUsersTable(), { ...baseUsersTable(), tableName: 'sessions' }]; + const { mutatedTables, diff } = applyProposedDdl(tables, ['DROP TABLE sessions'], ConnectionTypesEnum.postgres); + t.is(mutatedTables.length, 1); + t.is(mutatedTables[0].tableName, 'users'); + t.deepEqual([...diff.droppedTables], ['sessions']); +}); + +test('applyProposedDdl: ALTER TABLE ADD FOREIGN KEY constraint produces FK in diff', (t) => { + const tables = [ + baseUsersTable(), + { + tableName: 'posts', + structure: [ + { + column_name: 'id', + data_type: 'integer', + data_type_params: '', + udt_name: 'int4', + allow_null: false, + character_maximum_length: null, + column_default: null, + }, + { + column_name: 'user_id', + data_type: 'integer', + data_type_params: '', + udt_name: 'int4', + allow_null: true, + character_maximum_length: null, + column_default: null, + }, + ], + primaryColumns: [{ column_name: 'id', data_type: 'integer' }], + foreignKeys: [], + }, + ]; + const { mutatedTables, diff } = applyProposedDdl( + tables, + ['ALTER TABLE posts ADD CONSTRAINT fk_posts_user FOREIGN KEY (user_id) REFERENCES users(id)'], + ConnectionTypesEnum.postgres, + ); + const posts = mutatedTables.find((tt) => tt.tableName === 'posts')!; + t.is(posts.foreignKeys.length, 1); + t.is(posts.foreignKeys[0].column_name, 'user_id'); + t.is(posts.foreignKeys[0].referenced_table_name, 'users'); + t.deepEqual([...(diff.addedForeignKeys.get('posts') ?? new Set())], ['user_id->users.id']); +}); + +test('applyProposedDdl: parse errors are reported per-statement and do not abort the batch', (t) => { + const tables = [baseUsersTable()]; + const { diff } = applyProposedDdl( + tables, + ['ALTER TABLE users ADD COLUMN good_col INTEGER', 'THIS IS NOT VALID SQL'], + ConnectionTypesEnum.postgres, + ); + t.is(diff.statementResults.length, 2); + t.is(diff.statementResults[0].status, 'applied'); + t.is(diff.statementResults[1].status, 'error'); + t.regex(String(diff.statementResults[1].message), /parse error/); +}); + +test('applyProposedDdl: empty statements are skipped, not errored', (t) => { + const tables = [baseUsersTable()]; + const { diff } = applyProposedDdl(tables, ['', ' '], ConnectionTypesEnum.postgres); + t.is(diff.statementResults.length, 2); + t.true(diff.statementResults.every((r) => r.status === 'skipped')); +}); + +test('applyProposedDdl: ALTER on unknown table is skipped with reason', (t) => { + const tables = [baseUsersTable()]; + const { diff } = applyProposedDdl( + tables, + ['ALTER TABLE nonexistent ADD COLUMN x INTEGER'], + ConnectionTypesEnum.postgres, + ); + t.is(diff.statementResults[0].status, 'skipped'); + t.regex(String(diff.statementResults[0].message), /does not exist/); +}); + +test('buildMermaidErDiagram: highlight adds classDef and class lines for added tables', (t) => { + const tables: MermaidTableInput[] = [ + baseUsersTable(), + { + tableName: 'posts', + structure: [ + { + column_name: 'id', + data_type: 'serial', + data_type_params: '', + udt_name: 'serial', + allow_null: false, + character_maximum_length: null, + column_default: null, + }, + ], + primaryColumns: [{ column_name: 'id', data_type: 'serial' }], + foreignKeys: [], + }, + ]; + const { diagram } = buildMermaidErDiagram('mydb', tables, { + addedTables: new Set(['posts']), + addedColumns: new Map([['posts', new Set(['id'])]]), + addedForeignKeys: new Map(), + }); + t.true(diagram.includes('classDef addedEntity')); + t.true(diagram.includes('fill:#d4edda')); + t.true(diagram.includes('class posts addedEntity')); + t.false(diagram.includes('class users addedEntity')); +}); + +test('buildMermaidErDiagram: highlight tags new columns with NEW marker', (t) => { + const tables: MermaidTableInput[] = [ + { + ...baseUsersTable(), + structure: [ + ...baseUsersTable().structure, + { + column_name: 'nickname', + data_type: 'varchar', + data_type_params: '', + udt_name: 'varchar', + allow_null: true, + character_maximum_length: 50, + column_default: null, + }, + ], + }, + ]; + const { diagram } = buildMermaidErDiagram('mydb', tables, { + addedTables: new Set(), + addedColumns: new Map([['users', new Set(['nickname'])]]), + addedForeignKeys: new Map(), + }); + t.regex(diagram, /varchar nickname[^"\n]*NEW/); +}); + +test('buildMermaidErDiagram: highlight tags new FK relationships with [NEW]', (t) => { + const tables: MermaidTableInput[] = [ + baseUsersTable(), + { + tableName: 'posts', + structure: [ + { + column_name: 'user_id', + data_type: 'integer', + data_type_params: '', + udt_name: 'int4', + allow_null: true, + character_maximum_length: null, + column_default: null, + }, + ], + primaryColumns: [], + foreignKeys: [ + { + column_name: 'user_id', + referenced_table_name: 'users', + referenced_column_name: 'id', + constraint_name: 'fk_posts_user', + }, + ], + }, + ]; + const { diagram } = buildMermaidErDiagram('mydb', tables, { + addedTables: new Set(), + addedColumns: new Map(), + addedForeignKeys: new Map([['posts', new Set(['user_id->users.id'])]]), + }); + t.regex(diagram, /posts \}o--\|\| users : "user_id -> id \[NEW\]"/); +}); + +test('buildMermaidErDiagram: without highlight, behavior matches the legacy non-preview output', (t) => { + const tables = [baseUsersTable()]; + const { diagram } = buildMermaidErDiagram('mydb', tables); + t.false(diagram.includes('classDef')); + t.false(diagram.includes('NEW')); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-connection-diagram-preview-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-connection-diagram-preview-e2e.test.ts new file mode 100644 index 000000000..a7a677301 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-connection-diagram-preview-e2e.test.ts @@ -0,0 +1,288 @@ +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Messages } from '../../../src/exceptions/text/messages.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getRandomTestTableName } from '../../utils/get-random-test-table-name.js'; +import { getTestKnex } from '../../utils/get-test-knex.js'; +import { registerUserAndReturnUserInfo } from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; + +let parentTableName: string; +let childTableName: string; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }).compile(); + _testUtils = moduleFixture.get(TestUtils); + + app = moduleFixture.createNestApplication() as any; + app.use(cookieParser()); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + app.getHttpServer().listen(0); + + const postgresConnection = mockFactory.generateConnectionToTestPostgresDBInDocker(); + parentTableName = getRandomTestTableName(); + childTableName = getRandomTestTableName(); + const knex = getTestKnex(postgresConnection); + await knex.schema.dropTableIfExists(childTableName); + await knex.schema.dropTableIfExists(parentTableName); + await knex.schema.createTable(parentTableName, (table) => { + table.increments('id').primary(); + table.string('name', 100).notNullable(); + }); + await knex.schema.createTable(childTableName, (table) => { + table.increments('id').primary(); + table.string('label', 100); + table.integer('parent_id').references('id').inTable(parentTableName); + }); +}); + +test.after(async () => { + try { + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +test.serial( + 'POST /connection/diagram/:connectionId/preview > marks an ADD COLUMN as new and highlights diagram', + async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionDto = mockFactory.generateConnectionToTestPostgresDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ sqlCommands: [`ALTER TABLE ${parentTableName} ADD COLUMN nickname VARCHAR(50)`] }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 201, previewResponse.text); + const body = previewResponse.body; + + t.is(body.connectionId, created.id); + t.is(body.databaseType, 'postgres'); + t.true(body.diagram.startsWith('erDiagram')); + t.true(body.diagram.includes('nickname'), 'diagram should include the new column'); + t.true(body.diagram.includes('NEW'), 'diagram should mark the new column with NEW'); + t.deepEqual(body.diff.addedColumns[parentTableName], ['nickname']); + t.deepEqual(body.diff.addedTables, []); + t.deepEqual(body.diff.droppedTables, []); + t.is(body.diff.statementResults.length, 1); + t.is(body.diff.statementResults[0].status, 'applied'); + t.true(body.description.includes('new columns: nickname')); + }, +); + +test.serial( + 'POST /connection/diagram/:connectionId/preview > marks a CREATE TABLE as added and styles it green', + async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionDto = mockFactory.generateConnectionToTestPostgresDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const newTable = getRandomTestTableName(); + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ + sqlCommands: [ + `CREATE TABLE ${newTable} (id SERIAL PRIMARY KEY, parent_ref INTEGER REFERENCES ${parentTableName}(id), payload TEXT)`, + ], + }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 201, previewResponse.text); + const body = previewResponse.body; + + t.true(body.diagram.includes(newTable), 'diagram should mention the new table'); + t.true(body.diagram.includes('classDef addedEntity'), 'diagram should declare the addedEntity class'); + t.true(body.diagram.includes('fill:#d4edda'), 'classDef should use a green fill'); + t.true(body.diagram.includes(`class ${newTable} addedEntity`), 'diagram should attach class to new table alias'); + t.deepEqual(body.diff.addedTables, [newTable]); + t.truthy(body.diff.addedForeignKeys[newTable]); + t.is(body.diff.statementResults[0].status, 'applied'); + }, +); + +test.serial('POST /connection/diagram/:connectionId/preview > marks a DROP COLUMN in the diff', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionDto = mockFactory.generateConnectionToTestPostgresDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ sqlCommands: [`ALTER TABLE ${childTableName} DROP COLUMN label`] }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 201, previewResponse.text); + const body = previewResponse.body; + + t.deepEqual(body.diff.droppedColumns[childTableName], ['label']); + t.is(body.diff.statementResults[0].status, 'applied'); + t.notRegex( + body.diagram, + /\s+(varchar|character\s*varying)\s+label\b/i, + 'dropped column should not appear in the diagram body', + ); +}); + +test.serial( + 'POST /connection/diagram/:connectionId/preview > reports parse errors per statement without failing the request', + async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionDto = mockFactory.generateConnectionToTestPostgresDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ + sqlCommands: [`ALTER TABLE ${parentTableName} ADD COLUMN good_col INTEGER`, `THIS IS NOT VALID SQL`], + }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 201, previewResponse.text); + const body = previewResponse.body; + t.is(body.diff.statementResults.length, 2); + t.is(body.diff.statementResults[0].status, 'applied'); + t.is(body.diff.statementResults[1].status, 'error'); + t.regex(body.diff.statementResults[1].message, /parse error/); + t.deepEqual(body.diff.addedColumns[parentTableName], ['good_col']); + }, +); + +test.serial( + 'POST /connection/diagram/:connectionId/preview > validates body and rejects empty sqlCommands array', + async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionDto = mockFactory.generateConnectionToTestPostgresDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ sqlCommands: [] }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 400); + }, +); + +test.serial('POST /connection/diagram/:connectionId/preview > rejects non-SQL connection types with 400', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const mongoDto = mockFactory.generateConnectionToTestMongoDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(mongoDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ sqlCommands: ['CREATE TABLE x (id SERIAL PRIMARY KEY)'] }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 400); + t.is(previewResponse.body.message, Messages.DIAGRAM_NOT_SUPPORTED_FOR_CONNECTION_TYPE); +}); + +test.serial('POST /connection/diagram/:connectionId/preview > rejects unauthenticated requests', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionDto = mockFactory.generateConnectionToTestPostgresDBInDocker(); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionDto) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createConnectionResponse.status, 201); + const created = JSON.parse(createConnectionResponse.text); + + const previewResponse = await request(app.getHttpServer()) + .post(`/connection/diagram/${created.id}/preview`) + .send({ sqlCommands: ['CREATE TABLE x (id SERIAL PRIMARY KEY)'] }) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(previewResponse.status, 401); +});