From 38715752b7f496a1e971c2661c7c6373dfce56c9 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Fri, 13 Feb 2026 15:14:53 +0000 Subject: [PATCH 1/2] feat: add schema hash functionality and caching - Introduced `getSchemaHash` method across various data access objects to compute and cache schema hashes. - Added `DEFAULT_SCHEMA_HASH_CACHE_OPTIONS` in caching constants for schema hash caching. - Implemented schema hash validation in `LRUStorage` to manage schema changes effectively. - Updated `OperationTypeEnum` to include `getSchemaHash` operation type. - Enhanced error handling and logging for schema hash retrieval. - Created `validateSchemaCache` utility function to validate schema cache across data access objects. - Updated message texts to reflect new functionality and improve user guidance. --- .../find-tables-in-connection.use.case.ts | 4 + .../get-row-by-primary-key.use.case.ts | 4 + .../use-cases/get-table-rows.use.case.ts | 3 + .../use-cases/get-table-structure.use.case.ts | 4 + .../src/command/command-executor.ts | 559 +++++++++--------- .../src/enums/operation-type.enum.ts | 43 +- rocketadmin-agent/src/text/messages.ts | 163 ++--- shared-code/src/caching/caching-constants.ts | 7 +- shared-code/src/caching/lru-storage.ts | 49 ++ .../src/caching/schema-cache-validator.ts | 31 + .../data-access-object-agent.ts | 39 ++ .../data-access-object-clickhouse.ts | 63 ++ .../data-access-object-ibmdb2.ts | 34 ++ .../data-access-object-mssql.ts | 56 ++ .../data-access-object-mysql.ts | 83 +++ .../data-access-object-oracle.ts | 35 ++ .../data-access-object-postgres.ts | 62 ++ .../enums/data-access-object-commands.enum.ts | 1 + .../data-access-object-agent.interface.ts | 2 + .../data-access-object.interface.ts | 2 + 20 files changed, 867 insertions(+), 377 deletions(-) create mode 100644 shared-code/src/caching/schema-cache-validator.ts diff --git a/backend/src/entities/table/use-cases/find-tables-in-connection.use.case.ts b/backend/src/entities/table/use-cases/find-tables-in-connection.use.case.ts index 3433046de..eb4b154e4 100644 --- a/backend/src/entities/table/use-cases/find-tables-in-connection.use.case.ts +++ b/backend/src/entities/table/use-cases/find-tables-in-connection.use.case.ts @@ -2,6 +2,7 @@ import { HttpException, HttpStatus, Inject, Injectable } from '@nestjs/common'; import { getDataAccessObject } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/create-data-access-object.js'; import { TableDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/table.ds.js'; import { TableStructureDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/table-structure.ds.js'; +import { validateSchemaCache } from '@rocketadmin/shared-code/dist/src/caching/schema-cache-validator.js'; import * as Sentry from '@sentry/node'; import PQueue from 'p-queue'; import AbstractUseCase from '../../../common/abstract-use.case.js'; @@ -78,6 +79,9 @@ export class FindTablesInConnectionUseCase if (isConnectionTypeAgent(connection.type)) { userEmail = await this._dbContext.userRepository.getUserEmailOrReturnNull(userId); } + + await validateSchemaCache(dao, userEmail); + let tables: Array; try { tables = await dao.getTablesFromDB(userEmail); diff --git a/backend/src/entities/table/use-cases/get-row-by-primary-key.use.case.ts b/backend/src/entities/table/use-cases/get-row-by-primary-key.use.case.ts index 925863931..951464bff 100644 --- a/backend/src/entities/table/use-cases/get-row-by-primary-key.use.case.ts +++ b/backend/src/entities/table/use-cases/get-row-by-primary-key.use.case.ts @@ -7,6 +7,7 @@ import { ReferencedTableNamesAndColumnsDS } from '@rocketadmin/shared-code/dist/ import { buildDAOsTableSettingsDs } from '@rocketadmin/shared-code/dist/src/helpers/data-structures-builders/table-settings.ds.builder.js'; import { IDataAccessObject } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object.interface.js'; import { IDataAccessObjectAgent } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object-agent.interface.js'; +import { validateSchemaCache } from '@rocketadmin/shared-code/dist/src/caching/schema-cache-validator.js'; import JSON5 from 'json5'; import AbstractUseCase from '../../../common/abstract-use.case.js'; import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; @@ -70,6 +71,9 @@ export class GetRowByPrimaryKeyUseCase if (isConnectionTypeAgent(connection.type)) { userEmail = await this._dbContext.userRepository.getUserEmailOrReturnNull(userId); } + + await validateSchemaCache(dao, userEmail); + let [ tableStructure, tableWidgets, diff --git a/backend/src/entities/table/use-cases/get-table-rows.use.case.ts b/backend/src/entities/table/use-cases/get-table-rows.use.case.ts index 52fd1d514..65ff216f5 100644 --- a/backend/src/entities/table/use-cases/get-table-rows.use.case.ts +++ b/backend/src/entities/table/use-cases/get-table-rows.use.case.ts @@ -8,6 +8,7 @@ import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/en import { IDataAccessObject } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object.interface.js'; import { IDataAccessObjectAgent } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object-agent.interface.js'; import { FoundRowsDS } from '@rocketadmin/shared-code/src/data-access-layer/shared/data-structures/found-rows.ds.js'; +import { validateSchemaCache } from '@rocketadmin/shared-code/dist/src/caching/schema-cache-validator.js'; import Sentry from '@sentry/minimal'; import JSON5 from 'json5'; import AbstractUseCase from '../../../common/abstract-use.case.js'; @@ -88,6 +89,8 @@ export class GetTableRowsUseCase extends AbstractUseCase { - const dao = createDao(this.connectionConfig); - const { - operationType, - tableName, - row, - primaryKey, - tableSettings, - page, - perPage, - searchedFieldValue, - filteringFields, - autocompleteFields, - email, - referencedFieldName, - identityColumnName, - fieldValues, - tableStructure, - } = messageData.data; - let operationStatusResult = OperationResultStatusEnum.unknown; - switch (operationType) { - case OperationTypeEnum.addRowInTable: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.addRowInTable(tableName, row); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_ADD_ROW_IN_TABLE); - } finally { - Logger.createLogRecord(row, tableName, email, LogOperationTypeEnum.addRow, operationStatusResult, null); - } - break; - case OperationTypeEnum.deleteRowInTable: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.deleteRowInTable(tableName, primaryKey); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_DELETE_ROW_IN_TABLE); - } finally { - Logger.createLogRecord( - primaryKey, - tableName, - email, - LogOperationTypeEnum.deleteRow, - operationStatusResult, - null, - ); - } - break; - case OperationTypeEnum.getRowByPrimaryKey: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.getRowByPrimaryKey(tableName, primaryKey, tableSettings); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_ROW_FROM_TABLE); - } finally { - Logger.createLogRecord( - primaryKey, - tableName, - email, - LogOperationTypeEnum.rowReceived, - operationStatusResult, - null, - ); - } - break; - case OperationTypeEnum.bulkGetRowsFromTableByPrimaryKeys: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.bulkGetRowsFromTableByPrimaryKeys(tableName, primaryKey, tableSettings); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_ROWS_FROM_TABLE); - } finally { - Logger.createLogRecord( - primaryKey, - tableName, - email, - LogOperationTypeEnum.rowsReceived, - operationStatusResult, - null, - ); - } - break; - case OperationTypeEnum.getRowsFromTable: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.getRowsFromTable( - tableName, - tableSettings, - page, - perPage, - searchedFieldValue, - filteringFields, - autocompleteFields, - tableStructure, - ); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_ROWS_FROM_TABLE); - } finally { - Logger.createLogRecord( - null, - tableName, - email, - LogOperationTypeEnum.rowsReceived, - operationStatusResult, - null, - ); - } - break; - case OperationTypeEnum.getTableForeignKeys: - try { - return await dao.getTableForeignKeys(tableName); - } catch (e) { - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_TABLE_FOREIGN_KEYS); - } - case OperationTypeEnum.getTablePrimaryColumns: - try { - return await dao.getTablePrimaryColumns(tableName); - } catch (e) { - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_TABLE_PRIMARY_COLUMNS); - } - case OperationTypeEnum.getTableStructure: - try { - return await dao.getTableStructure(tableName); - } catch (e) { - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_TABLE_STRUCTURE); - } - case OperationTypeEnum.getTablesFromDB: - try { - return await dao.getTablesFromDB(); - } catch (e) { - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_TABLE_STRUCTURE); - } - case OperationTypeEnum.testConnect: - try { - return await dao.testConnect(); - } catch (e) { - console.log(Messages.FAIL_MESSAGE(e.message)); - return false; - } - case OperationTypeEnum.updateRowInTable: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.updateRowInTable(tableName, row, primaryKey); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_UPDATE_ROW); - } finally { - Logger.createLogRecord(row, tableName, email, LogOperationTypeEnum.updateRow, operationStatusResult, null); - } + async executeCommand(messageData: IMessageData): Promise { + const dao = createDao(this.connectionConfig); + const { + operationType, + tableName, + row, + primaryKey, + tableSettings, + page, + perPage, + searchedFieldValue, + filteringFields, + autocompleteFields, + email, + referencedFieldName, + identityColumnName, + fieldValues, + tableStructure, + } = messageData.data; + let operationStatusResult = OperationResultStatusEnum.unknown; + switch (operationType) { + case OperationTypeEnum.addRowInTable: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.addRowInTable(tableName, row); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_ADD_ROW_IN_TABLE); + } finally { + Logger.createLogRecord(row, tableName, email, LogOperationTypeEnum.addRow, operationStatusResult, null); + } + break; + case OperationTypeEnum.deleteRowInTable: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.deleteRowInTable(tableName, primaryKey); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_DELETE_ROW_IN_TABLE); + } finally { + Logger.createLogRecord( + primaryKey, + tableName, + email, + LogOperationTypeEnum.deleteRow, + operationStatusResult, + null, + ); + } + break; + case OperationTypeEnum.getRowByPrimaryKey: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.getRowByPrimaryKey(tableName, primaryKey, tableSettings); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_ROW_FROM_TABLE); + } finally { + Logger.createLogRecord( + primaryKey, + tableName, + email, + LogOperationTypeEnum.rowReceived, + operationStatusResult, + null, + ); + } + break; + case OperationTypeEnum.bulkGetRowsFromTableByPrimaryKeys: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.bulkGetRowsFromTableByPrimaryKeys(tableName, primaryKey, tableSettings); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_ROWS_FROM_TABLE); + } finally { + Logger.createLogRecord( + primaryKey, + tableName, + email, + LogOperationTypeEnum.rowsReceived, + operationStatusResult, + null, + ); + } + break; + case OperationTypeEnum.getRowsFromTable: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.getRowsFromTable( + tableName, + tableSettings, + page, + perPage, + searchedFieldValue, + filteringFields, + autocompleteFields, + tableStructure, + ); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_ROWS_FROM_TABLE); + } finally { + Logger.createLogRecord( + null, + tableName, + email, + LogOperationTypeEnum.rowsReceived, + operationStatusResult, + null, + ); + } + break; + case OperationTypeEnum.getTableForeignKeys: + try { + return await dao.getTableForeignKeys(tableName); + } catch (e) { + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_TABLE_FOREIGN_KEYS); + } + case OperationTypeEnum.getTablePrimaryColumns: + try { + return await dao.getTablePrimaryColumns(tableName); + } catch (e) { + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_TABLE_PRIMARY_COLUMNS); + } + case OperationTypeEnum.getTableStructure: + try { + return await dao.getTableStructure(tableName); + } catch (e) { + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_TABLE_STRUCTURE); + } + case OperationTypeEnum.getTablesFromDB: + try { + return await dao.getTablesFromDB(); + } catch (e) { + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_TABLE_STRUCTURE); + } + case OperationTypeEnum.testConnect: + try { + return await dao.testConnect(); + } catch (e) { + console.log(Messages.FAIL_MESSAGE(e.message)); + return false; + } + case OperationTypeEnum.updateRowInTable: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.updateRowInTable(tableName, row, primaryKey); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_UPDATE_ROW); + } finally { + Logger.createLogRecord(row, tableName, email, LogOperationTypeEnum.updateRow, operationStatusResult, null); + } - case OperationTypeEnum.validateSettings: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.validateSettings(tableSettings, tableName); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_VALIDATE_TABLE_SETTINGS); - } - case OperationTypeEnum.getIdentityColumns: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.getIdentityColumns(tableName, referencedFieldName, identityColumnName, fieldValues); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_TO_GET_IDENTITY_COLUMNS); - } + case OperationTypeEnum.validateSettings: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.validateSettings(tableSettings, tableName); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_VALIDATE_TABLE_SETTINGS); + } + case OperationTypeEnum.getIdentityColumns: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.getIdentityColumns(tableName, referencedFieldName, identityColumnName, fieldValues); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_TO_GET_IDENTITY_COLUMNS); + } - case OperationTypeEnum.getReferencedTableNamesAndColumns: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.getReferencedTableNamesAndColumns(tableName); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_TO_GET_IDENTITY_COLUMNS); - } - case OperationTypeEnum.isView: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.isView(tableName); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_TO_CHECK_IS_VIEW); - } + case OperationTypeEnum.getReferencedTableNamesAndColumns: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.getReferencedTableNamesAndColumns(tableName); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_TO_GET_IDENTITY_COLUMNS); + } + case OperationTypeEnum.isView: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.isView(tableName); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_TO_CHECK_IS_VIEW); + } - case OperationTypeEnum.getRowsAsStream: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.getRowsFromTable( - tableName, - tableSettings, - page, - perPage, - searchedFieldValue, - filteringFields, - autocompleteFields, - null, - ); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_GET_ROWS_FROM_TABLE); - } finally { - Logger.createLogRecord( - null, - tableName, - email, - LogOperationTypeEnum.rowsReceived, - operationStatusResult, - null, - ); - } - break; - case OperationTypeEnum.bulkUpdateRowsInTable: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.bulkUpdateRowsInTable(tableName, row, primaryKey); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_TO_UPDATE_ROWS); - } finally { - Logger.createLogRecord(row, tableName, email, LogOperationTypeEnum.updateRow, operationStatusResult, null); - } - break; - case OperationTypeEnum.bulkDeleteRowsInTable: - try { - operationStatusResult = OperationResultStatusEnum.successfully; - return await dao.bulkDeleteRowsInTable(tableName, primaryKey); - } catch (e) { - operationStatusResult = OperationResultStatusEnum.unsuccessfully; - console.log(Messages.FAIL_MESSAGE(e.message)); - return new Error(Messages.FAILED_TO_UPDATE_ROWS); - } finally { - Logger.createLogRecord( - primaryKey, - tableName, - email, - LogOperationTypeEnum.deleteRow, - operationStatusResult, - null, - ); - } - break; - case OperationTypeEnum.executeRawQuery: - try { - return await dao.executeRawQuery(row, tableName); - } catch (error) { - console.log(Messages.FAIL_MESSAGE(error.message)); - return new Error(Messages.FAILED_EXECUTE_RAW_QUERY); - } - default: - return new Error(Messages.UNKNOWN_OPERATION(operationType)); - } - } + case OperationTypeEnum.getRowsAsStream: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.getRowsFromTable( + tableName, + tableSettings, + page, + perPage, + searchedFieldValue, + filteringFields, + autocompleteFields, + null, + ); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_GET_ROWS_FROM_TABLE); + } finally { + Logger.createLogRecord( + null, + tableName, + email, + LogOperationTypeEnum.rowsReceived, + operationStatusResult, + null, + ); + } + break; + case OperationTypeEnum.bulkUpdateRowsInTable: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.bulkUpdateRowsInTable(tableName, row, primaryKey); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_TO_UPDATE_ROWS); + } finally { + Logger.createLogRecord(row, tableName, email, LogOperationTypeEnum.updateRow, operationStatusResult, null); + } + break; + case OperationTypeEnum.bulkDeleteRowsInTable: + try { + operationStatusResult = OperationResultStatusEnum.successfully; + return await dao.bulkDeleteRowsInTable(tableName, primaryKey); + } catch (e) { + operationStatusResult = OperationResultStatusEnum.unsuccessfully; + console.log(Messages.FAIL_MESSAGE(e.message)); + return new Error(Messages.FAILED_TO_UPDATE_ROWS); + } finally { + Logger.createLogRecord( + primaryKey, + tableName, + email, + LogOperationTypeEnum.deleteRow, + operationStatusResult, + null, + ); + } + break; + case OperationTypeEnum.executeRawQuery: + try { + return await dao.executeRawQuery(row, tableName); + } catch (error) { + console.log(Messages.FAIL_MESSAGE(error.message)); + return new Error(Messages.FAILED_EXECUTE_RAW_QUERY); + } + case OperationTypeEnum.getSchemaHash: + try { + const daoWithSchemaHash = dao as typeof dao & { getSchemaHash?: () => Promise }; + if (daoWithSchemaHash.getSchemaHash) { + return await daoWithSchemaHash.getSchemaHash(); + } + return ''; + } catch (error) { + console.log(Messages.FAIL_MESSAGE(error.message)); + return new Error(Messages.FAILED_GET_SCHEMA_HASH); + } + default: + return new Error(Messages.UNKNOWN_OPERATION(operationType)); + } + } } diff --git a/rocketadmin-agent/src/enums/operation-type.enum.ts b/rocketadmin-agent/src/enums/operation-type.enum.ts index f2b4c0d78..f43b4e429 100644 --- a/rocketadmin-agent/src/enums/operation-type.enum.ts +++ b/rocketadmin-agent/src/enums/operation-type.enum.ts @@ -1,23 +1,24 @@ export enum OperationTypeEnum { - addRowInTable = 'addRowInTable', - deleteRowInTable = 'deleteRowInTable', - getRowByPrimaryKey = 'getRowByPrimaryKey', - bulkGetRowsFromTableByPrimaryKeys = 'bulkGetRowsFromTableByPrimaryKeys', - getRowsFromTable = 'getRowsFromTable', - getTableForeignKeys = 'getTableForeignKeys', - getTablePrimaryColumns = 'getTablePrimaryColumns', - getTableStructure = 'getTableStructure', - getTablesFromDB = 'getTablesFromDB', - testConnect = 'testConnect', - updateRowInTable = 'updateRowInTable', - bulkUpdateRowsInTable = 'bulkUpdateRowsInTable', - bulkDeleteRowsInTable = 'bulkDeleteRowsInTable', - validateSettings = 'validateSettings', - initialConnection = 'initialConnection', - dataFromAgent = 'dataFromAgent', - getIdentityColumns = 'getIdentityColumns', - getReferencedTableNamesAndColumns = 'getReferencedTableNamesAndColumns', - isView = 'isView', - getRowsAsStream = 'getRowsAsStream', - executeRawQuery = 'executeRawQuery', + addRowInTable = 'addRowInTable', + deleteRowInTable = 'deleteRowInTable', + getRowByPrimaryKey = 'getRowByPrimaryKey', + bulkGetRowsFromTableByPrimaryKeys = 'bulkGetRowsFromTableByPrimaryKeys', + getRowsFromTable = 'getRowsFromTable', + getTableForeignKeys = 'getTableForeignKeys', + getTablePrimaryColumns = 'getTablePrimaryColumns', + getTableStructure = 'getTableStructure', + getTablesFromDB = 'getTablesFromDB', + testConnect = 'testConnect', + updateRowInTable = 'updateRowInTable', + bulkUpdateRowsInTable = 'bulkUpdateRowsInTable', + bulkDeleteRowsInTable = 'bulkDeleteRowsInTable', + validateSettings = 'validateSettings', + initialConnection = 'initialConnection', + dataFromAgent = 'dataFromAgent', + getIdentityColumns = 'getIdentityColumns', + getReferencedTableNamesAndColumns = 'getReferencedTableNamesAndColumns', + isView = 'isView', + getRowsAsStream = 'getRowsAsStream', + executeRawQuery = 'executeRawQuery', + getSchemaHash = 'getSchemaHash', } diff --git a/rocketadmin-agent/src/text/messages.ts b/rocketadmin-agent/src/text/messages.ts index fb37df87d..fca1f2e98 100644 --- a/rocketadmin-agent/src/text/messages.ts +++ b/rocketadmin-agent/src/text/messages.ts @@ -4,97 +4,98 @@ import { Constants } from '../helpers/constants/constants.js'; import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; export const Messages = { - APPLICATION_ENCRYPTION_PASSWORD_INVALID: `Invalid encryption password`, - CANT_LIST_AND_EXCLUDE: `You cannot select the same field names to view and exclude`, - CANT_ORDER_AND_EXCLUDE: `You cannot select the same field names to order and exclude`, - CONNECTION_TYPE_INVALID: `Unsupported database type. Now we supports ${enumToString(ConnectionTypesEnum)}`, - CONNECTION_HOST_INVALID: `Invalid connection host`, - CONNECTION_PORT_INVALID: `Invalid connection port`, - CONNECTION_USERNAME_INVALID: `Invalid connection username`, - CONNECTION_PASSWORD_INVALID: `Invalid connection password`, - CONNECTION_DATABASE_INVALID: `Invalid connection database`, - CONNECTION_SCHEMA_INVALID: `Invalid connection schema`, - CANT_EXCLUDE_PRIMARY_KEY: (key: string) => `You cannot exclude primary key ${key}`, - CONNECTION_TOKEN_MISSING: 'Connection token missing', - CONNECTION_TYPE_UNSUPPORTED: 'Connection to this type of database has not been implemented yet', - CORRUPTED_DATA: 'You data are corrupted. Please delete saved configuration file and re-enter the credentials', - CORRUPTED_DATA_OR_PASSWORD: `You password is incorrect or data is corrupted. Please check your credentials`, - CREDENTIALS_ACCEPTED: '-> Credentials accepted. Try to launch application', - DATABASE_MISSING: 'Database is missing', - FAIL_MESSAGE: (message: string) => `Method execution failed with message: "${message}".`, - FAILED_ESTABLISH_SSH_CONNECTION: `Failed to establish ssh connection`, - FAILED_ADD_ROW_IN_TABLE: `Failed to add row in table`, - FAILED_DELETE_ROW_IN_TABLE: `Failed to delete row from table`, - FAILED_GET_ROW_FROM_TABLE: `Failed to get row by primary key`, - FAILED_GET_ROWS_FROM_TABLE: `Failed to get rows from table`, - FAILED_GET_TABLE_FOREIGN_KEYS: `Failed to get table foreign keys`, - FAILED_GET_TABLE_PRIMARY_COLUMNS: `Failed to get table primary columns`, - FAILED_GET_TABLE_STRUCTURE: `Failed to get table structure`, - FAILED_GET_TABLES: `Failed to get tables from database`, - FAILED_UPDATE_ROW: `Failed to update row in table`, - FAILED_TO_UPDATE_ROWS: `Failed to update rows in table`, - FAILED_VALIDATE_TABLE_SETTINGS: `Failed validate table settings`, - FAILED_TO_GET_IDENTITY_COLUMNS: `Failed to get identity columns`, - FAILED_TO_CHECK_IS_VIEW: `Failed to check is view`, - FAILED_EXECUTE_RAW_QUERY: `Failed to execute raw query`, - HOST_MISSING: 'Host is missing', - LIST_PER_PAGE_INCORRECT: `You can't display less than one row per page`, - MUST_BE_ARRAY: (fieldName: string) => `The field "${fieldName}" must be an array`, - NO_SUCH_FIELDS_IN_TABLES: (fields: Array, tableName: string) => - `There are no such fields: ${fields.join(', ')} - in the table "${tableName}"`, - ORDERING_FIELD_INCORRECT: `Value of sorting order is incorrect. You can choose from values ${enumToString( - QueryOrderingEnum, - )}`, - PORT_FORMAT_INCORRECT: 'Port value must be a number', - PORT_MISSING: 'Port value is invalid', - SSH_FORMAT_INCORRECT: 'Ssh value must be a boolean', - SSL_FORMAT_INCORRECT: 'Ssl value must be a boolean', - SSH_HOST_MISSING: 'Ssh host is missing', - SSH_PORT_MISSING: 'Ssh port is missing', - SSH_USERNAME_MISSING: 'Ssh username is missing', - TYPE_MISSING: 'Type is missing', - USERNAME_MISSING: 'Username is missing', - UNKNOWN_OPERATION: (operation: string): string => `Received unsupported operation ${operation}.`, - SOCKET_WAS_DISCONNECTED: 'Socket is closed. Reconnect will be attempted in 1 second.', - SOCKET_ENCOUNTERED_ERROR: (message: string): string => - `Socket connection error -> ${message ? message : ``}, Closing socket`, + APPLICATION_ENCRYPTION_PASSWORD_INVALID: `Invalid encryption password`, + CANT_LIST_AND_EXCLUDE: `You cannot select the same field names to view and exclude`, + CANT_ORDER_AND_EXCLUDE: `You cannot select the same field names to order and exclude`, + CONNECTION_TYPE_INVALID: `Unsupported database type. Now we supports ${enumToString(ConnectionTypesEnum)}`, + CONNECTION_HOST_INVALID: `Invalid connection host`, + CONNECTION_PORT_INVALID: `Invalid connection port`, + CONNECTION_USERNAME_INVALID: `Invalid connection username`, + CONNECTION_PASSWORD_INVALID: `Invalid connection password`, + CONNECTION_DATABASE_INVALID: `Invalid connection database`, + CONNECTION_SCHEMA_INVALID: `Invalid connection schema`, + CANT_EXCLUDE_PRIMARY_KEY: (key: string) => `You cannot exclude primary key ${key}`, + CONNECTION_TOKEN_MISSING: 'Connection token missing', + CONNECTION_TYPE_UNSUPPORTED: 'Connection to this type of database has not been implemented yet', + CORRUPTED_DATA: 'You data are corrupted. Please delete saved configuration file and re-enter the credentials', + CORRUPTED_DATA_OR_PASSWORD: `You password is incorrect or data is corrupted. Please check your credentials`, + CREDENTIALS_ACCEPTED: '-> Credentials accepted. Try to launch application', + DATABASE_MISSING: 'Database is missing', + FAIL_MESSAGE: (message: string) => `Method execution failed with message: "${message}".`, + FAILED_ESTABLISH_SSH_CONNECTION: `Failed to establish ssh connection`, + FAILED_ADD_ROW_IN_TABLE: `Failed to add row in table`, + FAILED_DELETE_ROW_IN_TABLE: `Failed to delete row from table`, + FAILED_GET_ROW_FROM_TABLE: `Failed to get row by primary key`, + FAILED_GET_ROWS_FROM_TABLE: `Failed to get rows from table`, + FAILED_GET_TABLE_FOREIGN_KEYS: `Failed to get table foreign keys`, + FAILED_GET_TABLE_PRIMARY_COLUMNS: `Failed to get table primary columns`, + FAILED_GET_TABLE_STRUCTURE: `Failed to get table structure`, + FAILED_GET_TABLES: `Failed to get tables from database`, + FAILED_UPDATE_ROW: `Failed to update row in table`, + FAILED_TO_UPDATE_ROWS: `Failed to update rows in table`, + FAILED_VALIDATE_TABLE_SETTINGS: `Failed validate table settings`, + FAILED_TO_GET_IDENTITY_COLUMNS: `Failed to get identity columns`, + FAILED_TO_CHECK_IS_VIEW: `Failed to check is view`, + FAILED_EXECUTE_RAW_QUERY: `Failed to execute raw query`, + FAILED_GET_SCHEMA_HASH: `Failed to get schema hash`, + HOST_MISSING: 'Host is missing', + LIST_PER_PAGE_INCORRECT: `You can't display less than one row per page`, + MUST_BE_ARRAY: (fieldName: string) => `The field "${fieldName}" must be an array`, + NO_SUCH_FIELDS_IN_TABLES: (fields: Array, tableName: string) => + `There are no such fields: ${fields.join(', ')} - in the table "${tableName}"`, + ORDERING_FIELD_INCORRECT: `Value of sorting order is incorrect. You can choose from values ${enumToString( + QueryOrderingEnum, + )}`, + PORT_FORMAT_INCORRECT: 'Port value must be a number', + PORT_MISSING: 'Port value is invalid', + SSH_FORMAT_INCORRECT: 'Ssh value must be a boolean', + SSL_FORMAT_INCORRECT: 'Ssl value must be a boolean', + SSH_HOST_MISSING: 'Ssh host is missing', + SSH_PORT_MISSING: 'Ssh port is missing', + SSH_USERNAME_MISSING: 'Ssh username is missing', + TYPE_MISSING: 'Type is missing', + USERNAME_MISSING: 'Username is missing', + UNKNOWN_OPERATION: (operation: string): string => `Received unsupported operation ${operation}.`, + SOCKET_WAS_DISCONNECTED: 'Socket is closed. Reconnect will be attempted in 1 second.', + SOCKET_ENCOUNTERED_ERROR: (message: string): string => + `Socket connection error -> ${message ? message : ``}, Closing socket`, - INTRO_MESSAGES: { - WELCOME_MESSAGE: - 'Welcome! First of all, we need to configure the connection to your database. \n' + - 'Please, enter you unique connection token ' + - 'what you received when you created an agent-connection on the autoadmin website: \n ->', - CONNECTION_TYPE_MESSAGE: `Please choose the type of connection. + INTRO_MESSAGES: { + WELCOME_MESSAGE: + 'Welcome! First of all, we need to configure the connection to your database. \n' + + 'Please, enter you unique connection token ' + + 'what you received when you created an agent-connection on the autoadmin website: \n ->', + CONNECTION_TYPE_MESSAGE: `Please choose the type of connection. We support the following types: `, - CONNECTION_HOST_MESSAGE: `Please enter the host, where your database is located: \n ->`, - CONNECTION_PORT_MESSAGE: `Please enter the port, what your database is listening on your host: \n ->`, - CONNECTION_USERNAME_MESSAGE: `Please enter the login username of your database: \n ->`, - CONNECTION_PASSWORD_MESSAGE: `Please enter the password of your database: \n ->`, - CONNECTION_DATABASE_MESSAGE: `Please enter the database name: \n ->`, - CONNECTION_SCHEMA_MESSAGE: `Please enter the schema name. If it not exists leave this field empty: \n ->`, - CONNECTION_SID_MESSAGE: `You selected database type "Oracle Database", if it have instance identifier (SID) please enter it's name + CONNECTION_HOST_MESSAGE: `Please enter the host, where your database is located: \n ->`, + CONNECTION_PORT_MESSAGE: `Please enter the port, what your database is listening on your host: \n ->`, + CONNECTION_USERNAME_MESSAGE: `Please enter the login username of your database: \n ->`, + CONNECTION_PASSWORD_MESSAGE: `Please enter the password of your database: \n ->`, + CONNECTION_DATABASE_MESSAGE: `Please enter the database name: \n ->`, + CONNECTION_SCHEMA_MESSAGE: `Please enter the schema name. If it not exists leave this field empty: \n ->`, + CONNECTION_SID_MESSAGE: `You selected database type "Oracle Database", if it have instance identifier (SID) please enter it's name or leave this field empty, if it doesn't exist': \n ->`, - CONNECTION_DATACENTER_MESSAGE: `You selected database type "Cassandra", if it have data center name please enter it's name + CONNECTION_DATACENTER_MESSAGE: `You selected database type "Cassandra", if it have data center name please enter it's name or leave this field empty, if it doesn't exist': \n ->`, - CONNECTION_AUTH_SOURCE_MESSAGE: `You selected database type "MongoDB", if it have auth source name please enter it's name + CONNECTION_AUTH_SOURCE_MESSAGE: `You selected database type "MongoDB", if it have auth source name please enter it's name or leave this field empty, if it doesn't exist': \n ->`, - CONNECTION_AZURE_ENCRYPTION_MESSAGE: `Azure encryption option. + CONNECTION_AZURE_ENCRYPTION_MESSAGE: `Azure encryption option. If your database located in Microsoft Azure cloud, and requires encryption choose "Yes" or "No" - if it doesn't: \n ->`, - CONNECTION_SSL_OPTION_MESSAGE: `SSL option. Choose "Yes" if your database support ssl connections and + CONNECTION_SSL_OPTION_MESSAGE: `SSL option. Choose "Yes" if your database support ssl connections and if you want to use ssl connection to your database (before answering "YES", please put the ssl certificate in the application directory and name it "cert.pem"), "No" - in another cases. \n ->`, - APPLICATION_PORT_MESSAGE: `Enter port number, when this application will launch. Default is 3000. (You can left this value empty): \n ->`, - APPLICATION_CONFIG_SAVE_MESSAGE: `If you want to save application configuration to file in application folder choose "Yes" or "No" if you don't want to save \n ->`, - APPLICATION_CONFIG_ENCRYPT_MESSAGE: `If you want to encrypt file with saved configuration + APPLICATION_PORT_MESSAGE: `Enter port number, when this application will launch. Default is 3000. (You can left this value empty): \n ->`, + APPLICATION_CONFIG_SAVE_MESSAGE: `If you want to save application configuration to file in application folder choose "Yes" or "No" if you don't want to save \n ->`, + APPLICATION_CONFIG_ENCRYPT_MESSAGE: `If you want to encrypt file with saved configuration (you will need to enter the password after restarting the application) choose "Yes" or "No" if you don't want to do it \n ->`, - APPLICATION_SAVE_LOGS_OPTION: `If you want to save application logs to file chose "Yes", or "No" if you don't want to save`, - APPLICATION_CONFIG_ENCRYPT_PASSWORD_MESSAGE: `Please enter your encryption passphrase. We will not store it. + APPLICATION_SAVE_LOGS_OPTION: `If you want to save application logs to file chose "Yes", or "No" if you don't want to save`, + APPLICATION_CONFIG_ENCRYPT_PASSWORD_MESSAGE: `Please enter your encryption passphrase. We will not store it. If you forget your password, you will have to delete configuration file in your application folder and enter the connection parameters again \n ->`, - ASK_ENCRYPTION_PASSWORD_MESSAGE: `Please enter your encryption password \n ->`, - APPLICATION_CLI_QUIT: 'Quit the application', - APPLICATION_ATTEMPTS_QUIT: `You entered incorrect value ${Constants.CLI_ATTEMPTS_COUNT} times. Please check your parameters and try again. Quit the application`, - YOU_CHOOSE: (option: string) => `You choose ${option}`, - }, + ASK_ENCRYPTION_PASSWORD_MESSAGE: `Please enter your encryption password \n ->`, + APPLICATION_CLI_QUIT: 'Quit the application', + APPLICATION_ATTEMPTS_QUIT: `You entered incorrect value ${Constants.CLI_ATTEMPTS_COUNT} times. Please check your parameters and try again. Quit the application`, + YOU_CHOOSE: (option: string) => `You choose ${option}`, + }, }; diff --git a/shared-code/src/caching/caching-constants.ts b/shared-code/src/caching/caching-constants.ts index 6fe1ea954..599fe36f4 100644 --- a/shared-code/src/caching/caching-constants.ts +++ b/shared-code/src/caching/caching-constants.ts @@ -76,6 +76,11 @@ export const CACHING_CONSTANTS = { }, DEFAULT_TABLE_STRUCTURE_ELEMENTS_CACHE_OPTIONS: { max: 150, - ttl: 1000 * 60, + ttl: 1000 * 60 * 15, + }, + + DEFAULT_SCHEMA_HASH_CACHE_OPTIONS: { + max: 200, + ttl: 1000 * 30, }, }; diff --git a/shared-code/src/caching/lru-storage.ts b/shared-code/src/caching/lru-storage.ts index ed86698cb..522f460d6 100644 --- a/shared-code/src/caching/lru-storage.ts +++ b/shared-code/src/caching/lru-storage.ts @@ -1,3 +1,4 @@ +/** biome-ignore-all lint/complexity/noStaticOnlyClass: */ import { Client } from 'cassandra-driver'; import { Database } from 'ibm_db'; import { Knex } from 'knex'; @@ -20,6 +21,7 @@ const tableStructureCache = new LRUCache(CACHING_CONSTANTS.DEFAULT_TABLE_STRUCTU const tableForeignKeysCache = new LRUCache(CACHING_CONSTANTS.DEFAULT_TABLE_STRUCTURE_ELEMENTS_CACHE_OPTIONS); const tablePrimaryKeysCache = new LRUCache(CACHING_CONSTANTS.DEFAULT_TABLE_STRUCTURE_ELEMENTS_CACHE_OPTIONS); const redisClientCache = new LRUCache(CACHING_CONSTANTS.DEFAULT_REDIS_CLIENT_CACHE_OPTIONS); +const schemaHashCache = new LRUCache(CACHING_CONSTANTS.DEFAULT_SCHEMA_HASH_CACHE_OPTIONS); export class LRUStorage { public static getRedisClientCache(connection: ConnectionParams): any | null { const cachedClient = redisClientCache.get(LRUStorage.getConnectionIdentifier(connection)); @@ -225,4 +227,51 @@ export class LRUStorage { } return false; } + + public static getSchemaHashCache(connection: ConnectionParams | ConnectionAgentParams): string | null { + const connectionId = LRUStorage.getConnectionIdentifier(connection); + const cachedHash = schemaHashCache.get(connectionId); + return cachedHash ?? null; + } + + public static setSchemaHashCache(connection: ConnectionParams | ConnectionAgentParams, hash: string): void { + const connectionId = LRUStorage.getConnectionIdentifier(connection); + schemaHashCache.set(connectionId, hash); + } + + public static validateSchemaHashAndInvalidate( + connection: ConnectionParams | ConnectionAgentParams, + currentHash: string, + ): boolean { + const cachedHash = LRUStorage.getSchemaHashCache(connection); + if (cachedHash && cachedHash !== currentHash) { + LRUStorage.invalidateConnectionTableMetadata(connection); + LRUStorage.setSchemaHashCache(connection, currentHash); + return false; + } + if (!cachedHash) { + LRUStorage.setSchemaHashCache(connection, currentHash); + } + return true; + } + + public static invalidateConnectionTableMetadata(connection: ConnectionParams | ConnectionAgentParams): void { + const connectionCopy = { ...connection }; + const connectionStr = JSON.stringify({ connectionCopy }); + for (const key of tableStructureCache.keys()) { + if (typeof key === 'string' && key.includes(connectionStr.slice(0, -1))) { + tableStructureCache.delete(key); + } + } + for (const key of tableForeignKeysCache.keys()) { + if (typeof key === 'string' && key.includes(connectionStr.slice(0, -1))) { + tableForeignKeysCache.delete(key); + } + } + for (const key of tablePrimaryKeysCache.keys()) { + if (typeof key === 'string' && key.includes(connectionStr.slice(0, -1))) { + tablePrimaryKeysCache.delete(key); + } + } + } } diff --git a/shared-code/src/caching/schema-cache-validator.ts b/shared-code/src/caching/schema-cache-validator.ts new file mode 100644 index 000000000..9d826b9c0 --- /dev/null +++ b/shared-code/src/caching/schema-cache-validator.ts @@ -0,0 +1,31 @@ +import { IDataAccessObject } from '../shared/interfaces/data-access-object.interface.js'; +import { IDataAccessObjectAgent } from '../shared/interfaces/data-access-object-agent.interface.js'; + +export async function validateSchemaCache( + dao: IDataAccessObject | IDataAccessObjectAgent, + userEmail?: string, +): Promise { + try { + if ('getSchemaHash' in dao && typeof dao.getSchemaHash === 'function') { + if (isAgentDao(dao)) { + if (userEmail) { + const hash = await (dao as IDataAccessObjectAgent).getSchemaHash(userEmail); + return hash; + } + return null; + } + + const hash = await (dao as IDataAccessObject).getSchemaHash!(); + return hash; + } + + return null; + } catch (error) { + console.warn('Schema hash validation failed, proceeding without cache validation:', error?.message || error); + return null; + } +} + +function isAgentDao(dao: IDataAccessObject | IDataAccessObjectAgent): dao is IDataAccessObjectAgent { + return dao.getTableStructure.length >= 2; +} diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts index 46864ffe5..6668c9fb5 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-agent.ts @@ -771,6 +771,45 @@ export class DataAccessObjectAgent implements IDataAccessObjectAgent { }); } + public async getSchemaHash(userEmail: string): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const jwtAuthToken = this.generateJWT(this.connection.token); + axios.defaults.headers.common.Authorization = `Bearer ${jwtAuthToken}`; + + return this.executeWithRetry(async () => { + try { + const { data: { commandResult } = {} } = await axios.post(this.serverAddress, { + operationType: DataAccessObjectCommandsEnum.getSchemaHash, + email: userEmail, + }); + + if (commandResult instanceof Error) { + throw new Error(commandResult.message); + } + + if (!commandResult) { + throw new Error(ERROR_MESSAGES.NO_DATA_RETURNED_FROM_AGENT); + } + + const hash = commandResult; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash); + + return hash; + } catch (e) { + if (axios.isAxiosError(e)) { + this.checkIsErrorLocalAndThrowException(e); + throw new Error(this.extractAxiosErrorMessage(e)); + } + throw e; + } + }); + } + private generateJWT(connectionToken: string): string { const exp = new Date(); exp.setDate(exp.getDate() + 60); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts index 4f8300597..b3058c9d9 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts @@ -633,6 +633,69 @@ export class DataAccessObjectClickHouse extends BasicDataAccessObject implements } } + public async getSchemaHash(): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const client = await this.getClickHouseClient(); + const database = this.connection.database; + + try { + const query = ` + SELECT lower(hex(MD5( + arrayStringConcat( + arraySort( + groupArray(concat(table_name, '=', table_hash)) + ), + '|' + ) + ))) AS schema_hash + FROM ( + SELECT + t.name AS table_name, + lower(hex(MD5(concat( + 'TABLE|', t.name, '|', + 'ENGINE=', t.engine, '|', + 'COLUMNS=', ifNull(( + SELECT arrayStringConcat( + arraySort( + groupArray(concat( + toString(c.position), '#', + c.name, '#', + c.type, '#', + toString(c.default_kind) + )) + ), + '|' + ) + FROM system.columns c + WHERE c.database = '${database}' AND c.table = t.name + ), '') + )))) AS table_hash + FROM system.tables t + WHERE t.database = '${database}' + AND t.engine NOT IN ('View', 'MaterializedView') + ) + `; + + const result = await client.query({ + query, + format: 'JSONEachRow', + }); + + const rows = await result.json<{ schema_hash: string }>(); + const hash = rows?.[0]?.schema_hash || ''; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash); + + return hash; + } finally { + await client.close(); + } + } + private escapeIdentifier(identifier: string): string { this.validateIdentifier(identifier); return `\`${identifier.replace(/`/g, '``')}\``; diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts index b40141d29..358ee7a14 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts @@ -709,6 +709,40 @@ ORDER BY return result as Record[]; } + public async getSchemaHash(): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const connectionToDb = await this.getConnectionToDatabase(); + const schema = this.connection.schema?.toUpperCase() ?? this.connection.username.toUpperCase(); + + const query = ` + SELECT + COALESCE(table_count, 0) || '|' || + COALESCE(column_count, 0) || '|' || + COALESCE(index_count, 0) || '|' || + COALESCE(column_checksum, 0) AS SCHEMA_HASH + FROM ( + SELECT + (SELECT COUNT(*) FROM SYSCAT.TABLES WHERE TABSCHEMA = '${schema}' AND TYPE = 'T') AS table_count, + (SELECT COUNT(*) FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '${schema}') AS column_count, + (SELECT COUNT(*) FROM SYSCAT.INDEXES WHERE TABSCHEMA = '${schema}') AS index_count, + (SELECT COALESCE(SUM(COLNO + LENGTH(COLNAME) + LENGTH(TYPENAME)), 0) + FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '${schema}') AS column_checksum + FROM SYSIBM.SYSDUMMY1 + ) + `; + + const result = await connectionToDb.query(query); + const hash = (result as Array>)?.[0]?.SCHEMA_HASH || ''; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash as string); + + return hash as string; + } + private getConnectionToDatabase(): Promise { if (this.connection.ssh) { return this.createTunneledConnection(this.connection); diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts index 2bca06f9d..a3499f76b 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mssql.ts @@ -639,6 +639,62 @@ WHERE TABLE_TYPE = 'VIEW' return await knex.raw(query); } + public async getSchemaHash(): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const knex = await this.configureKnex(); + const schema = this.connection.schema ?? 'dbo'; + + const query = ` + WITH per_table AS ( + SELECT + t.name COLLATE DATABASE_DEFAULT AS table_name, + CONVERT(VARCHAR(32), HASHBYTES('MD5', + CONCAT( + 'TABLE|', t.name COLLATE DATABASE_DEFAULT, '|', + 'COLUMNS=', ISNULL(cols.columns_sig, ''), '|', + 'INDEXES=', ISNULL(idx.indexes_sig, '') + ) + ), 2) AS table_hash + FROM sys.tables t + OUTER APPLY ( + SELECT STRING_AGG( + CAST(CONCAT(c.column_id, '#', c.name COLLATE DATABASE_DEFAULT, '#', tp.name COLLATE DATABASE_DEFAULT, '#', c.is_nullable, '#', ISNULL(CAST(dc.definition AS NVARCHAR(MAX)), '')) AS NVARCHAR(MAX)), + '|' + ) WITHIN GROUP (ORDER BY c.column_id) AS columns_sig + FROM sys.columns c + JOIN sys.types tp ON tp.user_type_id = c.user_type_id + LEFT JOIN sys.default_constraints dc ON dc.object_id = c.default_object_id + WHERE c.object_id = t.object_id + ) cols + OUTER APPLY ( + SELECT STRING_AGG( + CAST(CONCAT(i.name COLLATE DATABASE_DEFAULT, ':', i.is_unique, ':', i.type_desc COLLATE DATABASE_DEFAULT) AS NVARCHAR(MAX)), + '|' + ) WITHIN GROUP (ORDER BY i.name) AS indexes_sig + FROM sys.indexes i + WHERE i.object_id = t.object_id AND i.name IS NOT NULL + ) idx + WHERE t.schema_id = SCHEMA_ID(?) + ) + SELECT CONVERT(VARCHAR(32), HASHBYTES('MD5', + STRING_AGG(CAST(CONCAT(table_name, '=', table_hash) AS NVARCHAR(MAX)), '|') + WITHIN GROUP (ORDER BY table_name) + ), 2) AS schema_hash + FROM per_table + `; + + const result = await knex.raw(query, [schema]); + const hash = result?.[0]?.schema_hash || ''; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash); + + return hash; + } + private async getSchemaName(tableName: string): Promise { if (this.connection.schema) { return `[${this.connection.schema}]`; diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts index 114dda571..8bc453d3a 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-mysql.ts @@ -691,6 +691,89 @@ export class DataAccessObjectMysql extends BasicDataAccessObject implements IDat return await knex.raw(query); } + public async getSchemaHash(): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const knex = await this.configureKnex(); + const { database } = this.connection; + + await knex.raw('SET SESSION group_concat_max_len = 1000000'); + + const query = ` + WITH per_table AS ( + SELECT + t.table_name, + MD5( + CONCAT( + 'TABLE|', t.table_name, '|', + 'ENGINE=', COALESCE(t.engine,''), '|', + 'COLLATION=', COALESCE(t.table_collation,''), '|', + 'COLUMNS=', COALESCE(c.columns_sig,''), '|', + 'INDEXES=', COALESCE(s.indexes_sig,'') + ) + ) AS table_md5 + FROM information_schema.tables t + LEFT JOIN ( + SELECT + table_name, + GROUP_CONCAT( + CONCAT_WS('#', + ordinal_position, + column_name, + column_type, + is_nullable, + COALESCE(column_default,''), + extra, + COALESCE(character_set_name,''), + COALESCE(collation_name,'') + ) + ORDER BY ordinal_position + SEPARATOR '|' + ) AS columns_sig + FROM information_schema.columns + WHERE table_schema = ? + GROUP BY table_name + ) c ON c.table_name = t.table_name + LEFT JOIN ( + SELECT + table_name, + GROUP_CONCAT( + CONCAT(index_name, ':', non_unique, ':', index_type, ':', cols) + ORDER BY index_name + SEPARATOR '|' + ) AS indexes_sig + FROM ( + SELECT + table_name, + index_name, + non_unique, + index_type, + GROUP_CONCAT(column_name ORDER BY seq_in_index SEPARATOR ',') AS cols + FROM information_schema.statistics + WHERE table_schema = ? + GROUP BY table_name, index_name, non_unique, index_type + ) idx_cols + GROUP BY table_name + ) s ON s.table_name = t.table_name + WHERE t.table_schema = ? + AND t.table_type = 'BASE TABLE' + ) + SELECT + MD5(GROUP_CONCAT(CONCAT(table_name, '=', table_md5) ORDER BY table_name SEPARATOR '|')) AS schema_hash + FROM per_table + `; + + const result = await knex.raw(query, [database, database, database]); + const hash = result[0]?.[0]?.schema_hash || ''; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash); + + return hash; + } + private async getRowsCount( knex: Knex, countRowsQB: Knex.QueryBuilder | null, diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts index 4cb934713..b97f97f04 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-oracle.ts @@ -852,6 +852,41 @@ export class DataAccessObjectOracle extends BasicDataAccessObject implements IDa return await knex.raw(query); } + public async getSchemaHash(): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const knex = await this.configureKnex(); + const schema = this.connection.schema ?? this.connection.username.toUpperCase(); + + const query = ` + SELECT + STANDARD_HASH( + table_count || '|' || column_count || '|' || index_count || '|' || table_hash_sum || '|' || column_hash_sum, + 'MD5' + ) AS schema_hash + FROM ( + SELECT + (SELECT COUNT(*) FROM all_tables WHERE owner = :schema) AS table_count, + (SELECT COUNT(*) FROM all_tab_columns WHERE owner = :schema) AS column_count, + (SELECT COUNT(*) FROM all_indexes WHERE owner = :schema) AS index_count, + (SELECT NVL(SUM(ORA_HASH(table_name)), 0) FROM all_tables WHERE owner = :schema) AS table_hash_sum, + (SELECT NVL(SUM(ORA_HASH(table_name || column_name || data_type || nullable || data_length)), 0) + FROM all_tab_columns WHERE owner = :schema) AS column_hash_sum + FROM DUAL + ) + `; + + const result = await knex.raw(query, { schema }); + const hash = result?.[0]?.SCHEMA_HASH || ''; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash); + + return hash; + } + private setupPagination(page: number, perPage: number, settings: TableSettingsDS) { if (!page || page <= 0) { page = DAO_CONSTANTS.DEFAULT_PAGINATION.page; diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts index 04ea86bca..186846279 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-postgres.ts @@ -712,6 +712,68 @@ export class DataAccessObjectPostgres extends BasicDataAccessObject implements I return knex.raw(query); } + public async getSchemaHash(): Promise { + const cachedHash = LRUStorage.getSchemaHashCache(this.connection); + if (cachedHash) { + return cachedHash; + } + + const knex = await this.configureKnex(); + const schema = this.connection.schema ?? 'public'; + + const query = ` + WITH per_table AS ( + SELECT + c.relname AS table_name, + MD5( + CONCAT( + 'TABLE|', c.relname, '|', + 'COLUMNS=', COALESCE(cols.columns_sig, ''), '|', + 'INDEXES=', COALESCE(idx.indexes_sig, '') + ) + ) AS table_md5 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN LATERAL ( + SELECT STRING_AGG( + CONCAT_WS('#', + a.attnum::text, + a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + a.attnotnull::text, + COALESCE(pg_get_expr(d.adbin, d.adrelid), '') + ), + '|' ORDER BY a.attnum + ) AS columns_sig + FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + ) cols ON true + LEFT JOIN LATERAL ( + SELECT STRING_AGG( + CONCAT(ic.relname, ':', i.indisunique::text, ':', am.amname), + '|' ORDER BY ic.relname + ) AS indexes_sig + FROM pg_index i + JOIN pg_class ic ON ic.oid = i.indexrelid + JOIN pg_am am ON am.oid = ic.relam + WHERE i.indrelid = c.oid + ) idx ON true + WHERE n.nspname = ? + AND c.relkind = 'r' + ) + SELECT MD5(STRING_AGG(CONCAT(table_name, '=', table_md5), '|' ORDER BY table_name)) AS schema_hash + FROM per_table + `; + + const result = await knex.raw(query, [schema]); + const hash = result.rows?.[0]?.schema_hash || ''; + + LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash); + + return hash; + } + private async getRowsCount( knex: Knex, countRowsQB: Knex.QueryBuilder | null, diff --git a/shared-code/src/shared/enums/data-access-object-commands.enum.ts b/shared-code/src/shared/enums/data-access-object-commands.enum.ts index 2aa747693..f3916746f 100644 --- a/shared-code/src/shared/enums/data-access-object-commands.enum.ts +++ b/shared-code/src/shared/enums/data-access-object-commands.enum.ts @@ -18,4 +18,5 @@ export enum DataAccessObjectCommandsEnum { isView = 'isView', getRowsAsStream = 'getRowsAsStream', executeRawQuery = 'executeRawQuery', + getSchemaHash = 'getSchemaHash', } diff --git a/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts b/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts index eb6ce5533..683a7e7bf 100644 --- a/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts +++ b/shared-code/src/shared/interfaces/data-access-object-agent.interface.ts @@ -109,4 +109,6 @@ export interface IDataAccessObjectAgent { importCSVInTable(file: Express.Multer.File, tableName: string, userEmail: string): Promise; executeRawQuery(query: string, tableName: string, userEmail: string): Promise>>; + + getSchemaHash(userEmail: string): Promise; } diff --git a/shared-code/src/shared/interfaces/data-access-object.interface.ts b/shared-code/src/shared/interfaces/data-access-object.interface.ts index a62c9ccc4..8326660d4 100644 --- a/shared-code/src/shared/interfaces/data-access-object.interface.ts +++ b/shared-code/src/shared/interfaces/data-access-object.interface.ts @@ -88,4 +88,6 @@ export interface IDataAccessObject { importCSVInTable(file: Express.Multer.File, tableName: string): Promise; executeRawQuery(query: string, tableName: string): Promise>>; + + getSchemaHash?(): Promise; } From e50dd12ada97a292a3e418f1d38d96de17b4900c Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Fri, 13 Feb 2026 16:06:05 +0000 Subject: [PATCH 2/2] fix: escape database value and parameterize schema queries in IBM Db2 data access object --- .../data-access-object-clickhouse.ts | 2 +- .../data-access-objects/data-access-object-ibmdb2.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts index b3058c9d9..be6bc3f88 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-clickhouse.ts @@ -640,7 +640,7 @@ export class DataAccessObjectClickHouse extends BasicDataAccessObject implements } const client = await this.getClickHouseClient(); - const database = this.connection.database; + const database = this.escapeValue(this.connection.database || 'default'); try { const query = ` diff --git a/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts b/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts index 358ee7a14..9668d97fa 100644 --- a/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts +++ b/shared-code/src/data-access-layer/data-access-objects/data-access-object-ibmdb2.ts @@ -726,16 +726,16 @@ ORDER BY COALESCE(column_checksum, 0) AS SCHEMA_HASH FROM ( SELECT - (SELECT COUNT(*) FROM SYSCAT.TABLES WHERE TABSCHEMA = '${schema}' AND TYPE = 'T') AS table_count, - (SELECT COUNT(*) FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '${schema}') AS column_count, - (SELECT COUNT(*) FROM SYSCAT.INDEXES WHERE TABSCHEMA = '${schema}') AS index_count, + (SELECT COUNT(*) FROM SYSCAT.TABLES WHERE TABSCHEMA = ? AND TYPE = 'T') AS table_count, + (SELECT COUNT(*) FROM SYSCAT.COLUMNS WHERE TABSCHEMA = ?) AS column_count, + (SELECT COUNT(*) FROM SYSCAT.INDEXES WHERE TABSCHEMA = ?) AS index_count, (SELECT COALESCE(SUM(COLNO + LENGTH(COLNAME) + LENGTH(TYPENAME)), 0) - FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '${schema}') AS column_checksum + FROM SYSCAT.COLUMNS WHERE TABSCHEMA = ?) AS column_checksum FROM SYSIBM.SYSDUMMY1 ) `; - const result = await connectionToDb.query(query); + const result = await connectionToDb.query(query, [schema, schema, schema, schema]); const hash = (result as Array>)?.[0]?.SCHEMA_HASH || ''; LRUStorage.validateSchemaHashAndInvalidate(this.connection, hash as string);