diff --git a/backend/src/common/data-injection.tokens.ts b/backend/src/common/data-injection.tokens.ts index 6d6ae5c40..a4a7907ef 100644 --- a/backend/src/common/data-injection.tokens.ts +++ b/backend/src/common/data-injection.tokens.ts @@ -187,6 +187,7 @@ export enum UseCaseType { AGENTS_GET_AI_TABLE_STRUCTURE = 'AGENTS_GET_AI_TABLE_STRUCTURE', AGENTS_EXECUTE_AI_RAW_QUERY = 'AGENTS_EXECUTE_AI_RAW_QUERY', AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE = 'AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE', + AGENTS_GET_AI_SAMPLE_ROWS = 'AGENTS_GET_AI_SAMPLE_ROWS', AGENTS_SCAN_AND_CREATE_SETTINGS = 'AGENTS_SCAN_AND_CREATE_SETTINGS', AGENTS_GET_COMPANY_SUBSCRIPTION_INFO = 'AGENTS_GET_COMPANY_SUBSCRIPTION_INFO', diff --git a/backend/src/microservices/agents-microservice/agents.controller.ts b/backend/src/microservices/agents-microservice/agents.controller.ts index 36ad33e80..9222d946d 100644 --- a/backend/src/microservices/agents-microservice/agents.controller.ts +++ b/backend/src/microservices/agents-microservice/agents.controller.ts @@ -12,6 +12,7 @@ import { AiConnectionContextRO, AiConnectionTablesRO, AiQueryResultRO, + AiSampleRowsRO, CompanySubscriptionInfoRO, PermissionAllowedRO, ValidatedUserTokenRO, @@ -20,6 +21,7 @@ import { AiDataRequestBaseDto, ExecuteAiAggregationPipelineDto, ExecuteAiRawQueryDto, + GetAiSampleRowsDto, GetAiTableStructureDto, } from './dto/agents-ai-data.dtos.js'; import { ValidateConnectionEditDto, ValidateTableAiRequestDto, ValidateUserTokenDto } from './dto/agents-auth.dtos.js'; @@ -29,6 +31,7 @@ import { IExecuteAiRawQuery, IGetAiConnectionContext, IGetAiConnectionTables, + IGetAiSampleRows, IGetAiTableStructure, IGetCompanySubscriptionInfo, IScanAndCreateSettings, @@ -61,6 +64,8 @@ export class AgentsController { private readonly executeAiRawQueryUseCase: IExecuteAiRawQuery, @Inject(UseCaseType.AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE) private readonly executeAiAggregationPipelineUseCase: IExecuteAiAggregationPipeline, + @Inject(UseCaseType.AGENTS_GET_AI_SAMPLE_ROWS) + private readonly getAiSampleRowsUseCase: IGetAiSampleRows, @Inject(UseCaseType.AGENTS_SCAN_AND_CREATE_SETTINGS) private readonly scanAndCreateSettingsUseCase: IScanAndCreateSettings, @Inject(UseCaseType.AGENTS_GET_COMPANY_SUBSCRIPTION_INFO) @@ -167,6 +172,27 @@ export class AgentsController { ); } + @ApiOperation({ summary: 'Fetch permission-filtered sample rows and a row count (grounds website generation)' }) + @ApiResponse({ status: 201, type: AiSampleRowsRO }) + @ApiBody({ type: GetAiSampleRowsDto }) + @Timeout(!isTest() ? TimeoutDefaults.EXTENDED : TimeoutDefaults.EXTENDED_TEST) + @Post('/ai/data/:connectionId/sample-rows') + public async getAiSampleRows( + @SlugUuid('connectionId') connectionId: string, + @Body() body: GetAiSampleRowsDto, + ): Promise { + return await this.getAiSampleRowsUseCase.execute( + { + connectionId, + userId: body.userId, + masterPassword: body.masterPassword ?? null, + tableName: body.tableName, + limit: body.limit ?? null, + }, + InTransactionEnum.OFF, + ); + } + @ApiOperation({ summary: 'Validate and execute a read-only MongoDB aggregation pipeline for the AI tool loop' }) @ApiResponse({ status: 201, type: AiQueryResultRO }) @ApiBody({ type: ExecuteAiAggregationPipelineDto }) diff --git a/backend/src/microservices/agents-microservice/agents.module.ts b/backend/src/microservices/agents-microservice/agents.module.ts index 414a7b33f..a202ae558 100644 --- a/backend/src/microservices/agents-microservice/agents.module.ts +++ b/backend/src/microservices/agents-microservice/agents.module.ts @@ -8,6 +8,7 @@ import { ExecuteAiAggregationPipelineUseCase } from './use-cases/execute-ai-aggr import { ExecuteAiRawQueryUseCase } from './use-cases/execute-ai-raw-query.use.case.js'; import { GetAiConnectionContextUseCase } from './use-cases/get-ai-connection-context.use.case.js'; import { GetAiConnectionTablesUseCase } from './use-cases/get-ai-connection-tables.use.case.js'; +import { GetAiSampleRowsUseCase } from './use-cases/get-ai-sample-rows.use.case.js'; import { GetAiTableStructureUseCase } from './use-cases/get-ai-table-structure.use.case.js'; import { GetCompanySubscriptionInfoUseCase } from './use-cases/get-company-subscription-info.use.case.js'; import { ScanAndCreateSettingsUseCase } from './use-cases/scan-and-create-settings.use.case.js'; @@ -54,6 +55,10 @@ import { ValidateUserTokenUseCase } from './use-cases/validate-user-token.use.ca provide: UseCaseType.AGENTS_EXECUTE_AI_AGGREGATION_PIPELINE, useClass: ExecuteAiAggregationPipelineUseCase, }, + { + provide: UseCaseType.AGENTS_GET_AI_SAMPLE_ROWS, + useClass: GetAiSampleRowsUseCase, + }, { provide: UseCaseType.AGENTS_SCAN_AND_CREATE_SETTINGS, useClass: ScanAndCreateSettingsUseCase, diff --git a/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts b/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts index c2e282643..54cc45895 100644 --- a/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts +++ b/backend/src/microservices/agents-microservice/data-structures/agents-responses.ds.ts @@ -41,6 +41,21 @@ export class AiQueryResultRO { result: unknown; } +export class AiSampleRowsRO { + @ApiProperty({ + type: 'array', + items: { type: 'object', additionalProperties: true }, + description: 'Sample rows, filtered to the columns the user may read.', + }) + rows: Array>; + + @ApiProperty({ description: 'Total row count in the table (may be an estimate for large datasets).' }) + rowCount: number; + + @ApiProperty({ description: 'When true, rowCount is an estimate rather than an exact COUNT.' }) + largeDataset: boolean; +} + export class AiConnectionTablesRO { @ApiProperty({ type: [String], description: 'Table names the user is permitted to read on the connection.' }) tables: Array; diff --git a/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts b/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts index aae703cb9..cb07527a6 100644 --- a/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts +++ b/backend/src/microservices/agents-microservice/data-structures/agents.ds.ts @@ -26,6 +26,11 @@ export class ExecuteAiRawQueryDs extends AiDataRequestDs { query: string; } +export class GetAiSampleRowsDs extends AiDataRequestDs { + tableName: string; + limit: number | null; +} + export class ExecuteAiAggregationPipelineDs extends AiDataRequestDs { tableName: string; pipeline: string; diff --git a/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts b/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts index 5f79edec5..8985dad4b 100644 --- a/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts +++ b/backend/src/microservices/agents-microservice/dto/agents-ai-data.dtos.ts @@ -1,5 +1,5 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; -import { IsNotEmpty, IsOptional, IsString, IsUUID } from 'class-validator'; +import { IsInt, IsNotEmpty, IsOptional, IsString, IsUUID, Min } from 'class-validator'; export class AiDataRequestBaseDto { @ApiProperty() @@ -33,6 +33,19 @@ export class ExecuteAiRawQueryDto extends AiDataRequestBaseDto { query: string; } +export class GetAiSampleRowsDto extends AiDataRequestBaseDto { + @ApiProperty() + @IsString() + @IsNotEmpty() + tableName: string; + + @ApiPropertyOptional({ description: 'Max sample rows to return. Clamped server-side to 5.' }) + @IsOptional() + @IsInt() + @Min(1) + limit?: number; +} + export class ExecuteAiAggregationPipelineDto extends AiDataRequestBaseDto { @ApiProperty() @IsString() diff --git a/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts b/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts index 851c5ea9d..24823e759 100644 --- a/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts +++ b/backend/src/microservices/agents-microservice/use-cases/agents-use-cases.interface.ts @@ -3,6 +3,7 @@ import { AiDataRequestDs, ExecuteAiAggregationPipelineDs, ExecuteAiRawQueryDs, + GetAiSampleRowsDs, GetAiTableStructureDs, GetCompanySubscriptionInfoDs, ScanAndCreateSettingsDs, @@ -13,6 +14,7 @@ import { AiConnectionContextRO, AiConnectionTablesRO, AiQueryResultRO, + AiSampleRowsRO, CompanySubscriptionInfoRO, PermissionAllowedRO, ValidatedUserTokenRO, @@ -42,6 +44,10 @@ export interface IGetAiConnectionTables { execute(inputData: AiDataRequestDs, inTransaction: InTransactionEnum): Promise; } +export interface IGetAiSampleRows { + execute(inputData: GetAiSampleRowsDs, inTransaction: InTransactionEnum): Promise; +} + export interface IExecuteAiRawQuery { execute(inputData: ExecuteAiRawQueryDs, inTransaction: InTransactionEnum): Promise; } diff --git a/backend/src/microservices/agents-microservice/use-cases/get-ai-sample-rows.use.case.ts b/backend/src/microservices/agents-microservice/use-cases/get-ai-sample-rows.use.case.ts new file mode 100644 index 000000000..b98afb312 --- /dev/null +++ b/backend/src/microservices/agents-microservice/use-cases/get-ai-sample-rows.use.case.ts @@ -0,0 +1,72 @@ +import { Inject, Injectable, Scope } from '@nestjs/common'; +import { buildDAOsTableSettingsDs } from '@rocketadmin/shared-code/dist/src/helpers/data-structures-builders/table-settings.ds.builder.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { CedarPermissionsService } from '../../../entities/cedar-authorization/cedar-permissions.service.js'; +import { buildCommonTableSettingsInput } from '../../../entities/table/utils/build-common-table-settings-input.util.js'; +import { GetAiSampleRowsDs } from '../data-structures/agents.ds.js'; +import { AiSampleRowsRO } from '../data-structures/agents-responses.ds.js'; +import { assertUserCanReadTables, setupAiConnection } from '../utils/ai-data-access.helpers.js'; +import { IGetAiSampleRows } from './agents-use-cases.interface.js'; + +export const AI_SAMPLE_ROWS_MAX_LIMIT = 5; + +@Injectable({ scope: Scope.REQUEST }) +export class GetAiSampleRowsUseCase + extends AbstractUseCase + implements IGetAiSampleRows +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly cedarPermissions: CedarPermissionsService, + ) { + super(); + } + + protected async implementation(inputData: GetAiSampleRowsDs): Promise { + const { connectionId, userId, masterPassword, tableName, limit } = inputData; + + const { foundConnection, dataAccessObject, userEmail } = await setupAiConnection( + this._dbContext, + connectionId, + masterPassword, + userId, + ); + + await assertUserCanReadTables(this.cedarPermissions, [tableName], userId, foundConnection.id); + + const tableStructure = await dataAccessObject.getTableStructure(tableName, userEmail); + const readableColumns = await this.cedarPermissions.getReadableColumns( + userId, + foundConnection.id, + tableName, + tableStructure.map((column) => column.column_name), + ); + + const perPage = Math.min(limit ?? AI_SAMPLE_ROWS_MAX_LIMIT, AI_SAMPLE_ROWS_MAX_LIMIT); + const settings = buildDAOsTableSettingsDs(buildCommonTableSettingsInput(null), null); + const foundRows = await dataAccessObject.getRowsFromTable( + tableName, + settings, + 1, + perPage, + '', + [], + { fields: [], value: '' }, + tableStructure, + userEmail, + ); + + const rows = foundRows.data.map((row) => + Object.fromEntries(Object.entries(row).filter(([columnName]) => readableColumns.has(columnName))), + ); + + return { + rows, + rowCount: foundRows.pagination.total, + largeDataset: foundRows.large_dataset, + }; + } +} diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-agents-microservice-sample-rows-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-agents-microservice-sample-rows-e2e.test.ts new file mode 100644 index 000000000..a4180c22f --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-agents-microservice-sample-rows-e2e.test.ts @@ -0,0 +1,220 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +/* eslint-disable security/detect-object-injection */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import jwt from 'jsonwebtoken'; +import request from 'supertest'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { AccessLevelEnum } from '../../../src/enums/access-level.enum.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { appConfig } from '../../../src/shared/config/app-config.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { createTestTable } from '../../utils/create-test-table.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + inviteUserInCompanyAndAcceptInvitation, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; +let currentTest; + +// Microservice JWT (request_id claim) for the internal agents controller. +function microserviceAuthHeader(): string { + const secret = appConfig.auth.microserviceJwtSecret as string; + return `Bearer ${jwt.sign({ request_id: faker.string.uuid() }, secret, { expiresIn: '1h' })}`; +} + +function userIdFromCookieToken(cookieToken: string): string { + const decoded = jwt.decode(cookieToken.split('=')[1]) as { id: string }; + return decoded.id; +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }).compile(); + app = moduleFixture.createNestApplication(); + _testUtils = moduleFixture.get(TestUtils); + + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnectionAndTable(): Promise<{ + token: string; + userId: string; + connectionId: string; + testTableName: string; + testTableColumnName: string; + testTableSecondColumnName: string; +}> { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const token = (await registerUserAndReturnUserInfo(app)).token; + const userId = userIdFromCookieToken(token); + const { testTableName, testTableColumnName, testTableSecondColumnName } = await createTestTable(connectionToTestDB); + testTables.push(testTableName); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + return { + token, + userId, + connectionId: JSON.parse(createConnectionResponse.text).id, + testTableName, + testTableColumnName, + testTableSecondColumnName, + }; +} + +function sampleRowsRequest(connectionId: string, body: Record, withAuth = true): request.Test { + const req = request(app.getHttpServer()) + .post(`/internal/agents/ai/data/${connectionId}/sample-rows`) + .send(body) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + return withAuth ? req.set('Authorization', microserviceAuthHeader()) : req; +} + +currentTest = 'POST /internal/agents/ai/data/:connectionId/sample-rows (internal, microservice JWT)'; + +test.serial(`${currentTest} returns sample rows and a row count for the connection owner`, async (t) => { + const { userId, connectionId, testTableName, testTableColumnName, testTableSecondColumnName } = + await createConnectionAndTable(); + + const response = await sampleRowsRequest(connectionId, { userId, tableName: testTableName }); + t.is(response.status, 201); + const ro = JSON.parse(response.text); + t.is(ro.rows.length, 5); + t.is(ro.rowCount, 42); + t.is(typeof ro.largeDataset, 'boolean'); + for (const row of ro.rows) { + t.is(Object.hasOwn(row, testTableColumnName), true); + t.is(Object.hasOwn(row, testTableSecondColumnName), true); + } +}); + +test.serial(`${currentTest} clamps the requested limit to 5`, async (t) => { + const { userId, connectionId, testTableName } = await createConnectionAndTable(); + + const clamped = await sampleRowsRequest(connectionId, { userId, tableName: testTableName, limit: 50 }); + t.is(clamped.status, 201); + t.is(JSON.parse(clamped.text).rows.length, 5); + + const small = await sampleRowsRequest(connectionId, { userId, tableName: testTableName, limit: 2 }); + t.is(small.status, 201); + t.is(JSON.parse(small.text).rows.length, 2); +}); + +test.serial(`${currentTest} rejects calls without the microservice JWT (401)`, async (t) => { + const { userId, connectionId, testTableName } = await createConnectionAndTable(); + + const response = await sampleRowsRequest(connectionId, { userId, tableName: testTableName }, false); + t.is(response.status, 401); +}); + +test.serial(`${currentTest} denies a user with no permissions on the connection (403)`, async (t) => { + const { connectionId, testTableName } = await createConnectionAndTable(); + const foreignUserToken = (await registerUserAndReturnUserInfo(app)).token; + const foreignUserId = userIdFromCookieToken(foreignUserToken); + + const response = await sampleRowsRequest(connectionId, { userId: foreignUserId, tableName: testTableName }); + t.is(response.status, 403); +}); + +test.serial(`${currentTest} filters row columns to the user's readable-column whitelist`, async (t) => { + const { token, connectionId, testTableName, testTableColumnName, testTableSecondColumnName } = + await createConnectionAndTable(); + + // Invite a second user into a new group that may read only one column of the table. + const createGroupResponse = await request(app.getHttpServer()) + .post(`/connection/group/${connectionId}`) + .send(mockFactory.generateCreateGroupDto1()) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(createGroupResponse.status, 201); + const groupId = JSON.parse(createGroupResponse.text).id; + + const permissions = { + connection: { connectionId, accessLevel: AccessLevelEnum.none }, + group: { groupId, accessLevel: AccessLevelEnum.none }, + tables: [ + { + tableName: testTableName, + accessLevel: { visibility: true, readonly: true, add: false, delete: false, edit: false }, + readableColumns: [testTableColumnName], + }, + ], + }; + const savePermissionsResponse = await request(app.getHttpServer()) + .put(`/permissions/${groupId}?connectionId=${connectionId}`) + .send({ permissions }) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(savePermissionsResponse.status, 200); + + const invitedUser = await inviteUserInCompanyAndAcceptInvitation(token, undefined, app, groupId); + const findInvitedUserResponse = await request(app.getHttpServer()) + .get('/user/') + .set('Cookie', invitedUser.token) + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + t.is(findInvitedUserResponse.status, 200); + const invitedUserId = JSON.parse(findInvitedUserResponse.text).id; + + const response = await sampleRowsRequest(connectionId, { userId: invitedUserId, tableName: testTableName }); + t.is(response.status, 201); + const ro = JSON.parse(response.text); + t.true(ro.rows.length > 0); + for (const row of ro.rows) { + t.is(Object.hasOwn(row, testTableColumnName), true); + t.is(Object.hasOwn(row, testTableSecondColumnName), false); + t.is(Object.hasOwn(row, 'id'), false); + } +}); diff --git a/docker-compose.yml b/docker-compose.yml index c6202314e..b864ed6f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,11 +104,13 @@ services: rocketadmin-private-microservice: build: - context: ../rocketadmin-saas - dockerfile: Dockerfile.test + # rocketadmin-saas now lives in the sitenova monorepo; its Dockerfile.test + # needs the monorepo root as build context (root pnpm lockfile/workspace). + context: ../sitenova + dockerfile: rocketadmin-saas/Dockerfile.test ports: - 3001:3001 - env_file: ../rocketadmin-saas/.env + env_file: ../sitenova/rocketadmin-saas/.env environment: # Path to the Go proxy's contract fixture (mounted below). Lets the # SaaS contract test find it regardless of on-disk layout.