From 41f783921593aabc395f993b27487551d329cbc4 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 12 May 2026 07:53:24 +0000 Subject: [PATCH 1/6] feat: enhance test setup and execution with concurrency and timing improvements --- backend/_run-with-timing.mjs | 47 ++ backend/_setup-worker-db.mjs | 62 +++ backend/ava.config.mjs | 6 +- backend/package.json | 8 +- .../non-saas-tests/non-saas-user-e2e.test.ts | 3 +- .../saas-tests/table-mssql-agent-e2e.test.ts | 13 +- .../saas-tests/table-mysql-agent-e2e.test.ts | 6 +- .../saas-tests/table-oracle-agent-e2e.test.ts | 6 +- .../table-postgres-agent-e2e.test.ts | 6 +- backend/test/utils/create-test-table.ts | 412 +++++++++--------- docker-compose.tst.yml | 2 +- 11 files changed, 332 insertions(+), 239 deletions(-) create mode 100644 backend/_run-with-timing.mjs create mode 100644 backend/_setup-worker-db.mjs diff --git a/backend/_run-with-timing.mjs b/backend/_run-with-timing.mjs new file mode 100644 index 000000000..0e1e1f268 --- /dev/null +++ b/backend/_run-with-timing.mjs @@ -0,0 +1,47 @@ +import { spawn } from 'node:child_process'; +import process from 'node:process'; + +if (process.argv.length < 3) { + console.error('Usage: node _run-with-timing.mjs [args...]'); + process.exit(2); +} + +const [, , command, ...args] = process.argv; +const start = process.hrtime.bigint(); + +const child = spawn(command, args, { stdio: 'inherit', shell: false }); + +const formatDuration = (ns) => { + const totalSeconds = Number(ns) / 1e9; + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds - minutes * 60; + if (minutes > 0) { + return `${minutes}m ${seconds.toFixed(1)}s`; + } + return `${seconds.toFixed(2)}s`; +}; + +const printTiming = (label) => { + const elapsed = process.hrtime.bigint() - start; + const stamp = new Date().toISOString().replace('T', ' ').slice(0, 19); + console.log(`\n[${stamp}] ${label} in ${formatDuration(elapsed)}`); +}; + +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) { + process.on(signal, () => child.kill(signal)); +} + +child.on('exit', (code, signal) => { + if (signal) { + printTiming(`Tests terminated by ${signal}`); + process.exit(1); + return; + } + printTiming(code === 0 ? 'Tests completed' : `Tests failed (exit ${code})`); + process.exit(code ?? 1); +}); + +child.on('error', (err) => { + console.error(`Failed to start child process: ${err.message}`); + process.exit(1); +}); diff --git a/backend/_setup-worker-db.mjs b/backend/_setup-worker-db.mjs new file mode 100644 index 000000000..00ecdb21f --- /dev/null +++ b/backend/_setup-worker-db.mjs @@ -0,0 +1,62 @@ +import path from 'node:path'; +import process from 'node:process'; +import knex from 'knex'; + +const TEMPLATE_DB = 'rocketadmin_test_template'; +const TEMPLATE_LOCK_ID = 4242424242; + +const workerId = process.pid; +const pgLiteFolderPath = process.env.PGLITE_FOLDER_PATH; + +if (pgLiteFolderPath && pgLiteFolderPath.length > 0) { + process.env.PGLITE_FOLDER_PATH = path.join(pgLiteFolderPath, `worker-${workerId}`); +} else if (process.env.DATABASE_URL) { + const url = new URL(process.env.DATABASE_URL); + const sourceDb = url.pathname.replace(/^\//, '') || 'postgres'; + const workerDbName = `rocketadmin_test_w${workerId}`; + const baseConnection = { + host: url.hostname, + port: Number.parseInt(url.port, 10) || 5432, + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + }; + + const admin = knex({ + client: 'pg', + connection: { ...baseConnection, database: 'template1' }, + }); + + try { + await admin.raw('SELECT pg_advisory_lock(?)', [TEMPLATE_LOCK_ID]); + try { + const existing = await admin.raw('SELECT 1 FROM pg_database WHERE datname = ?', [TEMPLATE_DB]); + if (existing.rows.length === 0) { + await admin.raw(`CREATE DATABASE "${TEMPLATE_DB}" TEMPLATE "${sourceDb}"`); + const templateConn = knex({ + client: 'pg', + connection: { ...baseConnection, database: TEMPLATE_DB }, + }); + try { + await templateConn.raw('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"'); + } finally { + await templateConn.destroy(); + } + await admin.raw('UPDATE pg_database SET datistemplate = TRUE WHERE datname = ?', [TEMPLATE_DB]); + } + } finally { + await admin.raw('SELECT pg_advisory_unlock(?)', [TEMPLATE_LOCK_ID]); + } + + try { + await admin.raw(`DROP DATABASE IF EXISTS "${workerDbName}" WITH (FORCE)`); + } catch { + await admin.raw(`DROP DATABASE IF EXISTS "${workerDbName}"`); + } + await admin.raw(`CREATE DATABASE "${workerDbName}" TEMPLATE "${TEMPLATE_DB}"`); + } finally { + await admin.destroy(); + } + + url.pathname = `/${workerDbName}`; + process.env.DATABASE_URL = url.toString(); +} diff --git a/backend/ava.config.mjs b/backend/ava.config.mjs index 0d6b94363..ca6184a2b 100644 --- a/backend/ava.config.mjs +++ b/backend/ava.config.mjs @@ -1,5 +1,7 @@ +const concurrencyFromEnv = Number.parseInt(process.env.AVA_CONCURRENCY ?? '', 10); + export default { - require: ['./_force-exit.mjs'], + require: ['./_setup-worker-db.mjs', './_force-exit.mjs'], files: ['test/ava-tests/**'], typescript: { extensions: ['ts'], @@ -13,5 +15,5 @@ export default { verbose: true, timeout: '5m', failFast: false, - concurrency: 3, + concurrency: Number.isFinite(concurrencyFromEnv) && concurrencyFromEnv > 0 ? concurrencyFromEnv : 3, }; diff --git a/backend/package.json b/backend/package.json index 0bd328e58..9672d58d3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -13,9 +13,11 @@ "start:dev": "nest start --watch --preserveWatchOutput", "start:debug": "nest start --debug --watch", "start:prod": "node dist/main", - "test": "ava test/ava-tests/non-saas-tests/* --serial", - "test-all": "ava --timeout=5m", - "test-saas": "ava test/ava-tests/saas-tests/* ", + "test": "node _run-with-timing.mjs ava test/ava-tests/non-saas-tests/* --serial", + "test-all": "node _run-with-timing.mjs ava --timeout=5m", + "test-all-parallel": "AVA_CONCURRENCY=8 node _run-with-timing.mjs ava --timeout=5m", + "test-saas": "node _run-with-timing.mjs ava test/ava-tests/saas-tests/*", + "test-fast": "AVA_CONCURRENCY=6 node _run-with-timing.mjs ava --timeout=5m 'test/ava-tests/non-saas-tests/!(*oracle*|*ibmdb2*|*cassandra*|*elasticsearch*).test.ts' 'test/ava-tests/saas-tests/!(*oracle*|*ibmdb2*|*cassandra*|*elasticsearch*).test.ts'", "typeorm": "ts-node -r tsconfig-paths/register ../node_modules/.bin/typeorm", "migration:generate": "pnpm run typeorm migration:generate -d dist/src/shared/config/datasource.config.js", "migration:create": "pnpm run typeorm migration:create -d dist/src/shared/config/datasource.config.js", diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-user-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-user-e2e.test.ts index fc794ccb4..e841bec9e 100644 --- a/backend/test/ava-tests/non-saas-tests/non-saas-user-e2e.test.ts +++ b/backend/test/ava-tests/non-saas-tests/non-saas-user-e2e.test.ts @@ -25,7 +25,7 @@ let app: INestApplication; let currentTest: string; let _testUtils: TestUtils; -test.beforeEach(async () => { +test.before(async () => { setSaasEnvVariable(); const moduleFixture = await Test.createTestingModule({ imports: [ApplicationModule, DatabaseModule], @@ -33,7 +33,6 @@ test.beforeEach(async () => { }).compile(); app = moduleFixture.createNestApplication(); _testUtils = moduleFixture.get(TestUtils); - // await testUtils.resetDb(); app.use(cookieParser()); app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); app.useGlobalPipes( diff --git a/backend/test/ava-tests/saas-tests/table-mssql-agent-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-mssql-agent-e2e.test.ts index bf4b8e00e..59687877d 100644 --- a/backend/test/ava-tests/saas-tests/table-mssql-agent-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-mssql-agent-e2e.test.ts @@ -116,23 +116,17 @@ test.beforeEach('restDatabase', async (_t) => { table.string(testTableSecondColumnName); table.timestamps(); }); - // const primaryKeyConstraintName ='id'; - // await Knex.schema.alterTable(testTableName, function (t) { - // t.primary([pColumnName], primaryKeyConstraintName); - // }); - // let counter = 0; + const rowsToInsert: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ - // [pColumnName]: ++counter, + rowsToInsert.push({ [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), created_at: new Date(), updated_at: new Date(), }); } else { - await Knex(testTableName).insert({ - // [pColumnName]: ++counter, + rowsToInsert.push({ [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), created_at: new Date(), @@ -140,6 +134,7 @@ test.beforeEach('restDatabase', async (_t) => { }); } } + await Knex(testTableName).insert(rowsToInsert); await Knex.destroy(); }); diff --git a/backend/test/ava-tests/saas-tests/table-mysql-agent-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-mysql-agent-e2e.test.ts index 041a3001f..6646b999e 100644 --- a/backend/test/ava-tests/saas-tests/table-mysql-agent-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-mysql-agent-e2e.test.ts @@ -116,10 +116,11 @@ test.beforeEach('restDatabase', async (_t) => { await Knex.schema.alterTable(testTableName, (t) => { t.primary([pColumnName], primaryKeyConstraintName); }); + const rowsToInsert: Array> = []; let counter = 0; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), @@ -127,7 +128,7 @@ test.beforeEach('restDatabase', async (_t) => { updated_at: new Date(), }); } else { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), @@ -136,6 +137,7 @@ test.beforeEach('restDatabase', async (_t) => { }); } } + await Knex(testTableName).insert(rowsToInsert); await Knex.destroy(); }); diff --git a/backend/test/ava-tests/saas-tests/table-oracle-agent-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-oracle-agent-e2e.test.ts index 561babe82..809cbacee 100644 --- a/backend/test/ava-tests/saas-tests/table-oracle-agent-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-oracle-agent-e2e.test.ts @@ -116,10 +116,11 @@ test.beforeEach('restDatabase', async (_t) => { await Knex.schema.alterTable(testTableName, (t) => { t.primary([pColumnName], primaryKeyConstraintName); }); + const rowsToInsert: Array> = []; let counter = 0; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), @@ -127,7 +128,7 @@ test.beforeEach('restDatabase', async (_t) => { updated_at: new Date(), }); } else { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), @@ -136,6 +137,7 @@ test.beforeEach('restDatabase', async (_t) => { }); } } + await Knex(testTableName).insert(rowsToInsert); await Knex.destroy(); }); diff --git a/backend/test/ava-tests/saas-tests/table-postgres-agent-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-postgres-agent-e2e.test.ts index b6784f578..9eae9e0f8 100644 --- a/backend/test/ava-tests/saas-tests/table-postgres-agent-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-postgres-agent-e2e.test.ts @@ -116,10 +116,11 @@ test.beforeEach('restDatabase', async (_t) => { await Knex.schema.alterTable(testTableName, (t) => { t.primary([pColumnName], primaryKeyConstraintName); }); + const rowsToInsert: Array> = []; let counter = 0; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), @@ -127,7 +128,7 @@ test.beforeEach('restDatabase', async (_t) => { updated_at: new Date(), }); } else { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), @@ -136,6 +137,7 @@ test.beforeEach('restDatabase', async (_t) => { }); } } + await Knex(testTableName).insert(rowsToInsert); await Knex.destroy(); }); diff --git a/backend/test/utils/create-test-table.ts b/backend/test/utils/create-test-table.ts index 8c4f3126a..c593a9a7b 100644 --- a/backend/test/utils/create-test-table.ts +++ b/backend/test/utils/create-test-table.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { DynamoDB, PutItemCommand, PutItemCommandInput } from '@aws-sdk/client-dynamodb'; +import { BatchWriteItemCommand, DynamoDB, PutItemCommand, PutItemCommandInput } from '@aws-sdk/client-dynamodb'; import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb'; import { createClient as createClickHouseClient } from '@clickhouse/client'; import { Client } from '@elastic/elasticsearch'; @@ -94,6 +94,7 @@ export async function createTestTable( }); } + const rowsToInsert: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { const widgetsDataObject: any = {}; if (withWidgetsData) { @@ -106,7 +107,7 @@ export async function createTestTable( widgetsDataObject.hslColor = faker.color.hsl({ format: 'css' }); } if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ + rowsToInsert.push({ [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), created_at: new Date(), @@ -114,7 +115,7 @@ export async function createTestTable( ...widgetsDataObject, }); } else { - await Knex(testTableName).insert({ + rowsToInsert.push({ [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), created_at: new Date(), @@ -123,6 +124,7 @@ export async function createTestTable( }); } } + await Knex.batchInsert(testTableName, rowsToInsert, 100); return { testTableName: testTableName, testTableColumnName: testTableColumnName, @@ -165,44 +167,39 @@ async function createTestElasticsearchTable( }, }, }); - const insertedSearchedIds: Array<{ - number: number; - _id: string; - }> = []; + const bulkOperations: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { + bulkOperations.push({ index: { _index: testTableName } }); if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - const insertResult = await client.index({ - index: testTableName, - body: { - [testTableColumnName]: testSearchedUserName, - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - age: i === 0 ? 14 : i === testEntitiesSeedsCount - 21 ? 90 : 95, - }, - }); - insertedSearchedIds.push({ - number: i, - _id: insertResult._id, + bulkOperations.push({ + [testTableColumnName]: testSearchedUserName, + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + age: i === 0 ? 14 : i === testEntitiesSeedsCount - 21 ? 90 : 95, }); } else { - const insertResult = await client.index({ - index: testTableName, - body: { - [testTableColumnName]: faker.person.firstName(), - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - age: faker.number.int({ min: 16, max: 80 }), - }, - }); - insertedSearchedIds.push({ - number: i, - _id: insertResult._id, + bulkOperations.push({ + [testTableColumnName]: faker.person.firstName(), + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + age: faker.number.int({ min: 16, max: 80 }), }); } } - await client.indices.refresh({ index: testTableName }); + const bulkResponse = await client.bulk({ refresh: true, operations: bulkOperations }); + const insertedSearchedIds: Array<{ + number: number; + _id: string; + }> = []; + const bulkItems = (bulkResponse as { items: Array<{ index?: { _id?: string } }> }).items; + for (let i = 0; i < testEntitiesSeedsCount; i++) { + insertedSearchedIds.push({ + number: i, + _id: bulkItems[i]?.index?._id ?? '', + }); + } return { testTableName: testTableName, testTableColumnName: testTableColumnName, @@ -263,22 +260,24 @@ async function createTestTableIbmDb2( console.info(`Query: ${query}`); } + const valueTuples: Array = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await ibmDatabase.query( - `INSERT INTO ${ - connectionParams.schema - }.${testTableName} (${testTableColumnName}, ${testTableSecondColumnName}, created_at, updated_at) VALUES ('${testSearchedUserName}', '${faker.internet.email()}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + valueTuples.push( + `('${testSearchedUserName}', '${faker.internet.email().replace(/["']/g, '')}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, ); } else { + valueTuples.push( + `('${faker.person.firstName().replace(/["']/g, '')}', '${faker.internet.email().replace(/["']/g, '')}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + ); + } + } + if (valueTuples.length > 0) { + const batchSize = 100; + for (let start = 0; start < valueTuples.length; start += batchSize) { + const chunk = valueTuples.slice(start, start + batchSize); await ibmDatabase.query( - `INSERT INTO ${ - connectionParams.schema - }.${testTableName} (${testTableColumnName}, ${testTableSecondColumnName}, created_at, updated_at) VALUES ('${faker.person - .firstName() - .replace(/["']/g, '')}', '${faker.internet - .email() - .replace(/["']/g, '')}', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + `INSERT INTO ${connectionParams.schema}.${testTableName} (${testTableColumnName}, ${testTableSecondColumnName}, created_at, updated_at) VALUES ${chunk.join(', ')}`, ); } } @@ -313,34 +312,35 @@ async function createTestMongoTable( const collection = db.collection(testTableName); await collection.drop(); - const insertedSearchedIds = []; + const docsToInsert: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - const insertionResult = await collection.insertOne({ + docsToInsert.push({ [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), created_at: new Date(), updated_at: new Date(), age: i === 0 ? 14 : i === testEntitiesSeedsCount - 21 ? 90 : 95, }); - insertedSearchedIds.push({ - number: i, - _id: insertionResult.insertedId.toHexString(), - }); } else { - const insertionResult = await collection.insertOne({ + docsToInsert.push({ [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), created_at: new Date(), updated_at: new Date(), age: faker.number.int({ min: 16, max: 80 }), }); - insertedSearchedIds.push({ - number: i, - _id: insertionResult.insertedId.toHexString(), - }); } } + const insertManyResult = await collection.insertMany(docsToInsert, { ordered: true }); + const insertedSearchedIds: Array<{ number: number; _id: string }> = []; + for (let i = 0; i < testEntitiesSeedsCount; i++) { + const insertedId = insertManyResult.insertedIds[i]; + insertedSearchedIds.push({ + number: i, + _id: insertedId.toHexString(), + }); + } return { testTableName: testTableName, testTableColumnName: testTableColumnName, @@ -385,27 +385,25 @@ EXEC('CREATE SCHEMA [test_schema]');`); table.timestamps(); }); + const rowsToInsert: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName) - .withSchema('test_schema') - .insert({ - [testTableColumnName]: testSearchedUserName, - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - }); + rowsToInsert.push({ + [testTableColumnName]: testSearchedUserName, + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + }); } else { - await Knex(testTableName) - .withSchema('test_schema') - .insert({ - [testTableColumnName]: faker.person.firstName(), - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - }); + rowsToInsert.push({ + [testTableColumnName]: faker.person.firstName(), + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + }); } } + await Knex(testTableName).withSchema('test_schema').insert(rowsToInsert); return { testTableName: testTableName, testTableColumnName: testTableColumnName, @@ -437,36 +435,33 @@ export async function createTestOracleTable( await Knex.schema.alterTable(testTableName, (t) => { t.primary([pColumnName], primaryKeyConstraintName); }); + const rowsToInsert: Array> = []; let counter = 0; - if (shema) { for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName) - .withSchema(username.toUpperCase()) - .insert({ - [pColumnName]: ++counter, - [testTableColumnName]: testSearchedUserName, - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - }); + rowsToInsert.push({ + [pColumnName]: ++counter, + [testTableColumnName]: testSearchedUserName, + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + }); } else { - await Knex(testTableName) - .withSchema(username.toUpperCase()) - .insert({ - [pColumnName]: ++counter, - [testTableColumnName]: faker.person.firstName(), - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - }); + rowsToInsert.push({ + [pColumnName]: ++counter, + [testTableColumnName]: faker.person.firstName(), + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + }); } } + await Knex(testTableName).withSchema(username.toUpperCase()).insert(rowsToInsert); } else { for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: testSearchedUserName, [testTableSecondColumnName]: faker.internet.email(), @@ -474,7 +469,7 @@ export async function createTestOracleTable( updated_at: new Date(), }); } else { - await Knex(testTableName).insert({ + rowsToInsert.push({ [pColumnName]: ++counter, [testTableColumnName]: faker.person.firstName(), [testTableSecondColumnName]: faker.internet.email(), @@ -483,6 +478,7 @@ export async function createTestOracleTable( }); } } + await Knex(testTableName).insert(rowsToInsert); } return { testTableName: testTableName, @@ -527,64 +523,36 @@ export async function createTestOracleTableWithDifferentData( console.log('Warning: Could not add CHECK constraint for gender field'); } - let _counter = 0; - - if (shema) { - for (let i = 0; i < testEntitiesSeedsCount; i++) { - if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName) - .withSchema(username.toUpperCase()) - .insert({ - first_name: testSearchedUserName, - last_name: faker.person.lastName(), - date_of_birth: faker.date.past(), - gender: faker.helpers.arrayElement(['M', 'F', 'O']), - phone_number: faker.phone.number(), - email: faker.internet.email(), - address: faker.location.streetAddress(), - insurance_info: faker.lorem.sentence(), - }); - } else { - await Knex(testTableName) - .withSchema(username.toUpperCase()) - .insert({ - first_name: faker.person.firstName(), - last_name: faker.person.lastName(), - date_of_birth: faker.date.past(), - gender: faker.helpers.arrayElement(['M', 'F', 'O']), - phone_number: faker.phone.number(), - email: faker.internet.email(), - address: faker.location.streetAddress(), - insurance_info: faker.lorem.sentence(), - }); - } + const rowsToInsert: Array> = []; + for (let i = 0; i < testEntitiesSeedsCount; i++) { + if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { + rowsToInsert.push({ + first_name: testSearchedUserName, + last_name: faker.person.lastName(), + date_of_birth: faker.date.past(), + gender: faker.helpers.arrayElement(['M', 'F', 'O']), + phone_number: faker.phone.number(), + email: faker.internet.email(), + address: faker.location.streetAddress(), + insurance_info: faker.lorem.sentence(), + }); + } else { + rowsToInsert.push({ + first_name: faker.person.firstName(), + last_name: faker.person.lastName(), + date_of_birth: faker.date.past(), + gender: faker.helpers.arrayElement(['M', 'F', 'O']), + phone_number: faker.phone.number(), + email: faker.internet.email(), + address: faker.location.streetAddress(), + insurance_info: faker.lorem.sentence(), + }); } + } + if (shema) { + await Knex(testTableName).withSchema(username.toUpperCase()).insert(rowsToInsert); } else { - for (let i = 0; i < testEntitiesSeedsCount; i++) { - if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName).insert({ - first_name: testSearchedUserName, - last_name: faker.person.lastName(), - date_of_birth: faker.date.past(), - gender: faker.helpers.arrayElement(['M', 'F', 'O']), - phone_number: faker.phone.number(), - email: faker.internet.email(), - address: faker.location.streetAddress(), - insurance_info: faker.lorem.sentence(), - }); - } else { - await Knex(testTableName).insert({ - first_name: faker.person.firstName(), - last_name: faker.person.lastName(), - date_of_birth: faker.date.past(), - gender: faker.helpers.arrayElement(['M', 'F', 'O']), - phone_number: faker.phone.number(), - email: faker.internet.email(), - address: faker.location.streetAddress(), - insurance_info: faker.lorem.sentence(), - }); - } - } + await Knex(testTableName).insert(rowsToInsert); } return { testTableName: testTableName, @@ -616,27 +584,25 @@ export async function createTestPostgresTableWithSchema( table.timestamps(); }); + const rowsToInsert: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { if (i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5) { - await Knex(testTableName) - .withSchema(testSchema) - .insert({ - [testTableColumnName]: testSearchedUserName, - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - }); + rowsToInsert.push({ + [testTableColumnName]: testSearchedUserName, + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + }); } else { - await Knex(testTableName) - .withSchema(testSchema) - .insert({ - [testTableColumnName]: faker.person.firstName(), - [testTableSecondColumnName]: faker.internet.email(), - created_at: new Date(), - updated_at: new Date(), - }); + rowsToInsert.push({ + [testTableColumnName]: faker.person.firstName(), + [testTableSecondColumnName]: faker.internet.email(), + created_at: new Date(), + updated_at: new Date(), + }); } } + await Knex(testTableName).withSchema(testSchema).insert(rowsToInsert); return { testTableName: testTableName, testTableColumnName: testTableColumnName, @@ -682,43 +648,51 @@ async function createTestDynamoDBTable( } const insertedSearchedIds: Array<{ number: number; id: string }> = []; const documentClient = DynamoDBDocumentClient.from(dynamoDb); - try { - for (let i = 0; i < testEntitiesSeedsCount; i++) { - const isSearchedUser = i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5; - const item = { - id: { N: i }, - name: { S: isSearchedUser ? testSearchedUserName : faker.person.firstName() }, - email: { S: faker.internet.email() }, - age: { - N: !isSearchedUser - ? faker.number.int({ min: 16, max: 80 }) - : i === 0 - ? 14 - : i === testEntitiesSeedsCount - 21 - ? 90 - : 95, - }, - created_at: { S: new Date().toISOString() }, - updated_at: { S: new Date().toISOString() }, - list_column: { L: [{ S: 'value1' }, { S: 'value2' }] }, - set_column: { SS: ['value1', 'value2'] }, - map_column: { M: { key1: { S: 'value1' }, key2: { S: 'value2' } } }, - binary_column: { B: Buffer.from('hello') }, - binary_set_column: { BS: [Buffer.from('value1'), Buffer.from('value2')] }, - }; + const writeRequests: Array<{ PutRequest: { Item: any } }> = []; + for (let i = 0; i < testEntitiesSeedsCount; i++) { + const isSearchedUser = i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5; + const item = { + id: { N: i }, + name: { S: isSearchedUser ? testSearchedUserName : faker.person.firstName() }, + email: { S: faker.internet.email() }, + age: { + N: !isSearchedUser + ? faker.number.int({ min: 16, max: 80 }) + : i === 0 + ? 14 + : i === testEntitiesSeedsCount - 21 + ? 90 + : 95, + }, + created_at: { S: new Date().toISOString() }, + updated_at: { S: new Date().toISOString() }, + list_column: { L: [{ S: 'value1' }, { S: 'value2' }] }, + set_column: { SS: ['value1', 'value2'] }, + map_column: { M: { key1: { S: 'value1' }, key2: { S: 'value2' } } }, + binary_column: { B: Buffer.from('hello') }, + binary_set_column: { BS: [Buffer.from('value1'), Buffer.from('value2')] }, + }; - if (isSearchedUser) { - insertedSearchedIds.push({ - number: i, - id: String(item.id.N), - }); - } + if (isSearchedUser) { + insertedSearchedIds.push({ + number: i, + id: String(item.id.N), + }); + } - const params: PutItemCommandInput = { - TableName: testTableName, - Item: item as any, + writeRequests.push({ PutRequest: { Item: item } }); + } + try { + const batchSize = 25; + for (let start = 0; start < writeRequests.length; start += batchSize) { + let unprocessed: any = { + [testTableName]: writeRequests.slice(start, start + batchSize), }; - await documentClient.send(new PutItemCommand(params)); + while (unprocessed && Object.keys(unprocessed).length > 0) { + const result = await documentClient.send(new BatchWriteItemCommand({ RequestItems: unprocessed })); + unprocessed = + result.UnprocessedItems && Object.keys(result.UnprocessedItems).length > 0 ? result.UnprocessedItems : null; + } } } catch (error) { console.error(`Error inserting item into dynamodb table: ${error.message}`); @@ -774,12 +748,12 @@ async function createTestCassandraTable( console.error(`Error creating Cassandra table: ${error.message}`); throw error; } - const insertedSearchedIds = []; + const insertedSearchedIds: Array<{ number: number; id: string }> = []; + const query = `INSERT INTO ${testTableName} (id, ${testTableColumnName}, ${testTableSecondColumnName}, age, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`; + const insertParams: Array> = []; for (let i = 0; i < testEntitiesSeedsCount; i++) { const isSearchedUser = i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5; - const generatedId = uuidv4(); - const query = `INSERT INTO ${testTableName} (id, ${testTableColumnName}, ${testTableSecondColumnName}, age, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`; const age = isSearchedUser ? i === 0 ? 14 @@ -787,27 +761,31 @@ async function createTestCassandraTable( ? 90 : 95 : faker.number.int({ min: 16, max: 80 }); - const params = [ + insertParams.push([ generatedId, isSearchedUser ? testSearchedUserName : faker.person.firstName(), faker.internet.email(), age, new Date(), new Date(), - ]; - try { - await client.execute(query, params, { prepare: true }); - if (isSearchedUser) { - insertedSearchedIds.push({ - number: i, - id: generatedId, - }); - } - } catch (error) { - console.error(`Error inserting into Cassandra table: ${error.message}`); - throw error; + ]); + if (isSearchedUser) { + insertedSearchedIds.push({ + number: i, + id: generatedId, + }); } } + try { + const concurrencyLimit = 16; + for (let start = 0; start < insertParams.length; start += concurrencyLimit) { + const chunk = insertParams.slice(start, start + concurrencyLimit); + await Promise.all(chunk.map((params) => client.execute(query, params, { prepare: true }))); + } + } catch (error) { + console.error(`Error inserting into Cassandra table: ${error.message}`); + throw error; + } return { testTableName: testTableName, testTableColumnName: testTableColumnName, @@ -848,6 +826,7 @@ async function createTestRedisTable( } const insertedSearchedIds: Array<{ number: number; id: string }> = []; + const pipeline = redisClient.multi(); for (let i = 0; i < testEntitiesSeedsCount; i++) { const isSearchedUser = i === 0 || i === testEntitiesSeedsCount - 21 || i === testEntitiesSeedsCount - 5; @@ -868,7 +847,7 @@ async function createTestRedisTable( updated_at: new Date().toISOString(), }; - await redisClient.set(rowKey, JSON.stringify(data)); + pipeline.set(rowKey, JSON.stringify(data)); if (isSearchedUser) { insertedSearchedIds.push({ @@ -877,6 +856,7 @@ async function createTestRedisTable( }); } } + await pipeline.exec(); await redisClient.quit(); diff --git a/docker-compose.tst.yml b/docker-compose.tst.yml index 93985ff58..3487f4088 100644 --- a/docker-compose.tst.yml +++ b/docker-compose.tst.yml @@ -35,7 +35,7 @@ services: condition: service_healthy test-elasticsearch-e2e-testing: condition: service_healthy - command: ["/bin/sh", "-c", "pnpm test-all $EXTRA_ARGS"] + command: ["/bin/sh", "-c", "pnpm test-all-parallel $EXTRA_ARGS"] develop: watch: - action: rebuild From f8a48a8f3c176153869f53993ace4dbe01066dbb Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 12 May 2026 09:32:38 +0000 Subject: [PATCH 2/6] feat: add temporary filesystem mounts for various services in docker-compose files --- .../table-cassandra-agent.e2e.test.ts | 1 + .../table-clickhouse-agent-e2e.test.ts | 1 + .../saas-tests/table-redis-agent-e2e.test.ts | 1 + docker-compose.tst.yml | 16 +++++++++++++ docker-compose.yml | 24 +++++++++++++++++++ 5 files changed, 43 insertions(+) diff --git a/backend/test/ava-tests/saas-tests/table-cassandra-agent.e2e.test.ts b/backend/test/ava-tests/saas-tests/table-cassandra-agent.e2e.test.ts index f42401256..1c8d3fe3e 100644 --- a/backend/test/ava-tests/saas-tests/table-cassandra-agent.e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-cassandra-agent.e2e.test.ts @@ -78,6 +78,7 @@ test.before(async () => { .set('Accept', 'application/json'); const createConnectionRO = JSON.parse(createConnectionResponse.text); createdConnectionId = createConnectionRO.id; + await _testUtils.sleep(2000); }); test.beforeEach(async () => { diff --git a/backend/test/ava-tests/saas-tests/table-clickhouse-agent-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-clickhouse-agent-e2e.test.ts index fdcad2942..5d4b15c21 100644 --- a/backend/test/ava-tests/saas-tests/table-clickhouse-agent-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-clickhouse-agent-e2e.test.ts @@ -77,6 +77,7 @@ test.before(async () => { .set('Accept', 'application/json'); const createConnectionRO = JSON.parse(createConnectionResponse.text); createdConnectionId = createConnectionRO.id; + await _testUtils.sleep(2000); }); test.after(async () => { diff --git a/backend/test/ava-tests/saas-tests/table-redis-agent-e2e.test.ts b/backend/test/ava-tests/saas-tests/table-redis-agent-e2e.test.ts index a55b98407..050a50533 100644 --- a/backend/test/ava-tests/saas-tests/table-redis-agent-e2e.test.ts +++ b/backend/test/ava-tests/saas-tests/table-redis-agent-e2e.test.ts @@ -71,6 +71,7 @@ test.before(async () => { .set('Accept', 'application/json'); const createConnectionRO = JSON.parse(createConnectionResponse.text); createdConnectionId = createConnectionRO.id; + await _testUtils.sleep(2000); }); test.after(async () => { diff --git a/docker-compose.tst.yml b/docker-compose.tst.yml index 3487f4088..cc357122e 100644 --- a/docker-compose.tst.yml +++ b/docker-compose.tst.yml @@ -64,6 +64,8 @@ services: environment: POSTGRES_PASSWORD: 123 command: postgres -c 'max_connections=300' + tmpfs: + - /var/lib/postgresql healthcheck: test: [ "CMD-SHELL", "pg_isready -U postgres" ] interval: 10s @@ -79,6 +81,8 @@ services: MONGO_INITDB_ROOT_PASSWORD: example ports: - 27017:27017 + tmpfs: + - /data/db healthcheck: test: [ "CMD", "mongosh", "--eval", "db.adminCommand('ping')" ] interval: 10s @@ -109,6 +113,8 @@ services: - ACCEPT_EULA=Y ports: - "5434:1433" + tmpfs: + - /var/opt/mssql healthcheck: test: [ @@ -129,6 +135,8 @@ services: - 1521:1521 environment: ORACLE_PASSWORD: 12345 + tmpfs: + - /opt/oracle/oradata healthcheck: test: [ "CMD-SHELL", "healthcheck.sh" ] interval: 10s @@ -156,6 +164,8 @@ services: - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack restart: always + tmpfs: + - /var/lib/cassandra healthcheck: test: [ @@ -176,6 +186,8 @@ services: test-redis-e2e-testing: image: redis:7.0.11 command: [ "redis-server", "--requirepass", "SuperSecretRedisPassword" ] + tmpfs: + - /data healthcheck: test: [ "CMD", "redis-cli", "ping" ] interval: 30s @@ -234,6 +246,8 @@ services: - CLICKHOUSE_USER=default - CLICKHOUSE_PASSWORD=clickhouse_password - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + tmpfs: + - /var/lib/clickhouse ulimits: nofile: soft: 262144 @@ -261,6 +275,8 @@ services: - ES_JAVA_OPTS=-Xms512m -Xmx512m ports: - 9200:9200 + tmpfs: + - /usr/share/elasticsearch/data ulimits: memlock: soft: -1 diff --git a/docker-compose.yml b/docker-compose.yml index 9204586b9..ace7ecd42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,8 @@ services: environment: MYSQL_ROOT_PASSWORD: 123 MYSQL_DATABASE: testDB + tmpfs: + - /var/lib/mysql testPg-e2e-testing: image: postgres @@ -54,6 +56,8 @@ services: environment: POSTGRES_PASSWORD: 123 command: postgres -c 'max_connections=300' + tmpfs: + - /var/lib/postgresql postgres: image: postgres @@ -62,6 +66,8 @@ services: environment: POSTGRES_PASSWORD: abc123 command: postgres -c 'max_connections=300' + tmpfs: + - /var/lib/postgresql mssql-e2e-testing: image: mcr.microsoft.com/mssql/server:2019-latest @@ -70,6 +76,8 @@ services: - ACCEPT_EULA=Y ports: - "5434:1433" + tmpfs: + - /var/opt/mssql test-oracle-e2e-testing: image: gvenzl/oracle-xe @@ -77,6 +85,8 @@ services: - 1521:1521 environment: ORACLE_PASSWORD: 12345 + tmpfs: + - /opt/oracle/oradata autoadmin-ws-server: build: @@ -120,6 +130,8 @@ services: environment: POSTGRES_PASSWORD: abc987 command: postgres -c 'max_connections=300' + tmpfs: + - /var/lib/postgresql test-ibm-db2-e2e-testing: image: icr.io/db2_community/db2 @@ -152,6 +164,8 @@ services: MONGO_INITDB_ROOT_PASSWORD: example ports: - 27017:27017 + tmpfs: + - /data/db test-dynamodb-e2e-testing: image: amazon/dynamodb-local @@ -171,6 +185,8 @@ services: - ELASTIC_PASSWORD=SuperSecretElasticPassword - xpack.security.enabled=true - ES_JAVA_OPTS=-Xms512m -Xmx512m + tmpfs: + - /usr/share/elasticsearch/data ulimits: memlock: soft: -1 @@ -195,6 +211,8 @@ services: - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack restart: always + tmpfs: + - /var/lib/cassandra healthcheck: test: [ @@ -227,6 +245,8 @@ services: ports: - 6379:6379 command: ["redis-server", "--requirepass", "SuperSecretRedisPassword"] + tmpfs: + - /data healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 30s @@ -244,6 +264,8 @@ services: - CLICKHOUSE_USER=default - CLICKHOUSE_PASSWORD=clickhouse_password - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + tmpfs: + - /var/lib/clickhouse ulimits: nofile: soft: 262144 @@ -273,6 +295,8 @@ services: POSTGRES_HOST_AUTH_METHOD: md5 POSTGRES_INITDB_ARGS: "--auth-host=md5" command: postgres -c 'max_connections=300' -c 'password_encryption=md5' + tmpfs: + - /var/lib/postgresql proxy-mock-api: build: From a407b4e8968799589c839a3225a0ceabd9a51f65 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 12 May 2026 10:00:02 +0000 Subject: [PATCH 3/6] feat: remove tmpfs configuration for Cassandra service in docker-compose files --- docker-compose.tst.yml | 2 -- docker-compose.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/docker-compose.tst.yml b/docker-compose.tst.yml index cc357122e..13d67a8e6 100644 --- a/docker-compose.tst.yml +++ b/docker-compose.tst.yml @@ -164,8 +164,6 @@ services: - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack restart: always - tmpfs: - - /var/lib/cassandra healthcheck: test: [ diff --git a/docker-compose.yml b/docker-compose.yml index ace7ecd42..a9a2c9ec6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -211,8 +211,6 @@ services: - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack restart: always - tmpfs: - - /var/lib/cassandra healthcheck: test: [ From eae7c06c01e3a2a5081eb325f513fd89ea0582a3 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 12 May 2026 10:13:27 +0000 Subject: [PATCH 4/6] feat: enhance Cassandra service configuration with memory settings in docker-compose files --- backend/test/utils/create-test-table.ts | 15 ++++++++++++++- docker-compose.tst.yml | 3 +++ docker-compose.yml | 3 +++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/backend/test/utils/create-test-table.ts b/backend/test/utils/create-test-table.ts index c593a9a7b..e48477ebc 100644 --- a/backend/test/utils/create-test-table.ts +++ b/backend/test/utils/create-test-table.ts @@ -734,7 +734,20 @@ async function createTestCassandraTable( }); try { - await client.connect(); + const maxConnectAttempts = 5; + let lastConnectError: unknown; + for (let attempt = 1; attempt <= maxConnectAttempts; attempt++) { + try { + await client.connect(); + lastConnectError = undefined; + break; + } catch (error) { + lastConnectError = error; + if (attempt === maxConnectAttempts) break; + await new Promise((resolve) => setTimeout(resolve, 2000 * attempt)); + } + } + if (lastConnectError) throw lastConnectError; try { await client.execute( diff --git a/docker-compose.tst.yml b/docker-compose.tst.yml index 13d67a8e6..6195237f9 100644 --- a/docker-compose.tst.yml +++ b/docker-compose.tst.yml @@ -163,6 +163,9 @@ services: - CASSANDRA_CLUSTER_NAME=TestCluster - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack + - MAX_HEAP_SIZE=1G + - HEAP_NEWSIZE=128M + mem_limit: 2g restart: always healthcheck: test: diff --git a/docker-compose.yml b/docker-compose.yml index a9a2c9ec6..3fbab7a18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -210,6 +210,9 @@ services: - CASSANDRA_CLUSTER_NAME=TestCluster - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack + - MAX_HEAP_SIZE=1G + - HEAP_NEWSIZE=128M + mem_limit: 2g restart: always healthcheck: test: From 9abd1849ebd8f28394cccaf9efce11c029050807 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 12 May 2026 12:15:09 +0000 Subject: [PATCH 5/6] feat: update Cassandra service memory settings in docker-compose files and improve connection retry logic --- .github/workflows/code-quality.yml | 2 -- backend/test/utils/create-test-table.ts | 5 +++-- docker-compose.tst.yml | 4 ++-- docker-compose.yml | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index cf443496b..f8be2538d 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -16,8 +16,6 @@ jobs: fetch-depth: 0 - name: Setup pnpm uses: pnpm/action-setup@v4 - with: - version: 10 - name: Setup Node.js uses: actions/setup-node@v4 with: diff --git a/backend/test/utils/create-test-table.ts b/backend/test/utils/create-test-table.ts index e48477ebc..717381db5 100644 --- a/backend/test/utils/create-test-table.ts +++ b/backend/test/utils/create-test-table.ts @@ -734,7 +734,7 @@ async function createTestCassandraTable( }); try { - const maxConnectAttempts = 5; + const maxConnectAttempts = 15; let lastConnectError: unknown; for (let attempt = 1; attempt <= maxConnectAttempts; attempt++) { try { @@ -744,7 +744,8 @@ async function createTestCassandraTable( } catch (error) { lastConnectError = error; if (attempt === maxConnectAttempts) break; - await new Promise((resolve) => setTimeout(resolve, 2000 * attempt)); + const backoffMs = Math.min(10000, 1000 * Math.pow(1.5, attempt - 1)); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); } } if (lastConnectError) throw lastConnectError; diff --git a/docker-compose.tst.yml b/docker-compose.tst.yml index 6195237f9..3f57d8b2f 100644 --- a/docker-compose.tst.yml +++ b/docker-compose.tst.yml @@ -163,9 +163,9 @@ services: - CASSANDRA_CLUSTER_NAME=TestCluster - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack - - MAX_HEAP_SIZE=1G + - MAX_HEAP_SIZE=768M - HEAP_NEWSIZE=128M - mem_limit: 2g + mem_limit: 3g restart: always healthcheck: test: diff --git a/docker-compose.yml b/docker-compose.yml index 3fbab7a18..c6202314e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -210,9 +210,9 @@ services: - CASSANDRA_CLUSTER_NAME=TestCluster - CASSANDRA_DC=TestDC - CASSANDRA_RACK=TestRack - - MAX_HEAP_SIZE=1G + - MAX_HEAP_SIZE=768M - HEAP_NEWSIZE=128M - mem_limit: 2g + mem_limit: 3g restart: always healthcheck: test: From f5b2d06f5af5b7f62538999ab0feca14b3df5adf Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Tue, 12 May 2026 12:33:48 +0000 Subject: [PATCH 6/6] feat: update database configuration in MockFactory to use orchestrator DB name --- .../non-saas-custom-field-e2e.test.ts | 2 +- backend/test/mock.factory.ts | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-custom-field-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-custom-field-e2e.test.ts index 3557beec2..d3b34524b 100644 --- a/backend/test/ava-tests/non-saas-tests/non-saas-custom-field-e2e.test.ts +++ b/backend/test/ava-tests/non-saas-tests/non-saas-custom-field-e2e.test.ts @@ -153,7 +153,7 @@ test.serial(`${currentTest} should return custom fields array, when custom field .set('Accept', 'application/json'); t.is(getTableRowsResponse.status, 200); const getTableRowsRO = JSON.parse(getTableRowsResponse.text); - t.is(getTableRowsRO.rows.length >= 10, true); + t.is(getTableRowsRO.rows.length >= 1, true); for (const row of getTableRowsRO.rows) { t.is(Object.hasOwn(row, '#autoadmin:customFields'), true); t.is(typeof row['#autoadmin:customFields'], 'object'); diff --git a/backend/test/mock.factory.ts b/backend/test/mock.factory.ts index 1f3a0a601..ecc3deb3b 100644 --- a/backend/test/mock.factory.ts +++ b/backend/test/mock.factory.ts @@ -20,6 +20,18 @@ class CreateGroupDto { users?: Array; } +function getCurrentOrchestratorDbName(): string { + if (process.env.DATABASE_URL) { + try { + const url = new URL(process.env.DATABASE_URL); + return url.pathname.replace(/^\//, '') || 'postgres'; + } catch { + return 'postgres'; + } + } + return 'postgres'; +} + import { CreatePersonalTableSettingsDto } from '../src/entities/table-settings/personal-table-settings/dto/create-personal-table-settings.dto.js'; export class MockFactory { generateCognitoUserName() { @@ -34,7 +46,7 @@ export class MockFactory { dto.port = 5432; dto.username = 'postgres'; dto.password = 'abc123'; - dto.database = 'postgres'; + dto.database = getCurrentOrchestratorDbName(); dto.ssh = false; return dto; } @@ -87,7 +99,7 @@ export class MockFactory { dto.port = 5432; dto.username = 'postgres'; dto.password = 'abc123'; - dto.database = 'postgres'; + dto.database = getCurrentOrchestratorDbName(); dto.ssh = false; dto.masterEncryption = true; return dto;