Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ pnpm-lock.yaml
yarn-lock.json
.vscode/database.json
yarn-error.log
.history
3 changes: 2 additions & 1 deletion .vscodeignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ package-lock.json
*.vsix
dist/**/*.map
tsconfig.json
webpack.config.json
webpack.config.json
.history
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ Get it from [Visual Studio Marketplace](https://marketplace.visualstudio.com/ite
"patternName": {
// Your source database connection section
"source": {
"type": "postgres", // Or mssql
"user": string,
"password": string,
"host": string,
Expand Down Expand Up @@ -90,6 +91,7 @@ Get it from [Visual Studio Marketplace](https://marketplace.visualstudio.com/ite

// Your target database connection section
"target": {
"type": "postgres", // Or mssql
"user": string,
"password": string,
"host": string,
Expand Down
1,174 changes: 1,116 additions & 58 deletions package-lock.json

Large diffs are not rendered by default.

10 changes: 7 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@
"database",
"management",
"sync",
"postgresql"
"postgresql",
"mssql"
],
"contributes": {
"viewsContainers": {
Expand Down Expand Up @@ -307,7 +308,7 @@
"vscode:prepublish": "npm run vscode-desktop:publish && npm run vscode-web:publish",
"vscode-desktop:publish": "npm run esbuild-base -- --minify",
"vscode-web:publish": "npm run compile-web -- --mode production --devtool false",
"esbuild-base": "esbuild ./src/extension.ts --bundle --outfile=dist/extension.js --external:vscode --format=cjs --platform=node",
"esbuild-base": "esbuild ./src/extension.ts --bundle --outfile=dist/extension.js --external:vscode --format=cjs --platform=node --loader:.node=file",
"esbuild": "npm run esbuild-base -- --sourcemap",
"esbuild-watch": "npm run esbuild-base -- --sourcemap --watch",
"compile": "tsc -p ./",
Expand Down Expand Up @@ -338,6 +339,7 @@
"@types/glob": "^8.1.0",
"@types/lodash.groupby": "^4.6.7",
"@types/mocha": "^10.0.1",
"@types/mssql": "^9.1.7",
"@types/node": "^20.2.5",
"@types/pg": "^8.10.2",
"@types/pg-promise": "^5.4.3",
Expand Down Expand Up @@ -371,6 +373,8 @@
"diff": "^5.1.0",
"fs-extra": "^11.1.1",
"lodash.groupby": "^4.6.0",
"msnodesqlv8": "^5.1.1",
"mssql": "^12.0.0",
"pg": "^8.11.1",
"pg-iterator": "^0.3.0",
"pg-promise": "^11.5.0"
Expand All @@ -381,4 +385,4 @@
},
"publisher": "nguyenngoclong",
"license": "MIT"
}
}
18 changes: 8 additions & 10 deletions src/commands/actions/generatePlanAsync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ const writePlanOutput = async (filePath: string, lines: string[]): Promise<strin
export const generatePlanAsync = async (options: {
fileManager: FileManager;
tables: TableConfig[];
dbType: 'postgres' | 'mssql';
}): Promise<boolean> => {
const { fileManager, tables } = options;
const { fileManager, tables, dbType } = options;

// Read plan detail
const sessionPath = fileManager.getSessionPath();
Expand Down Expand Up @@ -85,7 +86,7 @@ export const generatePlanAsync = async (options: {
// Line start with '+' => insert
if (line.startsWith('+')) {
const values = JSON.parse(line.substring(1));
const insertQuery = makeInsertQuery(table, planDetail, values);
const insertQuery = makeInsertQuery(table, planDetail, values, dbType);
allPlanLines.push(insertQuery);
insertPlanLines.push(insertQuery);
planDetail.insert++;
Expand All @@ -95,7 +96,7 @@ export const generatePlanAsync = async (options: {
// Line start with '-' => remove
if (line.startsWith('-')) {
const values = JSON.parse(line.substring(1));
const deleteQuery = makeDeleteQuery(table, planDetail, values);
const deleteQuery = makeDeleteQuery(table, planDetail, values, dbType);
allPlanLines.push(deleteQuery);
deletePlanLines.push(deleteQuery);
planDetail.delete++;
Expand Down Expand Up @@ -130,14 +131,14 @@ export const generatePlanAsync = async (options: {
const currentRecord = JSON.parse(currentLine);
switch (operation) {
case '+': {
const insertQuery = makeInsertQuery(table, planDetail, currentRecord);
const insertQuery = makeInsertQuery(table, planDetail, currentRecord, dbType);
allPlanLines.push(insertQuery);
insertPlanLines.push(insertQuery);
planDetail.insert++;
break;
}
case '-': {
const deleteQuery = makeDeleteQuery(table, planDetail, currentRecord);
const deleteQuery = makeDeleteQuery(table, planDetail, currentRecord, dbType);
allPlanLines.push(deleteQuery);
deletePlanLines.push(deleteQuery);
planDetail.delete++;
Expand All @@ -160,13 +161,13 @@ export const generatePlanAsync = async (options: {
const sameCurrentLine = groupItems[1];
if (rawCurrentLine.startsWith('+') && sameCurrentLine.startsWith('-')) {
const record = JSON.parse(rawCurrentLine.substring(1));
const updateQuery = makeUpdateQuery(table, planDetail, record);
const updateQuery = makeUpdateQuery(table, planDetail, record, dbType);
allPlanLines.push(updateQuery);
updatePlanLines.push(updateQuery);
planDetail.update++;
} else if (rawCurrentLine.startsWith('-') && sameCurrentLine.startsWith('+')) {
const record = JSON.parse(sameCurrentLine.substring(1));
const updateQuery = makeUpdateQuery(table, planDetail, record);
const updateQuery = makeUpdateQuery(table, planDetail, record, dbType);
allPlanLines.push(updateQuery);
updatePlanLines.push(updateQuery);
planDetail.update++;
Expand All @@ -192,17 +193,14 @@ export const generatePlanAsync = async (options: {
await writePlanOutput(allPlanFilePath, allPlanLines);
logger.info(`The ${allPlanFilePath} was successfully generated.`);

// Output all plan files
const insertPlanFilePath = fileManager.getPlanOutputPath('insert');
await writePlanOutput(insertPlanFilePath, insertPlanLines);
logger.info(`The ${insertPlanFilePath} was successfully generated.`);

// Output all plan files
const updatePlanFilePath = fileManager.getPlanOutputPath('update');
await writePlanOutput(updatePlanFilePath, updatePlanLines);
logger.info(`The ${updatePlanFilePath} was successfully generated.`);

// Output all plan files
const deletePlanFilePath = fileManager.getPlanOutputPath('delete');
await writePlanOutput(deletePlanFilePath, deletePlanLines);
logger.info(`The ${deletePlanFilePath} was successfully generated.`);
Expand Down
98 changes: 29 additions & 69 deletions src/commands/actions/generateSnapshotsAsync.ts
Original file line number Diff line number Diff line change
@@ -1,48 +1,13 @@
import fs from 'fs-extra';
import { EOL } from 'node:os';
import pg from 'pg';
import { QueryIterablePool } from 'pg-iterator';
import { FileManager, SnapshotType } from '../../utils/fileManager';
import { logger } from '../../utils/logger';
import { PatternConfig, PatternSession, TableConfig, getDatabaseInfo, getTabWidth } from '../../utils/utils';

const getPrimaryKeys = async (options: { pool: pg.Pool; table: TableConfig }): Promise<string[]> => {
const { pool, table } = options;
const rawQuery = `
SELECT c.column_name
FROM information_schema.table_constraints t
JOIN information_schema.constraint_column_usage c
ON c.constraint_name = t.constraint_name
WHERE t.constraint_type = 'PRIMARY KEY' AND c.table_name = '${table.name}'`;
const result = await pool.query(rawQuery);
const primaryKeys = [];
for (const row of result.rows) {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { column_name } = row as { column_name: string };
primaryKeys.push(column_name);
}
return primaryKeys;
};

const getColumnNames = async (options: { pool: pg.Pool; table: TableConfig }): Promise<string[]> => {
const { pool, table } = options;
const rawQuery = `
SELECT column_name
FROM information_schema.columns
WHERE table_name = '${table.name}'
ORDER BY ordinal_position`;
const result = await pool.query(rawQuery);
const tableColumns = [];
for await (const row of result.rows) {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { column_name } = row as { column_name: string };
tableColumns.push(column_name);
}
return tableColumns;
};
import { PatternConfig, PatternSession, TableConfig, getTabWidth } from '../../utils/utils';
import { createDatabaseProvider } from '../../utils/database/databaseProviderFactory';
import { DatabaseProvider } from '../../utils/database/databaseProvider';

export type GenerateSnapshotOptions = {
pool: pg.Pool;
dbProvider: DatabaseProvider;
fileManager: FileManager;
snapshotType: SnapshotType;
formatLine?: boolean;
Expand All @@ -51,10 +16,12 @@ export type GenerateSnapshotOptions = {
};

export const createSnapshotFiles = async (options: GenerateSnapshotOptions): Promise<void> => {
const { pool, fileManager, tables, snapshotType, session, formatLine = false } = options;
const { dbProvider, fileManager, tables, snapshotType, session, formatLine = false } = options;
const tab = getTabWidth();
const qs = new QueryIterablePool(pool);

try {
await dbProvider.connect();

for (let i = 0; i < tables.length; i++) {
const table = tables[i];

Expand All @@ -70,14 +37,14 @@ export const createSnapshotFiles = async (options: GenerateSnapshotOptions): Pro
// Get primary key
let primaryKeys = table.primaryKeys;
if (!primaryKeys || primaryKeys.length <= 0) {
primaryKeys = await getPrimaryKeys({ pool, table });
primaryKeys = await dbProvider.getPrimaryKeys(table);
}
session.plan[table.name].primaryKeys = primaryKeys;

// Get columns for table
let tableColumns = table.columns;
if (!tableColumns || tableColumns.length <= 0) {
tableColumns = await getColumnNames({ pool, table });
tableColumns = await dbProvider.getColumnNames(table);
}
session.plan[table.name].columns = tableColumns;

Expand All @@ -88,8 +55,14 @@ export const createSnapshotFiles = async (options: GenerateSnapshotOptions): Pro
}

// Generate select query
const selectColumns = tableColumns.length > 0 ? tableColumns.map((tc) => `"${tc}"`).join(', ') : '*';
const selectQuery = `SELECT ${selectColumns} FROM ${table.name}`;
const selectColumns =
tableColumns.length > 0 ? tableColumns.map((tc) => dbProvider.escapeIdentifier(tc)).join(', ') : '*';

const tableIdentifier = table.schema
? `${dbProvider.escapeIdentifier(table.schema)}.${dbProvider.escapeIdentifier(table.name)}`
: dbProvider.escapeIdentifier(table.name);

const selectQuery = `SELECT ${selectColumns} FROM ${tableIdentifier}`;
const where = table.where ? `WHERE ${table.where}` : '';
const orderBy = table.orderBy ? `ORDER BY ${table.orderBy}` : '';
const rawQuery = [selectQuery, where, orderBy].filter(Boolean).join(' ');
Expand All @@ -99,39 +72,26 @@ export const createSnapshotFiles = async (options: GenerateSnapshotOptions): Pro
await fs.remove(snapshotPath);
await fs.ensureFile(snapshotPath);

// Get stream data from table
// Stream data from table to file
logger.info(`Execute '${rawQuery}' at '${table.name}' table to create snapshot.`);
const rows = qs.query(rawQuery);

// Stream to snapshot file
logger.info(`Stream ${table.name} data to ${snapshotPath} ...`);
for await (const row of rows) {
// TODO: format json line to easy diff with JSONL
// const rowContent = formatLine ? JSON.stringify(row, null, tab) : JSON.stringify(row);
for await (const row of dbProvider.queryStream(rawQuery)) {
const rowContent = JSON.stringify(row);
fs.appendFileSync(snapshotPath, rowContent.concat(EOL), { encoding: 'utf-8' });
}
logger.info(`The ${table.name} data was successfully streamed .`);

logger.info(`The ${table.name} data was successfully streamed.`);
}

// Save table detail
const sessionPath = fileManager.getSessionPath();
await fs.outputJSON(sessionPath, session, { spaces: tab });
logger.info(`The ${sessionPath} was successfully created.`);
} finally {
if (qs) {
qs.release();
}
await dbProvider.disconnect();
}
};

/**
* Generate snapshot files
* - migrations/<migrate-name>/snapshots/original
* - migrations/<migrate-name>/snapshots/modified
* Create new migrate info file
* - migrations/<migrate-name>/session.json
*/
export const generateSnapshotAsync = async (options: {
fileManager: FileManager;
selectedPattern: string;
Expand All @@ -150,11 +110,11 @@ export const generateSnapshotAsync = async (options: {
};

// Generate snapshot for original database (target apply)
const targetPool = new pg.Pool(target);
logger.info(`Generating target snapshot with db connection '${getDatabaseInfo(target)}'....`);
const targetProvider = createDatabaseProvider(target);
logger.info(`Generating target snapshot with db connection '${targetProvider.getDatabaseInfo()}'....`);
await createSnapshotFiles({
fileManager,
pool: targetPool,
dbProvider: targetProvider,
snapshotType: SnapshotType.original,
formatLine,
session,
Expand All @@ -163,11 +123,11 @@ export const generateSnapshotAsync = async (options: {
logger.info('The target snapshot files was successfully generated');

// Generate snapshot for modified database (source changed)
const sourcePool = new pg.Pool(source);
logger.info(`Generating source snapshot with db connection '${getDatabaseInfo(source)}'....`);
const sourceProvider = createDatabaseProvider(source);
logger.info(`Generating source snapshot with db connection '${sourceProvider.getDatabaseInfo()}'....`);
await createSnapshotFiles({
fileManager,
pool: sourcePool,
dbProvider: sourceProvider,
snapshotType: SnapshotType.modified,
formatLine,
session,
Expand Down
31 changes: 8 additions & 23 deletions src/commands/actions/tryConnectionAsync.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,17 @@
import { logger } from '../../utils/logger';
import { getDatabaseInfo } from '../../utils/utils';
import { Client, PoolConfig } from 'pg';
import { DatabaseConfig } from '../../utils/database/databaseProvider';
import { createDatabaseProvider } from '../../utils/database/databaseProviderFactory';

/**
* Check database connection
*/
export const tryConnectionAsync = async (poolConfig: PoolConfig): Promise<boolean> => {
let client: Client | undefined = undefined;
export const tryConnectionAsync = async (dbConfig: DatabaseConfig): Promise<boolean> => {
const dbProvider = createDatabaseProvider(dbConfig);
try {
client = new Client({
host: poolConfig.host,
user: poolConfig.user,
password: poolConfig.password,
port: poolConfig.port,
database: poolConfig.database || 'postgres'
});
await client.connect();

// Get current version
logger.info(`Connecting to the '${getDatabaseInfo(poolConfig)}' successful.`);
await dbProvider.connect();
logger.info(`Connecting to the '${dbProvider.getDatabaseInfo()}' successful.`);
return true;
} catch (err) {
logger.info(`Could not connect to the '${getDatabaseInfo(poolConfig)}'.`, err);
logger.info(`Could not connect to the '${dbProvider.getDatabaseInfo()}'.`, err);
return false;
} finally {
if (client) {
await client.end();
client = undefined;
}
await dbProvider.disconnect();
}
};
Loading