From f67db8e4233a62368fe774fe5f77401363f57d8b Mon Sep 17 00:00:00 2001 From: khatabwedaa Date: Wed, 25 Mar 2026 07:10:59 +0300 Subject: [PATCH 1/2] Adds Excel and CSV import functionality Introduces comprehensive import capabilities to complement existing export features, including: - Implements concern-based import pattern with ToArray, ToCollection interfaces for flexible data reception - Adds row processing concerns: WithHeadingRow, WithImportMapping, WithColumnMapping for data transformation - Provides validation system with custom rules and optional class-validator DTO support - Enables batch processing with WithBatchInserts for efficient large dataset handling - Includes error handling via SkipsOnError and SkipsEmptyRows concerns - Adds WithStartRow and WithLimit for granular control over data reading Exposes new service methods: import(), importFromBuffer(), toArray(), toCollection() for various use cases Updates documentation with detailed examples and API reference for all import concerns Marks class-validator and class-transformer as optional peer dependencies for DTO validation feature --- README.md | 229 +++- package.json | 13 +- pnpm-lock.yaml | 57 +- src/concerns/index.ts | 23 + src/concerns/skips-empty-rows.interface.ts | 8 + src/concerns/skips-on-error.interface.ts | 8 + src/concerns/to-array.interface.ts | 6 + src/concerns/to-collection.interface.ts | 9 + src/concerns/with-batch-inserts.interface.ts | 7 + src/concerns/with-column-mapping.interface.ts | 8 + src/concerns/with-heading-row.interface.ts | 10 + src/concerns/with-import-mapping.interface.ts | 12 + src/concerns/with-limit.interface.ts | 6 + src/concerns/with-start-row.interface.ts | 6 + src/concerns/with-validation.interface.ts | 18 + src/excel.reader.ts | 49 + src/excel.service.ts | 61 +- src/excel.sheet-reader.ts | 258 ++++ src/excel.writer.ts | 31 +- src/helpers/csv-settings.ts | 26 + src/helpers/index.ts | 2 + src/helpers/validate-row.ts | 82 ++ src/index.ts | 30 + src/interfaces/import-result.interface.ts | 15 + src/interfaces/index.ts | 5 + test/excel.import.spec.ts | 1045 +++++++++++++++++ test/excel.service.spec.ts | 48 + test/validate-row.spec.ts | 48 + 28 files changed, 2077 insertions(+), 43 deletions(-) create mode 100644 src/concerns/skips-empty-rows.interface.ts create mode 100644 src/concerns/skips-on-error.interface.ts create mode 100644 src/concerns/to-array.interface.ts create mode 100644 src/concerns/to-collection.interface.ts create mode 100644 src/concerns/with-batch-inserts.interface.ts create mode 100644 src/concerns/with-column-mapping.interface.ts create mode 100644 src/concerns/with-heading-row.interface.ts create mode 100644 src/concerns/with-import-mapping.interface.ts create mode 100644 src/concerns/with-limit.interface.ts create mode 100644 src/concerns/with-start-row.interface.ts create mode 100644 src/concerns/with-validation.interface.ts create mode 100644 src/excel.reader.ts create mode 100644 src/excel.sheet-reader.ts create mode 100644 src/helpers/csv-settings.ts create mode 100644 src/helpers/validate-row.ts create mode 100644 src/interfaces/import-result.interface.ts create mode 100644 test/excel.import.spec.ts create mode 100644 test/validate-row.spec.ts diff --git a/README.md b/README.md index 303f1c3..31d52a9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

@nestbolt/excel

-

Supercharged Excel and CSV exports for NestJS applications. Effortlessly create and download spreadsheets with powerful features and seamless integration.

+

Supercharged Excel and CSV exports and imports for NestJS applications. Effortlessly create, download, and import spreadsheets with powerful features and seamless integration.

@@ -56,6 +56,18 @@ return this.excelService.downloadAsStream(new UsersExport(), "users.xlsx"); - [WithFrozenRows / WithFrozenColumns](#withfrozenrows--withfrozencolumns) - [FromTemplate](#fromtemplate) - [WithTemplateData](#withtemplatedata) +- [Imports](#imports) + - [ToArray](#toarray) + - [ToCollection](#tocollection) + - [WithHeadingRow](#withheadingrow) + - [WithImportMapping](#withimportmapping) + - [WithColumnMapping](#withcolumnmapping) + - [WithValidation](#withvalidation) + - [WithBatchInserts](#withbatchinserts) + - [WithStartRow](#withstartrow) + - [WithLimit](#withlimit) + - [SkipsEmptyRows](#skipsemptyrows) + - [SkipsOnError](#skipsonerror) - [Using the Service Directly](#using-the-service-directly) - [Configuration Options](#configuration-options) - [Testing](#testing) @@ -476,10 +488,216 @@ class InvoiceExport implements FromTemplate, WithTemplateData { The `dataStartCell()` specifies where the first row of data is written. Each subsequent row is placed on the next row below. +## Imports + +Import classes use the same **concern-based** pattern as exports. Implement one or more interfaces to configure how data is read, transformed, and validated. + +### Quick Import Example + +```typescript +class UsersImport implements ToCollection, WithHeadingRow, WithValidation, SkipsOnError { + readonly hasHeadingRow = true as const; + readonly skipsOnError = true as const; + + handleCollection(rows: Record[]) { + // Process imported rows + } + + rules() { + return { + name: [{ validate: (v) => v?.length > 0, message: "Name is required" }], + email: [{ validate: (v) => /^.+@.+\..+$/.test(v), message: "Invalid email" }], + }; + } +} + +// In your controller +const result = await this.excelService.import(new UsersImport(), "users.xlsx"); +// result.rows, result.errors, result.skipped +``` + +### ToArray + +Receive imported data as a two-dimensional array. + +```typescript +class DataImport implements ToArray { + handleArray(rows: any[][]) { + console.log(rows); // [[1, "Alice"], [2, "Bob"]] + } +} +``` + +### ToCollection + +Receive imported data as an array of objects. Requires `WithHeadingRow` or `WithColumnMapping` to derive object keys. + +```typescript +class UsersImport implements ToCollection, WithHeadingRow { + readonly hasHeadingRow = true as const; + + handleCollection(rows: Record[]) { + console.log(rows); // [{ ID: 1, Name: "Alice" }, ...] + } +} +``` + +### WithHeadingRow + +Use a row in the spreadsheet as column headings. Defaults to row 1. + +```typescript +class ImportWithCustomHeading implements WithHeadingRow { + readonly hasHeadingRow = true as const; + + headingRow() { + return 2; // row 2 contains the headers + } +} +``` + +### WithImportMapping + +Transform each row after reading. + +```typescript +class MappedImport implements WithHeadingRow, WithImportMapping { + readonly hasHeadingRow = true as const; + + mapRow(row: Record) { + return { + fullName: row.first_name + " " + row.last_name, + email: row.email.toLowerCase(), + }; + } +} +``` + +### WithColumnMapping + +Map column letters or 1-based indices to named fields, useful for files without headers. + +```typescript +class NoHeaderImport implements WithColumnMapping { + columnMapping() { + return { name: "A", email: "C", age: 2 }; + } +} +``` + +### WithValidation + +Validate imported rows using custom rules or class-validator DTOs. + +**Custom rules:** + +```typescript +rules() { + return { + name: [ + { validate: (v) => v?.length > 0, message: "Name is required" }, + ], + email: [ + { validate: (v) => /^.+@.+\..+$/.test(v), message: "Invalid email" }, + ], + }; +} +``` + +**class-validator DTO:** + +```typescript +import { IsString, IsEmail } from "class-validator"; + +class UserDto { + @IsString() @IsNotEmpty() name: string; + @IsEmail() email: string; +} + +// In your import class +rules() { + return { dto: UserDto }; +} +``` + +> **Note:** DTO mode requires `class-validator` and `class-transformer` as peer dependencies: +> ```bash +> pnpm add class-validator class-transformer +> ``` + +The `ImportResult` returned from the service contains: + +```typescript +interface ImportResult { + rows: T[]; // valid rows + errors: ImportValidationError[]; // per-row validation errors + skipped: number; // count of skipped rows +} +``` + +### WithBatchInserts + +Insert imported rows in configurable batch sizes. + +```typescript +class BatchImport implements WithBatchInserts { + batchSize() { + return 100; + } + + async handleBatch(batch: any[]) { + await this.userRepo.save(batch); + } +} +``` + +### WithStartRow + +Skip rows before a given row number. + +```typescript +startRow() { + return 3; // start reading from row 3 +} +``` + +### WithLimit + +Limit the number of data rows read. + +```typescript +limit() { + return 1000; // only read first 1000 data rows +} +``` + +### SkipsEmptyRows + +Ignore blank rows during import. + +```typescript +class CleanImport implements SkipsEmptyRows { + readonly skipsEmptyRows = true as const; +} +``` + +### SkipsOnError + +Skip invalid rows instead of throwing. Without this concern, the first validation failure throws an error with all collected errors attached. + +```typescript +class TolerantImport implements WithValidation, SkipsOnError { + readonly skipsOnError = true as const; + rules() { /* ... */ } +} +``` + ## Using the Service Directly Inject `ExcelService` and call its methods: +### Export Methods + | Method | Returns | Description | | ----------------------------------------------- | --------------------- | ------------------------------------------------------------ | | `download(exportable, filename, type?)` | `ExcelDownloadResult` | Returns buffer + filename + content type | @@ -487,6 +705,15 @@ Inject `ExcelService` and call its methods: | `store(exportable, filePath, type?)` | `void` | Writes the export to a local file | | `raw(exportable, type)` | `Buffer` | Returns the raw file buffer | +### Import Methods + +| Method | Returns | Description | +| --------------------------------------------------- | ---------------------------- | -------------------------------------------- | +| `import(importable, filePath, type?)` | `ImportResult` | Read and process a local file | +| `importFromBuffer(importable, buffer, type?)` | `ImportResult` | Read and process a buffer | +| `toArray(filePath, type?)` | `any[][]` | Shorthand: returns raw 2D array | +| `toCollection(filePath, type?)` | `Record[]` | Shorthand: returns objects using row 1 as headings | + ## Configuration Options | Option | Type | Default | Description | diff --git a/package.json b/package.json index 9908d97..80adcd7 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@nestbolt/excel", "version": "0.1.0", "packageManager": "pnpm@9.15.4", - "description": "Supercharged Excel and CSV exports for NestJS applications. Effortlessly create and download spreadsheets with powerful features and seamless integration.", + "description": "Supercharged Excel and CSV exports and imports for NestJS applications. Effortlessly create, download, and import spreadsheets with powerful features and seamless integration.", "main": "dist/index.js", "types": "dist/index.d.ts", "exports": { @@ -31,6 +31,7 @@ "xlsx", "csv", "export", + "import", "spreadsheet", "download" ], @@ -53,7 +54,13 @@ "peerDependencies": { "@nestjs/common": "^10.0.0 || ^11.0.0", "@nestjs/core": "^10.0.0 || ^11.0.0", - "reflect-metadata": "^0.1.13 || ^0.2.0" + "reflect-metadata": "^0.1.13 || ^0.2.0", + "class-validator": "^0.14.0", + "class-transformer": "^0.5.0" + }, + "peerDependenciesMeta": { + "class-validator": { "optional": true }, + "class-transformer": { "optional": true } }, "devDependencies": { "@nestjs/common": "^11.0.0", @@ -64,6 +71,8 @@ "reflect-metadata": "^0.2.2", "rxjs": "^7.8.2", "typescript": "^5.5.0", + "class-transformer": "^0.5.1", + "class-validator": "^0.14.2", "vitest": "^4.1.0" } } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8354830..d3e7c24 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,19 +14,25 @@ importers: devDependencies: '@nestjs/common': specifier: ^11.0.0 - version: 11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2) + version: 11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nestjs/core': specifier: ^11.0.0 - version: 11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2) + version: 11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nestjs/testing': specifier: ^11.0.0 - version: 11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2)) + version: 11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2)) '@types/node': specifier: ^20.0.0 version: 20.19.37 '@vitest/coverage-v8': specifier: ^4.1.0 version: 4.1.1(vitest@4.1.1(@types/node@20.19.37)(vite@8.0.2(@types/node@20.19.37))) + class-transformer: + specifier: ^0.5.1 + version: 0.5.1 + class-validator: + specifier: ^0.14.2 + version: 0.14.4 reflect-metadata: specifier: ^0.2.2 version: 0.2.2 @@ -270,6 +276,9 @@ packages: '@types/node@20.19.37': resolution: {integrity: sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==} + '@types/validator@13.15.10': + resolution: {integrity: sha512-T8L6i7wCuyoK8A/ZeLYt1+q0ty3Zb9+qbSSvrIVitzT3YjZqkTZ40IbRsPanlB4h1QB3JVL1SYCdR6ngtFYcuA==} + '@vitest/coverage-v8@4.1.1': resolution: {integrity: sha512-nZ4RWwGCoGOQRMmU/Q9wlUY540RVRxJZ9lxFsFfy0QV7Zmo5VVBhB6Sl9Xa0KIp2iIs3zWfPlo9LcY1iqbpzCw==} peerDependencies: @@ -376,6 +385,12 @@ packages: chainsaw@0.1.0: resolution: {integrity: sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==} + class-transformer@0.5.1: + resolution: {integrity: sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==} + + class-validator@0.14.4: + resolution: {integrity: sha512-AwNusCCam51q703dW82x95tOqQp6oC9HNUl724KxJJOfnKscI8dOloXFgyez7LbTTKWuRBA37FScqVbJEoq8Yw==} + compress-commons@4.1.2: resolution: {integrity: sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==} engines: {node: '>= 10'} @@ -530,6 +545,9 @@ packages: resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} engines: {node: '>= 0.6.3'} + libphonenumber-js@1.12.40: + resolution: {integrity: sha512-HKGs7GowShNls3Zh+7DTr6wYpPk5jC78l508yQQY3e8ZgJChM3A9JZghmMJZuK+5bogSfuTafpjksGSR3aMIEg==} + lie@3.3.0: resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==} @@ -849,6 +867,10 @@ packages: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true + validator@13.15.26: + resolution: {integrity: sha512-spH26xU080ydGggxRyR1Yhcbgx+j3y5jbNXk/8L+iRvdIEQ4uTRH2Sgf2dokud6Q4oAtsbNvJ1Ft+9xmm6IZcA==} + engines: {node: '>= 0.10'} + vite@8.0.2: resolution: {integrity: sha512-1gFhNi+bHhRE/qKZOJXACm6tX4bA3Isy9KuKF15AgSRuRazNBOJfdDemPBU16/mpMxApDPrWvZ08DcLPEoRnuA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1014,7 +1036,7 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true - '@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2)': + '@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2)': dependencies: file-type: 21.3.2 iterare: 1.2.1 @@ -1023,12 +1045,15 @@ snapshots: rxjs: 7.8.2 tslib: 2.8.1 uid: 2.0.2 + optionalDependencies: + class-transformer: 0.5.1 + class-validator: 0.14.4 transitivePeerDependencies: - supports-color - '@nestjs/core@11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2)': + '@nestjs/core@11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2)': dependencies: - '@nestjs/common': 11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/common': 11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nuxt/opencollective': 0.4.1 fast-safe-stringify: 2.1.1 iterare: 1.2.1 @@ -1038,10 +1063,10 @@ snapshots: tslib: 2.8.1 uid: 2.0.2 - '@nestjs/testing@11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2))': + '@nestjs/testing@11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2))': dependencies: - '@nestjs/common': 11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2) - '@nestjs/core': 11.1.17(@nestjs/common@11.1.17(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/common': 11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 11.1.17(@nestjs/common@11.1.17(class-transformer@0.5.1)(class-validator@0.14.4)(reflect-metadata@0.2.2)(rxjs@7.8.2))(reflect-metadata@0.2.2)(rxjs@7.8.2) tslib: 2.8.1 '@nuxt/opencollective@0.4.1': @@ -1130,6 +1155,8 @@ snapshots: dependencies: undici-types: 6.21.0 + '@types/validator@13.15.10': {} + '@vitest/coverage-v8@4.1.1(vitest@4.1.1(@types/node@20.19.37)(vite@8.0.2(@types/node@20.19.37)))': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -1276,6 +1303,14 @@ snapshots: dependencies: traverse: 0.3.9 + class-transformer@0.5.1: {} + + class-validator@0.14.4: + dependencies: + '@types/validator': 13.15.10 + libphonenumber-js: 1.12.40 + validator: 13.15.26 + compress-commons@4.1.2: dependencies: buffer-crc32: 0.2.13 @@ -1424,6 +1459,8 @@ snapshots: dependencies: readable-stream: 2.3.8 + libphonenumber-js@1.12.40: {} + lie@3.3.0: dependencies: immediate: 3.0.6 @@ -1712,6 +1749,8 @@ snapshots: uuid@8.3.2: {} + validator@13.15.26: {} + vite@8.0.2(@types/node@20.19.37): dependencies: lightningcss: 1.32.0 diff --git a/src/concerns/index.ts b/src/concerns/index.ts index d1f3539..4f635bc 100644 --- a/src/concerns/index.ts +++ b/src/concerns/index.ts @@ -50,3 +50,26 @@ export { BeforeSheetEventPayload, AfterSheetEventPayload, } from "./with-events.interface"; + +// Import — data receivers +export { ToArray } from "./to-array.interface"; +export { ToCollection } from "./to-collection.interface"; + +// Import — row processing +export { WithHeadingRow } from "./with-heading-row.interface"; +export { WithImportMapping } from "./with-import-mapping.interface"; +export { WithColumnMapping } from "./with-column-mapping.interface"; + +// Import — validation +export { + WithValidation, + ValidationRules, + ValidationRule, +} from "./with-validation.interface"; +export { SkipsOnError } from "./skips-on-error.interface"; +export { SkipsEmptyRows } from "./skips-empty-rows.interface"; + +// Import — limits & batching +export { WithLimit } from "./with-limit.interface"; +export { WithStartRow } from "./with-start-row.interface"; +export { WithBatchInserts } from "./with-batch-inserts.interface"; diff --git a/src/concerns/skips-empty-rows.interface.ts b/src/concerns/skips-empty-rows.interface.ts new file mode 100644 index 0000000..c3ec50b --- /dev/null +++ b/src/concerns/skips-empty-rows.interface.ts @@ -0,0 +1,8 @@ +/** + * Ignore blank rows during import. + * + * Marker concern — set `skipsEmptyRows` to `true`. + */ +export interface SkipsEmptyRows { + readonly skipsEmptyRows: true; +} diff --git a/src/concerns/skips-on-error.interface.ts b/src/concerns/skips-on-error.interface.ts new file mode 100644 index 0000000..8da4fc7 --- /dev/null +++ b/src/concerns/skips-on-error.interface.ts @@ -0,0 +1,8 @@ +/** + * Skip invalid rows instead of throwing during validation. + * + * Marker concern — set `skipsOnError` to `true`. + */ +export interface SkipsOnError { + readonly skipsOnError: true; +} diff --git a/src/concerns/to-array.interface.ts b/src/concerns/to-array.interface.ts new file mode 100644 index 0000000..c42d84c --- /dev/null +++ b/src/concerns/to-array.interface.ts @@ -0,0 +1,6 @@ +/** + * Receive imported data as a two-dimensional array. + */ +export interface ToArray { + handleArray(rows: any[][]): void | Promise; +} diff --git a/src/concerns/to-collection.interface.ts b/src/concerns/to-collection.interface.ts new file mode 100644 index 0000000..3178845 --- /dev/null +++ b/src/concerns/to-collection.interface.ts @@ -0,0 +1,9 @@ +/** + * Receive imported data as an array of objects. + * + * Requires {@link WithHeadingRow} or {@link WithColumnMapping} to derive + * object keys. + */ +export interface ToCollection> { + handleCollection(rows: T[]): void | Promise; +} diff --git a/src/concerns/with-batch-inserts.interface.ts b/src/concerns/with-batch-inserts.interface.ts new file mode 100644 index 0000000..75bdb76 --- /dev/null +++ b/src/concerns/with-batch-inserts.interface.ts @@ -0,0 +1,7 @@ +/** + * Insert imported rows in configurable batch sizes. + */ +export interface WithBatchInserts { + batchSize(): number; + handleBatch(batch: T[]): void | Promise; +} diff --git a/src/concerns/with-column-mapping.interface.ts b/src/concerns/with-column-mapping.interface.ts new file mode 100644 index 0000000..ad81956 --- /dev/null +++ b/src/concerns/with-column-mapping.interface.ts @@ -0,0 +1,8 @@ +/** + * Map column letters or 1-based indices to named fields. + * + * Example: `{ name: 'A', email: 'C' }` or `{ name: 1, email: 3 }`. + */ +export interface WithColumnMapping { + columnMapping(): Record; +} diff --git a/src/concerns/with-heading-row.interface.ts b/src/concerns/with-heading-row.interface.ts new file mode 100644 index 0000000..e40a294 --- /dev/null +++ b/src/concerns/with-heading-row.interface.ts @@ -0,0 +1,10 @@ +/** + * Use a row in the spreadsheet as column headings to derive object keys. + * + * Set `hasHeadingRow` to `true` as a marker. Optionally implement + * `headingRow()` to specify a custom row number (defaults to 1). + */ +export interface WithHeadingRow { + readonly hasHeadingRow: true; + headingRow?(): number; +} diff --git a/src/concerns/with-import-mapping.interface.ts b/src/concerns/with-import-mapping.interface.ts new file mode 100644 index 0000000..3ff280b --- /dev/null +++ b/src/concerns/with-import-mapping.interface.ts @@ -0,0 +1,12 @@ +/** + * Transform each row after reading during import. + * + * Named separately from the export `WithMapping` to allow a single class + * to implement both import and export mapping. + */ +export interface WithImportMapping< + TIn = Record, + TOut = Record, +> { + mapRow(row: TIn): TOut; +} diff --git a/src/concerns/with-limit.interface.ts b/src/concerns/with-limit.interface.ts new file mode 100644 index 0000000..eaed21a --- /dev/null +++ b/src/concerns/with-limit.interface.ts @@ -0,0 +1,6 @@ +/** + * Limit the number of data rows read during import. + */ +export interface WithLimit { + limit(): number; +} diff --git a/src/concerns/with-start-row.interface.ts b/src/concerns/with-start-row.interface.ts new file mode 100644 index 0000000..7a5a071 --- /dev/null +++ b/src/concerns/with-start-row.interface.ts @@ -0,0 +1,6 @@ +/** + * Skip rows before a given 1-based row number during import. + */ +export interface WithStartRow { + startRow(): number; +} diff --git a/src/concerns/with-validation.interface.ts b/src/concerns/with-validation.interface.ts new file mode 100644 index 0000000..63e8fff --- /dev/null +++ b/src/concerns/with-validation.interface.ts @@ -0,0 +1,18 @@ +/** + * Validate imported rows using custom rules or a class-validator DTO. + * + * Return an object with a `dto` key for class-validator integration, + * or a `ValidationRules` map for custom rule functions. + */ +export interface WithValidation> { + rules(): ValidationRules | { dto: new (...args: any[]) => any }; +} + +export type ValidationRules> = { + [K in keyof T]?: ValidationRule[]; +}; + +export interface ValidationRule { + validate: (value: any, row: Record) => boolean; + message: string; +} diff --git a/src/excel.reader.ts b/src/excel.reader.ts new file mode 100644 index 0000000..5aac4bd --- /dev/null +++ b/src/excel.reader.ts @@ -0,0 +1,49 @@ +import { Workbook } from "exceljs"; +import { Readable } from "stream"; +import type { ExcelModuleOptions } from "./interfaces"; +import type { ImportResult } from "./interfaces"; +import { ExcelType } from "./excel.constants"; +import { resolveCsvSettings } from "./helpers/csv-settings"; +import { processSheet } from "./excel.sheet-reader"; + +export async function readImport( + importable: object, + source: string | Buffer, + type: ExcelType, + options: ExcelModuleOptions, +): Promise { + const workbook = new Workbook(); + + if (Buffer.isBuffer(source)) { + if (type === ExcelType.CSV) { + const csvOpts = resolveCsvSettings(importable, options); + await workbook.csv.read(Readable.from(source), { + parserOptions: { + delimiter: csvOpts.delimiter, + quote: csvOpts.quoteChar, + }, + }); + } else { + await workbook.xlsx.load(source as any); + } + } else { + if (type === ExcelType.CSV) { + const csvOpts = resolveCsvSettings(importable, options); + await workbook.csv.readFile(source, { + parserOptions: { + delimiter: csvOpts.delimiter, + quote: csvOpts.quoteChar, + }, + }); + } else { + await workbook.xlsx.readFile(source); + } + } + + const worksheet = workbook.worksheets[0]; + if (!worksheet) { + throw new Error("No worksheet found in the imported file."); + } + + return processSheet(worksheet, importable); +} diff --git a/src/excel.service.ts b/src/excel.service.ts index ee5ab26..0b5206b 100644 --- a/src/excel.service.ts +++ b/src/excel.service.ts @@ -2,9 +2,14 @@ import { Inject, Injectable, Logger, StreamableFile } from "@nestjs/common"; import * as fs from "fs"; import * as path from "path"; import { EXCEL_OPTIONS, ExcelType, CONTENT_TYPES } from "./excel.constants"; -import type { ExcelModuleOptions, ExcelDownloadResult } from "./interfaces"; +import type { + ExcelModuleOptions, + ExcelDownloadResult, + ImportResult, +} from "./interfaces"; import { detectType } from "./helpers"; import { writeExport } from "./excel.writer"; +import { readImport } from "./excel.reader"; @Injectable() export class ExcelService { @@ -92,6 +97,60 @@ export class ExcelService { return writeExport(exportable, writerType, this.options); } + /* ---------------------------------------------------------------- */ + /* Import */ + /* ---------------------------------------------------------------- */ + + /** + * Read and process a local file through the importable's concerns. + */ + async import( + importable: object, + filePath: string, + readerType?: ExcelType, + ): Promise { + const type = readerType ?? this.resolveType(path.basename(filePath)); + return readImport(importable, filePath, type, this.options); + } + + /** + * Read and process a buffer through the importable's concerns. + */ + async importFromBuffer( + importable: object, + buffer: Buffer, + readerType?: ExcelType, + ): Promise { + const type = readerType ?? ExcelType.XLSX; + return readImport(importable, buffer, type, this.options); + } + + /** + * Shorthand: read a file and return the raw 2D array. + */ + async toArray( + filePath: string, + readerType?: ExcelType, + ): Promise { + const type = readerType ?? this.resolveType(path.basename(filePath)); + const result = await readImport({}, filePath, type, this.options); + return result.rows; + } + + /** + * Shorthand: read a file and return an array of objects using row 1 + * as headings. + */ + async toCollection( + filePath: string, + readerType?: ExcelType, + ): Promise[]> { + const type = readerType ?? this.resolveType(path.basename(filePath)); + const importable = { hasHeadingRow: true as const }; + const result = await readImport(importable, filePath, type, this.options); + return result.rows; + } + /* ---------------------------------------------------------------- */ /* Internal */ /* ---------------------------------------------------------------- */ diff --git a/src/excel.sheet-reader.ts b/src/excel.sheet-reader.ts new file mode 100644 index 0000000..29f6d1c --- /dev/null +++ b/src/excel.sheet-reader.ts @@ -0,0 +1,258 @@ +import type { CellValue, Worksheet } from "exceljs"; +import type { + SkipsEmptyRows, + SkipsOnError, + ToArray, + ToCollection, + WithBatchInserts, + WithColumnMapping, + WithHeadingRow, + WithImportMapping, + WithLimit, + WithStartRow, + WithValidation, +} from "./concerns"; +import { columnLetterToNumber } from "./helpers"; +import { validateRow } from "./helpers/validate-row"; +import type { ImportResult, ImportValidationError } from "./interfaces"; + +/* ------------------------------------------------------------------ */ +/* Type guards */ +/* ------------------------------------------------------------------ */ + +function isToArray(obj: any): obj is ToArray { + return typeof obj.handleArray === "function"; +} + +function isToCollection(obj: any): obj is ToCollection { + return typeof obj.handleCollection === "function"; +} + +function isWithHeadingRow(obj: any): obj is WithHeadingRow { + return obj.hasHeadingRow === true; +} + +function isWithImportMapping(obj: any): obj is WithImportMapping { + return typeof obj.mapRow === "function"; +} + +function isWithColumnMapping(obj: any): obj is WithColumnMapping { + return typeof obj.columnMapping === "function"; +} + +function isWithValidation(obj: any): obj is WithValidation { + return typeof obj.rules === "function"; +} + +function isWithBatchInserts(obj: any): obj is WithBatchInserts { + return ( + typeof obj.batchSize === "function" && typeof obj.handleBatch === "function" + ); +} + +function isWithLimit(obj: any): obj is WithLimit { + return typeof obj.limit === "function"; +} + +function isWithStartRow(obj: any): obj is WithStartRow { + return typeof obj.startRow === "function"; +} + +function isSkipsOnError(obj: any): obj is SkipsOnError { + return obj.skipsOnError === true; +} + +function isSkipsEmptyRows(obj: any): obj is SkipsEmptyRows { + return obj.skipsEmptyRows === true; +} + +/* ------------------------------------------------------------------ */ +/* Helpers */ +/* ------------------------------------------------------------------ */ + +function extractCellValue(value: CellValue): any { + if (value === null || value === undefined) return null; + if (typeof value === "object" && "result" in value) { + return (value as any).result; + } + if (typeof value === "object" && "richText" in value) { + return (value as any).richText.map((rt: any) => rt.text).join(""); + } + return value; +} + +function isEmptyRow(row: any[]): boolean { + return row.every((v) => v === null || v === undefined || v === ""); +} + +/* ------------------------------------------------------------------ */ +/* Sheet reader */ +/* ------------------------------------------------------------------ */ + +export async function processSheet( + worksheet: Worksheet, + importable: object, +): Promise { + // --- extract all raw rows from worksheet -------------------------- + const rawRows: { rowNumber: number; values: any[] }[] = []; + + worksheet.eachRow((row, rowNumber) => { + const values: any[] = []; + row.eachCell({ includeEmpty: true }, (cell, colNumber) => { + // Expand values array to accommodate the column + while (values.length < colNumber) values.push(null); + values[colNumber - 1] = extractCellValue(cell.value); + }); + rawRows.push({ rowNumber, values }); + }); + + // --- determine heading row ---------------------------------------- + let headings: string[] | null = null; + let headingRowNumber = 0; + + if (isWithHeadingRow(importable)) { + headingRowNumber = + typeof importable.headingRow === "function" ? importable.headingRow() : 1; + + const headingEntry = rawRows.find((r) => r.rowNumber === headingRowNumber); + if (headingEntry) { + headings = headingEntry.values.map((v) => + v !== null && v !== undefined ? String(v) : "", + ); + } + } + + // --- determine data start row ------------------------------------- + let dataStartRow: number; + if (isWithStartRow(importable)) { + dataStartRow = importable.startRow(); + } else if (headingRowNumber > 0) { + dataStartRow = headingRowNumber + 1; + } else { + dataStartRow = 1; + } + + // --- filter to data rows ------------------------------------------ + let dataRows = rawRows.filter((r) => r.rowNumber >= dataStartRow); + + // --- skip empty rows ---------------------------------------------- + if (isSkipsEmptyRows(importable)) { + dataRows = dataRows.filter((r) => !isEmptyRow(r.values)); + } + + // --- apply limit -------------------------------------------------- + if (isWithLimit(importable)) { + dataRows = dataRows.slice(0, importable.limit()); + } + + // --- convert to objects if headings or column mapping present ------ + const useColumnMapping = isWithColumnMapping(importable); + const useObjects = headings !== null || useColumnMapping; + + let processedRows: any[]; + + if (useObjects) { + let columnMap: Record | null = null; + + if (useColumnMapping) { + const raw = (importable as WithColumnMapping).columnMapping(); + columnMap = {}; + for (const [fieldName, colRef] of Object.entries(raw)) { + if (typeof colRef === "number") { + columnMap[fieldName] = colRef - 1; // convert 1-based to 0-based + } else { + columnMap[fieldName] = columnLetterToNumber(colRef) - 1; + } + } + } + + processedRows = dataRows.map((r) => { + const obj: Record = {}; + if (columnMap) { + for (const [field, idx] of Object.entries(columnMap)) { + obj[field] = idx < r.values.length ? r.values[idx] : null; + } + } else if (headings) { + for (let i = 0; i < headings.length; i++) { + const key = headings[i] || `__col${i}`; + obj[key] = i < r.values.length ? r.values[i] : null; + } + } + return { rowNumber: r.rowNumber, data: obj }; + }); + } else { + processedRows = dataRows.map((r) => ({ + rowNumber: r.rowNumber, + data: r.values, + })); + } + + // --- apply import mapping ----------------------------------------- + if (isWithImportMapping(importable)) { + processedRows = processedRows.map((r) => ({ + rowNumber: r.rowNumber, + data: (importable as WithImportMapping).mapRow(r.data), + })); + } + + // --- validation --------------------------------------------------- + const validationErrors: ImportValidationError[] = []; + let skipped = 0; + const validRows: any[] = []; + + if (isWithValidation(importable)) { + const rules = importable.rules(); + const skipOnError = isSkipsOnError(importable); + + for (const row of processedRows) { + const error = await validateRow(row.data, rules, row.rowNumber); + if (error) { + validationErrors.push(error); + if (skipOnError) { + skipped++; + continue; + } + } + validRows.push(row.data); + } + + if (!skipOnError && validationErrors.length > 0) { + const err = new Error( + `Import validation failed with ${validationErrors.length} error(s).`, + ); + (err as any).validationErrors = validationErrors; + throw err; + } + } else { + for (const row of processedRows) { + validRows.push(row.data); + } + } + + // --- deliver to importable ---------------------------------------- + if (isWithBatchInserts(importable)) { + const size = importable.batchSize(); + for (let i = 0; i < validRows.length; i += size) { + const batch = validRows.slice(i, i + size); + await importable.handleBatch(batch); + } + } + + if (isToCollection(importable)) { + await importable.handleCollection(validRows); + } + + if (isToArray(importable)) { + // If rows are objects, convert back to arrays for ToArray + const arrayRows = useObjects + ? validRows.map((obj) => Object.values(obj)) + : validRows; + await importable.handleArray(arrayRows); + } + + return { + rows: validRows, + errors: validationErrors, + skipped, + }; +} diff --git a/src/excel.writer.ts b/src/excel.writer.ts index b1813a4..545238b 100644 --- a/src/excel.writer.ts +++ b/src/excel.writer.ts @@ -6,14 +6,13 @@ import type { WithMultipleSheets, WithProperties, WithEvents, - WithCsvSettings, - CsvSettings, FromTemplate, WithTemplateData, } from "./concerns"; import { ExcelExportEvent } from "./concerns"; import { populateSheet } from "./excel.sheet"; import { parseCellRef } from "./helpers"; +import { resolveCsvSettings } from "./helpers/csv-settings"; /* ------------------------------------------------------------------ */ /* Type guards */ @@ -31,10 +30,6 @@ function isWithEvents(obj: any): obj is WithEvents { return typeof obj.registerEvents === "function"; } -function isWithCsvSettings(obj: any): obj is WithCsvSettings { - return typeof obj.csvSettings === "function"; -} - function isFromTemplate(obj: any): obj is FromTemplate { return ( typeof obj.templatePath === "function" && @@ -160,30 +155,6 @@ export async function writeExport( return Buffer.from(arrayBuffer); } -/* ------------------------------------------------------------------ */ -/* CSV settings resolution */ -/* ------------------------------------------------------------------ */ - -function resolveCsvSettings( - exportable: object, - options: ExcelModuleOptions, -): Required { - const defaults: Required = { - delimiter: ",", - quoteChar: '"', - lineEnding: "\n", - useBom: false, - encoding: "utf-8", - }; - - const global = options.csv ?? {}; - const perExport = isWithCsvSettings(exportable) - ? exportable.csvSettings() - : {}; - - return { ...defaults, ...global, ...perExport }; -} - /* ------------------------------------------------------------------ */ /* Template export */ /* ------------------------------------------------------------------ */ diff --git a/src/helpers/csv-settings.ts b/src/helpers/csv-settings.ts new file mode 100644 index 0000000..ed498d5 --- /dev/null +++ b/src/helpers/csv-settings.ts @@ -0,0 +1,26 @@ +import type { ExcelModuleOptions } from "../interfaces"; +import type { CsvSettings, WithCsvSettings } from "../concerns"; + +function isWithCsvSettings(obj: any): obj is WithCsvSettings { + return typeof obj.csvSettings === "function"; +} + +export function resolveCsvSettings( + exportable: object, + options: ExcelModuleOptions, +): Required { + const defaults: Required = { + delimiter: ",", + quoteChar: '"', + lineEnding: "\n", + useBom: false, + encoding: "utf-8", + }; + + const global = options.csv ?? {}; + const perExport = isWithCsvSettings(exportable) + ? exportable.csvSettings() + : {}; + + return { ...defaults, ...global, ...perExport }; +} diff --git a/src/helpers/index.ts b/src/helpers/index.ts index a6393b3..37d61d3 100644 --- a/src/helpers/index.ts +++ b/src/helpers/index.ts @@ -1 +1,3 @@ export { detectType, parseCellRef, columnLetterToNumber, numberToColumnLetter } from "./file-type-detector"; +export { resolveCsvSettings } from "./csv-settings"; +export { validateRow } from "./validate-row"; diff --git a/src/helpers/validate-row.ts b/src/helpers/validate-row.ts new file mode 100644 index 0000000..3faa911 --- /dev/null +++ b/src/helpers/validate-row.ts @@ -0,0 +1,82 @@ +import type { ValidationRules } from "../concerns/with-validation.interface"; +import type { + FieldError, + ImportValidationError, +} from "../interfaces/import-result.interface"; + +export async function validateRow( + row: Record, + rulesOrDto: ValidationRules | { dto: new (...args: any[]) => any }, + rowNumber: number, +): Promise { + if ("dto" in rulesOrDto && typeof rulesOrDto.dto === "function") { + return validateWithDto(row, rulesOrDto.dto, rowNumber); + } + return validateWithRules(row, rulesOrDto as ValidationRules, rowNumber); +} + +function validateWithRules( + row: Record, + rules: ValidationRules, + rowNumber: number, +): ImportValidationError | null { + const fieldErrors: FieldError[] = []; + + for (const [field, fieldRules] of Object.entries(rules)) { + if (!fieldRules) continue; + const messages: string[] = []; + for (const rule of fieldRules) { + if (!rule.validate(row[field], row)) { + messages.push(rule.message); + } + } + if (messages.length > 0) { + fieldErrors.push({ field, messages }); + } + } + + return fieldErrors.length > 0 + ? { row: rowNumber, errors: fieldErrors } + : null; +} + +async function validateWithDto( + row: Record, + dto: new (...args: any[]) => any, + rowNumber: number, +): Promise { + let classValidator: any; + let classTransformer: any; + + try { + classValidator = await import("class-validator"); + classTransformer = await import("class-transformer"); + } catch { + throw new Error( + "WithValidation with DTO requires class-validator and class-transformer. " + + "Install them: pnpm add class-validator class-transformer", + ); + } + + const instance = classTransformer.plainToInstance(dto, row); + const errors: any[] = classValidator.validateSync(instance); + + return mapDtoErrors(errors, rowNumber); +} + +/** @internal Exported for testing only. */ +export function mapDtoErrors( + errors: any[], + rowNumber: number, +): ImportValidationError | null { + if (errors.length === 0) return null; + + const fieldErrors: FieldError[] = errors.map((err: any) => ({ + field: err.property, + messages: err.constraints + ? (Object.values(err.constraints) as string[]) + : [], + })); + + return { row: rowNumber, errors: fieldErrors }; +} diff --git a/src/index.ts b/src/index.ts index 222d07b..afaeafb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -60,3 +60,33 @@ export type { AfterSheetEventPayload, } from "./concerns"; export { ExcelExportEvent } from "./concerns"; + +// Concerns — import data receivers +export type { ToArray } from "./concerns"; +export type { ToCollection } from "./concerns"; + +// Concerns — import row processing +export type { WithHeadingRow } from "./concerns"; +export type { WithImportMapping } from "./concerns"; +export type { WithColumnMapping } from "./concerns"; + +// Concerns — import validation +export type { + WithValidation, + ValidationRules, + ValidationRule, +} from "./concerns"; +export type { SkipsOnError } from "./concerns"; +export type { SkipsEmptyRows } from "./concerns"; + +// Concerns — import limits & batching +export type { WithLimit } from "./concerns"; +export type { WithStartRow } from "./concerns"; +export type { WithBatchInserts } from "./concerns"; + +// Import result types +export type { + ImportResult, + ImportValidationError, + FieldError, +} from "./interfaces"; diff --git a/src/interfaces/import-result.interface.ts b/src/interfaces/import-result.interface.ts new file mode 100644 index 0000000..1d339ca --- /dev/null +++ b/src/interfaces/import-result.interface.ts @@ -0,0 +1,15 @@ +export interface ImportResult { + rows: T[]; + errors: ImportValidationError[]; + skipped: number; +} + +export interface ImportValidationError { + row: number; + errors: FieldError[]; +} + +export interface FieldError { + field: string; + messages: string[]; +} diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts index c245fb1..64da50a 100644 --- a/src/interfaces/index.ts +++ b/src/interfaces/index.ts @@ -3,3 +3,8 @@ export { ExcelAsyncOptions, } from "./excel-options.interface"; export { ExcelDownloadResult } from "./export-result.interface"; +export { + ImportResult, + ImportValidationError, + FieldError, +} from "./import-result.interface"; diff --git a/test/excel.import.spec.ts b/test/excel.import.spec.ts new file mode 100644 index 0000000..fc1e90f --- /dev/null +++ b/test/excel.import.spec.ts @@ -0,0 +1,1045 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { Test, TestingModule } from "@nestjs/testing"; +import { Workbook } from "exceljs"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { IsEmail, IsNotEmpty, IsString } from "class-validator"; +import { ExcelService } from "../src/excel.service"; +import { EXCEL_OPTIONS, ExcelType } from "../src/excel.constants"; +import type { + ToArray, + ToCollection, + WithHeadingRow, + WithImportMapping, + WithColumnMapping, + WithValidation, + WithBatchInserts, + WithLimit, + WithStartRow, + SkipsOnError, + SkipsEmptyRows, + WithCsvSettings, +} from "../src/concerns"; + +/* ------------------------------------------------------------------ */ +/* Helpers */ +/* ------------------------------------------------------------------ */ + +async function createService(options = {}): Promise { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + { provide: EXCEL_OPTIONS, useValue: options }, + ExcelService, + ], + }).compile(); + return module.get(ExcelService); +} + +async function createXlsxFile( + filePath: string, + data: any[][], + headings?: string[], +): Promise { + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + if (headings) { + ws.addRow(headings); + } + for (const row of data) { + ws.addRow(row); + } + await wb.xlsx.writeFile(filePath); +} + +async function createXlsxBuffer( + data: any[][], + headings?: string[], +): Promise { + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + if (headings) { + ws.addRow(headings); + } + for (const row of data) { + ws.addRow(row); + } + const arrayBuffer = await wb.xlsx.writeBuffer(); + return Buffer.from(arrayBuffer); +} + +function createCsvFile(filePath: string, text: string): void { + fs.writeFileSync(filePath, text, "utf-8"); +} + +/* ------------------------------------------------------------------ */ +/* Test suite */ +/* ------------------------------------------------------------------ */ + +describe("ExcelService — Import", () => { + let service: ExcelService; + let tmpDir: string; + + beforeEach(async () => { + service = await createService(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "excel-import-")); + }); + + afterEach(() => { + if (fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + /* ---------------------------------------------------------------- */ + /* Basic XLSX import */ + /* ---------------------------------------------------------------- */ + + describe("basic XLSX import", () => { + it("should import a simple XLSX file via toArray()", async () => { + const filePath = path.join(tmpDir, "simple.xlsx"); + await createXlsxFile(filePath, [ + [1, "Alice", "alice@test.com"], + [2, "Bob", "bob@test.com"], + ]); + + const rows = await service.toArray(filePath); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual([1, "Alice", "alice@test.com"]); + expect(rows[1]).toEqual([2, "Bob", "bob@test.com"]); + }); + + it("should import XLSX via toCollection() using row 1 as headings", async () => { + const filePath = path.join(tmpDir, "headings.xlsx"); + await createXlsxFile( + filePath, + [ + [1, "Alice", "alice@test.com"], + [2, "Bob", "bob@test.com"], + ], + ["ID", "Name", "Email"], + ); + + const rows = await service.toCollection(filePath); + expect(rows).toHaveLength(2); + expect(rows[0]).toEqual({ ID: 1, Name: "Alice", Email: "alice@test.com" }); + expect(rows[1]).toEqual({ ID: 2, Name: "Bob", Email: "bob@test.com" }); + }); + + it("should import from buffer", async () => { + const buffer = await createXlsxBuffer([ + [10, "Charlie"], + [20, "Diana"], + ]); + + const result = await service.importFromBuffer({}, buffer); + expect(result.rows).toHaveLength(2); + expect(result.rows[0]).toEqual([10, "Charlie"]); + }); + + it("should return ImportResult with errors and skipped", async () => { + const filePath = path.join(tmpDir, "result.xlsx"); + await createXlsxFile(filePath, [[1, "Test"]]); + + const result = await service.import({}, filePath); + expect(result.rows).toHaveLength(1); + expect(result.errors).toEqual([]); + expect(result.skipped).toBe(0); + }); + }); + + /* ---------------------------------------------------------------- */ + /* Basic CSV import */ + /* ---------------------------------------------------------------- */ + + describe("basic CSV import", () => { + it("should import a CSV file", async () => { + const filePath = path.join(tmpDir, "data.csv"); + createCsvFile(filePath, "1,Alice,alice@test.com\n2,Bob,bob@test.com\n"); + + const rows = await service.toArray(filePath); + expect(rows).toHaveLength(2); + expect(rows[0][1]).toBe("Alice"); + }); + + it("should import CSV from buffer", async () => { + const buffer = Buffer.from("a,b\n1,2\n3,4\n", "utf-8"); + const result = await service.importFromBuffer({}, buffer, ExcelType.CSV); + expect(result.rows).toHaveLength(3); // includes heading row as data + }); + + it("should respect WithCsvSettings on import", async () => { + const filePath = path.join(tmpDir, "semicolons.csv"); + createCsvFile(filePath, "1;Alice\n2;Bob\n"); + + class SemicolonImport implements WithCsvSettings { + csvSettings() { + return { delimiter: ";" }; + } + } + + const result = await service.import( + new SemicolonImport(), + filePath, + ExcelType.CSV, + ); + expect(result.rows[0][0]).toBe(1); + expect(result.rows[0][1]).toBe("Alice"); + }); + }); + + /* ---------------------------------------------------------------- */ + /* ToArray concern */ + /* ---------------------------------------------------------------- */ + + describe("ToArray", () => { + it("should call handleArray with data", async () => { + let captured: any[][] = []; + + class ArrayImport implements ToArray { + handleArray(rows: any[][]) { + captured = rows; + } + } + + const filePath = path.join(tmpDir, "arr.xlsx"); + await createXlsxFile(filePath, [ + [1, "A"], + [2, "B"], + ]); + + await service.import(new ArrayImport(), filePath); + expect(captured).toHaveLength(2); + expect(captured[0]).toEqual([1, "A"]); + }); + }); + + /* ---------------------------------------------------------------- */ + /* ToCollection concern */ + /* ---------------------------------------------------------------- */ + + describe("ToCollection", () => { + it("should call handleCollection with objects", async () => { + let captured: any[] = []; + + class CollectionImport implements ToCollection, WithHeadingRow { + readonly hasHeadingRow = true as const; + handleCollection(rows: Record[]) { + captured = rows; + } + } + + const filePath = path.join(tmpDir, "coll.xlsx"); + await createXlsxFile(filePath, [[1, "Alice"]], ["ID", "Name"]); + + await service.import(new CollectionImport(), filePath); + expect(captured).toHaveLength(1); + expect(captured[0]).toEqual({ ID: 1, Name: "Alice" }); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithHeadingRow */ + /* ---------------------------------------------------------------- */ + + describe("WithHeadingRow", () => { + it("should use row 1 as default heading row", async () => { + class DefaultHeading implements WithHeadingRow { + readonly hasHeadingRow = true as const; + } + + const filePath = path.join(tmpDir, "heading.xlsx"); + await createXlsxFile(filePath, [[10, "Val"]], ["Col1", "Col2"]); + + const result = await service.import(new DefaultHeading(), filePath); + expect(result.rows[0]).toEqual({ Col1: 10, Col2: "Val" }); + }); + + it("should support custom heading row number", async () => { + class CustomHeading implements WithHeadingRow { + readonly hasHeadingRow = true as const; + headingRow() { + return 2; + } + } + + // Row 1: title, Row 2: headings, Row 3+: data + const filePath = path.join(tmpDir, "custom-heading.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["Report Title", ""]); + ws.addRow(["Name", "Score"]); + ws.addRow(["Alice", 95]); + ws.addRow(["Bob", 88]); + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new CustomHeading(), filePath); + expect(result.rows).toHaveLength(2); + expect(result.rows[0]).toEqual({ Name: "Alice", Score: 95 }); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithImportMapping */ + /* ---------------------------------------------------------------- */ + + describe("WithImportMapping", () => { + it("should transform each row", async () => { + class MappedImport implements WithHeadingRow, WithImportMapping { + readonly hasHeadingRow = true as const; + mapRow(row: Record) { + return { + fullName: String(row.Name).toUpperCase(), + score: Number(row.Score) * 2, + }; + } + } + + const filePath = path.join(tmpDir, "mapped.xlsx"); + await createXlsxFile(filePath, [["Alice", 50]], ["Name", "Score"]); + + const result = await service.import(new MappedImport(), filePath); + expect(result.rows[0]).toEqual({ fullName: "ALICE", score: 100 }); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithStartRow */ + /* ---------------------------------------------------------------- */ + + describe("WithStartRow", () => { + it("should skip rows before startRow", async () => { + class SkipFirst implements WithStartRow { + startRow() { + return 3; + } + } + + const filePath = path.join(tmpDir, "startrow.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["Title"]); + ws.addRow(["Subtitle"]); + ws.addRow([1, "Data1"]); + ws.addRow([2, "Data2"]); + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new SkipFirst(), filePath); + expect(result.rows).toHaveLength(2); + expect(result.rows[0]).toEqual([1, "Data1"]); + }); + + it("should combine with WithHeadingRow", async () => { + class HeadingAtRow2 implements WithHeadingRow, WithStartRow { + readonly hasHeadingRow = true as const; + headingRow() { + return 2; + } + startRow() { + return 3; + } + } + + const filePath = path.join(tmpDir, "headstart.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["Junk"]); + ws.addRow(["Name", "Age"]); + ws.addRow(["Alice", 30]); + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new HeadingAtRow2(), filePath); + expect(result.rows[0]).toEqual({ Name: "Alice", Age: 30 }); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithLimit */ + /* ---------------------------------------------------------------- */ + + describe("WithLimit", () => { + it("should limit number of rows read", async () => { + class LimitedImport implements WithLimit { + limit() { + return 2; + } + } + + const filePath = path.join(tmpDir, "limit.xlsx"); + await createXlsxFile(filePath, [ + [1, "A"], + [2, "B"], + [3, "C"], + [4, "D"], + ]); + + const result = await service.import(new LimitedImport(), filePath); + expect(result.rows).toHaveLength(2); + }); + }); + + /* ---------------------------------------------------------------- */ + /* SkipsEmptyRows */ + /* ---------------------------------------------------------------- */ + + describe("SkipsEmptyRows", () => { + it("should filter out blank rows", async () => { + class SkipEmpty implements SkipsEmptyRows { + readonly skipsEmptyRows = true as const; + } + + const filePath = path.join(tmpDir, "empty.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow([1, "Alice"]); + ws.addRow([null, null]); + ws.addRow([2, "Bob"]); + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new SkipEmpty(), filePath); + expect(result.rows).toHaveLength(2); + expect(result.rows[0]).toEqual([1, "Alice"]); + expect(result.rows[1]).toEqual([2, "Bob"]); + }); + + it("should keep rows with at least one non-empty cell", async () => { + class SkipEmpty implements SkipsEmptyRows { + readonly skipsEmptyRows = true as const; + } + + const filePath = path.join(tmpDir, "partial.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow([null, "Partial"]); + ws.addRow([null, null]); + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new SkipEmpty(), filePath); + expect(result.rows).toHaveLength(1); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithColumnMapping */ + /* ---------------------------------------------------------------- */ + + describe("WithColumnMapping", () => { + it("should map column letters to field names", async () => { + class ColMapped implements WithColumnMapping { + columnMapping() { + return { name: "A", email: "C" }; + } + } + + const filePath = path.join(tmpDir, "colmap.xlsx"); + await createXlsxFile(filePath, [ + ["Alice", 25, "alice@test.com"], + ]); + + const result = await service.import(new ColMapped(), filePath); + expect(result.rows[0]).toEqual({ + name: "Alice", + email: "alice@test.com", + }); + }); + + it("should map 1-based column indices to field names", async () => { + class IdxMapped implements WithColumnMapping { + columnMapping() { + return { id: 1, score: 3 }; + } + } + + const filePath = path.join(tmpDir, "idxmap.xlsx"); + await createXlsxFile(filePath, [[100, "ignored", 95]]); + + const result = await service.import(new IdxMapped(), filePath); + expect(result.rows[0]).toEqual({ id: 100, score: 95 }); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithValidation — custom rules */ + /* ---------------------------------------------------------------- */ + + describe("WithValidation — custom rules", () => { + it("should pass valid rows through", async () => { + class ValidImport implements WithHeadingRow, WithValidation { + readonly hasHeadingRow = true as const; + rules() { + return { + name: [ + { + validate: (v: any) => typeof v === "string" && v.length > 0, + message: "Name is required", + }, + ], + }; + } + } + + const filePath = path.join(tmpDir, "valid.xlsx"); + await createXlsxFile(filePath, [["Alice"]], ["name"]); + + const result = await service.import(new ValidImport(), filePath); + expect(result.rows).toHaveLength(1); + expect(result.errors).toHaveLength(0); + }); + + it("should throw for invalid rows without SkipsOnError", async () => { + class StrictImport implements WithHeadingRow, WithValidation { + readonly hasHeadingRow = true as const; + rules() { + return { + email: [ + { + validate: (v: any) => /^.+@.+\..+$/.test(String(v)), + message: "Invalid email", + }, + ], + }; + } + } + + const filePath = path.join(tmpDir, "invalid.xlsx"); + await createXlsxFile(filePath, [["not-an-email"]], ["email"]); + + await expect( + service.import(new StrictImport(), filePath), + ).rejects.toThrow("Import validation failed"); + }); + + it("should collect multiple field errors per row", async () => { + class MultiRule + implements WithHeadingRow, WithValidation, SkipsOnError + { + readonly hasHeadingRow = true as const; + readonly skipsOnError = true as const; + rules() { + return { + name: [ + { + validate: (v: any) => typeof v === "string" && v.length > 0, + message: "Name required", + }, + ], + age: [ + { + validate: (v: any) => typeof v === "number" && v > 0, + message: "Age must be positive", + }, + ], + }; + } + } + + const filePath = path.join(tmpDir, "multi-err.xlsx"); + await createXlsxFile(filePath, [[null, -5]], ["name", "age"]); + + const result = await service.import(new MultiRule(), filePath); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].errors).toHaveLength(2); + expect(result.skipped).toBe(1); + expect(result.rows).toHaveLength(0); + }); + + it("should include row number in validation errors", async () => { + class RowNumCheck + implements WithHeadingRow, WithValidation, SkipsOnError + { + readonly hasHeadingRow = true as const; + readonly skipsOnError = true as const; + rules() { + return { + val: [ + { + validate: (v: any) => v !== null && v !== "", + message: "Required", + }, + ], + }; + } + } + + const filePath = path.join(tmpDir, "rownum.xlsx"); + await createXlsxFile( + filePath, + [ + ["good"], + [""], + ["also good"], + ], + ["val"], + ); + + const result = await service.import(new RowNumCheck(), filePath); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].row).toBe(3); // heading=row1, data starts row2, bad row is row3 + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithValidation — DTO mode */ + /* ---------------------------------------------------------------- */ + + describe("WithValidation — DTO mode", () => { + class UserDto { + @IsString() + @IsNotEmpty() + name!: string; + + @IsEmail() + email!: string; + } + + it("should validate valid DTO rows", async () => { + class DtoImport implements WithHeadingRow, WithValidation { + readonly hasHeadingRow = true as const; + rules() { + return { dto: UserDto }; + } + } + + const filePath = path.join(tmpDir, "dto-valid.xlsx"); + await createXlsxFile( + filePath, + [["Alice", "alice@example.com"]], + ["name", "email"], + ); + + const result = await service.import(new DtoImport(), filePath); + expect(result.rows).toHaveLength(1); + expect(result.errors).toHaveLength(0); + }); + + it("should collect errors for invalid DTO rows", async () => { + class DtoSkipImport + implements WithHeadingRow, WithValidation, SkipsOnError + { + readonly hasHeadingRow = true as const; + readonly skipsOnError = true as const; + rules() { + return { dto: UserDto }; + } + } + + const filePath = path.join(tmpDir, "dto-invalid.xlsx"); + await createXlsxFile( + filePath, + [["", "not-email"]], + ["name", "email"], + ); + + const result = await service.import(new DtoSkipImport(), filePath); + expect(result.rows).toHaveLength(0); + expect(result.errors).toHaveLength(1); + expect(result.skipped).toBe(1); + + const fieldNames = result.errors[0].errors.map((e) => e.field); + expect(fieldNames).toContain("name"); + expect(fieldNames).toContain("email"); + }); + }); + + /* ---------------------------------------------------------------- */ + /* SkipsOnError */ + /* ---------------------------------------------------------------- */ + + describe("SkipsOnError", () => { + it("should skip invalid rows and continue", async () => { + class SkippingImport + implements WithHeadingRow, WithValidation, SkipsOnError + { + readonly hasHeadingRow = true as const; + readonly skipsOnError = true as const; + rules() { + return { + score: [ + { + validate: (v: any) => typeof v === "number" && v >= 0, + message: "Score must be non-negative", + }, + ], + }; + } + } + + const filePath = path.join(tmpDir, "skip.xlsx"); + await createXlsxFile( + filePath, + [ + ["Alice", 100], + ["Bob", -5], + ["Charlie", 85], + ], + ["name", "score"], + ); + + const result = await service.import(new SkippingImport(), filePath); + expect(result.rows).toHaveLength(2); + expect(result.skipped).toBe(1); + expect(result.errors).toHaveLength(1); + }); + }); + + /* ---------------------------------------------------------------- */ + /* WithBatchInserts */ + /* ---------------------------------------------------------------- */ + + describe("WithBatchInserts", () => { + it("should deliver rows in batches", async () => { + const batches: any[][] = []; + + class BatchImport implements WithBatchInserts { + batchSize() { + return 2; + } + handleBatch(batch: any[]) { + batches.push([...batch]); + } + } + + const filePath = path.join(tmpDir, "batch.xlsx"); + await createXlsxFile(filePath, [ + [1, "A"], + [2, "B"], + [3, "C"], + [4, "D"], + [5, "E"], + ]); + + await service.import(new BatchImport(), filePath); + expect(batches).toHaveLength(3); // 2, 2, 1 + expect(batches[0]).toHaveLength(2); + expect(batches[1]).toHaveLength(2); + expect(batches[2]).toHaveLength(1); + }); + + it("should support async handleBatch", async () => { + const results: number[] = []; + + class AsyncBatch implements WithBatchInserts { + batchSize() { + return 3; + } + async handleBatch(batch: any[]) { + results.push(batch.length); + } + } + + const filePath = path.join(tmpDir, "async-batch.xlsx"); + await createXlsxFile(filePath, [ + [1], + [2], + [3], + [4], + ]); + + await service.import(new AsyncBatch(), filePath); + expect(results).toEqual([3, 1]); + }); + }); + + /* ---------------------------------------------------------------- */ + /* Combined concerns */ + /* ---------------------------------------------------------------- */ + + describe("combined concerns", () => { + it("should run full pipeline: heading + mapping + validation + skip", async () => { + class FullPipeline + implements + WithHeadingRow, + WithImportMapping, + WithValidation, + SkipsOnError + { + readonly hasHeadingRow = true as const; + readonly skipsOnError = true as const; + + mapRow(row: Record) { + return { + name: String(row.name).trim(), + score: Number(row.score), + }; + } + + rules() { + return { + name: [ + { + validate: (v: any) => v.length > 0, + message: "Name required", + }, + ], + score: [ + { + validate: (v: any) => !isNaN(v) && v >= 0, + message: "Score must be non-negative number", + }, + ], + }; + } + } + + const filePath = path.join(tmpDir, "full.xlsx"); + await createXlsxFile( + filePath, + [ + [" Alice ", 95], + ["", 50], + ["Charlie", -10], + ["Diana", 88], + ], + ["name", "score"], + ); + + const result = await service.import(new FullPipeline(), filePath); + expect(result.rows).toHaveLength(2); + expect(result.rows[0]).toEqual({ name: "Alice", score: 95 }); + expect(result.rows[1]).toEqual({ name: "Diana", score: 88 }); + expect(result.skipped).toBe(2); + expect(result.errors).toHaveLength(2); + }); + + it("should combine StartRow + Limit + SkipsEmptyRows", async () => { + class CombinedLimits implements WithStartRow, WithLimit, SkipsEmptyRows { + readonly skipsEmptyRows = true as const; + startRow() { + return 2; + } + limit() { + return 3; + } + } + + const filePath = path.join(tmpDir, "combined-limits.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["Header"]); // row 1, skipped by startRow + ws.addRow([1, "A"]); // row 2 + ws.addRow([null, null]); // row 3, empty → skipped + ws.addRow([2, "B"]); // row 4 + ws.addRow([3, "C"]); // row 5 + ws.addRow([4, "D"]); // row 6 + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new CombinedLimits(), filePath); + // After skipping row 1 (startRow=2), filtering empties, then limit 3 + expect(result.rows).toHaveLength(3); + expect(result.rows[0]).toEqual([1, "A"]); + }); + + it("should import and re-export round-trip", async () => { + // Export + const exportFilePath = path.join(tmpDir, "roundtrip.xlsx"); + await createXlsxFile( + exportFilePath, + [ + [1, "Alice"], + [2, "Bob"], + ], + ["ID", "Name"], + ); + + // Import + const data = await service.toCollection(exportFilePath); + expect(data).toHaveLength(2); + expect(data[0]).toEqual({ ID: 1, Name: "Alice" }); + + // Re-export + class ReExport { + collection() { + return data.map((r) => [r.ID, r.Name]); + } + } + + const buffer = await service.raw(new ReExport(), ExcelType.XLSX); + expect(buffer.length).toBeGreaterThan(0); + }); + }); + + /* ---------------------------------------------------------------- */ + /* Edge cases */ + /* ---------------------------------------------------------------- */ + + describe("edge cases", () => { + it("should return empty result for empty worksheet", async () => { + const filePath = path.join(tmpDir, "empty-ws.xlsx"); + const wb = new Workbook(); + wb.addWorksheet("Empty"); + await wb.xlsx.writeFile(filePath); + + const result = await service.import({}, filePath); + expect(result.rows).toHaveLength(0); + expect(result.errors).toEqual([]); + expect(result.skipped).toBe(0); + }); + + it("should handle heading row with no data rows", async () => { + class EmptyData implements WithHeadingRow { + readonly hasHeadingRow = true as const; + } + + const filePath = path.join(tmpDir, "heading-only.xlsx"); + await createXlsxFile(filePath, [], ["Col1", "Col2"]); + + const result = await service.import(new EmptyData(), filePath); + expect(result.rows).toHaveLength(0); + }); + + it("should handle rich text cells", async () => { + const filePath = path.join(tmpDir, "richtext.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.getCell("A1").value = { + richText: [ + { text: "Hello " }, + { font: { bold: true }, text: "World" }, + ], + } as any; + await wb.xlsx.writeFile(filePath); + + const result = await service.import({}, filePath); + expect(result.rows[0][0]).toBe("Hello World"); + }); + + it("should convert objects to arrays for ToArray with headings", async () => { + let captured: any[][] = []; + + class ArrayWithHeadings implements ToArray, WithHeadingRow { + readonly hasHeadingRow = true as const; + handleArray(rows: any[][]) { + captured = rows; + } + } + + const filePath = path.join(tmpDir, "arr-head.xlsx"); + await createXlsxFile(filePath, [[1, "Alice"]], ["ID", "Name"]); + + await service.import(new ArrayWithHeadings(), filePath); + expect(captured).toHaveLength(1); + expect(captured[0]).toEqual([1, "Alice"]); + }); + + it("should handle formula cells by extracting result", async () => { + const filePath = path.join(tmpDir, "formula.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.getCell("A1").value = 10; + ws.getCell("B1").value = { formula: "A1*2", result: 20 } as any; + await wb.xlsx.writeFile(filePath); + + const result = await service.import({}, filePath); + expect(result.rows[0][0]).toBe(10); + expect(result.rows[0][1]).toBe(20); + }); + + it("should throw when no worksheet exists in buffer", async () => { + // Create a workbook with no worksheets — ExcelJS always adds one on + // xlsx.writeBuffer, so we manipulate the workbook after creation. + const wb = new Workbook(); + wb.addWorksheet("TempSheet"); + const buf = await wb.xlsx.writeBuffer(); + const wb2 = new Workbook(); + await wb2.xlsx.load(Buffer.from(buf)); + // Remove all worksheets + while (wb2.worksheets.length > 0) { + wb2.removeWorksheet(wb2.worksheets[0].id); + } + const emptyBuf = Buffer.from(await wb2.xlsx.writeBuffer()); + + await expect( + service.importFromBuffer({}, emptyBuf), + ).rejects.toThrow("No worksheet found"); + }); + + it("should use __col fallback for empty heading names", async () => { + class EmptyHeading implements WithHeadingRow { + readonly hasHeadingRow = true as const; + } + + const filePath = path.join(tmpDir, "empty-heading.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["Name", null, "Email"]); // middle heading is null → empty + ws.addRow(["Alice", 25, "alice@test.com"]); + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new EmptyHeading(), filePath); + expect(result.rows[0]).toHaveProperty("Name", "Alice"); + expect(result.rows[0]).toHaveProperty("__col1"); // fallback key + expect(result.rows[0]).toHaveProperty("Email", "alice@test.com"); + }); + + it("should return null for column mapping index beyond row length", async () => { + class WideMapping implements WithColumnMapping { + columnMapping() { + return { name: "A", missing: "Z" }; // column Z won't exist in data + } + } + + const filePath = path.join(tmpDir, "wide-map.xlsx"); + await createXlsxFile(filePath, [["Alice"]]); + + const result = await service.import(new WideMapping(), filePath); + expect(result.rows[0].name).toBe("Alice"); + expect(result.rows[0].missing).toBeNull(); + }); + + it("should return null for heading beyond row value length", async () => { + class WideHeading implements WithHeadingRow { + readonly hasHeadingRow = true as const; + } + + const filePath = path.join(tmpDir, "wide-heading.xlsx"); + const wb = new Workbook(); + const ws = wb.addWorksheet("Sheet1"); + ws.addRow(["Col1", "Col2", "Col3"]); // 3 headings + ws.addRow(["A"]); // only 1 value — Col2 and Col3 should be null + await wb.xlsx.writeFile(filePath); + + const result = await service.import(new WideHeading(), filePath); + expect(result.rows[0]).toEqual({ Col1: "A", Col2: null, Col3: null }); + }); + + it("should handle heading row beyond worksheet range", async () => { + class FarHeading implements WithHeadingRow { + readonly hasHeadingRow = true as const; + headingRow() { + return 999; // way beyond actual data + } + } + + const filePath = path.join(tmpDir, "far-heading.xlsx"); + await createXlsxFile(filePath, [["data"]]); + + // Should still work — no headings detected, data returned as arrays + const result = await service.import(new FarHeading(), filePath); + // Row 1 is before startRow (999+1=1000), so no data + expect(result.rows).toHaveLength(0); + }); + + it("should attach validationErrors to thrown error", async () => { + class StrictValidation implements WithHeadingRow, WithValidation { + readonly hasHeadingRow = true as const; + rules() { + return { + val: [{ validate: () => false, message: "Always fails" }], + }; + } + } + + const filePath = path.join(tmpDir, "throw-err.xlsx"); + await createXlsxFile(filePath, [["test"]], ["val"]); + + try { + await service.import(new StrictValidation(), filePath); + expect.fail("Should have thrown"); + } catch (err: any) { + expect(err.validationErrors).toBeDefined(); + expect(err.validationErrors).toHaveLength(1); + expect(err.validationErrors[0].row).toBe(2); + } + }); + }); +}); diff --git a/test/excel.service.spec.ts b/test/excel.service.spec.ts index 82989fe..75c6843 100644 --- a/test/excel.service.spec.ts +++ b/test/excel.service.spec.ts @@ -1860,6 +1860,54 @@ describe("ExcelService", () => { ]); }); + it("should handle template with partial properties", async () => { + class PartialPropsTemplate implements FromTemplate, WithProperties { + templatePath() { + return templatePath; + } + bindings() { + return { "{{company}}": "TestCo" }; + } + properties() { + return { creator: "OnlyCreator" }; + } + } + + const buffer = await service.raw( + new PartialPropsTemplate(), + ExcelType.XLSX, + ); + const wb = await readXlsx(buffer); + expect(wb.creator).toBe("OnlyCreator"); + // Other props should remain default/empty + expect(wb.title).toBeFalsy(); + }); + + it("should skip non-string cells in template placeholder replacement", async () => { + const numericPath = path.join(tmpDir, "numeric-tpl.xlsx"); + const nwb = new Workbook(); + const nws = nwb.addWorksheet("Sheet1"); + nws.getCell("A1").value = "{{name}}"; + nws.getCell("B1").value = 12345; // numeric cell — not a string + nws.getCell("C1").value = true; // boolean cell + await nwb.xlsx.writeFile(numericPath); + + class NumericTemplate implements FromTemplate { + templatePath() { + return numericPath; + } + bindings() { + return { "{{name}}": "Replaced" }; + } + } + + const buffer = await service.raw(new NumericTemplate(), ExcelType.XLSX); + const wb = await readXlsx(buffer); + expect(wb.worksheets[0].getCell("A1").value).toBe("Replaced"); + expect(wb.worksheets[0].getCell("B1").value).toBe(12345); + expect(wb.worksheets[0].getCell("C1").value).toBe(true); + }); + // cleanup afterEach(() => { if (fs.existsSync(tmpDir)) { diff --git a/test/validate-row.spec.ts b/test/validate-row.spec.ts new file mode 100644 index 0000000..32e23fc --- /dev/null +++ b/test/validate-row.spec.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from "vitest"; +import { validateRow, mapDtoErrors } from "../src/helpers/validate-row"; +import type { ValidationRules } from "../src/concerns/with-validation.interface"; + +describe("validateRow", () => { + it("should skip undefined rule entries", async () => { + const rules: ValidationRules = { + name: [{ validate: (v) => v === "ok", message: "fail" }], + age: undefined, // should be skipped gracefully + }; + + const result = await validateRow({ name: "ok", age: 10 }, rules, 1); + expect(result).toBeNull(); + }); +}); + +describe("mapDtoErrors", () => { + it("should return null for empty errors array", () => { + expect(mapDtoErrors([], 1)).toBeNull(); + }); + + it("should map errors with constraints", () => { + const errors = [ + { property: "email", constraints: { isEmail: "email must be valid" } }, + ]; + const result = mapDtoErrors(errors, 3); + expect(result).not.toBeNull(); + expect(result!.row).toBe(3); + expect(result!.errors[0].field).toBe("email"); + expect(result!.errors[0].messages).toEqual(["email must be valid"]); + }); + + it("should return empty messages when constraints is undefined", () => { + const errors = [{ property: "name", constraints: undefined }]; + const result = mapDtoErrors(errors, 2); + expect(result).not.toBeNull(); + expect(result!.errors[0].field).toBe("name"); + expect(result!.errors[0].messages).toEqual([]); + }); + + it("should return empty messages when constraints is null", () => { + const errors = [{ property: "age", constraints: null }]; + const result = mapDtoErrors(errors, 5); + expect(result).not.toBeNull(); + expect(result!.errors[0].field).toBe("age"); + expect(result!.errors[0].messages).toEqual([]); + }); +}); From 3ccf2e7cb06fd4eb2a15b2e79ad8dcc25cc1798a Mon Sep 17 00:00:00 2001 From: khatabwedaa Date: Wed, 25 Mar 2026 07:20:08 +0300 Subject: [PATCH 2/2] Ensures deterministic column ordering for object rows Fixes issue where object keys were returned in arbitrary order when converting back to arrays, causing column misalignment with the original spreadsheet structure. Maintains consistent key ordering by: - Sorting column mappings by their index position - Preserving heading order when available - Applying this order when converting objects back to arrays Adds validation for batch size to prevent invalid configurations (zero or negative values). Optimizes DTO validation by caching imported dependencies to avoid repeated dynamic imports. Updates documentation examples to include proper TypeScript definite assignment assertions and missing imports. --- README.md | 6 +++--- src/excel.sheet-reader.ts | 36 +++++++++++++++++++++++++++--------- src/helpers/validate-row.ts | 30 +++++++++++++++++++----------- test/excel.import.spec.ts | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 31d52a9..f87945d 100644 --- a/README.md +++ b/README.md @@ -607,11 +607,11 @@ rules() { **class-validator DTO:** ```typescript -import { IsString, IsEmail } from "class-validator"; +import { IsString, IsEmail, IsNotEmpty } from "class-validator"; class UserDto { - @IsString() @IsNotEmpty() name: string; - @IsEmail() email: string; + @IsString() @IsNotEmpty() name!: string; + @IsEmail() email!: string; } // In your import class diff --git a/src/excel.sheet-reader.ts b/src/excel.sheet-reader.ts index 29f6d1c..f4a6cc4 100644 --- a/src/excel.sheet-reader.ts +++ b/src/excel.sheet-reader.ts @@ -150,6 +150,7 @@ export async function processSheet( const useObjects = headings !== null || useColumnMapping; let processedRows: any[]; + let objectKeyOrder: string[] | null = null; if (useObjects) { let columnMap: Record | null = null; @@ -157,13 +158,23 @@ export async function processSheet( if (useColumnMapping) { const raw = (importable as WithColumnMapping).columnMapping(); columnMap = {}; + const entries: [string, number][] = []; for (const [fieldName, colRef] of Object.entries(raw)) { - if (typeof colRef === "number") { - columnMap[fieldName] = colRef - 1; // convert 1-based to 0-based - } else { - columnMap[fieldName] = columnLetterToNumber(colRef) - 1; - } + const idx = + typeof colRef === "number" + ? colRef - 1 + : columnLetterToNumber(colRef) - 1; + columnMap[fieldName] = idx; + entries.push([fieldName, idx]); } + // Deterministic key order: sorted by column index + objectKeyOrder = entries + .sort((a, b) => a[1] - b[1]) + .map(([k]) => k); + } + + if (!objectKeyOrder && headings) { + objectKeyOrder = headings.map((h, i) => h || `__col${i}`); } processedRows = dataRows.map((r) => { @@ -232,6 +243,11 @@ export async function processSheet( // --- deliver to importable ---------------------------------------- if (isWithBatchInserts(importable)) { const size = importable.batchSize(); + if (!Number.isInteger(size) || size < 1) { + throw new Error( + `WithBatchInserts.batchSize() must return a positive integer, got ${size}.`, + ); + } for (let i = 0; i < validRows.length; i += size) { const batch = validRows.slice(i, i + size); await importable.handleBatch(batch); @@ -243,10 +259,12 @@ export async function processSheet( } if (isToArray(importable)) { - // If rows are objects, convert back to arrays for ToArray - const arrayRows = useObjects - ? validRows.map((obj) => Object.values(obj)) - : validRows; + // If rows are objects, convert back to arrays for ToArray using + // deterministic key order so column ordering matches the spreadsheet. + const arrayRows = + useObjects && objectKeyOrder + ? validRows.map((obj) => objectKeyOrder!.map((k) => obj[k])) + : validRows; await importable.handleArray(arrayRows); } diff --git a/src/helpers/validate-row.ts b/src/helpers/validate-row.ts index 3faa911..bd808f6 100644 --- a/src/helpers/validate-row.ts +++ b/src/helpers/validate-row.ts @@ -40,26 +40,34 @@ function validateWithRules( : null; } -async function validateWithDto( - row: Record, - dto: new (...args: any[]) => any, - rowNumber: number, -): Promise { - let classValidator: any; - let classTransformer: any; +let cachedValidator: any; +let cachedTransformer: any; +async function loadDtoDeps(): Promise<{ validator: any; transformer: any }> { + if (cachedValidator && cachedTransformer) { + return { validator: cachedValidator, transformer: cachedTransformer }; + } try { - classValidator = await import("class-validator"); - classTransformer = await import("class-transformer"); + cachedValidator = await import("class-validator"); + cachedTransformer = await import("class-transformer"); } catch { throw new Error( "WithValidation with DTO requires class-validator and class-transformer. " + "Install them: pnpm add class-validator class-transformer", ); } + return { validator: cachedValidator, transformer: cachedTransformer }; +} + +async function validateWithDto( + row: Record, + dto: new (...args: any[]) => any, + rowNumber: number, +): Promise { + const { validator, transformer } = await loadDtoDeps(); - const instance = classTransformer.plainToInstance(dto, row); - const errors: any[] = classValidator.validateSync(instance); + const instance = transformer.plainToInstance(dto, row); + const errors: any[] = validator.validateSync(instance); return mapDtoErrors(errors, rowNumber); } diff --git a/test/excel.import.spec.ts b/test/excel.import.spec.ts index fc1e90f..9724c1f 100644 --- a/test/excel.import.spec.ts +++ b/test/excel.import.spec.ts @@ -738,6 +738,38 @@ describe("ExcelService — Import", () => { await service.import(new AsyncBatch(), filePath); expect(results).toEqual([3, 1]); }); + + it("should throw when batchSize is zero", async () => { + class ZeroBatch implements WithBatchInserts { + batchSize() { + return 0; + } + handleBatch() {} + } + + const filePath = path.join(tmpDir, "zero-batch.xlsx"); + await createXlsxFile(filePath, [[1]]); + + await expect( + service.import(new ZeroBatch(), filePath), + ).rejects.toThrow("batchSize() must return a positive integer"); + }); + + it("should throw when batchSize is negative", async () => { + class NegBatch implements WithBatchInserts { + batchSize() { + return -5; + } + handleBatch() {} + } + + const filePath = path.join(tmpDir, "neg-batch.xlsx"); + await createXlsxFile(filePath, [[1]]); + + await expect( + service.import(new NegBatch(), filePath), + ).rejects.toThrow("batchSize() must return a positive integer"); + }); }); /* ---------------------------------------------------------------- */