From 7c07e1a4d55f8130fef2f992c46db6c470c6c13e Mon Sep 17 00:00:00 2001 From: Mehdi Date: Wed, 11 Feb 2026 14:49:21 +0000 Subject: [PATCH 1/4] test: enforce test catalog consistency and discovery --- package.json | 3 +- scripts/validate-test-catalog.js | 224 ++++++++++++++++++ tests/catalog.md | 47 ++-- .../scripts/validate-test-catalog.test.js | 139 +++++++++++ 4 files changed, 392 insertions(+), 21 deletions(-) create mode 100644 scripts/validate-test-catalog.js create mode 100644 tests/unit/scripts/validate-test-catalog.test.js diff --git a/package.json b/package.json index 915847b..bece518 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "predev": "npm run build:ts && node scripts/clean-dev-assets.js", "dev": "node scripts/index.js dev", "clear-assets": "rimraf src/renderer/bundle.js src/renderer/bundle.js.map src/renderer/bundle.js.LICENSE.txt src/renderer/output.css", - "lint": "npm run format:check && cross-env ESLINT_USE_FLAT_CONFIG=false eslint src tests --ext .js,.jsx,.ts,.tsx --cache && npm run lint:md && npm run changelog:validate", + "lint": "npm run format:check && cross-env ESLINT_USE_FLAT_CONFIG=false eslint src tests --ext .js,.jsx,.ts,.tsx --cache && npm run lint:md && npm run test:catalog && npm run changelog:validate", "lint:md": "npm run lint:md:links && npm run lint:md:style", "lint:md:links": "node scripts/lint-markdown-links.js", "lint:md:style": "markdownlint \"**/*.{md,mdx}\" --config .markdownlint.json --ignore node_modules --ignore dist", @@ -28,6 +28,7 @@ "test": "jest --config jest.config.js --passWithNoTests", "test:watch": "jest --watch --config jest.config.js --passWithNoTests", "test:stress": "jest --config jest.config.js --runInBand --testMatch=\"**/tests/stress/**/*.{js,jsx,ts,tsx}\" --verbose", + "test:catalog": "node scripts/validate-test-catalog.js", "test:gitignore": "jest --config jest.config.js --testMatch=\"**/tests/unit/gitignore-parser.test.{js,ts}\" --verbose", "test:binary": "jest --config jest.config.js --testMatch=\"**/tests/unit/binary-detection.test.{js,ts}\" --verbose", "test:patterns": "jest --config jest.config.js --testMatch=\"**/tests/**/*pattern*.test.{js,ts}\" --verbose", diff --git a/scripts/validate-test-catalog.js b/scripts/validate-test-catalog.js new file mode 100644 index 0000000..e2b9b39 --- /dev/null +++ b/scripts/validate-test-catalog.js @@ -0,0 +1,224 @@ +#!/usr/bin/env node + +const fs = require('fs'); +const path = require('path'); +const { minimatch } = require('minimatch'); + +const ROOT_DIR = path.join(__dirname, '..'); +const DEFAULT_CATALOG_PATH = path.join(ROOT_DIR, 'tests', 'catalog.md'); +const DEFAULT_JEST_CONFIG_PATH = path.join(ROOT_DIR, 'jest.config.js'); +const CATALOG_PATH_REFERENCE_PATTERN = /`(tests\/[^`\s]+)`/g; +const EXECUTABLE_TEST_FILE_PATTERN = /\.(test|spec)\.(js|jsx|ts|tsx)$/; + +function toPosixPath(value) { + return value.replace(/\\/g, '/'); +} + +function extractCatalogPathReferences(content) { + const references = new Set(); + + for (const match of content.matchAll(CATALOG_PATH_REFERENCE_PATTERN)) { + references.add(toPosixPath(match[1])); + } + + return Array.from(references).sort(); +} + +function collectFilesRecursively(directoryPath) { + if (!fs.existsSync(directoryPath)) { + return []; + } + + const files = []; + const entries = fs.readdirSync(directoryPath, { withFileTypes: true }); + + for (const entry of entries) { + const absolutePath = path.join(directoryPath, entry.name); + if (entry.isDirectory()) { + files.push(...collectFilesRecursively(absolutePath)); + continue; + } + + files.push(absolutePath); + } + + return files; +} + +function loadJestConfig(jestConfigPath = DEFAULT_JEST_CONFIG_PATH) { + const resolvedPath = path.resolve(jestConfigPath); + delete require.cache[resolvedPath]; + const loaded = require(resolvedPath); + return loaded && typeof loaded === 'object' ? loaded : {}; +} + +function normalizeTestMatchPatterns(testMatchPatterns = [], rootDir = ROOT_DIR) { + return testMatchPatterns.map((pattern) => + toPosixPath(pattern).replace('/', '').replace('', '') + ); +} + +function compileIgnorePatterns(patterns = []) { + return patterns + .map((pattern) => { + try { + return new RegExp(pattern); + } catch (_error) { + return null; + } + }) + .filter(Boolean); +} + +function isIgnoredByJest(absoluteFilePath, ignorePatterns) { + const normalizedAbsolutePath = toPosixPath(absoluteFilePath); + return ignorePatterns.some((pattern) => pattern.test(normalizedAbsolutePath)); +} + +function isMatchedByJest(relativeFilePath, testMatchPatterns) { + if (testMatchPatterns.length === 0) { + return true; + } + + return testMatchPatterns.some((pattern) => minimatch(relativeFilePath, pattern, { dot: true })); +} + +function listExecutableTestFiles(rootDir = ROOT_DIR) { + const testsRoot = path.join(rootDir, 'tests'); + const files = collectFilesRecursively(testsRoot); + + return files + .map((absolutePath) => toPosixPath(path.relative(rootDir, absolutePath))) + .filter((relativePath) => EXECUTABLE_TEST_FILE_PATTERN.test(relativePath)) + .sort(); +} + +function validateTestCatalog({ + rootDir = ROOT_DIR, + catalogPath = DEFAULT_CATALOG_PATH, + jestConfigPath = DEFAULT_JEST_CONFIG_PATH, + jestConfig, +} = {}) { + let catalogContent = ''; + const errors = []; + + try { + catalogContent = fs.readFileSync(catalogPath, 'utf8'); + } catch (error) { + return { + isValid: false, + errors: [`Unable to read test catalog at ${catalogPath}: ${error.message}`], + catalogPathReferences: [], + missingCatalogPaths: [], + discoveredTestFiles: [], + unlistedDiscoveredTestFiles: [], + listedButNotDiscoveredTestFiles: [], + }; + } + + const effectiveJestConfig = jestConfig || loadJestConfig(jestConfigPath); + const testMatchPatterns = normalizeTestMatchPatterns(effectiveJestConfig.testMatch, rootDir); + const ignorePatterns = compileIgnorePatterns(effectiveJestConfig.testPathIgnorePatterns); + const catalogPathReferences = extractCatalogPathReferences(catalogContent); + + const missingCatalogPaths = catalogPathReferences + .filter((relativePath) => !relativePath.includes('*')) + .filter((relativePath) => !fs.existsSync(path.join(rootDir, relativePath))) + .sort(); + + const executableTestFiles = listExecutableTestFiles(rootDir); + const discoveredTestFiles = executableTestFiles + .filter((relativePath) => { + const absolutePath = path.join(rootDir, relativePath); + return ( + !isIgnoredByJest(absolutePath, ignorePatterns) && + isMatchedByJest(relativePath, testMatchPatterns) + ); + }) + .sort(); + + const discoveredTestFileSet = new Set(discoveredTestFiles); + const catalogReferenceSet = new Set(catalogPathReferences); + + const unlistedDiscoveredTestFiles = discoveredTestFiles + .filter((relativePath) => !catalogReferenceSet.has(relativePath)) + .sort(); + + const listedButNotDiscoveredTestFiles = catalogPathReferences + .filter((relativePath) => EXECUTABLE_TEST_FILE_PATTERN.test(relativePath)) + .filter((relativePath) => fs.existsSync(path.join(rootDir, relativePath))) + .filter((relativePath) => { + const absolutePath = path.join(rootDir, relativePath); + if (isIgnoredByJest(absolutePath, ignorePatterns)) { + return false; + } + + return !discoveredTestFileSet.has(relativePath); + }) + .sort(); + + if (missingCatalogPaths.length > 0) { + errors.push( + `Catalog references missing paths: ${missingCatalogPaths.map((item) => `\`${item}\``).join(', ')}` + ); + } + + if (unlistedDiscoveredTestFiles.length > 0) { + errors.push( + `Discovered tests missing from catalog: ${unlistedDiscoveredTestFiles.map((item) => `\`${item}\``).join(', ')}` + ); + } + + if (listedButNotDiscoveredTestFiles.length > 0) { + errors.push( + `Catalog lists tests not discovered by Jest: ${listedButNotDiscoveredTestFiles.map((item) => `\`${item}\``).join(', ')}` + ); + } + + return { + isValid: errors.length === 0, + errors, + catalogPathReferences, + missingCatalogPaths, + discoveredTestFiles, + unlistedDiscoveredTestFiles, + listedButNotDiscoveredTestFiles, + }; +} + +function run() { + const catalogArg = process.argv[2]; + const jestConfigArg = process.argv[3]; + const result = validateTestCatalog({ + catalogPath: catalogArg ? path.resolve(process.cwd(), catalogArg) : DEFAULT_CATALOG_PATH, + jestConfigPath: jestConfigArg + ? path.resolve(process.cwd(), jestConfigArg) + : DEFAULT_JEST_CONFIG_PATH, + }); + + if (!result.isValid) { + console.error('Test catalog validation failed:'); + for (const error of result.errors) { + console.error(`- ${error}`); + } + process.exit(1); + } + + console.log( + `Test catalog validation passed (${result.catalogPathReferences.length} references, ${result.discoveredTestFiles.length} discovered tests).` + ); +} + +if (require.main === module) { + run(); +} + +module.exports = { + CATALOG_PATH_REFERENCE_PATTERN, + EXECUTABLE_TEST_FILE_PATTERN, + extractCatalogPathReferences, + isMatchedByJest, + listExecutableTestFiles, + normalizeTestMatchPatterns, + validateTestCatalog, +}; diff --git a/tests/catalog.md b/tests/catalog.md index b09f303..bb78f79 100644 --- a/tests/catalog.md +++ b/tests/catalog.md @@ -5,6 +5,7 @@ Purpose: quick map of what is covered, why it exists, and which command to run. ## Core Commands - Full tests: `npm test -- --runInBand` +- Test catalog consistency (path + discovery checks): `npm run test:catalog` - Stress metrics summary (+ optional Pushgateway publish): `npm run stress:metrics` - Stress publish verification in Prometheus: `npm run prometheus:verify` - End-to-end perf metrics job (`TOOLS_DOMAIN` aware): `npm run perf:test` or `make perf-test` @@ -19,26 +20,32 @@ Purpose: quick map of what is covered, why it exists, and which command to run. ## Unit Tests -| File | Primary Target | Key Use Cases | -| ----------------------------------------------- | --------------------------------------- | --------------------------------------------------------------------------------------------- | -| `tests/unit/components/app.test.tsx` | `src/renderer/components/App.tsx` | Tab switching, config load, directory selection, processing flow, error handling | -| `tests/unit/components/config-tab.test.tsx` | `src/renderer/components/ConfigTab.tsx` | Config toggles/inputs, callback wiring, directory picker trigger | -| `tests/unit/components/file-tree.test.tsx` | `src/renderer/components/FileTree.tsx` | Tree render, folder expand/collapse, select all, empty-state behavior | -| `tests/unit/file-analyzer.test.ts` | `src/utils/file-analyzer.ts` | Include/exclude rules, gitignore behavior, binary handling, error cases | -| `tests/unit/gitignore-parser.test.ts` | `src/utils/gitignore-parser.ts` | Pattern parsing, negation behavior, caching, nested path handling | -| `tests/unit/binary-detection.test.ts` | `src/utils/file-analyzer.ts` | Binary signature detection, control-char thresholds, fallback-on-error behavior | -| `tests/unit/utils/filter-utils.test.ts` | `src/utils/filter-utils.ts` | Path normalization, extension filtering, custom excludes, gitignore precedence | -| `tests/unit/utils/secret-scanner.test.ts` | `src/utils/secret-scanner.ts` | Sensitive path detection, secret-pattern scanning, default-on safety toggles | -| `tests/unit/utils/fnmatch.test.ts` | `src/utils/fnmatch.ts` | Glob semantics: wildcards, classes, double-star, braces, path anchors | -| `tests/unit/utils/export-format.test.ts` | `src/utils/export-format.ts` | Export format normalization, XML attribute escaping, CDATA-safe sanitization | -| `tests/unit/utils/content-processor.test.ts` | `src/utils/content-processor.ts` | Content assembly, binary skip logic, malformed input handling | -| `tests/unit/utils/config-manager.test.ts` | `src/utils/config-manager.ts` | Default config load, parse failures, graceful fallback behavior | -| `tests/unit/utils/token-counter.test.ts` | `src/utils/token-counter.ts` | Token counting basics, empty/null input handling | -| `tests/unit/scripts/security.test.js` | `scripts/lib/security.js` | Command safety validation, Windows path acceptance for approved executables | -| `tests/unit/scripts/actions-freshness.test.js` | `scripts/lib/actions-freshness.js` | Workflow `uses:` reference parsing, pinning classification, freshness markdown report output | -| `tests/unit/scripts/validate-changelog.test.js` | `scripts/validate-changelog.js` | Release heading/date format checks, allowed section headings, latest release section coverage | -| `tests/unit/main/updater.test.ts` | `src/main/updater.ts` | Alpha/stable channel selection, platform gating, update-check result handling | -| `tests/unit/main/feature-flags.test.ts` | `src/main/feature-flags.ts` | OpenFeature normalization, env/remote merge rules, secure remote fetch behavior | +| File | Primary Target | Key Use Cases | +| ------------------------------------------------------ | --------------------------------------- | --------------------------------------------------------------------------------------------- | +| `tests/unit/components/app.test.tsx` | `src/renderer/components/App.tsx` | Tab switching, config load, directory selection, processing flow, error handling | +| `tests/unit/components/config-tab.test.tsx` | `src/renderer/components/ConfigTab.tsx` | Config toggles/inputs, callback wiring, directory picker trigger | +| `tests/unit/components/file-tree.test.tsx` | `src/renderer/components/FileTree.tsx` | Tree render, folder expand/collapse, select all, empty-state behavior | +| `tests/unit/components/source-tab.test.tsx` | `src/renderer/components/SourceTab.tsx` | Source tab input state, filter toggles, and event forwarding behavior | +| `tests/unit/file-analyzer.test.ts` | `src/utils/file-analyzer.ts` | Include/exclude rules, gitignore behavior, binary handling, error cases | +| `tests/unit/gitignore-parser.test.ts` | `src/utils/gitignore-parser.ts` | Pattern parsing, negation behavior, caching, nested path handling | +| `tests/unit/binary-detection.test.ts` | `src/utils/file-analyzer.ts` | Binary signature detection, control-char thresholds, fallback-on-error behavior | +| `tests/unit/utils/filter-utils.test.ts` | `src/utils/filter-utils.ts` | Path normalization, extension filtering, custom excludes, gitignore precedence | +| `tests/unit/utils/secret-scanner.test.ts` | `src/utils/secret-scanner.ts` | Sensitive path detection, secret-pattern scanning, default-on safety toggles | +| `tests/unit/utils/fnmatch.test.ts` | `src/utils/fnmatch.ts` | Glob semantics: wildcards, classes, double-star, braces, path anchors | +| `tests/unit/utils/export-format.test.ts` | `src/utils/export-format.ts` | Export format normalization, XML attribute escaping, CDATA-safe sanitization | +| `tests/unit/utils/content-processor.test.ts` | `src/utils/content-processor.ts` | Content assembly, binary skip logic, malformed input handling | +| `tests/unit/utils/config-manager.test.ts` | `src/utils/config-manager.ts` | Default config load, parse failures, graceful fallback behavior | +| `tests/unit/utils/token-counter.test.ts` | `src/utils/token-counter.ts` | Token counting basics, empty/null input handling | +| `tests/unit/scripts/security.test.js` | `scripts/lib/security.js` | Command safety validation, Windows path acceptance for approved executables | +| `tests/unit/scripts/actions-freshness.test.js` | `scripts/lib/actions-freshness.js` | Workflow `uses:` reference parsing, pinning classification, freshness markdown report output | +| `tests/unit/scripts/sonar-options.test.js` | `scripts/lib/sonar-options.js` | Sonar scanner option merge behavior and CPD exclusion defaults | +| `tests/unit/scripts/publish-stress-metrics.test.js` | `scripts/publish-stress-metrics.js` | Prometheus payload generation and Pushgateway publication safeguards | +| `tests/unit/scripts/verify-prometheus-metrics.test.js` | `scripts/verify-prometheus-metrics.js` | Prometheus scrape verification retries, timeouts, and parsing | +| `tests/unit/scripts/perf-metrics-job.test.js` | `scripts/run-perf-metrics-job.js` | End-to-end performance job orchestration (stress, publish, verify) | +| `tests/unit/scripts/validate-test-catalog.test.js` | `scripts/validate-test-catalog.js` | Catalog path validity and Jest discovery coverage checks | +| `tests/unit/scripts/validate-changelog.test.js` | `scripts/validate-changelog.js` | Release heading/date format checks, allowed section headings, latest release section coverage | +| `tests/unit/main/updater.test.ts` | `src/main/updater.ts` | Alpha/stable channel selection, platform gating, update-check result handling | +| `tests/unit/main/feature-flags.test.ts` | `src/main/feature-flags.ts` | OpenFeature normalization, env/remote merge rules, secure remote fetch behavior | ## Integration Tests diff --git a/tests/unit/scripts/validate-test-catalog.test.js b/tests/unit/scripts/validate-test-catalog.test.js new file mode 100644 index 0000000..da6567b --- /dev/null +++ b/tests/unit/scripts/validate-test-catalog.test.js @@ -0,0 +1,139 @@ +jest.unmock('fs'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { + extractCatalogPathReferences, + validateTestCatalog, +} = require('../../../scripts/validate-test-catalog'); + +function writeFile(rootDir, relativePath, content = '') { + const absolutePath = path.join(rootDir, relativePath); + fs.mkdirSync(path.dirname(absolutePath), { recursive: true }); + fs.writeFileSync(absolutePath, content); +} + +function createWorkspace() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'validate-test-catalog-')); +} + +describe('validate-test-catalog script', () => { + test('extracts unique test catalog references from markdown content', () => { + const content = ` +# Test Catalog +- \`tests/unit/a.test.ts\` +- \`tests/unit/a.test.ts\` +- \`tests/integration/b.test.ts\` +`; + + expect(extractCatalogPathReferences(content)).toEqual([ + 'tests/integration/b.test.ts', + 'tests/unit/a.test.ts', + ]); + }); + + test('passes when catalog references existing paths and discovered tests', () => { + const rootDir = createWorkspace(); + + writeFile(rootDir, 'tests/catalog.md', [ + '`tests/unit/a.test.ts`', + '`tests/integration/b.test.ts`', + '`tests/e2e/e2e.spec.ts`', + ].join('\n')); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + writeFile(rootDir, 'tests/integration/b.test.ts', 'test("b", () => expect(true).toBe(true));'); + writeFile(rootDir, 'tests/e2e/e2e.spec.ts', 'test("e2e", () => expect(true).toBe(true));'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: ['/tests/e2e/'], + }, + }); + + expect(report.isValid).toBe(true); + expect(report.errors).toEqual([]); + expect(report.discoveredTestFiles).toEqual([ + 'tests/integration/b.test.ts', + 'tests/unit/a.test.ts', + ]); + + fs.rmSync(rootDir, { recursive: true, force: true }); + }); + + test('fails when catalog references missing paths', () => { + const rootDir = createWorkspace(); + + writeFile(rootDir, 'tests/catalog.md', '`tests/unit/missing.test.ts`'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.missingCatalogPaths).toEqual(['tests/unit/missing.test.ts']); + expect(report.errors[0]).toContain('Catalog references missing paths'); + + fs.rmSync(rootDir, { recursive: true, force: true }); + }); + + test('fails when discovered tests are not listed in the catalog', () => { + const rootDir = createWorkspace(); + + writeFile(rootDir, 'tests/catalog.md', '`tests/unit/a.test.ts`'); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + writeFile(rootDir, 'tests/unit/forgotten.test.ts', 'test("forgotten", () => expect(true).toBe(true));'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.unlistedDiscoveredTestFiles).toEqual(['tests/unit/forgotten.test.ts']); + expect(report.errors[0]).toContain('Discovered tests missing from catalog'); + + fs.rmSync(rootDir, { recursive: true, force: true }); + }); + + test('fails when cataloged test locations are not discovered by Jest patterns', () => { + const rootDir = createWorkspace(); + + writeFile(rootDir, 'tests/catalog.md', [ + '`tests/unit/a.test.ts`', + '`tests/stress/out-of-scope.test.ts`', + ].join('\n')); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + writeFile( + rootDir, + 'tests/stress/out-of-scope.test.ts', + 'test("stress", () => expect(true).toBe(true));' + ); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/{unit,integration}/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.listedButNotDiscoveredTestFiles).toEqual(['tests/stress/out-of-scope.test.ts']); + expect(report.errors[0]).toContain('Catalog lists tests not discovered by Jest'); + + fs.rmSync(rootDir, { recursive: true, force: true }); + }); +}); From b980ec9657e4761e5253d5faf9ff8b4e2f1a2fd3 Mon Sep 17 00:00:00 2001 From: Mehdi Date: Wed, 11 Feb 2026 14:55:23 +0000 Subject: [PATCH 2/4] test: harden catalog validator test coverage --- .../scripts/validate-test-catalog.test.js | 227 +++++++++++------- 1 file changed, 138 insertions(+), 89 deletions(-) diff --git a/tests/unit/scripts/validate-test-catalog.test.js b/tests/unit/scripts/validate-test-catalog.test.js index da6567b..10de037 100644 --- a/tests/unit/scripts/validate-test-catalog.test.js +++ b/tests/unit/scripts/validate-test-catalog.test.js @@ -17,6 +17,26 @@ function createWorkspace() { return fs.mkdtempSync(path.join(os.tmpdir(), 'validate-test-catalog-')); } +function withWorkspace(testFn) { + const rootDir = createWorkspace(); + const cleanup = () => { + fs.rmSync(rootDir, { recursive: true, force: true }); + }; + + try { + const result = testFn(rootDir); + if (result && typeof result.then === 'function') { + return result.finally(cleanup); + } + + cleanup(); + return result; + } catch (error) { + cleanup(); + throw error; + } +} + describe('validate-test-catalog script', () => { test('extracts unique test catalog references from markdown content', () => { const content = ` @@ -32,108 +52,137 @@ describe('validate-test-catalog script', () => { ]); }); - test('passes when catalog references existing paths and discovered tests', () => { - const rootDir = createWorkspace(); - - writeFile(rootDir, 'tests/catalog.md', [ - '`tests/unit/a.test.ts`', - '`tests/integration/b.test.ts`', - '`tests/e2e/e2e.spec.ts`', - ].join('\n')); - writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); - writeFile(rootDir, 'tests/integration/b.test.ts', 'test("b", () => expect(true).toBe(true));'); - writeFile(rootDir, 'tests/e2e/e2e.spec.ts', 'test("e2e", () => expect(true).toBe(true));'); - - const report = validateTestCatalog({ - rootDir, - catalogPath: path.join(rootDir, 'tests/catalog.md'), - jestConfig: { - testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], - testPathIgnorePatterns: ['/tests/e2e/'], - }, - }); - - expect(report.isValid).toBe(true); - expect(report.errors).toEqual([]); - expect(report.discoveredTestFiles).toEqual([ - 'tests/integration/b.test.ts', - 'tests/unit/a.test.ts', - ]); + test('extracts wildcard and non-test catalog references as-is', () => { + const content = ` +# Test Catalog +- \`tests/unit/*.test.ts\` +- \`tests/README.md\` +`; - fs.rmSync(rootDir, { recursive: true, force: true }); + expect(extractCatalogPathReferences(content)).toEqual(['tests/README.md', 'tests/unit/*.test.ts']); }); - test('fails when catalog references missing paths', () => { - const rootDir = createWorkspace(); - - writeFile(rootDir, 'tests/catalog.md', '`tests/unit/missing.test.ts`'); - - const report = validateTestCatalog({ - rootDir, - catalogPath: path.join(rootDir, 'tests/catalog.md'), - jestConfig: { - testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], - testPathIgnorePatterns: [], - }, + test('returns a failure result when the test catalog cannot be read', () => { + return withWorkspace((rootDir) => { + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests', 'missing-catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.errors).toEqual( + expect.arrayContaining([expect.stringContaining('Unable to read test catalog')]) + ); + expect(report.catalogPathReferences).toEqual([]); + expect(report.missingCatalogPaths).toEqual([]); + expect(report.discoveredTestFiles).toEqual([]); + expect(report.unlistedDiscoveredTestFiles).toEqual([]); + expect(report.listedButNotDiscoveredTestFiles).toEqual([]); }); + }); - expect(report.isValid).toBe(false); - expect(report.missingCatalogPaths).toEqual(['tests/unit/missing.test.ts']); - expect(report.errors[0]).toContain('Catalog references missing paths'); + test('passes when catalog references existing paths and discovered tests', () => { + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', [ + '`tests/unit/a.test.ts`', + '`tests/integration/b.test.ts`', + '`tests/e2e/e2e.spec.ts`', + ].join('\n')); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + writeFile(rootDir, 'tests/integration/b.test.ts', 'test("b", () => expect(true).toBe(true));'); + writeFile(rootDir, 'tests/e2e/e2e.spec.ts', 'test("e2e", () => expect(true).toBe(true));'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: ['/tests/e2e/'], + }, + }); + + expect(report.isValid).toBe(true); + expect(report.errors).toEqual([]); + expect(report.discoveredTestFiles).toEqual([ + 'tests/integration/b.test.ts', + 'tests/unit/a.test.ts', + ]); + }); + }); - fs.rmSync(rootDir, { recursive: true, force: true }); + test('fails when catalog references missing paths', () => { + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', '`tests/unit/missing.test.ts`'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.missingCatalogPaths).toEqual(['tests/unit/missing.test.ts']); + expect(report.errors[0]).toContain('Catalog references missing paths'); + }); }); test('fails when discovered tests are not listed in the catalog', () => { - const rootDir = createWorkspace(); - - writeFile(rootDir, 'tests/catalog.md', '`tests/unit/a.test.ts`'); - writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); - writeFile(rootDir, 'tests/unit/forgotten.test.ts', 'test("forgotten", () => expect(true).toBe(true));'); - - const report = validateTestCatalog({ - rootDir, - catalogPath: path.join(rootDir, 'tests/catalog.md'), - jestConfig: { - testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], - testPathIgnorePatterns: [], - }, + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', '`tests/unit/a.test.ts`'); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + writeFile( + rootDir, + 'tests/unit/forgotten.test.ts', + 'test("forgotten", () => expect(true).toBe(true));' + ); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.unlistedDiscoveredTestFiles).toEqual(['tests/unit/forgotten.test.ts']); + expect(report.errors[0]).toContain('Discovered tests missing from catalog'); }); - - expect(report.isValid).toBe(false); - expect(report.unlistedDiscoveredTestFiles).toEqual(['tests/unit/forgotten.test.ts']); - expect(report.errors[0]).toContain('Discovered tests missing from catalog'); - - fs.rmSync(rootDir, { recursive: true, force: true }); }); test('fails when cataloged test locations are not discovered by Jest patterns', () => { - const rootDir = createWorkspace(); - - writeFile(rootDir, 'tests/catalog.md', [ - '`tests/unit/a.test.ts`', - '`tests/stress/out-of-scope.test.ts`', - ].join('\n')); - writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); - writeFile( - rootDir, - 'tests/stress/out-of-scope.test.ts', - 'test("stress", () => expect(true).toBe(true));' - ); - - const report = validateTestCatalog({ - rootDir, - catalogPath: path.join(rootDir, 'tests/catalog.md'), - jestConfig: { - testMatch: ['/tests/{unit,integration}/**/*.{js,jsx,ts,tsx}'], - testPathIgnorePatterns: [], - }, + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', [ + '`tests/unit/a.test.ts`', + '`tests/stress/out-of-scope.test.ts`', + ].join('\n')); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + writeFile( + rootDir, + 'tests/stress/out-of-scope.test.ts', + 'test("stress", () => expect(true).toBe(true));' + ); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/{unit,integration}/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(false); + expect(report.listedButNotDiscoveredTestFiles).toEqual(['tests/stress/out-of-scope.test.ts']); + expect(report.errors[0]).toContain('Catalog lists tests not discovered by Jest'); }); - - expect(report.isValid).toBe(false); - expect(report.listedButNotDiscoveredTestFiles).toEqual(['tests/stress/out-of-scope.test.ts']); - expect(report.errors[0]).toContain('Catalog lists tests not discovered by Jest'); - - fs.rmSync(rootDir, { recursive: true, force: true }); }); }); From 5680c8c24341198250101d53b3d20e54a333c63a Mon Sep 17 00:00:00 2001 From: Mehdi Date: Wed, 11 Feb 2026 15:01:51 +0000 Subject: [PATCH 3/4] fix: harden test catalog validator path handling --- scripts/validate-test-catalog.js | 70 +++++++++++---- .../scripts/validate-test-catalog.test.js | 89 +++++++++++++++++++ 2 files changed, 144 insertions(+), 15 deletions(-) diff --git a/scripts/validate-test-catalog.js b/scripts/validate-test-catalog.js index e2b9b39..254e7a5 100644 --- a/scripts/validate-test-catalog.js +++ b/scripts/validate-test-catalog.js @@ -9,6 +9,10 @@ const DEFAULT_CATALOG_PATH = path.join(ROOT_DIR, 'tests', 'catalog.md'); const DEFAULT_JEST_CONFIG_PATH = path.join(ROOT_DIR, 'jest.config.js'); const CATALOG_PATH_REFERENCE_PATTERN = /`(tests\/[^`\s]+)`/g; const EXECUTABLE_TEST_FILE_PATTERN = /\.(test|spec)\.(js|jsx|ts|tsx)$/; +const JEST_DEFAULT_TEST_MATCH_PATTERNS = [ + '**/__tests__/**/*.[jt]s?(x)', + '**/?(*.)+(spec|test).[jt]s?(x)', +]; function toPosixPath(value) { return value.replace(/\\/g, '/'); @@ -52,18 +56,19 @@ function loadJestConfig(jestConfigPath = DEFAULT_JEST_CONFIG_PATH) { return loaded && typeof loaded === 'object' ? loaded : {}; } -function normalizeTestMatchPatterns(testMatchPatterns = [], rootDir = ROOT_DIR) { +function normalizeTestMatchPatterns(testMatchPatterns = []) { return testMatchPatterns.map((pattern) => toPosixPath(pattern).replace('/', '').replace('', '') ); } -function compileIgnorePatterns(patterns = []) { +function compileIgnorePatterns(patterns = [], onInvalidPattern = () => {}) { return patterns .map((pattern) => { try { return new RegExp(pattern); - } catch (_error) { + } catch (error) { + onInvalidPattern(pattern, error); return null; } }) @@ -76,11 +81,10 @@ function isIgnoredByJest(absoluteFilePath, ignorePatterns) { } function isMatchedByJest(relativeFilePath, testMatchPatterns) { - if (testMatchPatterns.length === 0) { - return true; - } + const effectivePatterns = + testMatchPatterns.length > 0 ? testMatchPatterns : JEST_DEFAULT_TEST_MATCH_PATTERNS; - return testMatchPatterns.some((pattern) => minimatch(relativeFilePath, pattern, { dot: true })); + return effectivePatterns.some((pattern) => minimatch(relativeFilePath, pattern, { dot: true })); } function listExecutableTestFiles(rootDir = ROOT_DIR) { @@ -101,6 +105,7 @@ function validateTestCatalog({ } = {}) { let catalogContent = ''; const errors = []; + const warnings = []; try { catalogContent = fs.readFileSync(catalogPath, 'utf8'); @@ -108,6 +113,7 @@ function validateTestCatalog({ return { isValid: false, errors: [`Unable to read test catalog at ${catalogPath}: ${error.message}`], + warnings, catalogPathReferences: [], missingCatalogPaths: [], discoveredTestFiles: [], @@ -117,8 +123,13 @@ function validateTestCatalog({ } const effectiveJestConfig = jestConfig || loadJestConfig(jestConfigPath); - const testMatchPatterns = normalizeTestMatchPatterns(effectiveJestConfig.testMatch, rootDir); - const ignorePatterns = compileIgnorePatterns(effectiveJestConfig.testPathIgnorePatterns); + const testMatchPatterns = normalizeTestMatchPatterns(effectiveJestConfig.testMatch); + const ignorePatterns = compileIgnorePatterns( + effectiveJestConfig.testPathIgnorePatterns, + (pattern, error) => { + warnings.push(`Invalid Jest ignore pattern \`${pattern}\`: ${error.message}`); + } + ); const catalogPathReferences = extractCatalogPathReferences(catalogContent); const missingCatalogPaths = catalogPathReferences @@ -178,6 +189,7 @@ function validateTestCatalog({ return { isValid: errors.length === 0, errors, + warnings, catalogPathReferences, missingCatalogPaths, discoveredTestFiles, @@ -186,16 +198,42 @@ function validateTestCatalog({ }; } +function resolvePathWithinRoot(inputPath, defaultPath, label) { + const resolvedPath = inputPath ? path.resolve(process.cwd(), inputPath) : defaultPath; + const relativeToRoot = path.relative(ROOT_DIR, resolvedPath); + const isOutsideRoot = relativeToRoot.startsWith('..') || path.isAbsolute(relativeToRoot); + + if (isOutsideRoot) { + throw new Error(`${label} must resolve inside the repository: ${resolvedPath}`); + } + + return resolvedPath; +} + function run() { - const catalogArg = process.argv[2]; - const jestConfigArg = process.argv[3]; + const [catalogArg, jestConfigArg] = process.argv.slice(2); + + let catalogPath = DEFAULT_CATALOG_PATH; + let jestConfigPath = DEFAULT_JEST_CONFIG_PATH; + + try { + catalogPath = resolvePathWithinRoot(catalogArg, DEFAULT_CATALOG_PATH, 'Catalog path'); + jestConfigPath = resolvePathWithinRoot(jestConfigArg, DEFAULT_JEST_CONFIG_PATH, 'Jest config path'); + } catch (error) { + console.error('Test catalog validation failed:'); + console.error(`- ${error.message}`); + process.exit(1); + } + const result = validateTestCatalog({ - catalogPath: catalogArg ? path.resolve(process.cwd(), catalogArg) : DEFAULT_CATALOG_PATH, - jestConfigPath: jestConfigArg - ? path.resolve(process.cwd(), jestConfigArg) - : DEFAULT_JEST_CONFIG_PATH, + catalogPath, + jestConfigPath, }); + for (const warning of result.warnings) { + console.warn(`Warning: ${warning}`); + } + if (!result.isValid) { console.error('Test catalog validation failed:'); for (const error of result.errors) { @@ -220,5 +258,7 @@ module.exports = { isMatchedByJest, listExecutableTestFiles, normalizeTestMatchPatterns, + resolvePathWithinRoot, + compileIgnorePatterns, validateTestCatalog, }; diff --git a/tests/unit/scripts/validate-test-catalog.test.js b/tests/unit/scripts/validate-test-catalog.test.js index 10de037..fdfc982 100644 --- a/tests/unit/scripts/validate-test-catalog.test.js +++ b/tests/unit/scripts/validate-test-catalog.test.js @@ -3,7 +3,9 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const { + compileIgnorePatterns, extractCatalogPathReferences, + resolvePathWithinRoot, validateTestCatalog, } = require('../../../scripts/validate-test-catalog'); @@ -82,6 +84,7 @@ describe('validate-test-catalog script', () => { expect(report.discoveredTestFiles).toEqual([]); expect(report.unlistedDiscoveredTestFiles).toEqual([]); expect(report.listedButNotDiscoveredTestFiles).toEqual([]); + expect(report.warnings).toEqual([]); }); }); @@ -111,6 +114,7 @@ describe('validate-test-catalog script', () => { 'tests/integration/b.test.ts', 'tests/unit/a.test.ts', ]); + expect(report.warnings).toEqual([]); }); }); @@ -130,6 +134,7 @@ describe('validate-test-catalog script', () => { expect(report.isValid).toBe(false); expect(report.missingCatalogPaths).toEqual(['tests/unit/missing.test.ts']); expect(report.errors[0]).toContain('Catalog references missing paths'); + expect(report.warnings).toEqual([]); }); }); @@ -155,6 +160,7 @@ describe('validate-test-catalog script', () => { expect(report.isValid).toBe(false); expect(report.unlistedDiscoveredTestFiles).toEqual(['tests/unit/forgotten.test.ts']); expect(report.errors[0]).toContain('Discovered tests missing from catalog'); + expect(report.warnings).toEqual([]); }); }); @@ -183,6 +189,89 @@ describe('validate-test-catalog script', () => { expect(report.isValid).toBe(false); expect(report.listedButNotDiscoveredTestFiles).toEqual(['tests/stress/out-of-scope.test.ts']); expect(report.errors[0]).toContain('Catalog lists tests not discovered by Jest'); + expect(report.warnings).toEqual([]); + }); + }); + + test('uses Jest default discovery patterns when testMatch is omitted', () => { + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', '`tests/unit/uses-default.spec.ts`'); + writeFile( + rootDir, + 'tests/unit/uses-default.spec.ts', + 'test("default-pattern", () => expect(true).toBe(true));' + ); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testPathIgnorePatterns: [], + }, + }); + + expect(report.isValid).toBe(true); + expect(report.discoveredTestFiles).toEqual(['tests/unit/uses-default.spec.ts']); + }); + }); + + test('collects warnings for invalid ignore patterns', () => { + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', '`tests/unit/a.test.ts`'); + writeFile(rootDir, 'tests/unit/a.test.ts', 'test("a", () => expect(true).toBe(true));'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: ['['], + }, + }); + + expect(report.isValid).toBe(true); + expect(report.warnings).toHaveLength(1); + expect(report.warnings[0]).toContain('Invalid Jest ignore pattern'); + }); + }); + + test('compiles valid ignore patterns and reports invalid ones', () => { + const invalidPatternWarnings = []; + const patterns = compileIgnorePatterns(['[', '/tests/e2e/'], (pattern) => { + invalidPatternWarnings.push(pattern); + }); + + expect(patterns).toHaveLength(1); + expect(patterns[0].test('/tmp/tests/e2e/example.spec.ts')).toBe(true); + expect(invalidPatternWarnings).toEqual(['[']); + }); + + test('rejects CLI paths that resolve outside the repository root', () => { + expect(() => + resolvePathWithinRoot(path.resolve('/tmp/outside-catalog.md'), '/unused', 'Catalog path') + ).toThrow('must resolve inside the repository'); + + const resolved = resolvePathWithinRoot('tests/catalog.md', '/unused', 'Catalog path'); + expect(resolved.endsWith(path.join('tests', 'catalog.md'))).toBe(true); + expect(path.isAbsolute(resolved)).toBe(true); + }); + + test('does not flag cataloged tests that are intentionally ignored by Jest', () => { + return withWorkspace((rootDir) => { + writeFile(rootDir, 'tests/catalog.md', '`tests/e2e/ignored.spec.ts`'); + writeFile(rootDir, 'tests/e2e/ignored.spec.ts', 'test("ignored", () => expect(true).toBe(true));'); + + const report = validateTestCatalog({ + rootDir, + catalogPath: path.join(rootDir, 'tests/catalog.md'), + jestConfig: { + testMatch: ['/tests/**/*.{js,jsx,ts,tsx}'], + testPathIgnorePatterns: ['/tests/e2e/'], + }, + }); + + expect(report.isValid).toBe(true); + expect(report.listedButNotDiscoveredTestFiles).toEqual([]); }); }); }); From 57437c106f9975b9e4c90c0ad9c897aeb325196c Mon Sep 17 00:00:00 2001 From: Mehdi Date: Wed, 11 Feb 2026 15:06:01 +0000 Subject: [PATCH 4/4] test: remove sonar hotspot paths from validator tests --- tests/unit/scripts/validate-test-catalog.test.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/unit/scripts/validate-test-catalog.test.js b/tests/unit/scripts/validate-test-catalog.test.js index fdfc982..8d50b6c 100644 --- a/tests/unit/scripts/validate-test-catalog.test.js +++ b/tests/unit/scripts/validate-test-catalog.test.js @@ -242,16 +242,22 @@ describe('validate-test-catalog script', () => { }); expect(patterns).toHaveLength(1); - expect(patterns[0].test('/tmp/tests/e2e/example.spec.ts')).toBe(true); + expect(patterns[0].test('/workspace/tests/e2e/example.spec.ts')).toBe(true); expect(invalidPatternWarnings).toEqual(['[']); }); test('rejects CLI paths that resolve outside the repository root', () => { + const fallbackCatalogPath = path.join(process.cwd(), 'tests', 'catalog.md'); + expect(() => - resolvePathWithinRoot(path.resolve('/tmp/outside-catalog.md'), '/unused', 'Catalog path') + resolvePathWithinRoot( + path.resolve(process.cwd(), '..', 'outside-catalog.md'), + fallbackCatalogPath, + 'Catalog path' + ) ).toThrow('must resolve inside the repository'); - const resolved = resolvePathWithinRoot('tests/catalog.md', '/unused', 'Catalog path'); + const resolved = resolvePathWithinRoot('tests/catalog.md', fallbackCatalogPath, 'Catalog path'); expect(resolved.endsWith(path.join('tests', 'catalog.md'))).toBe(true); expect(path.isAbsolute(resolved)).toBe(true); });