From 60372c9d781623dbadd1cf40f8242538a405b33b Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Mon, 16 Mar 2026 14:36:19 +0100 Subject: [PATCH 01/15] Enabled running mixed Vitest and Karma tests. --- .../ckeditor5-dev-tests/bin/testautomated.js | 11 +- .../lib/tasks/runautomatedtests.js | 332 +++++++++++--- .../tests/tasks/runautomatedtests.js | 434 ++++++++++++++++++ 3 files changed, 699 insertions(+), 78 deletions(-) diff --git a/packages/ckeditor5-dev-tests/bin/testautomated.js b/packages/ckeditor5-dev-tests/bin/testautomated.js index 8ebb79fac..4f2151e00 100755 --- a/packages/ckeditor5-dev-tests/bin/testautomated.js +++ b/packages/ckeditor5-dev-tests/bin/testautomated.js @@ -17,8 +17,15 @@ if ( options.files.length === 0 ) { options.files = [ '*' ]; } -// "Lark" is the default theme for tests. -options.themePath = fileURLToPath( import.meta.resolve( '@ckeditor/ckeditor5-theme-lark' ) ); +// "Lark" is the default theme for tests. The resolution may fail when +// ckeditor5-dev-tests is installed outside the CKEditor 5 workspace +// (e.g. in ckeditor5-commercial where the theme lives in external/ckeditor5). +try { + options.themePath = fileURLToPath( import.meta.resolve( '@ckeditor/ckeditor5-theme-lark' ) ); +} catch { + // Theme unavailable — Karma tests that import styles will still work + // if the loader can resolve the theme through other means. +} if ( fs.existsSync( '.env' ) ) { loadEnvFile( '.env' ); diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 0c7f8cb49..764287cc0 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -4,6 +4,7 @@ */ import fs from 'node:fs'; +import { spawn } from 'node:child_process'; import { styleText } from 'node:util'; import { logger } from '@ckeditor/ckeditor5-dev-utils'; import getKarmaConfig from '../utils/automated-tests/getkarmaconfig.js'; @@ -24,89 +25,181 @@ const IGNORE_GLOBS = [ upath.join( '**', 'tests', '**', '_utils', '**', '*.{js,ts}' ) ]; -// An absolute path to the entry file that will be passed to Karma. -const ENTRY_FILE_PATH = upath.join( process.cwd(), 'build', '.automated-tests', 'entry-point.js' ); - -export default function runAutomatedTests( options ) { - return Promise.resolve().then( () => { - if ( !options.production ) { - console.warn( styleText( - 'yellow', - '⚠ You\'re running tests in dev mode - some error protections are loose. Use the `--production` flag ' + - 'to use strictest verification methods.' - ) ); - } +const VITEST_COVERAGE_DIRECTORY = 'coverage-vitest'; - const globPatterns = transformFilesToTestGlob( options.files ); +export default async function runAutomatedTests( options ) { + if ( !options.production ) { + console.warn( styleText( + 'yellow', + '⚠ You\'re running tests in dev mode - some error protections are loose. Use the `--production` flag ' + + 'to use strictest verification methods.' + ) ); + } - createEntryFile( globPatterns, options.production ); + const globPatterns = resolveTestGlobs( options.files ); + const testFiles = collectTestFiles( globPatterns ); + const { karmaFiles, vitestProjects } = partitionByRunner( testFiles ); - const optionsForKarma = Object.assign( {}, options, { - entryFile: ENTRY_FILE_PATH, - globPatterns - } ); + if ( !karmaFiles.length && !vitestProjects.length ) { + throw new Error( 'Not found files to tests. Specified patterns are invalid.' ); + } - return runKarma( optionsForKarma ); - } ); + if ( karmaFiles.length && vitestProjects.length && options.watch ) { + throw new Error( + 'Watch mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch mode separately for Karma and Vitest packages.' + ); + } + + const errors = []; + + if ( karmaFiles.length ) { + try { + await runKarmaTests( options, karmaFiles ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( vitestProjects.length ) { + try { + await spawnVitest( options, vitestProjects ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( errors.length ) { + throw aggregateErrors( errors ); + } + + if ( options.coverage ) { + mergeCoverageReports( karmaFiles.length > 0, vitestProjects.length > 0 ); + } } -function transformFilesToTestGlob( files ) { +// -- Glob resolution & file collection ----------------------------------------------------------- + +function resolveTestGlobs( files ) { if ( !Array.isArray( files ) || files.length === 0 ) { throw new Error( 'Karma requires files to tests. `options.files` has to be non-empty array.' ); } const globMap = {}; - for ( const singleFile of files ) { - globMap[ singleFile ] = transformFileOptionToTestGlob( singleFile ); + for ( const file of files ) { + globMap[ file ] = transformFileOptionToTestGlob( file ); } return globMap; } -function createEntryFile( globPatterns, production ) { - mkdirp.sync( upath.dirname( ENTRY_FILE_PATH ) ); +function collectTestFiles( globPatterns ) { karmaLogger.setupFromConfig( { logLevel: 'INFO' } ); const log = karmaLogger.create( 'config' ); const allFiles = []; - for ( const singlePattern of Object.keys( globPatterns ) ) { + for ( const [ pattern, resolvedGlobs ] of Object.entries( globPatterns ) ) { let hasFiles = false; - for ( const resolvedPattern of globPatterns[ singlePattern ] ) { - const files = globSync( resolvedPattern ).map( filePath => upath.normalize( filePath ) ); + for ( const glob of resolvedGlobs ) { + const files = globSync( glob ).map( f => upath.normalize( f ) ); if ( files.length ) { hasFiles = true; } allFiles.push( - ...files.filter( file => !IGNORE_GLOBS.some( globPattern => minimatch( file, globPattern ) ) ) + ...files.filter( file => !IGNORE_GLOBS.some( ignore => minimatch( file, ignore ) ) ) ); } if ( !hasFiles ) { - log.warn( 'Pattern "%s" does not match any file.', singlePattern ); + log.warn( 'Pattern "%s" does not match any file.', pattern ); } } - if ( !allFiles.length ) { - throw new Error( 'Not found files to tests. Specified patterns are invalid.' ); + return allFiles; +} + +// -- Runner partitioning -------------------------------------------------------------------------- + +function partitionByRunner( testFiles ) { + const karmaFiles = []; + const vitestProjects = new Set(); + const runnerCache = new Map(); + + for ( const filePath of testFiles ) { + const packageRoot = getPackageRoot( filePath ); + + if ( !runnerCache.has( packageRoot ) ) { + runnerCache.set( packageRoot, detectPackageRunner( packageRoot ) ); + } + + const { runner, projectName } = runnerCache.get( packageRoot ); + + if ( runner === 'vitest' ) { + vitestProjects.add( projectName ); + } else { + karmaFiles.push( filePath ); + } } + return { karmaFiles, vitestProjects: [ ...vitestProjects ] }; +} + +function getPackageRoot( filePath ) { + const normalized = upath.normalize( filePath ); + const testsIndex = normalized.lastIndexOf( '/tests/' ); + + if ( testsIndex === -1 ) { + throw new Error( `Cannot determine package root for "${ filePath }".` ); + } + + return normalized.slice( 0, testsIndex ); +} + +function detectPackageRunner( packageRoot ) { + const projectName = upath.basename( packageRoot ).replace( /^ckeditor5-/, '' ); + const packageJson = JSON.parse( fs.readFileSync( upath.join( packageRoot, 'package.json' ), 'utf8' ) ); + const runner = /\bvitest\b/.test( packageJson?.scripts?.test || '' ) ? 'vitest' : 'karma'; + + return { projectName, runner }; +} + +// -- Karma runner --------------------------------------------------------------------------------- + +async function runKarmaTests( options, karmaFiles ) { + const entryFilePath = upath.join( process.cwd(), 'build', '.automated-tests', 'entry-point.js' ); + + createKarmaEntryFile( entryFilePath, karmaFiles, options.production ); + + // Build globPatterns from karmaFiles only, so the coverage loader instruments + // just the Karma packages' source code — not Vitest packages that happen to be + // imported transitively. + return startKarmaServer( { + ...options, + entryFile: entryFilePath, + globPatterns: { karma: karmaFiles } + } ); +} + +function createKarmaEntryFile( entryFilePath, files, production ) { + const utilsDir = upath.join( import.meta.dirname, '..', 'utils', 'automated-tests' ); + const testImports = [ ...files ]; + // Set global license key in the `before` hook. - allFiles.unshift( upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'licensekeybefore.js' ) ); + testImports.unshift( upath.join( utilsDir, 'licensekeybefore.js' ) ); // Inject the leak detector root hooks. Need to be split into two parts due to #598. - allFiles.splice( 0, 0, upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'leaksdetectorbefore.js' ) ); - allFiles.push( upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'leaksdetectorafter.js' ) ); + testImports.splice( 0, 0, upath.join( utilsDir, 'leaksdetectorbefore.js' ) ); + testImports.push( upath.join( utilsDir, 'leaksdetectorafter.js' ) ); - const entryFileContent = allFiles - .map( file => 'import "' + file + '";' ); + const entryLines = testImports.map( file => `import "${ file }";` ); // Inject the custom chai assertions. See ckeditor/ckeditor5#9668. - const assertionsDir = upath.join( import.meta.dirname, '..', 'utils', 'automated-tests', 'assertions' ); + const assertionsDir = upath.join( utilsDir, 'assertions' ); const customAssertions = fs.readdirSync( assertionsDir ).map( assertionFileName => { return [ assertionFileName, @@ -116,17 +209,18 @@ function createEntryFile( globPatterns, production ) { // Two loops are needed to achieve correct order in `ckeditor5/build/.automated-tests/entry-point.js`. for ( const [ fileName, functionName ] of customAssertions ) { - entryFileContent.push( `import ${ functionName }Factory from "${ assertionsDir }/${ fileName }";` ); + entryLines.push( `import ${ functionName }Factory from "${ assertionsDir }/${ fileName }";` ); } for ( const [ , functionName ] of customAssertions ) { - entryFileContent.push( `${ functionName }Factory( chai );` ); + entryLines.push( `${ functionName }Factory( chai );` ); } if ( production ) { - entryFileContent.unshift( assertConsoleUsageToThrowErrors() ); + entryLines.unshift( assertConsoleUsageToThrowErrors() ); } - fs.writeFileSync( ENTRY_FILE_PATH, entryFileContent.join( '\n' ) + '\n' ); + mkdirp.sync( upath.dirname( entryFilePath ) ); + fs.writeFileSync( entryFilePath, entryLines.join( '\n' ) + '\n' ); // Webpack watcher compiles the file in a loop. It causes to Karma that runs tests multiple times in watch mode. // A ugly hack blocks the loop and tests are executed once. @@ -134,9 +228,128 @@ function createEntryFile( globPatterns, production ) { const now = Date.now() / 1000; // 10 sec is default value of FS_ACCURENCY (which is hardcoded in Webpack watcher). const then = now - 10; - fs.utimesSync( ENTRY_FILE_PATH, then, then ); + fs.utimesSync( entryFilePath, then, then ); } +function startKarmaServer( options ) { + return new Promise( ( resolve, reject ) => { + const KarmaServer = karma.Server; + const parseConfig = karma.config.parseConfig; + + const config = getKarmaConfig( options ); + const parsedConfig = parseConfig( null, config, { throwErrors: true } ); + + const server = new KarmaServer( parsedConfig, exitCode => { + if ( exitCode === 0 ) { + resolve(); + } else { + reject( new Error( `Karma finished with "${ exitCode }" code.` ) ); + } + } ); + + if ( options.coverage ) { + const coveragePath = upath.join( process.cwd(), 'coverage' ); + + server.on( 'run_complete', () => { + // Use timeout to not write to the console in the middle of Karma's status. + setTimeout( () => { + const log = logger(); + + log.info( `Coverage report saved in '${ styleText( 'cyan', coveragePath ) }'.` ); + } ); + } ); + } + + server.start(); + } ); +} + +// -- Vitest runner -------------------------------------------------------------------------------- + +function spawnVitest( options, vitestProjects ) { + return new Promise( ( resolve, reject ) => { + const args = [ 'vitest' ]; + + args.push( options.watch ? '--watch' : '--run' ); + + if ( options.coverage ) { + const coverageDir = upath.join( process.cwd(), VITEST_COVERAGE_DIRECTORY ); + args.push( '--coverage', '--coverage.reportsDirectory', coverageDir ); + } + + for ( const project of vitestProjects ) { + args.push( '--project', project ); + } + + const child = spawn( 'pnpm', args, { + stdio: 'inherit', + cwd: process.cwd(), + shell: process.platform === 'win32' + } ); + + child.on( 'error', reject ); + + child.on( 'close', exitCode => { + if ( exitCode === 0 ) { + resolve(); + } else { + reject( new Error( `Vitest finished with "${ exitCode }" code.` ) ); + } + } ); + } ); +} + +// -- Coverage merging ----------------------------------------------------------------------------- + +function mergeCoverageReports( hasKarmaResults, hasVitestResults ) { + const coverageDir = upath.join( process.cwd(), 'coverage' ); + const mergedFilePath = upath.join( coverageDir, 'lcov.info' ); + const chunks = []; + + if ( hasKarmaResults ) { + const karmaCoverageFiles = globSync( upath.join( coverageDir, '**', 'lcov.info' ) ) + .map( f => upath.normalize( f ) ) + .filter( f => f !== upath.normalize( mergedFilePath ) ); + + for ( const file of karmaCoverageFiles ) { + chunks.push( fs.readFileSync( file, 'utf8' ) ); + } + } + + if ( hasVitestResults ) { + const vitestCoverageFile = upath.join( process.cwd(), VITEST_COVERAGE_DIRECTORY, 'lcov.info' ); + + if ( fs.existsSync( vitestCoverageFile ) ) { + const content = fs.readFileSync( vitestCoverageFile, 'utf8' ); + const cwdPrefix = upath.normalize( process.cwd() ) + '/'; + + // Vitest reports absolute SF: paths — strip the cwd prefix to make them + // relative, matching Karma's output format. + chunks.push( content.replaceAll( `SF:${ cwdPrefix }`, 'SF:' ) ); + } + } + + if ( !chunks.length ) { + return; + } + + mkdirp.sync( coverageDir ); + fs.writeFileSync( mergedFilePath, chunks.join( '\n' ) ); +} + +// -- Error handling ------------------------------------------------------------------------------- + +function aggregateErrors( errors ) { + if ( errors.length === 1 ) { + return errors[ 0 ]; + } + + const details = errors.map( e => `- ${ e.message }` ).join( '\n' ); + return new Error( `Test execution failed in multiple runners:\n${ details }` ); +} + +// -- Console assertion (production mode) ---------------------------------------------------------- + function assertConsoleUsageToThrowErrors() { const functionString = makeConsoleUsageToThrowErrors.toString(); @@ -186,36 +399,3 @@ function makeConsoleUsageToThrowErrors() { } ); } ); } - -function runKarma( options ) { - return new Promise( ( resolve, reject ) => { - const KarmaServer = karma.Server; - const parseConfig = karma.config.parseConfig; - - const config = getKarmaConfig( options ); - const parsedConfig = parseConfig( null, config, { throwErrors: true } ); - - const server = new KarmaServer( parsedConfig, exitCode => { - if ( exitCode === 0 ) { - resolve(); - } else { - reject( new Error( `Karma finished with "${ exitCode }" code.` ) ); - } - } ); - - if ( options.coverage ) { - const coveragePath = upath.join( process.cwd(), 'coverage' ); - - server.on( 'run_complete', () => { - // Use timeout to not write to the console in the middle of Karma's status. - setTimeout( () => { - const log = logger(); - - log.info( `Coverage report saved in '${ styleText( 'cyan', coveragePath ) }'.` ); - } ); - } ); - } - - server.start(); - } ); -} diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 395346502..4fd6a23f7 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import fs from 'node:fs'; +import { spawn } from 'node:child_process'; import { styleText } from 'node:util'; import { globSync } from 'glob'; import { mkdirp } from 'mkdirp'; @@ -26,9 +27,31 @@ const stubs = vi.hoisted( () => ( { on: vi.fn(), start: vi.fn() } + }, + spawn: { + call: vi.fn() } } ) ); +vi.mock( 'node:child_process', () => ( { + spawn: vi.fn( ( ...args ) => { + stubs.spawn.call( ...args ); + + const callbacks = {}; + + return { + on: ( eventName, callback ) => { + callbacks[ eventName ] = callback; + }, + emit: ( eventName, ...eventArgs ) => { + if ( callbacks[ eventName ] ) { + callbacks[ eventName ]( ...eventArgs ); + } + } + }; + } ) +} ) ); + vi.mock( 'karma', () => ( { default: { Server: class KarmaServer { @@ -66,6 +89,10 @@ describe( 'runAutomatedTests()', () => { beforeEach( async () => { vi.spyOn( process, 'cwd' ).mockReturnValue( '/workspace' ); + stubs.spawn.call.mockReset(); + + // Default: return empty JSON for package.json reads (no scripts → Karma runner). + vi.mocked( fs ).readFileSync.mockReturnValue( '{}' ); vi.mocked( karmaLogger ).create.mockImplementation( name => { expect( name ).to.equal( 'config' ); @@ -76,6 +103,8 @@ describe( 'runAutomatedTests()', () => { runAutomatedTests = ( await import( '../../lib/tasks/runautomatedtests.js' ) ).default; } ); + // -- Karma-only tests ------------------------------------------------------------------------- + it( 'should create an entry file before tests execution', async () => { const options = { files: [ @@ -473,4 +502,409 @@ describe( 'runAutomatedTests()', () => { ].join( '\n' ) ) ); } ); + + // -- Vitest-only tests ------------------------------------------------------------------------ + + it( 'should run only Vitest when all selected packages use Vitest', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 0 ); + + await promise; + + expect( stubs.karma.server.constructor ).not.toHaveBeenCalled(); + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ 'vitest', '--run', '--project', 'engine' ], + { stdio: 'inherit', cwd: '/workspace', shell: false } + ); + expect( vi.mocked( fs ).writeFileSync ).not.toHaveBeenCalledWith( + '/workspace/build/.automated-tests/entry-point.js', + expect.any( String ) + ); + } ); + + it( 'should pass --watch flag to Vitest when watch mode is enabled', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: true, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 0 ); + + await promise; + + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ 'vitest', '--watch', '--project', 'engine' ], + expect.any( Object ) + ); + } ); + + it( 'should pass coverage flags to Vitest when coverage is enabled', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ) + // For mergeCoverageReports — no Karma coverage files. + .mockReturnValue( [] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 0 ); + + await promise; + + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ 'vitest', '--run', '--coverage', '--coverage.reportsDirectory', '/workspace/coverage-vitest', '--project', 'engine' ], + expect.any( Object ) + ); + } ); + + it( 'should reject when Vitest process exits with non-zero code', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 1 ); + + await expect( promise ).rejects.toThrow( 'Vitest finished with "1" code.' ); + } ); + + // -- Mixed Karma + Vitest tests --------------------------------------------------------------- + + it( 'should route mixed package selection to Karma and Vitest', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + watch: false + }; + + vi.mocked( fs ).readdirSync.mockReturnValue( [] ); + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + return '{}'; + } ); + + const promise = runAutomatedTests( options ); + + setTimeout( () => { + const [ firstCall ] = stubs.karma.server.constructor.mock.calls; + const [ , exitCallback ] = firstCall; + + exitCallback( 0 ); + + setTimeout( () => { + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + + subprocess.emit( 'close', 0 ); + } ); + } ); + + await promise; + + expect( stubs.karma.server.constructor ).toHaveBeenCalledOnce(); + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ 'vitest', '--run', '--project', 'utils' ], + { stdio: 'inherit', cwd: '/workspace', shell: false } + ); + } ); + + it( 'should throw when watch mode is used with mixed Karma + Vitest packages', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + watch: true + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + return '{}'; + } ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Watch mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch mode separately for Karma and Vitest packages.' + ); + } ); + + it( 'should aggregate errors when both runners fail', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + watch: false + }; + + vi.mocked( fs ).readdirSync.mockReturnValue( [] ); + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: {} } ); + } + + return '{}'; + } ); + + vi.mocked( karma ).config.parseConfig.mockImplementation( () => { + throw new Error( 'Karma finished with "1" code.' ); + } ); + + const promise = runAutomatedTests( options ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 2 ); + + await expect( promise ).rejects.toThrow( /Test execution failed in multiple runners/ ); + } ); + + // -- Coverage merging tests ------------------------------------------------------------------- + + it( 'should merge coverage for mixed Karma + Vitest run', async () => { + const options = { + files: [ '*' ], + production: true, + coverage: true, + watch: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/packages/ckeditor5-utils/tests/**/*.js' + ] ); + vi.mocked( fs ).readdirSync.mockReturnValue( [] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/first.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/coverage/ChromeHeadless 146.0.0.0 (Mac OS 10.15.7)/lcov.info' + ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + if ( path === '/workspace/coverage/ChromeHeadless 146.0.0.0 (Mac OS 10.15.7)/lcov.info' ) { + return 'TN:karma\nSF:/workspace/packages/ckeditor5-utils/src/index.js\nend_of_record\n'; + } + + if ( path === '/workspace/coverage-vitest/lcov.info' ) { + return 'TN:vitest\nSF:/workspace/packages/ckeditor5-engine/src/index.js\nend_of_record\n'; + } + + return '{}'; + } ); + vi.mocked( fs ).existsSync.mockImplementation( path => path === '/workspace/coverage-vitest/lcov.info' ); + + const promise = runAutomatedTests( options ); + + setTimeout( () => { + const [ firstCall ] = stubs.karma.server.constructor.mock.calls; + const [ , exitCallback ] = firstCall; + + exitCallback( 0 ); + + setTimeout( () => { + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + + subprocess.emit( 'close', 0 ); + } ); + } ); + + await promise; + + expect( stubs.karma.server.constructor ).toHaveBeenCalledOnce(); + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ + 'vitest', + '--run', + '--coverage', + '--coverage.reportsDirectory', + '/workspace/coverage-vitest', + '--project', + 'engine' + ], + { stdio: 'inherit', cwd: '/workspace', shell: false } + ); + + // Verify merged coverage: Vitest absolute paths are converted to relative. + expect( vi.mocked( fs ).writeFileSync ).toHaveBeenCalledWith( + '/workspace/coverage/lcov.info', + expect.stringContaining( 'TN:karma' ) + ); + expect( vi.mocked( fs ).writeFileSync ).toHaveBeenCalledWith( + '/workspace/coverage/lcov.info', + expect.stringContaining( 'SF:packages/ckeditor5-engine/src/index.js' ) + ); + } ); + + // -- Multiple Vitest projects test ------------------------------------------------------------ + + it( 'should run all Vitest projects in a single process from cwd', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 0 ); + + await promise; + + // All Vitest projects should be passed to a single process spawned from cwd. + expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + 'pnpm', + [ 'vitest', '--run', '--project', 'utils', '--project', 'engine' ], + { stdio: 'inherit', cwd: '/workspace', shell: false } + ); + } ); } ); From 9b7313445b0d31e56ad1137e6d7e6a73794463b5 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Mon, 16 Mar 2026 17:40:34 +0100 Subject: [PATCH 02/15] Removed merging `lcov.info` coverage reports, as they are not used on the CI. --- .../lib/tasks/runautomatedtests.js | 46 +--------- .../tests/tasks/runautomatedtests.js | 89 ------------------- 2 files changed, 1 insertion(+), 134 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 764287cc0..1819479ca 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -25,8 +25,6 @@ const IGNORE_GLOBS = [ upath.join( '**', 'tests', '**', '_utils', '**', '*.{js,ts}' ) ]; -const VITEST_COVERAGE_DIRECTORY = 'coverage-vitest'; - export default async function runAutomatedTests( options ) { if ( !options.production ) { console.warn( styleText( @@ -72,10 +70,6 @@ export default async function runAutomatedTests( options ) { if ( errors.length ) { throw aggregateErrors( errors ); } - - if ( options.coverage ) { - mergeCoverageReports( karmaFiles.length > 0, vitestProjects.length > 0 ); - } } // -- Glob resolution & file collection ----------------------------------------------------------- @@ -273,7 +267,7 @@ function spawnVitest( options, vitestProjects ) { args.push( options.watch ? '--watch' : '--run' ); if ( options.coverage ) { - const coverageDir = upath.join( process.cwd(), VITEST_COVERAGE_DIRECTORY ); + const coverageDir = upath.join( process.cwd(), 'coverage-vitest' ); args.push( '--coverage', '--coverage.reportsDirectory', coverageDir ); } @@ -299,44 +293,6 @@ function spawnVitest( options, vitestProjects ) { } ); } -// -- Coverage merging ----------------------------------------------------------------------------- - -function mergeCoverageReports( hasKarmaResults, hasVitestResults ) { - const coverageDir = upath.join( process.cwd(), 'coverage' ); - const mergedFilePath = upath.join( coverageDir, 'lcov.info' ); - const chunks = []; - - if ( hasKarmaResults ) { - const karmaCoverageFiles = globSync( upath.join( coverageDir, '**', 'lcov.info' ) ) - .map( f => upath.normalize( f ) ) - .filter( f => f !== upath.normalize( mergedFilePath ) ); - - for ( const file of karmaCoverageFiles ) { - chunks.push( fs.readFileSync( file, 'utf8' ) ); - } - } - - if ( hasVitestResults ) { - const vitestCoverageFile = upath.join( process.cwd(), VITEST_COVERAGE_DIRECTORY, 'lcov.info' ); - - if ( fs.existsSync( vitestCoverageFile ) ) { - const content = fs.readFileSync( vitestCoverageFile, 'utf8' ); - const cwdPrefix = upath.normalize( process.cwd() ) + '/'; - - // Vitest reports absolute SF: paths — strip the cwd prefix to make them - // relative, matching Karma's output format. - chunks.push( content.replaceAll( `SF:${ cwdPrefix }`, 'SF:' ) ); - } - } - - if ( !chunks.length ) { - return; - } - - mkdirp.sync( coverageDir ); - fs.writeFileSync( mergedFilePath, chunks.join( '\n' ) ); -} - // -- Error handling ------------------------------------------------------------------------------- function aggregateErrors( errors ) { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 4fd6a23f7..dd840395f 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -767,95 +767,6 @@ describe( 'runAutomatedTests()', () => { await expect( promise ).rejects.toThrow( /Test execution failed in multiple runners/ ); } ); - // -- Coverage merging tests ------------------------------------------------------------------- - - it( 'should merge coverage for mixed Karma + Vitest run', async () => { - const options = { - files: [ '*' ], - production: true, - coverage: true, - watch: false - }; - - vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ - '/workspace/packages/ckeditor5-engine/tests/**/*.js', - '/workspace/packages/ckeditor5-utils/tests/**/*.js' - ] ); - vi.mocked( fs ).readdirSync.mockReturnValue( [] ); - vi.mocked( globSync ) - .mockReturnValueOnce( [ - '/workspace/packages/ckeditor5-engine/tests/model/model.js' - ] ) - .mockReturnValueOnce( [ - '/workspace/packages/ckeditor5-utils/tests/first.js' - ] ) - .mockReturnValueOnce( [ - '/workspace/coverage/ChromeHeadless 146.0.0.0 (Mac OS 10.15.7)/lcov.info' - ] ); - vi.mocked( fs ).readFileSync.mockImplementation( path => { - if ( path.includes( 'ckeditor5-engine/package.json' ) ) { - return JSON.stringify( { scripts: { test: 'vitest run' } } ); - } - - if ( path.includes( 'ckeditor5-utils/package.json' ) ) { - return JSON.stringify( { scripts: { test: 'karma start' } } ); - } - - if ( path === '/workspace/coverage/ChromeHeadless 146.0.0.0 (Mac OS 10.15.7)/lcov.info' ) { - return 'TN:karma\nSF:/workspace/packages/ckeditor5-utils/src/index.js\nend_of_record\n'; - } - - if ( path === '/workspace/coverage-vitest/lcov.info' ) { - return 'TN:vitest\nSF:/workspace/packages/ckeditor5-engine/src/index.js\nend_of_record\n'; - } - - return '{}'; - } ); - vi.mocked( fs ).existsSync.mockImplementation( path => path === '/workspace/coverage-vitest/lcov.info' ); - - const promise = runAutomatedTests( options ); - - setTimeout( () => { - const [ firstCall ] = stubs.karma.server.constructor.mock.calls; - const [ , exitCallback ] = firstCall; - - exitCallback( 0 ); - - setTimeout( () => { - const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); - - subprocess.emit( 'close', 0 ); - } ); - } ); - - await promise; - - expect( stubs.karma.server.constructor ).toHaveBeenCalledOnce(); - expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( - 'pnpm', - [ - 'vitest', - '--run', - '--coverage', - '--coverage.reportsDirectory', - '/workspace/coverage-vitest', - '--project', - 'engine' - ], - { stdio: 'inherit', cwd: '/workspace', shell: false } - ); - - // Verify merged coverage: Vitest absolute paths are converted to relative. - expect( vi.mocked( fs ).writeFileSync ).toHaveBeenCalledWith( - '/workspace/coverage/lcov.info', - expect.stringContaining( 'TN:karma' ) - ); - expect( vi.mocked( fs ).writeFileSync ).toHaveBeenCalledWith( - '/workspace/coverage/lcov.info', - expect.stringContaining( 'SF:packages/ckeditor5-engine/src/index.js' ) - ); - } ); - // -- Multiple Vitest projects test ------------------------------------------------------------ it( 'should run all Vitest projects in a single process from cwd', async () => { From 3e191a133520f9a391190a43f73fa63f3a2fb638 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Tue, 17 Mar 2026 10:51:05 +0100 Subject: [PATCH 03/15] Fixed runner-agnostic error message in resolveTestGlobs(). --- packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js | 2 +- packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 1819479ca..d1d3e6f15 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -76,7 +76,7 @@ export default async function runAutomatedTests( options ) { function resolveTestGlobs( files ) { if ( !Array.isArray( files ) || files.length === 0 ) { - throw new Error( 'Karma requires files to tests. `options.files` has to be non-empty array.' ); + throw new Error( 'Test runner requires files to test. `options.files` has to be a non-empty array.' ); } const globMap = {}; diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index dd840395f..e669f5265 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -154,7 +154,7 @@ describe( 'runAutomatedTests()', () => { it( 'throws when files are not specified', async () => { await expect( runAutomatedTests( { production: true } ) ) - .rejects.toThrow( 'Karma requires files to tests. `options.files` has to be non-empty array.' ); + .rejects.toThrow( 'Test runner requires files to test. `options.files` has to be a non-empty array.' ); } ); it( 'throws when specified files are invalid', async () => { From d6a8587136f02b52e9bc199a3426a5feec56eae2 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Tue, 17 Mar 2026 10:55:07 +0100 Subject: [PATCH 04/15] Fixed grammar in the "no test files found" error message. --- packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js | 2 +- packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index d1d3e6f15..d2ad78b96 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -39,7 +39,7 @@ export default async function runAutomatedTests( options ) { const { karmaFiles, vitestProjects } = partitionByRunner( testFiles ); if ( !karmaFiles.length && !vitestProjects.length ) { - throw new Error( 'Not found files to tests. Specified patterns are invalid.' ); + throw new Error( 'No test files found. Specified patterns are invalid.' ); } if ( karmaFiles.length && vitestProjects.length && options.watch ) { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index e669f5265..e012cb509 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -181,7 +181,7 @@ describe( 'runAutomatedTests()', () => { vi.mocked( globSync ).mockReturnValue( [] ); await expect( runAutomatedTests( options ) ) - .rejects.toThrow( 'Not found files to tests. Specified patterns are invalid.' ); + .rejects.toThrow( 'No test files found. Specified patterns are invalid.' ); expect( stubs.log.warn ).toHaveBeenCalledTimes( 2 ); expect( stubs.log.warn ).toHaveBeenCalledWith( 'Pattern "%s" does not match any file.', 'basic-foo' ); From 4454c7118dd0c480584b93499d340c916df9f874 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Tue, 17 Mar 2026 11:35:59 +0100 Subject: [PATCH 05/15] Fixed platform-dependent shell assertion in spawn tests. --- .../ckeditor5-dev-tests/tests/tasks/runautomatedtests.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index e012cb509..9ae88b575 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -535,7 +535,7 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', [ 'vitest', '--run', '--project', 'engine' ], - { stdio: 'inherit', cwd: '/workspace', shell: false } + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); expect( vi.mocked( fs ).writeFileSync ).not.toHaveBeenCalledWith( '/workspace/build/.automated-tests/entry-point.js', @@ -690,7 +690,7 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', [ 'vitest', '--run', '--project', 'utils' ], - { stdio: 'inherit', cwd: '/workspace', shell: false } + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); } ); @@ -815,7 +815,7 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', [ 'vitest', '--run', '--project', 'utils', '--project', 'engine' ], - { stdio: 'inherit', cwd: '/workspace', shell: false } + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); } ); } ); From ff13b524dd64779af08ca90ed78d944423ebde53 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Tue, 17 Mar 2026 11:38:40 +0100 Subject: [PATCH 06/15] Removed stale mock setup referencing removed mergeCoverageReport. --- .../tests/tasks/runautomatedtests.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 9ae88b575..22729dee2 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -587,16 +587,12 @@ describe( 'runAutomatedTests()', () => { vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ '/workspace/packages/ckeditor5-engine/tests/**/*.js' ] ); - vi.mocked( globSync ) - .mockReturnValueOnce( [ - '/workspace/packages/ckeditor5-engine/tests/model/model.js' - ] ) - // For mergeCoverageReports — no Karma coverage files. - .mockReturnValue( [] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { scripts: { test: 'vitest --run' } } ) ); - vi.mocked( fs ).existsSync.mockReturnValue( false ); const promise = runAutomatedTests( options ); await new Promise( resolve => setTimeout( resolve ) ); From bc86463c654d954147537389d68b80dbb6949d69 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Tue, 17 Mar 2026 13:08:57 +0100 Subject: [PATCH 07/15] Reintroduced a `-f` compound file support for Vitest. --- .../lib/tasks/runautomatedtests.js | 32 ++++++---- .../tests/tasks/runautomatedtests.js | 63 ++++++++++++++++--- 2 files changed, 74 insertions(+), 21 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index d2ad78b96..6bbafc714 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -36,13 +36,13 @@ export default async function runAutomatedTests( options ) { const globPatterns = resolveTestGlobs( options.files ); const testFiles = collectTestFiles( globPatterns ); - const { karmaFiles, vitestProjects } = partitionByRunner( testFiles ); + const { karmaFiles, vitestSelection } = partitionByRunner( testFiles ); - if ( !karmaFiles.length && !vitestProjects.length ) { + if ( !karmaFiles.length && !vitestSelection.length ) { throw new Error( 'No test files found. Specified patterns are invalid.' ); } - if ( karmaFiles.length && vitestProjects.length && options.watch ) { + if ( karmaFiles.length && vitestSelection.length && options.watch ) { throw new Error( 'Watch mode cannot be used in a mixed Karma + Vitest run. ' + 'Run watch mode separately for Karma and Vitest packages.' @@ -59,9 +59,9 @@ export default async function runAutomatedTests( options ) { } } - if ( vitestProjects.length ) { + if ( vitestSelection.length ) { try { - await spawnVitest( options, vitestProjects ); + await spawnVitest( options, vitestSelection ); } catch ( error ) { errors.push( error ); } @@ -121,7 +121,7 @@ function collectTestFiles( globPatterns ) { function partitionByRunner( testFiles ) { const karmaFiles = []; - const vitestProjects = new Set(); + const vitestSelection = new Map(); const runnerCache = new Map(); for ( const filePath of testFiles ) { @@ -134,13 +134,15 @@ function partitionByRunner( testFiles ) { const { runner, projectName } = runnerCache.get( packageRoot ); if ( runner === 'vitest' ) { - vitestProjects.add( projectName ); + const files = vitestSelection.get( projectName ) || []; + files.push( filePath ); + vitestSelection.set( projectName, files ); } else { karmaFiles.push( filePath ); } } - return { karmaFiles, vitestProjects: [ ...vitestProjects ] }; + return { karmaFiles, vitestSelection: [ ...vitestSelection.entries() ] }; } function getPackageRoot( filePath ) { @@ -260,7 +262,13 @@ function startKarmaServer( options ) { // -- Vitest runner -------------------------------------------------------------------------------- -function spawnVitest( options, vitestProjects ) { +async function spawnVitest( options, vitestSelection ) { + for ( const [ project, selectedFiles ] of vitestSelection ) { + await spawnVitestProject( options, project, selectedFiles ); + } +} + +function spawnVitestProject( options, project, selectedFiles ) { return new Promise( ( resolve, reject ) => { const args = [ 'vitest' ]; @@ -271,8 +279,10 @@ function spawnVitest( options, vitestProjects ) { args.push( '--coverage', '--coverage.reportsDirectory', coverageDir ); } - for ( const project of vitestProjects ) { - args.push( '--project', project ); + args.push( '--project', project ); + + for ( const filePath of selectedFiles ) { + args.push( upath.relative( process.cwd(), filePath ) ); } const child = spawn( 'pnpm', args, { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 22729dee2..71cbe122c 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -534,7 +534,13 @@ describe( 'runAutomatedTests()', () => { expect( stubs.karma.server.constructor ).not.toHaveBeenCalled(); expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', - [ 'vitest', '--run', '--project', 'engine' ], + [ + 'vitest', + '--run', + '--project', + 'engine', + 'packages/ckeditor5-engine/tests/model/model.js' + ], { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); expect( vi.mocked( fs ).writeFileSync ).not.toHaveBeenCalledWith( @@ -571,7 +577,13 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', - [ 'vitest', '--watch', '--project', 'engine' ], + [ + 'vitest', + '--watch', + '--project', + 'engine', + 'packages/ckeditor5-engine/tests/model/model.js' + ], expect.any( Object ) ); } ); @@ -604,7 +616,16 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', - [ 'vitest', '--run', '--coverage', '--coverage.reportsDirectory', '/workspace/coverage-vitest', '--project', 'engine' ], + [ + 'vitest', + '--run', + '--coverage', + '--coverage.reportsDirectory', + '/workspace/coverage-vitest', + '--project', + 'engine', + 'packages/ckeditor5-engine/tests/model/model.js' + ], expect.any( Object ) ); } ); @@ -685,7 +706,7 @@ describe( 'runAutomatedTests()', () => { expect( stubs.karma.server.constructor ).toHaveBeenCalledOnce(); expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( 'pnpm', - [ 'vitest', '--run', '--project', 'utils' ], + [ 'vitest', '--run', '--project', 'utils', 'packages/ckeditor5-utils/tests/first.js' ], { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); } ); @@ -765,7 +786,7 @@ describe( 'runAutomatedTests()', () => { // -- Multiple Vitest projects test ------------------------------------------------------------ - it( 'should run all Vitest projects in a single process from cwd', async () => { + it( 'should run each Vitest project in a separate process with selected files', async () => { const options = { files: [ 'utils', 'engine' ], production: true, @@ -802,15 +823,37 @@ describe( 'runAutomatedTests()', () => { const promise = runAutomatedTests( options ); await new Promise( resolve => setTimeout( resolve ) ); - const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); - subprocess.emit( 'close', 0 ); + const [ firstSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + firstSubprocess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , secondSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + secondSubprocess.emit( 'close', 0 ); await promise; - // All Vitest projects should be passed to a single process spawned from cwd. - expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 1, + 'pnpm', + [ + 'vitest', + '--run', + '--project', + 'utils', + 'external/ckeditor5/packages/ckeditor5-utils/tests/first.js' + ], + { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } + ); + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 2, 'pnpm', - [ 'vitest', '--run', '--project', 'utils', '--project', 'engine' ], + [ + 'vitest', + '--run', + '--project', + 'engine', + 'external/ckeditor5/packages/ckeditor5-engine/tests/model.js' + ], { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); } ); From e7e72c1069a4c92f778a8c22110094dc6075389f Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 10:19:22 +0100 Subject: [PATCH 08/15] Added a changeset file. --- .changelog/20260318101839_i_4307_test_wrapper_v2.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .changelog/20260318101839_i_4307_test_wrapper_v2.md diff --git a/.changelog/20260318101839_i_4307_test_wrapper_v2.md b/.changelog/20260318101839_i_4307_test_wrapper_v2.md new file mode 100644 index 000000000..0dab99069 --- /dev/null +++ b/.changelog/20260318101839_i_4307_test_wrapper_v2.md @@ -0,0 +1,9 @@ +--- +type: Feature +scope: + - ckeditor5-dev-tests +--- + +Enabled running mixed Vitest and Karma tests in a single `runAutomatedTests()` invocation. The test runner now partitions test files accordingly, and executes both runners sequentially. + +Watch mode is restricted to single-runner selections to avoid interleaved output. From 8ef13cb1936720ffd7e2d583c377fbaeac1346f3 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 10:28:24 +0100 Subject: [PATCH 09/15] Added interrupt singnal handling for vitest watch mode and cleanup. --- packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 6bbafc714..d310ffd8f 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -159,7 +159,7 @@ function getPackageRoot( filePath ) { function detectPackageRunner( packageRoot ) { const projectName = upath.basename( packageRoot ).replace( /^ckeditor5-/, '' ); const packageJson = JSON.parse( fs.readFileSync( upath.join( packageRoot, 'package.json' ), 'utf8' ) ); - const runner = /\bvitest\b/.test( packageJson?.scripts?.test || '' ) ? 'vitest' : 'karma'; + const runner = packageJson.scripts?.test?.includes( 'vitest' ) ? 'vitest' : 'karma'; return { projectName, runner }; } @@ -294,7 +294,7 @@ function spawnVitestProject( options, project, selectedFiles ) { child.on( 'error', reject ); child.on( 'close', exitCode => { - if ( exitCode === 0 ) { + if ( exitCode === 0 || exitCode === 130 ) { resolve(); } else { reject( new Error( `Vitest finished with "${ exitCode }" code.` ) ); From 964ce09fff6302582849924d3e96420901d91560 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 12:19:10 +0100 Subject: [PATCH 10/15] Added producing merged coverage report for Vitest. --- .../lib/tasks/runautomatedtests.js | 52 +++- .../tests/tasks/runautomatedtests.js | 233 +++++++++++++++++- 2 files changed, 279 insertions(+), 6 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index d310ffd8f..a5eb908f7 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -266,6 +266,10 @@ async function spawnVitest( options, vitestSelection ) { for ( const [ project, selectedFiles ] of vitestSelection ) { await spawnVitestProject( options, project, selectedFiles ); } + + if ( options.coverage ) { + await mergeVitestCoverage( vitestSelection ); + } } function spawnVitestProject( options, project, selectedFiles ) { @@ -275,7 +279,7 @@ function spawnVitestProject( options, project, selectedFiles ) { args.push( options.watch ? '--watch' : '--run' ); if ( options.coverage ) { - const coverageDir = upath.join( process.cwd(), 'coverage-vitest' ); + const coverageDir = upath.join( process.cwd(), 'coverage-vitest', project ); args.push( '--coverage', '--coverage.reportsDirectory', coverageDir ); } @@ -303,6 +307,52 @@ function spawnVitestProject( options, project, selectedFiles ) { } ); } +function mergeVitestCoverage( vitestSelection ) { + const cwd = process.cwd(); + const coverageBaseDir = upath.join( cwd, 'coverage-vitest' ); + const nycOutputDir = upath.join( coverageBaseDir, '.nyc_output' ); + + mkdirp.sync( nycOutputDir ); + + // Copy each project's coverage-final.json into .nyc_output/ so nyc can merge them. + for ( const [ project ] of vitestSelection ) { + const sourceFile = upath.join( coverageBaseDir, project, 'coverage-final.json' ); + + if ( fs.existsSync( sourceFile ) ) { + fs.copyFileSync( sourceFile, upath.join( nycOutputDir, `${ project }.json` ) ); + } + } + + const log = logger(); + + return new Promise( ( resolve, reject ) => { + const child = spawn( 'pnpx', [ + 'nyc', 'report', + '--temp-dir', nycOutputDir, + '--report-dir', coverageBaseDir, + '--reporter', 'html', + '--reporter', 'json', + '--reporter', 'lcovonly', + '--reporter', 'text-summary' + ], { + stdio: 'inherit', + cwd, + shell: process.platform === 'win32' + } ); + + child.on( 'error', reject ); + + child.on( 'close', exitCode => { + if ( exitCode === 0 ) { + log.info( `Combined Vitest coverage report saved in '${ styleText( 'cyan', coverageBaseDir ) }'.` ); + resolve(); + } else { + reject( new Error( `nyc report finished with "${ exitCode }" code.` ) ); + } + } ); + } ); +} + // -- Error handling ------------------------------------------------------------------------------- function aggregateErrors( errors ) { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 71cbe122c..1b1269cd8 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -30,6 +30,11 @@ const stubs = vi.hoisted( () => ( { }, spawn: { call: vi.fn() + }, + devUtilsLogger: { + info: vi.fn(), + warning: vi.fn(), + error: vi.fn() } } ) ); @@ -81,6 +86,9 @@ vi.mock( 'node:fs' ); vi.mock( 'mkdirp' ); vi.mock( 'glob' ); vi.mock( 'karma/lib/logger.js' ); +vi.mock( '@ckeditor/ckeditor5-dev-utils', () => ( { + logger: vi.fn( () => stubs.devUtilsLogger ) +} ) ); vi.mock( '../../lib/utils/automated-tests/getkarmaconfig.js' ); vi.mock( '../../lib/utils/transformfileoptiontotestglob.js' ); @@ -588,7 +596,7 @@ describe( 'runAutomatedTests()', () => { ); } ); - it( 'should pass coverage flags to Vitest when coverage is enabled', async () => { + it( 'should pass coverage flags to Vitest and merge coverage with nyc', async () => { const options = { files: [ 'engine' ], production: true, @@ -605,29 +613,67 @@ describe( 'runAutomatedTests()', () => { vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { scripts: { test: 'vitest --run' } } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( true ); const promise = runAutomatedTests( options ); await new Promise( resolve => setTimeout( resolve ) ); - const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); - subprocess.emit( 'close', 0 ); + // First spawn: vitest project run. + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + // Second spawn: nyc report merge. + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'close', 0 ); await promise; - expect( stubs.spawn.call ).toHaveBeenCalledExactlyOnceWith( + // Vitest was called with per-project coverage directory. + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 1, 'pnpm', [ 'vitest', '--run', '--coverage', '--coverage.reportsDirectory', - '/workspace/coverage-vitest', + '/workspace/coverage-vitest/engine', '--project', 'engine', 'packages/ckeditor5-engine/tests/model/model.js' ], expect.any( Object ) ); + + // coverage-final.json was copied into .nyc_output. + expect( vi.mocked( fs ).existsSync ).toHaveBeenCalledWith( + '/workspace/coverage-vitest/engine/coverage-final.json' + ); + expect( vi.mocked( fs ).copyFileSync ).toHaveBeenCalledWith( + '/workspace/coverage-vitest/engine/coverage-final.json', + '/workspace/coverage-vitest/.nyc_output/engine.json' + ); + + // nyc report was called with correct reporters. + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 2, + 'pnpx', + [ + 'nyc', 'report', + '--temp-dir', '/workspace/coverage-vitest/.nyc_output', + '--report-dir', '/workspace/coverage-vitest', + '--reporter', 'html', + '--reporter', 'json', + '--reporter', 'lcovonly', + '--reporter', 'text-summary' + ], + expect.objectContaining( { stdio: 'inherit', cwd: '/workspace' } ) + ); + + // Log message was printed. + expect( stubs.devUtilsLogger.info ).toHaveBeenCalled(); } ); it( 'should reject when Vitest process exits with non-zero code', async () => { @@ -857,4 +903,181 @@ describe( 'runAutomatedTests()', () => { { stdio: 'inherit', cwd: '/workspace', shell: process.platform === 'win32' } ); } ); + + // -- Edge cases ------------------------------------------------------------------------------- + + it( 'should resolve when Vitest exits with code 130 (SIGINT)', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'close', 130 ); + + await promise; + } ); + + it( 'should reject when spawn emits an error event', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ subprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + subprocess.emit( 'error', new Error( 'spawn ENOENT' ) ); + + await expect( promise ).rejects.toThrow( 'spawn ENOENT' ); + } ); + + it( 'should skip copying coverage-final.json when the file does not exist', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'close', 0 ); + + await promise; + + expect( vi.mocked( fs ).copyFileSync ).not.toHaveBeenCalled(); + } ); + + it( 'should reject when nyc report fails', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'close', 1 ); + + await expect( promise ).rejects.toThrow( 'nyc report finished with "1" code.' ); + } ); + + it( 'should reject when nyc spawn emits an error', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/model/model.js' + ] ); + vi.mocked( fs ).readFileSync.mockReturnValue( JSON.stringify( { + scripts: { test: 'vitest --run' } + } ) ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ vitestProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + vitestProcess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + + const [ , nycProcess ] = vi.mocked( spawn ).mock.results.map( r => r.value ); + nycProcess.emit( 'error', new Error( 'nyc ENOENT' ) ); + + await expect( promise ).rejects.toThrow( 'nyc ENOENT' ); + } ); + + it( 'should throw when a test file path does not contain /tests/ segment', async () => { + const options = { + files: [ 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ).mockReturnValue( [ + '/workspace/packages/ckeditor5-engine/src/model.js' + ] ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Cannot determine package root for "/workspace/packages/ckeditor5-engine/src/model.js".' + ); + } ); } ); From a818470b3d44d46884cecf4022cc6d58cae9f573 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 12:42:12 +0100 Subject: [PATCH 11/15] Fixed Vitest watch hangs per project sequentially. --- .../lib/tasks/runautomatedtests.js | 7 ++++ .../tests/tasks/runautomatedtests.js | 42 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index a5eb908f7..e13ed6db8 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -49,6 +49,13 @@ export default async function runAutomatedTests( options ) { ); } + if ( options.watch && vitestSelection.length > 1 ) { + throw new Error( + 'Watch mode cannot be used for multiple Vitest projects in one run. ' + + 'Run watch mode separately for each Vitest project.' + ); + } + const errors = []; if ( karmaFiles.length ) { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 1b1269cd8..5606ad779 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -904,6 +904,48 @@ describe( 'runAutomatedTests()', () => { ); } ); + it( 'should throw when watch mode is used with multiple Vitest projects', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: true, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Watch mode cannot be used for multiple Vitest projects in one run. ' + + 'Run watch mode separately for each Vitest project.' + ); + + expect( stubs.spawn.call ).not.toHaveBeenCalled(); + } ); + // -- Edge cases ------------------------------------------------------------------------------- it( 'should resolve when Vitest exits with code 130 (SIGINT)', async () => { From 25dce5bc6767c6533bf72606441c04dc353f452d Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 12:44:52 +0100 Subject: [PATCH 12/15] Fixed Vitest stops after first project failure. --- .../lib/tasks/runautomatedtests.js | 17 ++++++- .../tests/tasks/runautomatedtests.js | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index e13ed6db8..21a85c958 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -270,8 +270,23 @@ function startKarmaServer( options ) { // -- Vitest runner -------------------------------------------------------------------------------- async function spawnVitest( options, vitestSelection ) { + const errors = []; + for ( const [ project, selectedFiles ] of vitestSelection ) { - await spawnVitestProject( options, project, selectedFiles ); + try { + await spawnVitestProject( options, project, selectedFiles ); + } catch ( error ) { + errors.push( error ); + } + } + + if ( errors.length ) { + if ( errors.length === 1 ) { + throw errors[ 0 ]; + } + + const details = errors.map( e => `- ${ e.message }` ).join( '\n' ); + throw new Error( `Vitest execution failed in multiple projects:\n${ details }` ); } if ( options.coverage ) { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 5606ad779..859318d41 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -946,6 +946,54 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).not.toHaveBeenCalled(); } ); + it( 'should continue running remaining Vitest projects after a project failure', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: false, + coverage: false + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ firstSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + firstSubprocess.emit( 'close', 1 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , secondSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + secondSubprocess.emit( 'close', 0 ); + + await expect( promise ).rejects.toThrow( 'Vitest finished with "1" code.' ); + expect( stubs.spawn.call ).toHaveBeenCalledTimes( 2 ); + } ); + // -- Edge cases ------------------------------------------------------------------------------- it( 'should resolve when Vitest exits with code 130 (SIGINT)', async () => { From 23a3e149f52f8b1cc1abfecb9125d08d3381f737 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 13:16:27 +0100 Subject: [PATCH 13/15] Fixed Vitest coverage flags conflict in CLI. --- packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js | 2 +- packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 21a85c958..257e48d26 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -302,7 +302,7 @@ function spawnVitestProject( options, project, selectedFiles ) { if ( options.coverage ) { const coverageDir = upath.join( process.cwd(), 'coverage-vitest', project ); - args.push( '--coverage', '--coverage.reportsDirectory', coverageDir ); + args.push( '--coverage.enabled', '--coverage.reportsDirectory', coverageDir ); } args.push( '--project', project ); diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 859318d41..6ac35f231 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -637,7 +637,7 @@ describe( 'runAutomatedTests()', () => { [ 'vitest', '--run', - '--coverage', + '--coverage.enabled', '--coverage.reportsDirectory', '/workspace/coverage-vitest/engine', '--project', From f62392c36ec6c196a787235d1a96c5fc118abfbd Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Wed, 18 Mar 2026 14:01:53 +0100 Subject: [PATCH 14/15] Fixed Mixed server mode hangs before Vitest execution. --- .../lib/tasks/runautomatedtests.js | 6 ++-- .../tests/tasks/runautomatedtests.js | 36 +++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 257e48d26..45acf76b9 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -42,10 +42,10 @@ export default async function runAutomatedTests( options ) { throw new Error( 'No test files found. Specified patterns are invalid.' ); } - if ( karmaFiles.length && vitestSelection.length && options.watch ) { + if ( karmaFiles.length && vitestSelection.length && ( options.watch || options.server ) ) { throw new Error( - 'Watch mode cannot be used in a mixed Karma + Vitest run. ' + - 'Run watch mode separately for Karma and Vitest packages.' + 'Watch/server mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch/server mode separately for Karma and Vitest packages.' ); } diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 6ac35f231..8ab59eb55 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -784,8 +784,40 @@ describe( 'runAutomatedTests()', () => { } ); await expect( runAutomatedTests( options ) ).rejects.toThrow( - 'Watch mode cannot be used in a mixed Karma + Vitest run. ' + - 'Run watch mode separately for Karma and Vitest packages.' + 'Watch/server mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch/server mode separately for Karma and Vitest packages.' + ); + } ); + + it( 'should throw when server mode is used with mixed Karma + Vitest packages', async () => { + const options = { + files: [ 'utils', 'emoji' ], + production: true, + coverage: false, + server: true + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/**/*.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/**/*.js' ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [ '/workspace/packages/ckeditor5-emoji/tests/emoji.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-emoji/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'karma start' } } ); + } + + return '{}'; + } ); + + await expect( runAutomatedTests( options ) ).rejects.toThrow( + 'Watch/server mode cannot be used in a mixed Karma + Vitest run. ' + + 'Run watch/server mode separately for Karma and Vitest packages.' ); } ); From cb6456c9219630c35b398adf32fe2a392d99ae31 Mon Sep 17 00:00:00 2001 From: Marcin Panek Date: Thu, 19 Mar 2026 09:35:01 +0100 Subject: [PATCH 15/15] Fixed Vitest coverage skipped on failed tests. --- .../lib/tasks/runautomatedtests.js | 12 ++-- .../tests/tasks/runautomatedtests.js | 59 +++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js index 45acf76b9..0df8f14af 100644 --- a/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/lib/tasks/runautomatedtests.js @@ -280,6 +280,14 @@ async function spawnVitest( options, vitestSelection ) { } } + if ( options.coverage ) { + try { + await mergeVitestCoverage( vitestSelection ); + } catch ( error ) { + errors.push( error ); + } + } + if ( errors.length ) { if ( errors.length === 1 ) { throw errors[ 0 ]; @@ -288,10 +296,6 @@ async function spawnVitest( options, vitestSelection ) { const details = errors.map( e => `- ${ e.message }` ).join( '\n' ); throw new Error( `Vitest execution failed in multiple projects:\n${ details }` ); } - - if ( options.coverage ) { - await mergeVitestCoverage( vitestSelection ); - } } function spawnVitestProject( options, project, selectedFiles ) { diff --git a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js index 8ab59eb55..1428c0863 100644 --- a/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js +++ b/packages/ckeditor5-dev-tests/tests/tasks/runautomatedtests.js @@ -1026,6 +1026,65 @@ describe( 'runAutomatedTests()', () => { expect( stubs.spawn.call ).toHaveBeenCalledTimes( 2 ); } ); + it( 'should merge Vitest coverage even when a project fails', async () => { + const options = { + files: [ 'utils', 'engine' ], + production: true, + watch: false, + coverage: true + }; + + vi.mocked( transformFileOptionToTestGlob ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-utils/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/**/*.js' + ] ) + .mockReturnValueOnce( [ + '/workspace/packages/ckeditor5-engine/tests/**/*.js', + '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/**/*.js' + ] ); + vi.mocked( globSync ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-utils/tests/first.js' ] ) + .mockReturnValueOnce( [] ) + .mockReturnValueOnce( [ '/workspace/external/ckeditor5/packages/ckeditor5-engine/tests/model.js' ] ); + vi.mocked( fs ).readFileSync.mockImplementation( path => { + if ( path.includes( 'ckeditor5-utils/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + if ( path.includes( 'ckeditor5-engine/package.json' ) ) { + return JSON.stringify( { scripts: { test: 'vitest run' } } ); + } + + return '{}'; + } ); + vi.mocked( fs ).existsSync.mockReturnValue( false ); + + const promise = runAutomatedTests( options ); + await new Promise( resolve => setTimeout( resolve ) ); + + const [ firstSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + firstSubprocess.emit( 'close', 1 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , secondSubprocess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + secondSubprocess.emit( 'close', 0 ); + + await new Promise( resolve => setTimeout( resolve ) ); + const [ , , nycProcess ] = vi.mocked( spawn ).mock.results.map( result => result.value ); + nycProcess.emit( 'close', 0 ); + + await expect( promise ).rejects.toThrow( 'Vitest finished with "1" code.' ); + expect( stubs.spawn.call ).toHaveBeenCalledTimes( 3 ); + expect( stubs.spawn.call ).toHaveBeenNthCalledWith( + 3, + 'pnpx', + expect.arrayContaining( [ 'nyc', 'report' ] ), + expect.objectContaining( { cwd: '/workspace' } ) + ); + } ); + // -- Edge cases ------------------------------------------------------------------------------- it( 'should resolve when Vitest exits with code 130 (SIGINT)', async () => {