From c1ec013fe230d0258bc0915634d248feb84e298a Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 14 Jul 2026 10:09:30 +0200 Subject: [PATCH 1/7] add-headless-option --- tests/helper.mjs | 35 +++++++++++++++++++++++++++-------- tests/run-end2end.mjs | 4 ++-- tests/run-unittests.mjs | 4 ++-- tests/server.mjs | 39 ++++++++++++++++++++++++++------------- 4 files changed, 57 insertions(+), 25 deletions(-) diff --git a/tests/helper.mjs b/tests/helper.mjs index fb25cbc3a..d95fe6e87 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -13,8 +13,9 @@ export const DEFAULT_RETRIES = 1; const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, - { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." }, + { name: "port", type: Number, defaultValue: 0, description: "Set the test-server port. The default value is 0 (dynamic port)." }, { name: "retry", type: Number, defaultValue: DEFAULT_RETRIES, description: "Number of retries for the tests on failure." }, + { name: "headless", type: Boolean, description: "Run browser in headless mode. Automatically enabled on Linux when $DISPLAY and $WAYLAND_DISPLAY are unset." }, { name: "help", alias: "h", description: "Print this help text." }, ]; @@ -36,19 +37,25 @@ function printHelp(message = "", exitStatus = 0) { process.exit(exitStatus); } +export function detectHeadless(options) { + return Boolean(options?.headless || (process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY)); +} + export default async function testSetup(helpText) { const options = commandLineArgs(optionDefinitions); if ("help" in options) printHelp(helpText); - const BROWSER = options?.browser; + const BROWSER = options?.browser || process.env.BROWSER; if (!BROWSER) printHelp("No browser specified, use $BROWSER or --browser", 1); if (options.retry < 0) printHelp("Number of retries cannot be negative", 1); + const isHeadless = detectHeadless(options); + let builder; switch (BROWSER) { case "safari": { @@ -57,23 +64,34 @@ export default async function testSetup(helpText) { break; } case "firefox": { - builder = new Builder().forBrowser(BROWSER).setFirefoxOptions(new firefox.Options().enableBidi()); + const firefoxOptions = new firefox.Options().enableBidi(); + if (isHeadless) + firefoxOptions.addArguments("--headless"); + + builder = new Builder().forBrowser(BROWSER).setFirefoxOptions(firefoxOptions); break; } case "chrome": { - builder = new Builder().forBrowser(BROWSER).setChromeOptions(new chrome.Options().enableBidi()); + const chromeOptions = new chrome.Options().enableBidi(); + if (isHeadless) + chromeOptions.addArguments("--headless"); + + builder = new Builder().forBrowser(BROWSER).setChromeOptions(chromeOptions); break; } case "edge": { - builder = new Builder().forBrowser(BROWSER).setEdgeOptions(new edge.Options().enableBidi()); + const edgeOptions = new edge.Options().enableBidi(); + if (isHeadless) + edgeOptions.addArguments("--headless"); + + builder = new Builder().forBrowser(BROWSER).setEdgeOptions(edgeOptions); break; } default: { printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); } } - const PORT = options.port; - const server = await serve(PORT); + const { server, port } = await serve(options.port); let driver, logInspector; process.on("unhandledRejection", (err) => { @@ -100,8 +118,9 @@ export default async function testSetup(helpText) { server.close(); if (logInspector) await logInspector.close(); + if (driver) driver.close(); } - return { driver, PORT, stop, retry: options.retry }; + return { driver, port, stop, retry: options.retry }; } diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index cdb81ebb4..d36b3b18d 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -9,13 +9,13 @@ This script runs end2end tests by invoking the benchmark via the main Speedometer page in /index.html. `.trim(); -const { driver, PORT, stop, retry } = await testSetup(HELP); +const { driver, port, stop, retry } = await testSetup(HELP); const suites = benchmarkConfigurator.suites; async function testPage(url) { console.log(`Testing: ${url}`); - await driver.get(`http://localhost:${PORT}/${url}`); + await driver.get(`http://localhost:${port}/${url}`); await driver.executeAsyncScript((callback) => { if (globalThis.benchmarkClient) diff --git a/tests/run-unittests.mjs b/tests/run-unittests.mjs index 5fd7fb86d..f80b1f418 100644 --- a/tests/run-unittests.mjs +++ b/tests/run-unittests.mjs @@ -8,11 +8,11 @@ This script runs the unittests located in tests/unittests/* through the mocha web interface located at tests/index.html `.trim(); -const { driver, PORT, stop } = await testSetup(HELP); +const { driver, port, stop } = await testSetup(HELP); async function test() { try { - await driver.get(`http://localhost:${PORT}/tests/index.html`); + await driver.get(`http://localhost:${port}/tests/index.html`); const { testResults, stats } = await driver.executeAsyncScript(function (callback) { const returnResults = () => diff --git a/tests/server.mjs b/tests/server.mjs index 4101d5c80..c28a287c3 100644 --- a/tests/server.mjs +++ b/tests/server.mjs @@ -12,8 +12,9 @@ import "lws-static"; const ROOT_DIR = path.join(process.cwd(), "./"); export default async function serve(port) { - if (!port) + if (!port && port !== 0) throw new Error("Port is required"); + const ws = await LocalWebServer.create({ port: port, directory: ROOT_DIR, @@ -22,30 +23,42 @@ export default async function serve(port) { logFormat: "dev", stack: ["lws-log", "lws-cors", "lws-static", "lws-index"], }); - await verifyStartup(ws, port); + const actualPort = await verifyStartup(ws); process.on("exit", () => ws.server.close()); return { + port: actualPort, + server: { + close() { + ws.server.close(); + }, + }, close() { ws.server.close(); }, }; } -async function verifyStartup(ws, port) { - await new Promise((resolve, reject) => { - ws.server.on("listening", () => { +async function verifyStartup(ws) { + return new Promise((resolve, reject) => { + const onListening = () => { + const actualPort = ws.server.address().port; console.log("Server started:"); - console.log(` http://localhost:${port}`); - console.log(` http://localhost:${port}?developerMode`); + console.log(` http://localhost:${actualPort}`); + console.log(` http://localhost:${actualPort}?developerMode`); console.log(""); - resolve(); - }); - ws.server.on("error", (e) => { - console.error("Error while starting the server", e); - reject(e); - }); + resolve(actualPort); + }; + if (ws.server.listening) { + onListening(); + } else { + ws.server.on("listening", onListening); + ws.server.on("error", (e) => { + console.error("Error while starting the server", e); + reject(e); + }); + } }); } From 3bdbe889687e301413cc88db0954d9fda5f5f1c8 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 14 Jul 2026 10:12:14 +0200 Subject: [PATCH 2/7] fx --- tests/helper.mjs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/helper.mjs b/tests/helper.mjs index d95fe6e87..aac1a8a6e 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -38,7 +38,13 @@ function printHelp(message = "", exitStatus = 0) { } export function detectHeadless(options) { - return Boolean(options?.headless || (process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY)); + if (options?.headless) + return true; + + if (process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY) + return true; + + return false; } export default async function testSetup(helpText) { @@ -47,7 +53,7 @@ export default async function testSetup(helpText) { if ("help" in options) printHelp(helpText); - const BROWSER = options?.browser || process.env.BROWSER; + const BROWSER = options?.browser; if (!BROWSER) printHelp("No browser specified, use $BROWSER or --browser", 1); From a18a09b9b548836d5e7e94ec18e5421820fc2851 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 14 Jul 2026 10:14:00 +0200 Subject: [PATCH 3/7] cleanup --- package.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index f0884b94b..2b0f45748 100644 --- a/package.json +++ b/package.json @@ -22,14 +22,14 @@ "pretty:fix": "prettier --write ./", "format": "node tests/format.mjs", "format:all": "node tests/format.mjs --all", - "test:chrome": "node tests/run-unittests.mjs --browser chrome", - "test:firefox": "node tests/run-unittests.mjs --browser firefox", - "test:safari": "node tests/run-unittests.mjs --browser safari", - "test:edge": "node tests/run-unittests.mjs --browser edge", - "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome", - "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox", - "test-e2e:safari": "node tests/run-end2end.mjs --browser safari", - "test-e2e:edge": "node tests/run-end2end.mjs --browser edge" + "test:chrome": "node tests/run-unittests.mjs --browser=chrome", + "test:firefox": "node tests/run-unittests.mjs --browser=firefox", + "test:safari": "node tests/run-unittests.mjs --browser=safari", + "test:edge": "node tests/run-unittests.mjs --browser=edge", + "test-e2e:chrome": "node tests/run-end2end.mjs --browser=chrome", + "test-e2e:firefox": "node tests/run-end2end.mjs --browser=firefox", + "test-e2e:safari": "node tests/run-end2end.mjs --browser=safari", + "test-e2e:edge": "node tests/run-end2end.mjs --browser=edge" }, "devDependencies": { "@babel/core": "^7.21.3", From 270a13220ce740463360692b278fef0f4dca2b3a Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 14 Jul 2026 10:20:16 +0200 Subject: [PATCH 4/7] add-test --- tests/helper.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/helper.mjs b/tests/helper.mjs index aac1a8a6e..df1f85761 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -65,6 +65,9 @@ export default async function testSetup(helpText) { let builder; switch (BROWSER) { case "safari": { + if (isHeadless) + console.warn("Warning: --headless is not supported with safari, running in windowed mode."); + builder = new Builder().forBrowser(BROWSER); // No bidi and log support in safari. break; From 770b819ed9640ad7cc9da0195dad945f0fa92a0c Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 15 Jul 2026 10:51:10 +0200 Subject: [PATCH 5/7] fx --- tests/unittests/suites.mjs | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/tests/unittests/suites.mjs b/tests/unittests/suites.mjs index 216ab68a0..97252f569 100644 --- a/tests/unittests/suites.mjs +++ b/tests/unittests/suites.mjs @@ -64,41 +64,6 @@ for (const [name, suites] of Object.entries(Suites)) { expect(suite.url.length).to.be.greaterThan(0); }); }); - it("should have resources.txt listing only valid files", async function () { - // validating all resource files can take a bit longer than the default timeout. - this.timeout(10000); - const baseUrl = `${window.location.origin}/`; - const brokenResourcesList = []; - for (const suite of suites) { - if (!suite.resources) - continue; - const resourcesUrl = new URL(suite.resources, baseUrl).href; - const res = await fetch(resourcesUrl); - if (!res.ok) - throw new Error(`Failed to load resources.txt for ${suite.name} at ${resourcesUrl}`); - - const text = await res.text(); - if (text.trim().length === 0) - throw new Error(`resources.txt for ${suite.name} is empty`); - - const files = text.trim().split("\n"); - for (const file of files) - expect(file.trim().length).to.be.greaterThan(0, `Found empty line in resources.txt for ${suite.name}`); - - await Promise.all( - files.map(async (file) => { - const fileUrl = new URL(file, resourcesUrl).href; - const fileRes = await fetch(fileUrl, { method: "HEAD" }); - if (!fileRes.ok) - brokenResourcesList.push(`${fileUrl} (listed in ${resourcesUrl})`); - else - expect(fileRes.ok).to.be(true); - }) - ); - } - if (brokenResourcesList.length > 0) - throw new Error(`Failed to load the following resources:\n${brokenResourcesList.join("\n")}`); - }); }); } From c5c3a664e9e73776ec35b4912c01c881beda83f7 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 15 Jul 2026 10:53:30 +0200 Subject: [PATCH 6/7] add-tests --- tests/unittests/resources.mjs | 63 +++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/unittests/resources.mjs diff --git a/tests/unittests/resources.mjs b/tests/unittests/resources.mjs new file mode 100644 index 000000000..aa08e2e8e --- /dev/null +++ b/tests/unittests/resources.mjs @@ -0,0 +1,63 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { ExperimentalSuites } from "../../suites-experimental/suites.mjs"; +import { DefaultSuites } from "../../suites/default-suites.mjs"; + +const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../"); + +const Suites = { + ExperimentalSuites, + DefaultSuites, +}; + +for (const [name, suites] of Object.entries(Suites)) { + describe(`${name}-resources`, () => { + it("should have resources.txt listing only valid files via local filesystem", async function () { + // validating all resource files can take a bit longer than the default timeout. + this.timeout(10000); + const brokenResourcesList = []; + for (const suite of suites) { + if (!suite.resources) + continue; + const resourcesPath = path.resolve(ROOT_DIR, suite.resources); + let text; + try { + text = await fs.readFile(resourcesPath, "utf-8"); + } catch (error) { + brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: ${error.message}]`); + continue; + } + + if (text.trim().length === 0) { + brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: resources.txt is empty]`); + continue; + } + + const files = text.trim().split("\n"); + for (const file of files) + expect(file.trim().length).to.be.greaterThan(0, `Found empty line in resources.txt for ${suite.name}`); + + await Promise.all( + files.map(async (file) => { + const cleanFile = file.trim(); + if (!cleanFile) + return; + const filePath = cleanFile.startsWith("/") + ? path.join(ROOT_DIR, cleanFile) + : path.resolve(path.dirname(resourcesPath), cleanFile); + try { + const stat = await fs.stat(filePath); + if (!stat.isFile()) + brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: not a file]`); + } catch (error) { + brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: ${error.message}]`); + } + }) + ); + } + if (brokenResourcesList.length > 0) + throw new Error(`Failed to check the following resources:\n${brokenResourcesList.join("\n")}`); + }); + }); +} From 05c07f6f94548c6b45202be9353d70e311d3c79e Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 15 Jul 2026 11:00:15 +0200 Subject: [PATCH 7/7] cleanup --- tests/unittests/resources.mjs | 93 ++++++++++++++++------------------- 1 file changed, 43 insertions(+), 50 deletions(-) diff --git a/tests/unittests/resources.mjs b/tests/unittests/resources.mjs index aa08e2e8e..d005d4113 100644 --- a/tests/unittests/resources.mjs +++ b/tests/unittests/resources.mjs @@ -6,58 +6,51 @@ import { DefaultSuites } from "../../suites/default-suites.mjs"; const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../"); -const Suites = { - ExperimentalSuites, - DefaultSuites, -}; +const Suites = [...ExperimentalSuites, ...DefaultSuites]; -for (const [name, suites] of Object.entries(Suites)) { - describe(`${name}-resources`, () => { - it("should have resources.txt listing only valid files via local filesystem", async function () { - // validating all resource files can take a bit longer than the default timeout. - this.timeout(10000); - const brokenResourcesList = []; - for (const suite of suites) { - if (!suite.resources) - continue; - const resourcesPath = path.resolve(ROOT_DIR, suite.resources); - let text; - try { - text = await fs.readFile(resourcesPath, "utf-8"); - } catch (error) { - brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: ${error.message}]`); - continue; - } +describe("resources", () => { + it("should have resources.txt listing only valid files via local filesystem", async function () { + // validating all resource files can take a bit longer than the default timeout. + this.timeout(10000); + const brokenResourcesList = []; + for (const suite of Suites) { + if (!suite.resources) + continue; + const resourcesPath = path.resolve(ROOT_DIR, suite.resources); + let text; + try { + text = await fs.readFile(resourcesPath, "utf-8"); + } catch (error) { + brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: ${error.message}]`); + continue; + } - if (text.trim().length === 0) { - brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: resources.txt is empty]`); - continue; - } + if (text.trim().length === 0) { + brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: resources.txt is empty]`); + continue; + } - const files = text.trim().split("\n"); - for (const file of files) - expect(file.trim().length).to.be.greaterThan(0, `Found empty line in resources.txt for ${suite.name}`); + const files = text.trim().split("\n"); + for (const file of files) + expect(file.trim().length).to.be.greaterThan(0, `Found empty line in resources.txt for ${suite.name}`); - await Promise.all( - files.map(async (file) => { - const cleanFile = file.trim(); - if (!cleanFile) - return; - const filePath = cleanFile.startsWith("/") - ? path.join(ROOT_DIR, cleanFile) - : path.resolve(path.dirname(resourcesPath), cleanFile); - try { - const stat = await fs.stat(filePath); - if (!stat.isFile()) - brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: not a file]`); - } catch (error) { - brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: ${error.message}]`); - } - }) - ); - } - if (brokenResourcesList.length > 0) - throw new Error(`Failed to check the following resources:\n${brokenResourcesList.join("\n")}`); - }); + await Promise.all( + files.map(async (file) => { + const cleanFile = file.trim(); + if (!cleanFile) + return; + const filePath = cleanFile.startsWith("/") ? path.join(ROOT_DIR, cleanFile) : path.resolve(path.dirname(resourcesPath), cleanFile); + try { + const stat = await fs.stat(filePath); + if (!stat.isFile()) + brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: not a file]`); + } catch (error) { + brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: ${error.message}]`); + } + }) + ); + } + if (brokenResourcesList.length > 0) + throw new Error(`Failed to check the following resources:\n${brokenResourcesList.join("\n")}`); }); -} +});