Improve end2end test runner

package.json

@@ -22,14 +22,14 @@
         "pretty:fix": "prettier --write ./",
         "format": "node tests/format.mjs",
         "format:all": "node tests/format.mjs --all",
-        "test:chrome": "node tests/run-unittests.mjs --browser chrome",
-        "test:firefox": "node tests/run-unittests.mjs --browser firefox",
-        "test:safari": "node tests/run-unittests.mjs --browser safari",
-        "test:edge": "node tests/run-unittests.mjs --browser edge",
-        "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome",
-        "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox",
-        "test-e2e:safari": "node tests/run-end2end.mjs --browser safari",
-        "test-e2e:edge": "node tests/run-end2end.mjs --browser edge",
+        "test:chrome": "node tests/run-unittests.mjs --browser=chrome",
+        "test:firefox": "node tests/run-unittests.mjs --browser=firefox",
+        "test:safari": "node tests/run-unittests.mjs --browser=safari",
+        "test:edge": "node tests/run-unittests.mjs --browser=edge",
+        "test-e2e:chrome": "node tests/run-end2end.mjs --browser=chrome",
+        "test-e2e:firefox": "node tests/run-end2end.mjs --browser=firefox",
+        "test-e2e:safari": "node tests/run-end2end.mjs --browser=safari",
+        "test-e2e:edge": "node tests/run-end2end.mjs --browser=edge",
         "test": "npm run test:node",
         "test:node": "mocha --require ./tests/setup-node.mjs \"tests/unittests/*.mjs\""
     },

tests/helper.mjs

@@ -13,8 +13,9 @@ export const DEFAULT_RETRIES = 1;
 
 const optionDefinitions = [
     { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." },
-    { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." },
+    { name: "port", type: Number, defaultValue: 0, description: "Set the test-server port. The default value is 0 (dynamic port)." },
     { name: "retry", type: Number, defaultValue: DEFAULT_RETRIES, description: "Number of retries for the tests on failure." },
+    { name: "headless", type: Boolean, description: "Run browser in headless mode. Automatically enabled on Linux when $DISPLAY and $WAYLAND_DISPLAY are unset." },
     { name: "help", alias: "h", description: "Print this help text." },
 ];
 
@@ -36,6 +37,16 @@ function printHelp(message = "", exitStatus = 0) {
     process.exit(exitStatus);
 }
 
+export function detectHeadless(options) {
+    if (options?.headless)
+        return true;
+
+    if (process.platform === "linux" && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY)
+        return true;
+
+    return false;
+}
+
 export default async function testSetup(helpText) {
     const options = commandLineArgs(optionDefinitions);
 
@@ -49,31 +60,47 @@ export default async function testSetup(helpText) {
     if (options.retry < 0)
         printHelp("Number of retries cannot be negative", 1);
 
+    const isHeadless = detectHeadless(options);
+
     let builder;
     switch (BROWSER) {
         case "safari": {
+            if (isHeadless)
+                console.warn("Warning: --headless is not supported with safari, running in windowed mode.");
+
             builder = new Builder().forBrowser(BROWSER);
             // No bidi and log support in safari.
             break;
         }
         case "firefox": {
-            builder = new Builder().forBrowser(BROWSER).setFirefoxOptions(new firefox.Options().enableBidi());
+            const firefoxOptions = new firefox.Options().enableBidi();
+            if (isHeadless)
+                firefoxOptions.addArguments("--headless");
+
+            builder = new Builder().forBrowser(BROWSER).setFirefoxOptions(firefoxOptions);
             break;
         }
         case "chrome": {
-            builder = new Builder().forBrowser(BROWSER).setChromeOptions(new chrome.Options().enableBidi());
+            const chromeOptions = new chrome.Options().enableBidi();
+            if (isHeadless)
+                chromeOptions.addArguments("--headless");
+
+            builder = new Builder().forBrowser(BROWSER).setChromeOptions(chromeOptions);
             break;
         }
         case "edge": {
-            builder = new Builder().forBrowser(BROWSER).setEdgeOptions(new edge.Options().enableBidi());
+            const edgeOptions = new edge.Options().enableBidi();
+            if (isHeadless)
+                edgeOptions.addArguments("--headless");
+
+            builder = new Builder().forBrowser(BROWSER).setEdgeOptions(edgeOptions);
             break;
         }
         default: {
             printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`);
         }
     }
-    const PORT = options.port;
-    const server = await serve(PORT);
+    const { server, port } = await serve(options.port);
     let driver, logInspector;
 
     process.on("unhandledRejection", (err) => {
@@ -100,8 +127,9 @@ export default async function testSetup(helpText) {
         server.close();
         if (logInspector)
             await logInspector.close();
+
         if (driver)
             driver.close();
     }
-    return { driver, PORT, stop, retry: options.retry };
+    return { driver, port, stop, retry: options.retry };
 }

tests/run-end2end.mjs

@@ -9,13 +9,13 @@ This script runs end2end tests by invoking the benchmark via the main
 Speedometer page in /index.html.
 `.trim();
 
-const { driver, PORT, stop, retry } = await testSetup(HELP);
+const { driver, port, stop, retry } = await testSetup(HELP);
 
 const suites = benchmarkConfigurator.suites;
 
 async function testPage(url) {
     console.log(`Testing: ${url}`);
-    await driver.get(`http://localhost:${PORT}/${url}`);
+    await driver.get(`http://localhost:${port}/${url}`);
 
     await driver.executeAsyncScript((callback) => {
         if (globalThis.benchmarkClient)

tests/run-unittests.mjs

@@ -8,11 +8,11 @@ This script runs the unittests located in tests/unittests/*
 through the mocha web interface located at tests/index.html
 `.trim();
 
-const { driver, PORT, stop } = await testSetup(HELP);
+const { driver, port, stop } = await testSetup(HELP);
 
 async function test() {
     try {
-        await driver.get(`http://localhost:${PORT}/tests/index.html`);
+        await driver.get(`http://localhost:${port}/tests/index.html`);
 
         const { testResults, stats } = await driver.executeAsyncScript(function (callback) {
             const returnResults = () =>

tests/server.mjs

@@ -12,8 +12,9 @@ import "lws-static";
 const ROOT_DIR = path.join(process.cwd(), "./");
 
 export default async function serve(port) {
-    if (!port)
+    if (!port && port !== 0)
         throw new Error("Port is required");
+
     const ws = await LocalWebServer.create({
         port: port,
         hostname: "127.0.0.1",
@@ -23,30 +24,42 @@ export default async function serve(port) {
         logFormat: "dev",
         stack: ["lws-log", "lws-cors", "lws-static", "lws-index"],
     });
-    await verifyStartup(ws, port);
+    const actualPort = await verifyStartup(ws);
 
     process.on("exit", () => ws.server.close());
 
     return {
+        port: actualPort,
+        server: {
+            close() {
+                ws.server.close();
+            },
+        },
         close() {
             ws.server.close();
         },
     };
 }
 
-async function verifyStartup(ws, port) {
-    await new Promise((resolve, reject) => {
-        ws.server.on("listening", () => {
+async function verifyStartup(ws) {
+    return new Promise((resolve, reject) => {
+        const onListening = () => {
+            const actualPort = ws.server.address().port;
             console.log("Server started:");
-            console.log(`  http://localhost:${port}`);
-            console.log(`  http://localhost:${port}?developerMode`);
+            console.log(`  http://localhost:${actualPort}`);
+            console.log(`  http://localhost:${actualPort}?developerMode`);
             console.log("");
-            resolve();
-        });
-        ws.server.on("error", (e) => {
-            console.error("Error while starting the server", e);
-            reject(e);
-        });
+            resolve(actualPort);
+        };
+        if (ws.server.listening) {
+            onListening();
+        } else {
+            ws.server.on("listening", onListening);
+            ws.server.on("error", (e) => {
+                console.error("Error while starting the server", e);
+                reject(e);
+            });
+        }
     });
 }
 

tests/unittests/resources.mjs

@@ -0,0 +1,56 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { ExperimentalSuites } from "../../suites-experimental/suites.mjs";
+import { DefaultSuites } from "../../suites/default-suites.mjs";
+
+const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../");
+
+const Suites = [...ExperimentalSuites, ...DefaultSuites];
+
+describe("resources", () => {
+    it("should have resources.txt listing only valid files via local filesystem", async function () {
+        // validating all resource files can take a bit longer than the default timeout.
+        this.timeout(10000);
+        const brokenResourcesList = [];
+        for (const suite of Suites) {
+            if (!suite.resources)
+                continue;
+            const resourcesPath = path.resolve(ROOT_DIR, suite.resources);
+            let text;
+            try {
+                text = await fs.readFile(resourcesPath, "utf-8");
+            } catch (error) {
+                brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: ${error.message}]`);
+                continue;
+            }
+
+            if (text.trim().length === 0) {
+                brokenResourcesList.push(`${suite.resources} (for ${suite.name}) [error: resources.txt is empty]`);
+                continue;
+            }
+
+            const files = text.trim().split("\n");
+            for (const file of files)
+                expect(file.trim().length).to.be.greaterThan(0, `Found empty line in resources.txt for ${suite.name}`);
+
+            await Promise.all(
+                files.map(async (file) => {
+                    const cleanFile = file.trim();
+                    if (!cleanFile)
+                        return;
+                    const filePath = cleanFile.startsWith("/") ? path.join(ROOT_DIR, cleanFile) : path.resolve(path.dirname(resourcesPath), cleanFile);
+                    try {
+                        const stat = await fs.stat(filePath);
+                        if (!stat.isFile())
+                            brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: not a file]`);
+                    } catch (error) {
+                        brokenResourcesList.push(`${cleanFile} (listed in ${suite.resources}) [error: ${error.message}]`);
+                    }
+                })
+            );
+        }
+        if (brokenResourcesList.length > 0)
+            throw new Error(`Failed to check the following resources:\n${brokenResourcesList.join("\n")}`);
+    });
+});

tests/unittests/suites.mjs

@@ -64,62 +64,6 @@ for (const [name, suites] of Object.entries(Suites)) {
                 expect(suite.url.length).to.be.greaterThan(0);
             });
         });
-        it("should have resources.txt listing only valid files", async function () {
-            // validating all resource files can take a bit longer than the default timeout.
-            this.timeout(10000);
-            const isNode = typeof window === "undefined";
-            const baseUrl = isNode ? new URL("../../", import.meta.url).href : `${window.location.origin}/`;
-            const brokenResourcesList = [];
-            for (const suite of suites) {
-                if (!suite.resources)
-                    continue;
-                const resourcesUrl = new URL(suite.resources, baseUrl).href;
-                let text;
-                if (isNode) {
-                    const fs = await import("node:fs/promises");
-                    try {
-                        text = await fs.readFile(new URL(resourcesUrl), "utf-8");
-                    } catch (e) {
-                        throw new Error(`Failed to load resources.txt for ${suite.name} at ${resourcesUrl}`);
-                    }
-                } else {
-                    const res = await fetch(resourcesUrl);
-                    if (!res.ok)
-                        throw new Error(`Failed to load resources.txt for ${suite.name} at ${resourcesUrl}`);
-                    text = await res.text();
-                }
-
-                if (text.trim().length === 0)
-                    throw new Error(`resources.txt for ${suite.name} is empty`);
-
-                const files = text.trim().split("\n");
-                for (const file of files)
-                    expect(file.trim().length).to.be.greaterThan(0, `Found empty line in resources.txt for ${suite.name}`);
-
-                await Promise.all(
-                    files.map(async (file) => {
-                        const fileUrl = new URL(file, resourcesUrl).href;
-                        if (isNode) {
-                            const fs = await import("node:fs/promises");
-                            try {
-                                await fs.stat(new URL(fileUrl));
-                                expect(true).to.be(true);
-                            } catch {
-                                brokenResourcesList.push(`${fileUrl} (listed in ${resourcesUrl})`);
-                            }
-                        } else {
-                            const fileRes = await fetch(fileUrl, { method: "HEAD" });
-                            if (!fileRes.ok)
-                                brokenResourcesList.push(`${fileUrl} (listed in ${resourcesUrl})`);
-                            else
-                                expect(fileRes.ok).to.be(true);
-                        }
-                    })
-                );
-            }
-            if (brokenResourcesList.length > 0)
-                throw new Error(`Failed to load the following resources:\n${brokenResourcesList.join("\n")}`);
-        });
     });
 }