diff --git a/index.html b/index.html index 31ba85ee4..6a26fc563 100644 --- a/index.html +++ b/index.html @@ -26,13 +26,20 @@
- +
About Speedometer Test Instructions
+
+ +
+
+
+
+
diff --git a/resources/benchmark-configurator.mjs b/resources/benchmark-configurator.mjs index caac91775..531f5fd03 100644 --- a/resources/benchmark-configurator.mjs +++ b/resources/benchmark-configurator.mjs @@ -56,6 +56,8 @@ export class BenchmarkConfigurator { this.#suites.forEach((suite) => { if (!suite.tags) suite.tags = []; + if (!("measurePrepare" in suite)) + suite.measurePrepare = false; if (suite.url.startsWith("experimental/")) suite.tags.unshift("all", "experimental"); else @@ -115,8 +117,10 @@ export class BenchmarkConfigurator { } _loadSuite(suite) { - suite.tags.forEach((tag) => this.#tags.add(tag)); - this.#suites.push(suite); + if (suite.tags) + suite.tags.forEach((tag) => this.#tags.add(tag)); + + this.#suites.push({ ...suite, tags: suite.tags ? [...suite.tags] : [] }); } enableSuites(names, tags) { diff --git a/resources/developer-mode.mjs b/resources/developer-mode.mjs index ec906d4a6..1fdf35c46 100644 --- a/resources/developer-mode.mjs +++ b/resources/developer-mode.mjs @@ -29,6 +29,7 @@ export function createDeveloperModeContainer() { settings.append(createUIForWaitAfterSuite()); settings.append(createUIForAsyncSteps()); settings.append(createUIForLayoutMode()); + settings.append(createUIForPreload()); settings.append(createUIForComplexity()); content.append(document.createElement("hr")); @@ -54,6 +55,12 @@ function createUIForWarmupSuite() { }); } +function createUIForPreload() { + return createCheckboxUI("Use service worker for resource preloading", params.preload, (isChecked) => { + params.preload = isChecked; + }); +} + function createUIForMeasurePrepare() { return createCheckboxUI("Measure Prepare", params.measurePrepare, (isChecked) => { params.measurePrepare = isChecked; diff --git a/resources/main.css b/resources/main.css index 86c8207ed..41678b76a 100644 --- a/resources/main.css +++ b/resources/main.css @@ -419,7 +419,17 @@ button.show-about { display: none; } -#progress { +#preload-progress { + display: none; +} + +:root[data-benchmark-state="PRELOADING"] #preload-progress, +:root[data-benchmark-state="PRELOADING"] #preload-info { + display: block; +} + +#progress, +#preload-progress { position: absolute; bottom: -6px; left: 60px; @@ -429,7 +439,8 @@ button.show-about { border-right: 6px solid var(--background); } -#progress-completed { +#progress-completed, +#preload-progress-completed { position: absolute; top: 0; left: 0; @@ -440,11 +451,13 @@ button.show-about { background-color: var(--inactive-color); } -#progress-completed::-webkit-progress-value { +#progress-completed::-webkit-progress-value, +#preload-progress-completed::-webkit-progress-value { background-color: var(--foreground); } -#progress-completed::-moz-progress-bar { +#progress-completed::-moz-progress-bar, +#preload-progress-completed::-moz-progress-bar { background-color: var(--foreground); } @@ -455,7 +468,9 @@ button.show-about { background-color: var(--background); } -#info { +#info, +#preload-info { + display: none; position: absolute; bottom: -25px; left: 60px; @@ -465,11 +480,13 @@ button.show-about { text-align: center; font-size: 12px; } -#info-label { +#info-label, +#preload-info-label { position: absolute; left: 6px; } -#info-progress { +#info-progress, +#preload-info-progress { position: absolute; right: 6px; text-align: right; @@ -945,3 +962,50 @@ section#about .note { color: #fff; stroke: #fff; } + +#preload-progress, +#preload-info { + display: none; +} + +[data-benchmark-state="PRELOADING"] { + cursor: wait; +} + +[data-benchmark-state="PRELOADING"] #preload-progress, +[data-benchmark-state="PRELOADING"] #preload-info { + display: block; +} + +[data-benchmark-state="RUNNING"] #info { + display: block; +} + +[data-benchmark-state="PRELOADING"] .start-tests-button { + color: var(--foreground); + background-image: linear-gradient(-45deg, var(--foreground) 3px, transparent 3px, transparent 50%, var(--foreground) 50%, var(--foreground) calc(50% + 3px), transparent calc(50% + 3px), transparent 100%); + background-size: 20px 20px; + animation: barber-pole 1s linear infinite; + pointer-events: none; +} + +[data-benchmark-state="PRELOADING"] .developer-mode .start-tests-button { + pointer-events: auto; + cursor: pointer; +} + +[data-benchmark-state="PRELOADING"] .start-tests-button > div { + background-color: var(--background); + border-radius: 10px; + padding: 0 0.3em; + display: inline-block; +} + +@keyframes barber-pole { + 0% { + background-position: 0 0, 0 0; + } + 100% { + background-position: 20px 0, 0 0; + } +} diff --git a/resources/main.mjs b/resources/main.mjs index d7884488e..f33fbf005 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -1,9 +1,167 @@ import { BenchmarkRunner } from "./benchmark-runner.mjs"; import * as Statistics from "./statistics.mjs"; +import { SW_MESSAGES } from "./shared/sw-messages.mjs"; import { renderMetricView } from "./metric-ui.mjs"; import { defaultParams, params } from "./shared/params.mjs"; import { createDeveloperModeContainer } from "./developer-mode.mjs"; +const PRELOAD_TIMEOUT_MS = 60_000; + +export class ResourcePreloader { + constructor() { + this._registration = null; + this._sw = null; + this._preloadParams = ""; + this._clientId = typeof crypto?.randomUUID === "function" ? crypto.randomUUID() : Math.random().toString(36).substring(2) + Date.now().toString(36); + } + + isCached() { + if (!params.preload) + return true; + return params.toSearchParams() === this._preloadParams; + } + + async setup() { + if (!params.preload) { + await this._unregisterOldServiceWorkers(); + this._registration = null; + this._sw = null; + return false; + } + + await this._registerServiceWorker(); + this._setupMessageListener(); + + return true; + } + + async _unregisterOldServiceWorkers() { + const existingRegistrations = await navigator.serviceWorker.getRegistrations(); + for (const existing of existingRegistrations) + await existing.unregister(); + } + + async _registerServiceWorker() { + this._registration = await navigator.serviceWorker.register("./sw.mjs", { type: "module" }); + await navigator.serviceWorker.ready; + this._sw = navigator.serviceWorker.controller || this._registration.active; + } + + _setupMessageListener() { + navigator.serviceWorker.addEventListener("message", (event) => { + if (event.data?.type === SW_MESSAGES.FATAL_ERROR) + globalThis.benchmarkClient?.handleError(new Error(event.data.message)); + }); + } + + _sendMessageWithReply(messageData, onProgress, timeoutMs = 0) { + if (!this._sw) + return Promise.resolve(); + + messageData.clientId = this._clientId; + + return new Promise((resolve, reject) => { + const channel = new MessageChannel(); + const port = channel.port1; + let timeoutId = null; + + // Prevent GC in Safari + this._activeChannels = this._activeChannels || new Set(); + this._activeChannels.add(channel); + + const cleanup = () => { + port.onmessage = null; + this._activeChannels.delete(channel); + if (timeoutId) + clearTimeout(timeoutId); + }; + + if (timeoutMs > 0) { + timeoutId = setTimeout(() => { + cleanup(); + console.error(`Service worker message timed out: ${messageData.type}`); + resolve(); // Resolve to not block execution + }, timeoutMs); + } + + port.onmessage = (event) => { + const data = event.data || {}; + if (data.type === SW_MESSAGES.PRELOAD_PROGRESS) { + onProgress?.(data); + return; + } + cleanup(); + if (data.type === "PRELOAD_ABORTED") { + resolve({ type: "PRELOAD_ABORTED" }); + return; + } + if (data.type === "ERROR") { + reject(new Error(data.message || "Unknown SW Error")); + return; + } + + resolve(data); + }; + port.start(); + + this._sw.postMessage(messageData, [channel.port2]); + }); + } + + async clearSw() { + if (!this._sw) + return; + await this._sendMessageWithReply({ type: SW_MESSAGES.CLEAR_CACHE }); + } + + async resetPreloading() { + if (!this._sw) + return; + await this._sendMessageWithReply({ type: SW_MESSAGES.RESET_PRELOADING }); + if (this._activePreloadPromise) + await this._activePreloadPromise; + } + + async preloadSuites(suites, resourceLoadDelay, clearCache = true, onProgress) { + const suitesData = suites + .filter((s) => s.resources) + .map((s) => ({ + name: s.name, + url: new URL(s.url, window.location.href).href, + resources: new URL(s.resources, window.location.href).href, + })); + + if (!this._sw || suitesData.length === 0) + return undefined; + + const startTime = performance.now(); + this._activePreloadPromise = this._sendMessageWithReply({ type: SW_MESSAGES.PRELOAD_SUITES, suites: suitesData, delay: resourceLoadDelay, clearCache }, onProgress, PRELOAD_TIMEOUT_MS); + const response = await this._activePreloadPromise; + this._activePreloadPromise = null; + + if (response?.type === "PRELOAD_ABORTED") + return "ABORTED"; + + if (response?.type === SW_MESSAGES.PRELOAD_DONE) { + const timeTakenMs = performance.now() - startTime; + const sizeMB = (response.transferredSize / (1024 * 1024)).toFixed(2); + const timeSec = (timeTakenMs / 1000).toFixed(2); + console.log(`Preloaded ${response.count} files (${sizeMB} MB transferred) in ${timeSec}s`); + } + this._preloadParams = params.toSearchParams(); + return undefined; + } +} + +const BENCHMARK_STATE = Object.freeze({ + IDLE: "IDLE", + PRELOADING: "PRELOADING", + READY: "READY", + RUNNING: "RUNNING", + DONE: "DONE", + ERROR: "ERROR", +}); + // FIXME(camillobruni): Add base class class MainBenchmarkClient { developerMode = false; @@ -14,8 +172,8 @@ class MainBenchmarkClient { _progressCompleted = null; _isRunning = false; _hasResults = false; - _developerModeContainer = null; _metrics = Object.create(null); + _resourcePreloader = new ResourcePreloader(); _steppingPromise = null; _steppingResolver = null; _benchmarkConfiguratorPromise = null; @@ -31,21 +189,21 @@ class MainBenchmarkClient { }); } - start() { + async start() { if (this._isStepping()) this._clearStepping(); - else if (this._startBenchmark()) + else if (await this._startBenchmark()) this._showSection("#running"); } - step() { + async step() { const currentSteppingResolver = this._steppingResolver; this._steppingPromise = new Promise((resolve) => { this._steppingResolver = resolve; }); currentSteppingResolver?.(); if (!this._isRunning) { - this._startBenchmark(); + await this._startBenchmark(); this._showSection("#running"); } } @@ -88,6 +246,19 @@ class MainBenchmarkClient { } if (!this._isStepping()) this._developerModeContainer?.remove(); + + await this._resourcePreloader.resetPreloading(); + const preloadResult = await this._setupResourcePreloader(benchmarkConfigurator); + if (preloadResult === "ABORTED") + return false; + + try { + await this._setBenchmarkState(BENCHMARK_STATE.RUNNING); + } catch (error) { + alert(error.message); + return false; + } + this._progressCompleted = document.getElementById("progress-completed"); if (params.iterationCount < 50) { const progressNode = document.getElementById("progress"); @@ -142,11 +313,20 @@ class MainBenchmarkClient { this._finishedTestCount = 0; } - didFinishLastIteration(metrics) { + async didFinishLastIteration(metrics) { console.assert(this._isRunning); this._isRunning = false; + + const response = await this._resourcePreloader._sendMessageWithReply({ type: SW_MESSAGES.GET_FAILED_REQUESTS }); + if (response?.requests && response.requests.length > 0) { + console.warn("The following requests failed during the benchmark and bypassed the cache:"); + console.warn(response.requests.join("\n")); + alert("Warning: The benchmark had missing resources that were not cached. Check the console for details."); + } + this._hasResults = true; this._metrics = metrics; + this._setBenchmarkState(BENCHMARK_STATE.DONE); const scoreResults = this._computeResults(this._measuredValuesList, "score"); if (scoreResults.isValid) @@ -167,7 +347,8 @@ class MainBenchmarkClient { this._isRunning = false; this._hasResults = true; this._metrics = Object.create(null); - this._populateInvalidScore(); + this._setBenchmarkState(BENCHMARK_STATE.ERROR); + this._populateInvalidScore(error.message); this.showResultsSummary(); throw error; } @@ -181,10 +362,20 @@ class MainBenchmarkClient { document.getElementById("confidence-number").textContent = `\u00b1 ${scoreResults.formattedDelta}`; } - _populateInvalidScore() { + _populateInvalidScore(errorText) { document.getElementById("summary").className = "invalid"; document.getElementById("result-number").textContent = "Error"; document.getElementById("confidence-number").textContent = ""; + + const invalidScoreText = document.getElementById("invalid-score-text"); + if (errorText) { + invalidScoreText.textContent = errorText; + } else { + invalidScoreText.innerHTML = ` + One or more subtests produced no duration.
+ Please check your browser settings and re-run the benchmark.
+ `; + } } _computeResults(measuredValuesList, displayUnit) { @@ -289,15 +480,15 @@ class MainBenchmarkClient { document.getElementById("metrics-results").innerHTML = html; const filePrefix = `speedometer-3-${new Date().toISOString()}`; - let jsonData = this._formattedJSONResult({ modern: false }); - let jsonLink = document.getElementById("download-classic-json"); - jsonLink.href = URL.createObjectURL(new Blob([jsonData], { type: "application/json" })); - jsonLink.setAttribute("download", `${filePrefix}.json`); + const classicJsonData = this._formattedJSONResult({ modern: false }); + const classicJsonLink = document.getElementById("download-classic-json"); + classicJsonLink.href = URL.createObjectURL(new Blob([classicJsonData], { type: "application/json" })); + classicJsonLink.setAttribute("download", `${filePrefix}.json`); - jsonLink = document.getElementById("download-full-json"); - jsonData = this._formattedJSONResult({ modern: true }); - jsonLink.href = URL.createObjectURL(new Blob([jsonData], { type: "application/json" })); - jsonLink.setAttribute("download", `${filePrefix}.json`); + const fullJsonLink = document.getElementById("download-full-json"); + const fullJsonData = this._formattedJSONResult({ modern: true }); + fullJsonLink.href = URL.createObjectURL(new Blob([fullJsonData], { type: "application/json" })); + fullJsonLink.setAttribute("download", `${filePrefix}.json`); const csvData = this._formattedCSVResult(); const csvLink = document.getElementById("download-csv"); @@ -344,6 +535,7 @@ class MainBenchmarkClient { document.getElementById("copy-csv").onclick = this.copyCSVResults.bind(this); document.querySelectorAll(".start-tests-button").forEach((button) => { button.onclick = this._startBenchmarkHandler.bind(this); + button.disabled = true; }); } @@ -358,10 +550,120 @@ class MainBenchmarkClient { document.body.append(this._developerModeContainer); } + await this._resourcePreloader.resetPreloading(); + if (params.developerMode) { + this._enableStartButtons(); + } else { + const preloadResult = await this._setupResourcePreloader(benchmarkConfigurator); + if (preloadResult === "ABORTED") + return; + } + if (params.startAutomatically) this.start(); } + async _setupResourcePreloader(benchmarkConfigurator) { + await this._resourcePreloader.setup(); + if (!this._resourcePreloader.isCached()) + return this._cacheResources(benchmarkConfigurator); + else + this._enableStartButtons(); + + return undefined; + } + + async _cacheResources(benchmarkConfigurator) { + const enabledSuites = benchmarkConfigurator.suites.filter((suite) => suite.enabled); + const clearCache = !params.isDefault(); + + this._resetPreloadUI(); + this._setBenchmarkState(BENCHMARK_STATE.PRELOADING); + await new Promise((resolve) => requestAnimationFrame(resolve)); + + this._preloadGeneration = (this._preloadGeneration || 0) + 1; + const currentGeneration = this._preloadGeneration; + const onProgress = (data) => { + if (this._preloadGeneration === currentGeneration) + this._updateCacheProgress(data); + }; + + try { + const result = await this._resourcePreloader.preloadSuites(enabledSuites, params.resourceLoadDelay, clearCache, onProgress); + if (result === "ABORTED") + return "ABORTED"; + this._enableStartButtons(); + return undefined; + } catch (error) { + console.error("Service Worker preload failed:", error); + this._setBenchmarkState(BENCHMARK_STATE.ERROR); + this._populateInvalidScore(error?.message); + this.showResultsSummary(); + throw error; + } + } + + _resetPreloadUI() { + this._latestProgressData = null; + this._progressUpdateScheduled = false; + document.getElementById("preload-progress-completed").value = 0; + document.getElementById("preload-info-label").textContent = ""; + document.getElementById("preload-info-progress").textContent = ""; + document.body.style.setProperty("--preload-progress", "0%"); + } + + _updateCacheProgress(progressData) { + this._latestProgressData = progressData; + if (this._progressUpdateScheduled) + return; + this._progressUpdateScheduled = true; + + requestAnimationFrame(() => { + this._progressUpdateScheduled = false; + const data = this._latestProgressData; + if (!data) + return; + const { loaded, total, url, suiteName } = data; + + document.body.style.setProperty("--preload-progress", `${total > 0 ? (loaded / total) * 100 : 100}%`); + const progress = document.getElementById("preload-progress-completed"); + progress.max = total; + progress.value = loaded; + let filename = url ? url.substring(url.lastIndexOf("/") + 1) : ""; + if (!filename && url) + filename = url.substring(url.lastIndexOf("/", url.length - 2) + 1); + const labelText = suiteName ? `${suiteName}: ${filename}` : filename; + document.getElementById("preload-info-label").textContent = labelText; + document.getElementById("preload-info-progress").textContent = `${loaded} / ${total}`; + }); + } + + _enableStartButtons() { + this._setBenchmarkState(BENCHMARK_STATE.READY); + document.querySelectorAll(".start-tests-button").forEach((button) => { + button.disabled = false; + }); + } + + async _setBenchmarkState(state) { + document.body.setAttribute("data-benchmark-state", state); + + const startButtons = document.querySelectorAll(".start-tests-button"); + if (state === BENCHMARK_STATE.PRELOADING) { + this._resetPreloadUI(); + startButtons.forEach((btn) => { + btn.innerHTML = "
Preloading
"; + }); + } else if (state !== BENCHMARK_STATE.RUNNING) { + document.body.style.removeProperty("--preload-progress"); + startButtons.forEach((btn) => { + btn.innerHTML = "
Start Test
"; + }); + if (state !== BENCHMARK_STATE.READY) + await this._resourcePreloader?.clearSw(); + } + } + _hashChangeHandler() { this._showSection(window.location.hash); } diff --git a/resources/shared/params.mjs b/resources/shared/params.mjs index f9e0a0541..698e80f5d 100644 --- a/resources/shared/params.mjs +++ b/resources/shared/params.mjs @@ -43,6 +43,10 @@ export class Params { measurePrepare = false; // External config url to override internal tests. config = ""; + // Resource load delay in ms for the service worker pre-caching. + resourceLoadDelay = 0; + // Use service worker for resource preloading. + preload = false; constructor(searchParams = undefined, warnUnused = false) { if (searchParams) @@ -88,6 +92,8 @@ export class Params { this.layoutMode = this._parseEnumParam(searchParams, "layoutMode", LAYOUT_MODES); this.measurePrepare = this._parseBooleanParam(searchParams, "measurePrepare"); this.config = this._parseConfig(searchParams); + this.resourceLoadDelay = this._parseIntParam(searchParams, "resourceLoadDelay", 0); + this.preload = this._parseBooleanParam(searchParams, "preload"); if (warnUnused) { const unused = Array.from(searchParams.keys()); diff --git a/resources/shared/sw-messages.mjs b/resources/shared/sw-messages.mjs new file mode 100644 index 000000000..0ec39ff53 --- /dev/null +++ b/resources/shared/sw-messages.mjs @@ -0,0 +1,10 @@ +export const SW_MESSAGES = Object.freeze({ + PRELOAD_SUITES: "PRELOAD_SUITES", + PRELOAD_PROGRESS: "PRELOAD_PROGRESS", + PRELOAD_DONE: "PRELOAD_DONE", + RESET_PRELOADING: "RESET_PRELOADING", + FATAL_ERROR: "FATAL_ERROR", + CLEAR_CACHE: "CLEAR_CACHE", + GET_FAILED_REQUESTS: "GET_FAILED_REQUESTS", + FAILED_REQUESTS: "FAILED_REQUESTS", +}); diff --git a/resources/suite-runner.mjs b/resources/suite-runner.mjs index 61cd18541..e38c3a4c6 100644 --- a/resources/suite-runner.mjs +++ b/resources/suite-runner.mjs @@ -107,7 +107,7 @@ export class SuiteRunner { const { suiteTotal, suitePrepare } = this.#suiteResults.total; if (suiteTotal === 0) throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`); - if (this.#params.measurePrepare && suitePrepare === 0) + if ((this.#params.measurePrepare || this.#suite.measurePrepare) && suitePrepare === 0) throw new Error(`Got invalid 0-time prepare time for suite ${this.#suite.name}: ${suitePrepare}`); } diff --git a/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs b/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs index 5e6246320..ee0476b4c 100644 --- a/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs +++ b/suites-experimental/javascript-wc-indexeddb/dist/src/speedometer-utils/params.mjs @@ -39,6 +39,10 @@ export class Params { measurePrepare = false; // External config url to override internal tests. config = ""; + // Resource load delay in ms for the service worker pre-caching. + resourceLoadDelay = 0; + // Use service worker for resource preloading. + preload = false; constructor(searchParams = undefined, warnUnused = false) { if (searchParams) @@ -74,6 +78,8 @@ export class Params { this.layoutMode = this._parseEnumParam(searchParams, "layoutMode", LAYOUT_MODES); this.measurePrepare = this._parseBooleanParam(searchParams, "measurePrepare"); this.config = this._parseConfig(searchParams); + this.resourceLoadDelay = this._parseIntParam(searchParams, "resourceLoadDelay", 0); + this.preload = this._parseBooleanParam(searchParams, "preload"); if (warnUnused) { const unused = Array.from(searchParams.keys()); @@ -211,6 +217,10 @@ export class Params { toSearchParams() { return this.toSearchParamsObject().toString(); } + + isDefault() { + return this === defaultParams; + } } function isValidJsonUrl(url) { diff --git a/suites-experimental/suites.mjs b/suites-experimental/suites.mjs index 2f740e4bd..2a7bc765b 100644 --- a/suites-experimental/suites.mjs +++ b/suites-experimental/suites.mjs @@ -4,6 +4,38 @@ import { getNumberOfItemsToAdd } from "../resources/shared/todomvc-utils.mjs"; import { freezeSuites } from "../resources/suites-helper.mjs"; export const ExperimentalSuites = freezeSuites([ + { + name: "TodoMVC-JavaScript-ES5-Cached", + url: "suites/todomvc/vanilla-examples/javascript-es5/dist/index.html", + resources: "suites/todomvc/vanilla-examples/javascript-es5/dist/resources.txt", + tags: ["todomvc", "experimental", "cached"], + async prepare(page) { + (await page.waitForElement(".new-todo")).focus(); + }, + tests: [ + new BenchmarkTestStep(`Adding${getNumberOfItemsToAdd()}Items`, (page) => { + const numberOfItemsToAdd = getNumberOfItemsToAdd(); + const newTodo = page.querySelector(".new-todo"); + for (let i = 0; i < numberOfItemsToAdd; i++) { + newTodo.setValue(getTodoText("ja", i)); + newTodo.dispatchEvent("change"); + newTodo.enter("keypress"); + } + }), + new BenchmarkTestStep("CompletingAllItems", (page) => { + const numberOfItemsToAdd = getNumberOfItemsToAdd(); + const checkboxes = page.querySelectorAll(".toggle"); + for (let i = 0; i < numberOfItemsToAdd; i++) + checkboxes[i].click(); + }), + new BenchmarkTestStep("DeletingAllItems", (page) => { + const numberOfItemsToAdd = getNumberOfItemsToAdd(); + const deleteButtons = page.querySelectorAll(".destroy"); + for (let i = numberOfItemsToAdd - 1; i >= 0; i--) + deleteButtons[i].click(); + }), + ], + }, { name: "TodoMVC-LocalStorage", url: "suites-experimental/todomvc-localstorage/dist/index.html", diff --git a/suites/default-suites.mjs b/suites/default-suites.mjs index 2e9522bc6..7ffede005 100644 --- a/suites/default-suites.mjs +++ b/suites/default-suites.mjs @@ -1006,7 +1006,8 @@ export const DefaultSuites = freezeSuites([ }, { name: "Perf-Dashboard", - url: "suites/perf.webkit.org/public/v3/#/charts/?since=1678991819934&paneList=((55-1974-null-null-(5-2.5-500)))", + url: "suites/perf.webkit.org/public/v3/index.html#/charts/?since=1678991819934&paneList=((55-1974-null-null-(5-2.5-500)))", + resources: "suites/perf.webkit.org/resources.txt", tags: ["default", "chart", "webcomponents"], async prepare(page) { await page.waitForElement("#app-is-ready"); diff --git a/suites/perf.webkit.org/resources.txt b/suites/perf.webkit.org/resources.txt new file mode 100644 index 000000000..661dcb374 --- /dev/null +++ b/suites/perf.webkit.org/resources.txt @@ -0,0 +1,144 @@ +public/api/analysis-tasks-platform=55-metric=1006 +public/api/analysis-tasks-platform=55-metric=1407 +public/api/analysis-tasks-platform=55-metric=1648 +public/api/analysis-tasks-platform=55-metric=1649 +public/api/analysis-tasks-platform=55-metric=1974 +public/api/measurement-set-platform=16-metric=1 +public/api/measurement-set-platform=16-metric=13 +public/api/measurement-set-platform=16-metric=17 +public/api/measurement-set-platform=16-metric=1726 +public/api/measurement-set-platform=16-metric=6 +public/api/measurement-set-platform=16-metric=7 +public/api/measurement-set-platform=55-metric=1 +public/api/measurement-set-platform=55-metric=1006 +public/api/measurement-set-platform=55-metric=13 +public/api/measurement-set-platform=55-metric=1407 +public/api/measurement-set-platform=55-metric=1648 +public/api/measurement-set-platform=55-metric=1649 +public/api/measurement-set-platform=55-metric=17 +public/api/measurement-set-platform=55-metric=1726 +public/api/measurement-set-platform=55-metric=1974 +public/api/measurement-set-platform=55-metric=6 +public/api/measurement-set-platform=55-metric=7 +public/common.css +public/data/manifest.json +public/data/measurement-set-16-1-1682812800000.json +public/data/measurement-set-16-1.json +public/data/measurement-set-16-13-1682812800000.json +public/data/measurement-set-16-13.json +public/data/measurement-set-16-17-1682812800000.json +public/data/measurement-set-16-17.json +public/data/measurement-set-16-1726-1682812800000.json +public/data/measurement-set-16-1726.json +public/data/measurement-set-16-6-1682812800000.json +public/data/measurement-set-16-6.json +public/data/measurement-set-16-7-1682812800000.json +public/data/measurement-set-16-7.json +public/data/measurement-set-55-1-1682812800000.json +public/data/measurement-set-55-1.json +public/data/measurement-set-55-13-1682812800000.json +public/data/measurement-set-55-13.json +public/data/measurement-set-55-1407-1682812800000.json +public/data/measurement-set-55-1407.json +public/data/measurement-set-55-1648-1682812800000.json +public/data/measurement-set-55-1648.json +public/data/measurement-set-55-1649-1682812800000.json +public/data/measurement-set-55-1649.json +public/data/measurement-set-55-17-1682812800000.json +public/data/measurement-set-55-17.json +public/data/measurement-set-55-1726-1682812800000.json +public/data/measurement-set-55-1726.json +public/data/measurement-set-55-1974-1682812800000.json +public/data/measurement-set-55-1974.json +public/data/measurement-set-55-6-1682812800000.json +public/data/measurement-set-55-6.json +public/data/measurement-set-55-7-1682812800000.json +public/data/measurement-set-55-7.json +public/shared/common-component-base.js +public/shared/common-remote.js +public/shared/statistics.js +public/v3/async-task.js +public/v3/bundled-scripts.js +public/v3/commit-set-range-bisector.js +public/v3/components/analysis-results-viewer.js +public/v3/components/analysis-task-bug-list.js +public/v3/components/bar-graph-group.js +public/v3/components/base.js +public/v3/components/button-base.js +public/v3/components/chart-pane-base.js +public/v3/components/chart-revision-range.js +public/v3/components/chart-status-evaluator.js +public/v3/components/chart-styles.js +public/v3/components/close-button.js +public/v3/components/combo-box.js +public/v3/components/commit-log-viewer.js +public/v3/components/custom-analysis-task-configurator.js +public/v3/components/custom-configuration-test-group-form.js +public/v3/components/customizable-test-group-form.js +public/v3/components/dashboard-chart-status-view.js +public/v3/components/editable-text.js +public/v3/components/expand-collapse-button.js +public/v3/components/freshness-indicator.js +public/v3/components/instant-file-uploader.js +public/v3/components/interactive-time-series-chart.js +public/v3/components/minus-button.js +public/v3/components/mutable-list-view.js +public/v3/components/owned-commit-viewer.js +public/v3/components/pane-selector.js +public/v3/components/plus-button.js +public/v3/components/ratio-bar-graph.js +public/v3/components/repetition-type-selection.js +public/v3/components/results-table.js +public/v3/components/spinner-icon.js +public/v3/components/test-group-form.js +public/v3/components/test-group-results-viewer.js +public/v3/components/test-group-revision-table.js +public/v3/components/time-series-chart.js +public/v3/components/warning-icon.js +public/v3/index.html +public/v3/instrumentation.js +public/v3/lazily-evaluated-function.js +public/v3/main.js +public/v3/models/analysis-results.js +public/v3/models/analysis-task.js +public/v3/models/bug-tracker.js +public/v3/models/bug.js +public/v3/models/build-request.js +public/v3/models/builder.js +public/v3/models/commit-log.js +public/v3/models/commit-set.js +public/v3/models/data-model.js +public/v3/models/manifest.js +public/v3/models/measurement-adaptor.js +public/v3/models/measurement-cluster.js +public/v3/models/measurement-set.js +public/v3/models/metric.js +public/v3/models/platform-group.js +public/v3/models/platform.js +public/v3/models/repository.js +public/v3/models/test-group.js +public/v3/models/test.js +public/v3/models/time-series.js +public/v3/models/triggerable.js +public/v3/models/uploaded-file.js +public/v3/pages/analysis-category-page.js +public/v3/pages/analysis-category-toolbar.js +public/v3/pages/analysis-task-page.js +public/v3/pages/build-request-queue-page.js +public/v3/pages/chart-pane-status-view.js +public/v3/pages/chart-pane.js +public/v3/pages/charts-page.js +public/v3/pages/charts-toolbar.js +public/v3/pages/create-analysis-task-page.js +public/v3/pages/dashboard-page.js +public/v3/pages/dashboard-toolbar.js +public/v3/pages/domain-control-toolbar.js +public/v3/pages/heading.js +public/v3/pages/page-router.js +public/v3/pages/page-with-heading.js +public/v3/pages/page.js +public/v3/pages/summary-page.js +public/v3/pages/test-freshness-page.js +public/v3/pages/toolbar.js +public/v3/privileged-api.js +public/v3/remote.js diff --git a/sw.mjs b/sw.mjs new file mode 100644 index 000000000..8365dbff7 --- /dev/null +++ b/sw.mjs @@ -0,0 +1,329 @@ +import { SW_MESSAGES } from "./resources/shared/sw-messages.mjs"; + +const CACHE_NAME = "speedometer-cache-v4.0"; +const DB_NAME = "SpeedometerStateDB"; +const STORE_NAME = "SpeedometerSWState"; +const DB_VERSION = 2; +let failedRequests = new Set(); + +class LockStore { + constructor() { + this.dbPromise = null; + } + + async _openDB() { + if (this.dbPromise) + return this.dbPromise; + + this.dbPromise = new Promise((resolve, reject) => { + const request = indexedDB.open(DB_NAME, DB_VERSION); + request.onupgradeneeded = (event) => { + const db = event.target.result; + if (!db.objectStoreNames.contains(STORE_NAME)) + db.createObjectStore(STORE_NAME); + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => { + this.dbPromise = null; + reject(request.error); + }; + }); + return this.dbPromise; + } + + async _runTransaction(mode, callback) { + try { + const db = await this._openDB(); + return await new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, mode); + const req = callback(tx.objectStore(STORE_NAME)); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + } catch (e) { + console.warn(`IndexedDB ${mode} failed`, e); + return null; + } + } + + async getOwner() { + const data = await this._runTransaction("readonly", (store) => store.get("stateData")); + return data?.clientId || null; + } + + async setOwner(clientId) { + await this._runTransaction("readwrite", (store) => store.put({ clientId }, "stateData")); + } + + async clear() { + await this._runTransaction("readwrite", (store) => store.delete("stateData")); + } + + async hasLock(clientId) { + const activeClientId = await this.getOwner(); + if (!activeClientId) + return true; + return activeClientId === clientId; + } +} + +const STORE = new LockStore(); + +function replyToClient(event, type, msg = {}) { + if (event.ports?.[0]) { + event.ports[0].start?.(); + event.ports[0].postMessage({ type, ...msg }); + } +} + +function replyError(event, message) { + replyToClient(event, "ERROR", { message }); +} + +function delayAsync(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +const MAX_CONCURRENT_REQUESTS = 20; + +class RequestLimiter { + constructor(limit = MAX_CONCURRENT_REQUESTS) { + this.limit = limit; + this.active = 0; + this.queue = []; + } + + async _processQueue() { + while (this.queue.length > 0) { + const task = this.queue.shift(); + try { + await task(); + } catch (e) { + // Individual task errors are handled by their respective promises + } + await delayAsync(1); // Yield to event loop + } + this.active--; + } + + schedule(fn) { + return new Promise((resolve, reject) => { + const task = () => fn().then(resolve, reject); + task.resolve = resolve; + this.queue.push(task); + + if (this.active < this.limit) { + this.active++; + this._processQueue(); // Intentionally not awaited + } + }); + } + + clear() { + for (const task of this.queue) + task.resolve?.(0); + + this.queue.length = 0; + } +} + +const requestLimiter = new RequestLimiter(MAX_CONCURRENT_REQUESTS); + +let currentPreloadEvent = null; +let currentPreloadId = 0; + +function handleResetPreloadingMessage(event) { + if (currentPreloadEvent) + replyToClient(currentPreloadEvent, "PRELOAD_ABORTED"); + + currentPreloadEvent = null; + currentPreloadId++; + requestLimiter.clear(); + if (event) + replyToClient(event, SW_MESSAGES.RESET_PRELOADING); +} + +async function handlePreloadSuitesMessage(event, clientId, { suites = [], delay = 0, clearCache = true }) { + try { + await STORE.getOwner(); + await updateActiveClient(clientId); + + handleResetPreloadingMessage(); // Call it without event to avoid replying to the PRELOAD_SUITES channel! + + currentPreloadEvent = event; + const preloadId = currentPreloadId; + + failedRequests.clear(); + if (clearCache) + await caches.delete(CACHE_NAME); + + const cache = await caches.open(CACHE_NAME); + const urlsToCache = await getUrlsToCache(suites); + const total = urlsToCache.length; + + let loaded = 0; + let transferredSize = 0; + + const promises = urlsToCache.map(async (item, index) => { + if (preloadId !== currentPreloadId) + return; + const size = await fetchAndCache(cache, item.url, delay * index); + if (preloadId !== currentPreloadId) + return; + transferredSize += size; + loaded++; + replyToClient(event, SW_MESSAGES.PRELOAD_PROGRESS, { loaded, total, url: item.url, suiteName: item.suiteName }); + }); + + await Promise.all(promises); + + if (preloadId !== currentPreloadId) + return; + + if (!await STORE.hasLock(clientId)) { + replyError(event, "Speedometer aborted: Another tab took over."); + return; + } + replyToClient(event, SW_MESSAGES.PRELOAD_DONE, { transferredSize, count: urlsToCache.length }); + } catch (error) { + console.error("Error during preload:", error); + replyError(event, error.message || "Failed to preload resources."); + } finally { + if (currentPreloadEvent === event) + currentPreloadEvent = null; + } +} + +async function getUrlsToCache(suites) { + const urlsToCache = []; + for (const suite of suites) { + if (suite.resources) + urlsToCache.push(...await parseSuiteResources(suite)); + } + + return urlsToCache; +} + +async function parseSuiteResources(suite) { + const response = await fetch(suite.resources); + if (!response.ok) { + console.error(`Failed to load resources for suite ${suite.name}: ${suite.resources} returned ${response.status}`); + throw new Error(`Failed to load resources for suite ${suite.name}: ${suite.resources} returned ${response.status}`); + } + const text = await response.text(); + return text + .trim() + .split("\n") + .map((resourceUrl) => ({ + url: new URL(resourceUrl.trim(), suite.url).href, + suiteName: suite.name, + })); +} + +function getResponseSize(response) { + const contentLength = response.headers.get("content-length"); + return contentLength ? parseInt(contentLength, 10) : 0; +} + +async function fetchAndCache(cache, url, delayMs) { + if (delayMs) + await delayAsync(delayMs); + return requestLimiter.schedule(async () => { + const request = new Request(url, { cache: "no-cache" }); + const existing = await cache.match(request, { ignoreSearch: true }); + if (existing) + return getResponseSize(existing); + + try { + const response = await fetch(request); + if (!response.ok) + return 0; + const size = getResponseSize(response); + await cache.put(request, response); + return size; + } catch (e) { + console.warn("Cache failed for", url, e); + return 0; + } + }); +} + +self.addEventListener("install", () => { + self.skipWaiting(); +}); + +self.addEventListener("activate", (event) => { + event.waitUntil(self.clients.claim()); +}); + +async function updateActiveClient(newClientId) { + const oldClientId = await STORE.getOwner(); + if (oldClientId === newClientId) + return false; + + await STORE.setOwner(newClientId); + + if (oldClientId) { + const oldClient = await self.clients.get(oldClientId); + oldClient?.postMessage({ type: SW_MESSAGES.FATAL_ERROR, message: "Speedometer aborted: Another tab took over." }); + } + return true; +} + +async function handleClearCacheMessage(event, clientId) { + try { + if (!await STORE.hasLock(clientId)) { + replyError(event, "Cannot clear SW: You do not own the lock."); + return; + } + await STORE.clear(); + replyToClient(event, "SUCCESS"); + } catch (e) { + replyError(event, e.message || "Failed to clear cache"); + } +} + +async function handleGetFailedRequestsMessage(event, clientId) { + replyToClient(event, SW_MESSAGES.FAILED_REQUESTS, { requests: Array.from(failedRequests) }); +} + +const MESSAGE_HANDLERS = Object.freeze({ + [SW_MESSAGES.PRELOAD_SUITES]: handlePreloadSuitesMessage, + [SW_MESSAGES.RESET_PRELOADING]: handleResetPreloadingMessage, + [SW_MESSAGES.CLEAR_CACHE]: handleClearCacheMessage, + [SW_MESSAGES.GET_FAILED_REQUESTS]: handleGetFailedRequestsMessage, +}); + +self.addEventListener("message", (event) => { + const { data } = event; + if (!data) + return; + const clientId = data.clientId || event.source?.id; + const handler = MESSAGE_HANDLERS[data.type]; + if (handler) + event.waitUntil(handler(event, clientId, data)); + else + console.error("Unknown service worker message type:", data.type); +}); + +self.addEventListener("fetch", (event) => { + if (event.request.cache === "only-if-cached" && event.request.mode !== "same-origin") + return; + + event.respondWith( + (async () => { + let requestToMatch = event.request; + const urlObj = new URL(event.request.url); + if (urlObj.pathname.endsWith("/")) { + urlObj.pathname += "index.html"; + requestToMatch = urlObj.href; + } + + const cachedResponse = await caches.match(requestToMatch, { cacheName: CACHE_NAME, ignoreSearch: true }); + if (cachedResponse) + return cachedResponse; + + return fetch(event.request); + })() + ); +}); diff --git a/tests/index.html b/tests/index.html index a6bc60a3b..821356444 100644 --- a/tests/index.html +++ b/tests/index.html @@ -28,6 +28,7 @@ }); await import("./unittests/benchmark-runner.mjs"); + await import("./unittests/benchmark-configurator.mjs"); await import("./unittests/params.mjs"); await import("./unittests/suites.mjs"); diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index cdb81ebb4..f10177ea4 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -97,6 +97,18 @@ async function testAll() { assert(metrics.Score.values.length === 1); } +async function testPreloadEnabled() { + const metrics = await testPage("index.html?iterationCount=1&suites=Perf-Dashboard&preload=true"); + assert("Perf-Dashboard" in metrics); + assert(metrics["Perf-Dashboard"].values.length === 1); +} + +async function testPreloadDisabled() { + const metrics = await testPage("index.html?iterationCount=1&suites=Perf-Dashboard&preload=false"); + assert("Perf-Dashboard" in metrics); + assert(metrics["Perf-Dashboard"].values.length === 1); +} + async function testDeveloperMode() { const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Perf-Dashboard"]; const metrics = await testPage(`index.html?${params.join("&")}`); @@ -140,6 +152,8 @@ async function tryTests() { await driver.manage().setTimeouts({ script: 60000 }); await testIterations(); await testAll(); + await testPreloadEnabled(); + await testPreloadDisabled(); await testDeveloperMode(); console.log("\nTests complete!"); } diff --git a/tests/unittests/benchmark-configurator.mjs b/tests/unittests/benchmark-configurator.mjs new file mode 100644 index 000000000..82585db5f --- /dev/null +++ b/tests/unittests/benchmark-configurator.mjs @@ -0,0 +1,42 @@ +import { BenchmarkConfigurator } from "../../resources/benchmark-configurator.mjs"; + +describe("BenchmarkConfigurator", () => { + it("should set measurePrepare to false on processed suites if not present", async () => { + const configurator = new BenchmarkConfigurator(); + await configurator.init(); + const suites = configurator.suites; + expect(suites.length).to.be.greaterThan(0); + suites.forEach((suite) => { + expect(suite).to.have.property("measurePrepare"); + expect(suite.measurePrepare).to.be.a("boolean"); + }); + }); + + it("should validate URLs correctly in _isValidUrl", () => { + const configurator = new BenchmarkConfigurator(); + expect(configurator._isValidUrl("http://example.com")).to.be(true); + expect(configurator._isValidUrl("suites/todomvc/index.html")).to.be(true); + expect(configurator._isValidUrl("")).to.be(false); + expect(configurator._isValidUrl(null)).to.be(false); + }); + + it("should enable suites by names", async () => { + const configurator = new BenchmarkConfigurator(); + await configurator.init(); + const firstSuiteName = configurator.suites[0].name; + configurator.enableSuites([firstSuiteName], []); + expect(configurator.suites[0].enabled).to.be(true); + // Assuming there are multiple suites + expect(configurator.suites[1].enabled).to.be(false); + }); + + it("should enable suites by tags", async () => { + const configurator = new BenchmarkConfigurator(); + await configurator.init(); + configurator.enableSuites([], ["todomvc"]); + const todomvcSuites = configurator.suites.filter((suite) => suite.tags.includes("todomvc")); + const otherSuites = configurator.suites.filter((suite) => !suite.tags.includes("todomvc")); + todomvcSuites.forEach((suite) => expect(suite.enabled).to.be(true)); + otherSuites.forEach((suite) => expect(suite.enabled).to.be(false)); + }); +}); diff --git a/tests/unittests/params.mjs b/tests/unittests/params.mjs index 0b3821445..e61fe6187 100644 --- a/tests/unittests/params.mjs +++ b/tests/unittests/params.mjs @@ -93,6 +93,24 @@ describe("Params", () => { }); }); + describe("isDefault", () => { + it("should return true for defaultParams", () => { + expect(defaultParams.isDefault()).to.be(true); + }); + it("should return false for newly instantiated empty Params", () => { + const params = new Params(); + expect(params.isDefault()).to.be(false); + }); + it("should return false for custom params", () => { + const params = new Params( + new URLSearchParams({ + iterationCount: "100", + }) + ); + expect(params.isDefault()).to.be(false); + }); + }); + describe("parse input params", () => { it("should parse custom viewport", () => { const params = new Params(