From 024d31d5346a5e629dbea4e91f7b04caa62b1400 Mon Sep 17 00:00:00 2001 From: Aditya Singh Date: Sat, 23 May 2026 07:50:34 -0700 Subject: [PATCH] feat(aria-snapshot): add numberSubstitution option for baseline generation Closes #40080. The aria-snapshot baseline generator currently rewrites numbers and dates into \d+ regexes, which is helpful for live UIs but unwanted for tests against mocked endpoints where the values are stable. This adds a numberSubstitution: 'regex' | 'static' option (default 'regex' so existing baselines do not change) that can be set per matcher or globally via the expect config. --- docs/src/api/class-locatorassertions.md | 14 ++ docs/src/api/class-pageassertions.md | 14 ++ docs/src/test-api/class-testconfig.md | 1 + docs/src/test-api/class-testproject.md | 1 + packages/injected/src/ariaSnapshot.ts | 12 +- packages/injected/src/injectedScript.ts | 4 +- packages/playwright/src/matchers/expect.ts | 1 + .../src/matchers/toMatchAriaSnapshot.ts | 6 +- packages/playwright/types/test.d.ts | 44 ++++++ .../update-aria-snapshot.spec.ts | 139 ++++++++++++++++++ 10 files changed, 228 insertions(+), 8 deletions(-) diff --git a/docs/src/api/class-locatorassertions.md b/docs/src/api/class-locatorassertions.md index dcfafda26a7fd..ea8bc160187bc 100644 --- a/docs/src/api/class-locatorassertions.md +++ b/docs/src/api/class-locatorassertions.md @@ -2383,6 +2383,13 @@ assertThat(page.locator("body")).matchesAriaSnapshot(""" ### option: LocatorAssertions.toMatchAriaSnapshot.timeout = %%-csharp-java-python-assertions-timeout-%% * since: v1.49 +### option: LocatorAssertions.toMatchAriaSnapshot.numberSubstitution +* since: v1.55 +* langs: js +- `numberSubstitution` <["regex"|"static"]> + +Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + ## async method: LocatorAssertions.toMatchAriaSnapshot#2 * since: v1.50 * langs: js @@ -2411,3 +2418,10 @@ Generates sequential names if not specified. ### option: LocatorAssertions.toMatchAriaSnapshot#2.timeout = %%-csharp-java-python-assertions-timeout-%% * since: v1.50 + +### option: LocatorAssertions.toMatchAriaSnapshot#2.numberSubstitution +* since: v1.55 +* langs: js +- `numberSubstitution` <["regex"|"static"]> + +Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. diff --git a/docs/src/api/class-pageassertions.md b/docs/src/api/class-pageassertions.md index ce5de2f09b212..4c189e6b7087c 100644 --- a/docs/src/api/class-pageassertions.md +++ b/docs/src/api/class-pageassertions.md @@ -186,6 +186,13 @@ assertThat(page).matchesAriaSnapshot(""" ### option: PageAssertions.toMatchAriaSnapshot.timeout = %%-csharp-java-python-assertions-timeout-%% * since: v1.60 +### option: PageAssertions.toMatchAriaSnapshot.numberSubstitution +* since: v1.55 +* langs: js +- `numberSubstitution` <["regex"|"static"]> + +Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + ## async method: PageAssertions.NotToMatchAriaSnapshot * since: v1.60 * langs: python @@ -225,6 +232,13 @@ Generates sequential names if not specified. ### option: PageAssertions.toMatchAriaSnapshot#2.timeout = %%-js-assertions-timeout-%% * since: v1.60 +### option: PageAssertions.toMatchAriaSnapshot#2.numberSubstitution +* since: v1.55 +* langs: js +- `numberSubstitution` <["regex"|"static"]> + +Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + ## async method: PageAssertions.toHaveScreenshot#1 * since: v1.23 * langs: js diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 83cae0624a263..c5e2f082b2868 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -84,6 +84,7 @@ The structure of the git commit metadata is subject to change. - `toMatchAriaSnapshot` ?<[Object]> Configuration for the [`method: LocatorAssertions.toMatchAriaSnapshot#2`] method. - `pathTemplate` ?<[string]> A template controlling location of the aria snapshots. See [`property: TestConfig.snapshotPathTemplate`] for details. - `children` ?<["contain" | "equal" | "deep-equal"]> Controls how children of the snapshot root are matched against the actual accessibility tree. This is equivalent to adding a `/children` property at the top of every aria snapshot template. Individual snapshots can override this by including an explicit `/children` property. + - `numberSubstitution` ?<["regex" | "static"]> Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. Individual assertions can override this via the per-call option. - `toMatchSnapshot` ?<[Object]> Configuration for the [`method: SnapshotAssertions.toMatchSnapshot#1`] method. - `maxDiffPixels` ?<[int]> An acceptable amount of pixels that could be different, unset by default. - `maxDiffPixelRatio` ?<[float]> An acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 6cb2a98e147c7..c70d5655868fb 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -103,6 +103,7 @@ export default defineConfig({ - `toMatchAriaSnapshot` ?<[Object]> Configuration for the [`method: LocatorAssertions.toMatchAriaSnapshot#2`] method. - `pathTemplate` ?<[string]> A template controlling location of the aria snapshots. See [`property: TestProject.snapshotPathTemplate`] for details. - `children` ?<["contain" | "equal" | "deep-equal"]> Controls how children of the snapshot root are matched against the actual accessibility tree. This is equivalent to adding a `/children` property at the top of every aria snapshot template. Individual snapshots can override this by including an explicit `/children` property. + - `numberSubstitution` ?<["regex" | "static"]> Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. Individual assertions can override this via the per-call option. - `toMatchSnapshot` ?<[Object]> Configuration for the [`method: SnapshotAssertions.toMatchSnapshot#1`] method. - `threshold` ?<[float]> an acceptable perceived color difference between the same pixel in compared images, ranging from `0` (strict) and `1` (lax). `"pixelmatch"` comparator computes color difference in [YIQ color space](https://en.wikipedia.org/wiki/YIQ) and defaults `threshold` value to `0.2`. - `maxDiffPixels` ?<[int]> an acceptable amount of pixels that could be different, unset by default. diff --git a/packages/injected/src/ariaSnapshot.ts b/packages/injected/src/ariaSnapshot.ts index 17b438dc13616..8d30b420adce0 100644 --- a/packages/injected/src/ariaSnapshot.ts +++ b/packages/injected/src/ariaSnapshot.ts @@ -42,6 +42,7 @@ export type AriaTreeOptions = { doNotRenderActive?: boolean; depth?: number; boxes?: boolean; + numberSubstitution?: 'regex' | 'static'; }; type InternalOptions = { @@ -74,8 +75,11 @@ function toInternalOptions(options: AriaTreeOptions): InternalOptions { return { visibility: 'ariaAndVisible', refs: 'none', renderBoxes }; } if (options.mode === 'codegen') { - // To generate aria assertion with regex heurisitcs. - return { visibility: 'aria', refs: 'none', renderStringsAsRegex: true, renderBoxes }; + // To generate aria assertion with regex heuristics. The numberSubstitution + // option (default 'regex') lets callers opt out of auto-converting numbers + // and dates into \d+ regexes when the baseline values are stable. + const renderStringsAsRegex = options.numberSubstitution !== 'static'; + return { visibility: 'aria', refs: 'none', renderStringsAsRegex, renderBoxes }; } // To match aria snapshot. return { visibility: 'aria', refs: 'none', renderBoxes }; @@ -390,14 +394,14 @@ export type MatcherReceived = { regex: string; }; -export function matchesExpectAriaTemplate(rootElement: Element, template: aria.AriaTemplateNode): { matches: aria.AriaNode[], received: MatcherReceived } { +export function matchesExpectAriaTemplate(rootElement: Element, template: aria.AriaTemplateNode, options?: { numberSubstitution?: 'regex' | 'static' }): { matches: aria.AriaNode[], received: MatcherReceived } { const snapshot = generateAriaTree(rootElement, { mode: 'default' }); const matches = matchesNodeDeep(snapshot.root, template, false, false); return { matches, received: { raw: renderAriaTree(snapshot, { mode: 'default' }).text, - regex: renderAriaTree(snapshot, { mode: 'codegen' }).text, + regex: renderAriaTree(snapshot, { mode: 'codegen', numberSubstitution: options?.numberSubstitution }).text, } }; } diff --git a/packages/injected/src/injectedScript.ts b/packages/injected/src/injectedScript.ts index e7b476fd1c6d9..d7f63d495a26f 100644 --- a/packages/injected/src/injectedScript.ts +++ b/packages/injected/src/injectedScript.ts @@ -1509,7 +1509,7 @@ export class InjectedScript { if (options.expression === 'to.match.aria' && !options.selector) { if (!this.document.body) return { matches: options.isNot, missingReceived: true }; - const result = matchesExpectAriaTemplate(this.document.body, options.expectedValue); + const result = matchesExpectAriaTemplate(this.document.body, options.expectedValue, options.expressionArg); return { received: result.received, matches: !!result.matches.length, @@ -1627,7 +1627,7 @@ export class InjectedScript { { if (expression === 'to.match.aria') { - const result = matchesExpectAriaTemplate(element, options.expectedValue); + const result = matchesExpectAriaTemplate(element, options.expectedValue, options.expressionArg); return { received: result.received, matches: !!result.matches.length, diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts index c4de97609df82..aaa7b3e0dae30 100644 --- a/packages/playwright/src/matchers/expect.ts +++ b/packages/playwright/src/matchers/expect.ts @@ -139,6 +139,7 @@ export type ExpectConfig = { toMatchAriaSnapshot?: { pathTemplate?: string; children?: 'contain' | 'equal' | 'deep-equal'; + numberSubstitution?: 'regex' | 'static'; }; toPass?: { timeout?: number; intervals?: number[] }; }; diff --git a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts index 5fa1965c28fb7..c4ca0f710cc2b 100644 --- a/packages/playwright/src/matchers/toMatchAriaSnapshot.ts +++ b/packages/playwright/src/matchers/toMatchAriaSnapshot.ts @@ -42,7 +42,7 @@ export async function toMatchAriaSnapshot( this: ExpectMatcherStateInternal, receiver: LocatorEx | Page, expectedParam?: ToMatchAriaSnapshotExpected, - options: { timeout?: number } = {}, + options: { timeout?: number, numberSubstitution?: 'regex' | 'static' } = {}, ): Promise> { const matcherName = 'toMatchAriaSnapshot'; expectTypes(receiver, ['Page', 'Locator'], matcherName); @@ -91,7 +91,9 @@ export async function toMatchAriaSnapshot( if (globalChildren && !expected.match(/^- \/children:/m)) expected = `- /children: ${globalChildren}\n` + expected; - const expectParams = { expectedValue: expected, isNot: this.isNot, timeout }; + const numberSubstitution = options.numberSubstitution ?? expectConfig().toMatchAriaSnapshot?.numberSubstitution ?? 'regex'; + const expressionArg = numberSubstitution === 'static' ? { numberSubstitution } : undefined; + const expectParams = { expectedValue: expected, expressionArg, isNot: this.isNot, timeout }; const { matches: pass, received, log, timedOut, errorMessage } = locator ? await (locator as LocatorEx)._expect('to.match.aria', expectParams) : await ((receiver as Page).mainFrame() as FrameEx)._expect('to.match.aria', expectParams); diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 1e292f153a35a..de9bb899b6300 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -313,6 +313,14 @@ interface TestProject { * including an explicit `/children` property. */ children?: "contain"|"equal"|"deep-equal"; + + /** + * Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to + * `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep + * the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + * Individual assertions can override this via the per-call option. + */ + numberSubstitution?: "regex"|"static"; }; /** @@ -1253,6 +1261,14 @@ interface TestConfig { * including an explicit `/children` property. */ children?: "contain"|"equal"|"deep-equal"; + + /** + * Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to + * `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep + * the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + * Individual assertions can override this via the per-call option. + */ + numberSubstitution?: "regex"|"static"; }; /** @@ -9717,6 +9733,13 @@ interface LocatorAssertions { * @param options */ toMatchAriaSnapshot(expected: string, options?: { + /** + * Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to + * `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep + * the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + */ + numberSubstitution?: "regex"|"static"; + /** * Time to retry the assertion for in milliseconds. Defaults to `timeout` in `TestConfig.expect`. */ @@ -9745,6 +9768,13 @@ interface LocatorAssertions { */ name?: string; + /** + * Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to + * `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep + * the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + */ + numberSubstitution?: "regex"|"static"; + /** * Time to retry the assertion for in milliseconds. Defaults to `timeout` in `TestConfig.expect`. */ @@ -9894,6 +9924,13 @@ interface PageAssertions { * @param options */ toMatchAriaSnapshot(expected: string, options?: { + /** + * Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to + * `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep + * the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + */ + numberSubstitution?: "regex"|"static"; + /** * Time to retry the assertion for in milliseconds. Defaults to `timeout` in `TestConfig.expect`. */ @@ -9922,6 +9959,13 @@ interface PageAssertions { */ name?: string; + /** + * Controls how numbers and dates are rendered when generating or updating the aria-snapshot baseline. Defaults to + * `'regex'`, which rewrites numeric runs into `\d+` patterns so live UIs match across runs. Set to `'static'` to keep + * the actual values in the baseline, which is useful when the UI is driven by mocked or otherwise stable data. + */ + numberSubstitution?: "regex"|"static"; + /** * Time to retry the assertion for in milliseconds. Defaults to `timeout` in `TestConfig.expect`. */ diff --git a/tests/playwright-test/update-aria-snapshot.spec.ts b/tests/playwright-test/update-aria-snapshot.spec.ts index ae11bfcaf35bc..71ee036251d83 100644 --- a/tests/playwright-test/update-aria-snapshot.spec.ts +++ b/tests/playwright-test/update-aria-snapshot.spec.ts @@ -223,6 +223,145 @@ test('should generate baseline with regex', async ({ runInlineTest }, testInfo) expect(result2.exitCode).toBe(0); }); +test('should generate baseline with static numbers when numberSubstitution is static (per-call)', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`
    +
  • Item 1
    • +
  • Time 15:30
    • +
  • Year 2022
    • +
  • Duration 12ms
    • +
  • Total 22
    • +
\`); + await expect(page.locator('body')).toMatchAriaSnapshot(\`\`, { numberSubstitution: 'static' }); + }); + ` + }); + + expect(result.exitCode).toBe(1); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + const data = fs.readFileSync(patchPath, 'utf-8'); + expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts +--- a/a.spec.ts ++++ b/a.spec.ts +@@ -8,6 +8,13 @@ +
  • Duration 12ms
    • +
  • Total 22
    • + \`); +- await expect(page.locator('body')).toMatchAriaSnapshot(\`\`, { numberSubstitution: 'static' }); ++ await expect(page.locator('body')).toMatchAriaSnapshot(\` ++ - list: ++ - listitem: Item 1 ++ - listitem: Time 15:30 ++ - listitem: Year 2022 ++ - listitem: Duration 12ms ++ - listitem: Total 22 ++ \`, { numberSubstitution: 'static' }); + }); + +\\ No newline at end of file +`); + + execSync(`patch -p1 < ${patchPath}`, { cwd: testInfo.outputPath() }); + const result2 = await runInlineTest({}); + expect(result2.exitCode).toBe(0); +}); + +test('should generate baseline with static numbers when numberSubstitution is static (global config)', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'playwright.config.ts': ` + export default { + expect: { toMatchAriaSnapshot: { numberSubstitution: 'static' } }, + }; + `, + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`
      +
    • Item 1
      • +
    • Time 15:30
      • +
    • Year 2022
      • +
    \`); + await expect(page.locator('body')).toMatchAriaSnapshot(\`\`); + }); + ` + }); + + expect(result.exitCode).toBe(1); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + const data = fs.readFileSync(patchPath, 'utf-8'); + expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts +--- a/a.spec.ts ++++ b/a.spec.ts +@@ -6,6 +6,11 @@ +
  • Time 15:30
    • +
  • Year 2022
    • + \`); +- await expect(page.locator('body')).toMatchAriaSnapshot(\`\`); ++ await expect(page.locator('body')).toMatchAriaSnapshot(\` ++ - list: ++ - listitem: Item 1 ++ - listitem: Time 15:30 ++ - listitem: Year 2022 ++ \`); + }); + +\\ No newline at end of file +`); + + execSync(`patch -p1 < ${patchPath}`, { cwd: testInfo.outputPath() }); + const result2 = await runInlineTest({}); + expect(result2.exitCode).toBe(0); +}); + +test('per-call numberSubstitution should override the global setting', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + '.git/marker': '', + 'playwright.config.ts': ` + export default { + expect: { toMatchAriaSnapshot: { numberSubstitution: 'static' } }, + }; + `, + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('test', async ({ page }) => { + await page.setContent(\`
      +
    • Total 22
      • +
    \`); + await expect(page.locator('body')).toMatchAriaSnapshot(\`\`, { numberSubstitution: 'regex' }); + }); + ` + }); + + expect(result.exitCode).toBe(1); + const patchPath = testInfo.outputPath('test-results/rebaselines.patch'); + const data = fs.readFileSync(patchPath, 'utf-8'); + expect(trimPatch(data)).toBe(`diff --git a/a.spec.ts b/a.spec.ts +--- a/a.spec.ts ++++ b/a.spec.ts +@@ -4,6 +4,9 @@ + await page.setContent(\`
      +
    • Total 22
      • +
    \`); +- await expect(page.locator('body')).toMatchAriaSnapshot(\`\`, { numberSubstitution: 'regex' }); ++ await expect(page.locator('body')).toMatchAriaSnapshot(\` ++ - list: ++ - listitem: /Total \\\\d+/ ++ \`, { numberSubstitution: 'regex' }); + }); + +\\ No newline at end of file +`); + + execSync(`patch -p1 < ${patchPath}`, { cwd: testInfo.outputPath() }); + const result2 = await runInlineTest({}); + expect(result2.exitCode).toBe(0); +}); + test('should generate baseline with special characters', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ '.git/marker': '',