CI: test stability

.github/workflows/run-testcafe-on-gh-pages.yml

@@ -78,7 +78,7 @@ jobs:
               working-directory: devextreme/apps/demos
               env:
                 CHANGEDFILEINFOSPATH: changed-files.json
-                BROWSERS: chrome:headless --window-size=1200,800 --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl="swiftshader" --disable-features=PaintHolding --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning
+                BROWSERS: chrome:headless --window-size=1200,800 --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl="swiftshader" --disable-features=PaintHolding,KeyboardFocusableScrollers --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning
                 #DEBUG: hammerhead:*,testcafe:*
                 CONCURRENCY: 4
                 TCQUARANTINE: true

.github/workflows/testcafe_tests.yml

@@ -193,6 +193,7 @@ jobs:
       if: ${{ !matrix.ARGS.styleDependent || needs.check-should-run.outputs.styles-changed == 'true' }}
       env:
         NODE_OPTIONS: --max-old-space-size=8192
+        RUN_LABEL: ${{ matrix.ARGS.name }}
       run: |
         if [ "${{ matrix.ARGS.theme }}" != "" ]; then
           THEME="--theme ${{ matrix.ARGS.theme }}"
@@ -203,7 +204,12 @@ jobs:
         [ "${{ matrix.ARGS.concurrency }}" != "" ] && CONCURRENCY="--concurrency ${{ matrix.ARGS.concurrency }}"
         [ "${{ matrix.ARGS.platform }}" != "" ] && PLATFORM="--platform ${{ matrix.ARGS.platform }}"
         [ "${{ matrix.ARGS.cache }}" == "true" ] && CACHE="--cache true"
-        all_args="--browsers=chrome:devextreme-shr2 --componentFolder ${{ matrix.ARGS.componentFolder }} $CONCURRENCY $INDICES $PLATFORM $THEME $CACHE"
+        TZVAL="${{ matrix.ARGS.timezone }}"
+        [ -z "$TZVAL" ] && TZVAL="GMT"
+        # A job stays green if it has no more than --deferThreshold failures:
+        # those tests are recorded and handed to the dedicated "retry-unstable"
+        # job. More failures than that fail the job as a real regression.
+        all_args="--browsers=chrome:devextreme-shr2 --componentFolder ${{ matrix.ARGS.componentFolder }} $CONCURRENCY $INDICES $PLATFORM $THEME $CACHE --deferThreshold 3 --timezone $TZVAL"
         echo "$all_args"
         pnpm run test $all_args
 
@@ -227,6 +233,91 @@ jobs:
         path: ${{ github.workspace }}/e2e/testcafe-devextreme/artifacts/failedtests/**/*
         if-no-files-found: ignore
 
+    - name: Upload failed test list
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v7
+      with:
+        name: failed-list-${{ env.JOB_NAME }}
+        path: ${{ github.workspace }}/e2e/testcafe-devextreme/artifacts/failed-tests/**/*
+        if-no-files-found: ignore
+
+  retry-unstable:
+    name: Retry unstable tests
+    runs-on: devextreme-shr2
+    needs: [check-should-run, build, testcafe]
+    if: ${{ always() && needs.check-should-run.outputs.should-run == 'true' }}
+    timeout-minutes: 30
+
+    steps:
+    - name: Get sources
+      uses: actions/checkout@v6
+
+    - name: Use Node.js
+      uses: actions/setup-node@v6
+      with:
+        node-version-file: '.node-version'
+
+    - name: Download failed test lists
+      uses: actions/download-artifact@v8
+      continue-on-error: true
+      with:
+        path: ${{ github.workspace }}/e2e/testcafe-devextreme/failed-lists
+        pattern: failed-list-*
+
+    - name: Decide retry strategy
+      id: decide
+      working-directory: ./e2e/testcafe-devextreme
+      run: node retry-unstable.mjs --mode decide --dir ./failed-lists --out ./retry-plan.json
+
+    - uses: pnpm/action-setup@v6
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      with:
+        run_install: false
+
+    - name: Get pnpm store directory
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      shell: bash
+      run: |
+        echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+    - uses: actions/cache/restore@v5
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      name: Restore pnpm cache
+      with:
+        path: ${{ env.STORE_PATH }}
+        key: ${{ runner.os }}-pnpm-cache-${{ hashFiles('**/pnpm-lock.yaml') }}
+        restore-keys: |
+          ${{ runner.os }}-pnpm-cache
+
+    - name: Install dependencies
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      run: pnpm install --frozen-lockfile
+
+    - name: Download build artifacts
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      uses: actions/download-artifact@v8
+      with:
+        name: devextreme-artifacts
+        path: ./packages/devextreme
+
+    - name: Unpack build artifacts
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      working-directory: ./packages/devextreme
+      run: 7z x artifacts.zip -aoa
+
+    - name: Setup Chrome
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      uses: ./.github/actions/setup-chrome
+      with:
+        chrome-version: '149.0.7827.114'
+
+    - name: Retry potentially-unstable tests
+      if: ${{ steps.decide.outputs.action == 'retry' }}
+      working-directory: ./e2e/testcafe-devextreme
+      env:
+        NODE_OPTIONS: --max-old-space-size=8192
+      run: node retry-unstable.mjs --mode retry --out ./retry-plan.json --attempts 2
+
   merge-artifacts:
     runs-on: devextreme-shr2
     needs: testcafe
@@ -252,7 +343,7 @@ jobs:
   notify:
     runs-on: devextreme-shr2
     name: Send notifications
-    needs: [build, testcafe]
+    needs: [build, testcafe, retry-unstable]
     if: github.event_name != 'pull_request' && contains(needs.*.result, 'failure')
 
     steps:

.github/workflows/visual-tests-demos.yml

@@ -717,7 +717,7 @@ jobs:
     - name: Set Chrome flags
       id: chrome-flags
       run: |
-        BASE_FLAGS="chrome:headless --window-size=1200,800 --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding --js-flags=--random-seed=2147483647"
+        BASE_FLAGS="chrome:headless --window-size=1200,800 --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding,KeyboardFocusableScrollers --js-flags=--random-seed=2147483647"
 
         # For Material theme, enable better font rendering to avoid instability
         if [[ "${{ matrix.THEME }}" != *"material"* ]]; then
@@ -902,7 +902,7 @@ jobs:
       working-directory: apps/demos
       env:
         NODE_OPTIONS: --max-old-space-size=8192
-        BROWSERS: chrome:headless --window-size=1200,800 --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning
+        BROWSERS: chrome:headless --window-size=1200,800 --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding,KeyboardFocusableScrollers --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning
         # DEBUG: hammerhead:*,testcafe:*
         CONCURRENCY: ${{ steps.set-concurrency.outputs.concurrency }}
         TCQUARANTINE: true
@@ -1029,7 +1029,7 @@ jobs:
         working-directory: apps/demos
         env:
           CHANGEDFILEINFOSPATH: changed-files.json
-          BROWSERS: chrome:headless --window-size=1200,800 --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning
+          BROWSERS: chrome:headless --window-size=1200,800 --disable-gpu --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding,KeyboardFocusableScrollers --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning
           # DEBUG: hammerhead:*,testcafe:*
           CONCURRENCY: 1
           TCQUARANTINE: true

apps/demos/utils/visual-tests/testcafe-runner.ts

@@ -1,7 +1,7 @@
 import createTestCafe, { ClientFunction } from 'testcafe';
 import fs from 'fs';
 
-const LAUNCH_RETRY_ATTEMPTS = 3;
+const LAUNCH_RETRY_ATTEMPTS = 0;
 const LAUNCH_RETRY_TIMEOUT = 10000;
 
 const wait = async (
@@ -96,7 +96,7 @@ async function main() {
 
   const failedCount = await retry(() => runner
     .reporter(reporters)
-    .browsers(process.env.BROWSERS || 'chrome --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning')
+    .browsers(process.env.BROWSERS || 'chrome --no-sandbox --disable-dev-shm-usage --disable-partial-raster --disable-skia-runtime-opts --run-all-compositor-stages-before-draw --disable-new-content-rendering-timeout --disable-threaded-animation --disable-threaded-scrolling --disable-checker-imaging --disable-image-animation-resync --use-gl=swiftshader --disable-features=PaintHolding,KeyboardFocusableScrollers --js-flags=--random-seed=2147483647 --font-render-hinting=none --disable-font-subpixel-positioning')
     .concurrency(concurrency || 1)
     .run({
       quarantineMode: getQuarantineMode(),

e2e/testcafe-devextreme/helpers/domUtils.ts

@@ -137,8 +137,8 @@ export const addFocusableElementBefore = ClientFunction((
   return button.id;
 });
 
-export const hasHorizontalScroll = ClientFunction((selector) => {
-  const element = selector();
+export const hasHorizontalScroll = ClientFunction((containerSelector) => {
+  const container = containerSelector();
 
-  return element.scrollWidth > element.clientWidth;
+  return container.scrollWidth > container.clientWidth;
 });