Samples screenshot regression

.gitattributes

@@ -37,6 +37,7 @@ glslangValidator filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.tar.gz filter=lfs diff=lfs merge=lfs -text
 *.vsix filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
 
 # 3D formats
 *.max filter=lfs diff=lfs merge=lfs -text

.github/workflows/test-samples-baselines.yml

@@ -0,0 +1,96 @@
+name: Test Samples (Update Baselines)
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        description: "Workflow run ID to pull capture artifacts from (the daily test-samples-screenshots run whose captures should become the new baselines). Empty = latest successful run."
+        type: string
+        default: ''
+      branch_name:
+        description: "Branch name to push the baseline update to (auto-generated if empty)."
+        type: string
+        default: ''
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  Update:
+    name: Refresh tests/Stride.Samples.Tests baselines
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+          # Need full history for the auto-generated branch name to be stable.
+          fetch-depth: 0
+
+      - name: Resolve target run id
+        id: resolve
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [ -n "${{ github.event.inputs.run_id }}" ]; then
+            echo "run_id=${{ github.event.inputs.run_id }}" >> "$GITHUB_OUTPUT"
+          else
+            # Latest successful run of the daily capture workflow.
+            run_id=$(gh run list \
+              --workflow=test-samples-screenshots.yml \
+              --status=success \
+              --limit=1 \
+              --json databaseId \
+              --jq '.[0].databaseId')
+            if [ -z "$run_id" ]; then
+              echo "No successful test-samples-screenshots run found." >&2
+              exit 1
+            fi
+            echo "run_id=$run_id" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Download D3D11 capture artifact from run ${{ steps.resolve.outputs.run_id }}
+        # Baselines are sourced from D3D11 only — that's the gating matrix entry. D3D12/Vulkan
+        # are best-effort; their captures should not become the baseline reference.
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p captures
+          gh run download ${{ steps.resolve.outputs.run_id }} \
+            --name screenshots-Direct3D11 \
+            --dir ./captures
+          # Layout: captures/screenshot-out/<sample>/screenshots/<frame>.png
+          find captures -maxdepth 4 -type d | head -40
+
+      - name: Refresh tests/Stride.Samples.Tests/<Sample>/<frame>.png from captures
+        run: |
+          set -e
+          changed=0
+          for sample_dir in captures/screenshot-out/*/; do
+            sample=$(basename "$sample_dir")
+            src="$sample_dir/screenshots"
+            [ -d "$src" ] || continue
+            dest="tests/Stride.Samples.Tests/$sample"
+            mkdir -p "$dest"
+            for png in "$src"/*.png; do
+              [ -f "$png" ] || continue
+              cp "$png" "$dest/"
+              changed=$((changed + 1))
+            done
+          done
+          echo "Replaced $changed baseline PNG(s)."
+
+      - name: Open PR with refreshed baselines
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: "tests/Stride.Samples.Tests: refresh screenshot baselines from run ${{ steps.resolve.outputs.run_id }}"
+          title: "Refresh sample screenshot baselines (run ${{ steps.resolve.outputs.run_id }})"
+          body: |
+            Automated baseline refresh from the captures of `test-samples-screenshots.yml`
+            run [#${{ steps.resolve.outputs.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.resolve.outputs.run_id }}).
+
+            Diff this PR against the previous baselines visually before merging — every PNG
+            replacement here becomes the new "approved" reference for daily comparison.
+          branch: ${{ github.event.inputs.branch_name || format('baselines/refresh-{0}', steps.resolve.outputs.run_id) }}
+          delete-branch: true
+          add-paths: tests/Stride.Samples.Tests/**/*.png

.github/workflows/test-samples-screenshots.yml

@@ -0,0 +1,168 @@
+name: Test Samples (Screenshots)
+
+on:
+  workflow_dispatch:
+    inputs:
+      build-type:
+        description: Build
+        default: Debug
+        type: choice
+        options:
+          - Debug
+          - Release
+  schedule:
+    # Daily at 04:00 UTC.
+    - cron: '0 4 * * *'
+
+concurrency:
+  group: test-samples-screenshots-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Run:
+    name: Capture & compare all samples (${{ matrix.graphics-api }}, ${{ github.event.inputs.build-type || 'Debug' }})
+    # Single job per graphics API: the harness + orchestrator + comparator are sample-agnostic, so a
+    # matrix-per-sample would build the engine 15× for the same artifacts. Sequential capture
+    # via xunit's DisableParallelization keeps total compute low (~1× engine build + ~5 min of
+    # actual sample runs) at comparable wall-clock to a 15-parallel matrix. Per-API parallelism
+    # is unavoidable (each API needs its own engine build).
+    strategy:
+      fail-fast: false
+      matrix:
+        graphics-api: [Direct3D11, Direct3D12, Vulkan]
+    runs-on: windows-2025-vs2026
+    # TODO: drop continue-on-error once D3D12 and Vulkan captures are stable.
+    continue-on-error: ${{ matrix.graphics-api != 'Direct3D11' }}
+    env:
+      # Software rendering is the default in the autotesting harness; set STRIDE_TESTS_GPU=1 to opt into the GPU.
+      STRIDE_TESTS_RENDERDOC: "error"
+      DOTNET_DbgEnableMiniDump: "1"
+      DOTNET_DbgMiniDumpType: "1"
+      DOTNET_DbgMiniDumpName: "${{ github.workspace }}\\crash-dumps\\dotnet_%p.dmp"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '10.0.x'
+
+      - name: Configure crash dumps
+        shell: pwsh
+        run: |
+          reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v DontShowUI /t REG_DWORD /d 1 /f
+          $dumpDir = "${{ github.workspace }}\crash-dumps"
+          New-Item -Path $dumpDir -ItemType Directory -Force | Out-Null
+          reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpFolder /t REG_EXPAND_SZ /d $dumpDir /f
+          reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpType /t REG_DWORD /d 1 /f
+          reg add "HKLM\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpCount /t REG_DWORD /d 10 /f
+
+      # nuget.config declares `bin/packages` as the `stride-local` source. NuGet validates every
+      # source path at restore time; on a fresh CI checkout that dir doesn't exist yet (it gets
+      # populated by Stride's auto-pack-deploy on first build), so we materialize it as empty.
+      - name: Pre-create bin/packages so restore doesn't fail on missing local source
+        shell: pwsh
+        run: New-Item -ItemType Directory -Path bin/packages -Force | Out-Null
+
+      # PackageStore at test runtime calls Settings.LoadDefaultSettings(null) which only loads
+      # user + machine configs, NOT the workspace's nuget.config (no walkup with null root).
+      # Locally this works because the dev's user config has Stride.AutoPackDeploy's
+      # %LocalAppData%\Stride\NugetDev as a source. CI's runneradmin user has no such config,
+      # so add bin/packages explicitly to the user config. --configfile forces the write to
+      # the user config rather than walking up to the workspace's nuget.config (which would
+      # conflict with the existing 'stride-local' source declared there).
+      - name: Register bin/packages with user NuGet config
+        shell: pwsh
+        run: |
+          New-Item -Path "$env:APPDATA\NuGet" -ItemType Directory -Force | Out-Null
+          dotnet nuget add source "${{ github.workspace }}\bin\packages" --name stride-local --configfile "$env:APPDATA\NuGet\NuGet.Config"
+
+      # Stride.Games.AutoTesting must be built explicitly so its auto-pack-deploy lands the .nupkg
+      # in bin/packages — the sample projects consume it via <PackageReference>, not via the
+      # ProjectReference graph the test project would otherwise transitively pull. The engine
+      # is built with StrideGraphicsApis pinned to the matrix entry so the resulting nupkg only
+      # contains binaries for that API; the regenerated samples then link against those.
+      - name: Build harness assembly (packs Stride.Games.AutoTesting into bin/packages)
+        run: |
+          dotnet build sources\engine\Stride.Games.AutoTesting\Stride.Games.AutoTesting.csproj `
+            -p:StrideNativeBuildMode=Clang `
+            -nr:false -v:m -p:WarningLevel=0 `
+            -p:Configuration=${{ github.event.inputs.build-type || 'Debug' }} `
+            -p:StrideGraphicsApi=${{ matrix.graphics-api }} `
+            -p:StrideGraphicsApis=${{ matrix.graphics-api }}
+
+      # Install Stride.Dependencies.Lavapipe and register its ICD with the Vulkan loader
+      # so the samples' Vulkan path resolves a software driver on the CI runner (no GPU).
+      # On Windows the loader reads HKLM\SOFTWARE\Khronos\Vulkan\Drivers — VK_DRIVER_FILES
+      # env var alone isn't reliable across runner Vulkan SDK versions. Mirrors the
+      # SwiftShader registration in test-windows-game.yml.
+      - name: Install Lavapipe and register its Vulkan ICD (Vulkan only)
+        if: matrix.graphics-api == 'Vulkan'
+        shell: pwsh
+        run: |
+          $tmpDir = "${{ runner.temp }}\lavapipe-fetch"
+          New-Item -ItemType Directory -Path $tmpDir -Force | Out-Null
+          Set-Content -Path "$tmpDir\LavapipeFetch.csproj" -Value '<Project Sdk="Microsoft.NET.Sdk"><PropertyGroup><TargetFramework>net10.0</TargetFramework></PropertyGroup></Project>'
+          dotnet add "$tmpDir\LavapipeFetch.csproj" package Stride.Dependencies.Lavapipe
+          $icd = Get-ChildItem -Recurse -Path "$env:USERPROFILE\.nuget\packages\stride.dependencies.lavapipe" -Filter "lvp_icd*.json" -ErrorAction SilentlyContinue | Where-Object { $_.DirectoryName -match 'win-x64' } | Select-Object -First 1
+          if (-not $icd) { throw "Lavapipe ICD not found in NuGet cache after restore." }
+          New-Item -Path "HKLM:\SOFTWARE\Khronos\Vulkan\Drivers" -Force | Out-Null
+          New-ItemProperty -Path "HKLM:\SOFTWARE\Khronos\Vulkan\Drivers" -Name $icd.FullName -Value 0 -PropertyType DWord -Force | Out-Null
+          Write-Host "Registered Lavapipe ICD: $($icd.FullName)"
+
+      # Builds the test project plus its transitive ProjectReferences. StrideGraphicsApi (singular)
+      # selects the test SDK output path; StrideGraphicsApis (plural) keeps the engine deps aligned
+      # with the harness build above.
+      - name: Build test project
+        run: |
+          dotnet build samples\Tests\Stride.Samples.Tests.csproj `
+            -p:StrideNativeBuildMode=Clang `
+            -nr:false -v:m -p:WarningLevel=0 `
+            -p:Configuration=${{ github.event.inputs.build-type || 'Debug' }} `
+            -p:StrideGraphicsApi=${{ matrix.graphics-api }} `
+            -p:StrideGraphicsApis=${{ matrix.graphics-api }}
+
+      - name: Run sample screenshot tests
+        env:
+          # Comparator opts a few non-deterministic tests (ParticlesSample, SpriteStudioDemo,
+          # AnimatedModel) into a Claude vision second-opinion when LPIPS is over threshold.
+          # Key is bound to a budget-capped Anthropic workspace; the fallback only runs on
+          # frames that already failed LPIPS so cost is bounded even when noisy.
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          dotnet test samples\Tests\Stride.Samples.Tests.csproj `
+            --no-build `
+            --filter "FullyQualifiedName~Stride.Samples.Tests.SampleScreenshotTests" `
+            --logger "trx;LogFileName=screenshots.trx" `
+            --results-directory TestResults `
+            -p:Configuration=${{ github.event.inputs.build-type || 'Debug' }} `
+            -p:StrideGraphicsApi=${{ matrix.graphics-api }}
+
+      - name: Publish test report
+        if: always()
+        uses: phoenix-actions/test-reporting@v15
+        with:
+          name: 'Sample screenshot regression (${{ matrix.graphics-api }})'
+          path: TestResults/*.trx
+          reporter: dotnet-trx
+          output-to: step-summary
+          list-tests: 'failed'
+
+      - name: Upload test artifacts (captures + done.json + TRX)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: screenshots-${{ matrix.graphics-api }}
+          path: |
+            screenshot-out/
+            TestResults/
+          if-no-files-found: warn
+
+      - name: Upload crash dumps
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: crash-dumps-${{ matrix.graphics-api }}
+          path: crash-dumps/
+          if-no-files-found: ignore

.gitignore

@@ -97,6 +97,11 @@ x64Release
 *.opendb
 screenshots
 samplesGenerated
+screenshot-regression-out
+screenshot-out
+# Auto-generated by samples/Directory.Build.targets when -p:StrideAutoTesting=true is passed
+# (force-loads Stride.Games.AutoTesting at startup so its [ModuleInitializer] runs).
+_AutoTestingBootstrap.g.cs
 
 # VS Code files
 .vscode/*

build/Stride.sln

@@ -188,12 +188,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.Graphics.Tests.10_0.
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.Graphics.Tests.11_0.Windows", "..\sources\engine\Stride.Graphics.Tests.11_0\Stride.Graphics.Tests.11_0.Windows.csproj", "{7CA99C7B-E3A2-4DE6-9D6C-314AE39BBBB7}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.SamplesTestServer", "..\sources\tools\Stride.SamplesTestServer\Stride.SamplesTestServer.csproj", "{75D71310-ECF7-4592-9E35-3FE540040982}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.Particles", "..\sources\engine\Stride.Particles\Stride.Particles.csproj", "{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.Games.Testing", "..\sources\engine\Stride.Games.Testing\Stride.Games.Testing.csproj", "{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.Native", "..\sources\engine\Stride.Native\Stride.Native.csproj", "{1DBBC150-F085-43EF-B41D-27C72D133770}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stride.Assets.Tests2", "..\sources\engine\Stride.Assets.Tests2\Stride.Assets.Tests2.csproj", "{370ADF53-DFFA-461E-B72A-1302C0A0DE00}"
@@ -978,18 +974,6 @@ Global
 		{7CA99C7B-E3A2-4DE6-9D6C-314AE39BBBB7}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{7CA99C7B-E3A2-4DE6-9D6C-314AE39BBBB7}.Release|Win32.ActiveCfg = Release|Any CPU
 		{7CA99C7B-E3A2-4DE6-9D6C-314AE39BBBB7}.Release|Win32.Build.0 = Release|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Debug|Win32.ActiveCfg = Debug|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Debug|Win32.Build.0 = Debug|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Release|Any CPU.Build.0 = Release|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Release|Mixed Platforms.Build.0 = Release|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Release|Win32.ActiveCfg = Release|Any CPU
-		{75D71310-ECF7-4592-9E35-3FE540040982}.Release|Win32.Build.0 = Release|Any CPU
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
@@ -1002,18 +986,6 @@ Global
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}.Release|Win32.ActiveCfg = Release|Any CPU
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31}.Release|Win32.Build.0 = Release|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Debug|Win32.ActiveCfg = Debug|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Debug|Win32.Build.0 = Debug|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Release|Any CPU.Build.0 = Release|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Release|Win32.ActiveCfg = Release|Any CPU
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9}.Release|Win32.Build.0 = Release|Any CPU
 		{1DBBC150-F085-43EF-B41D-27C72D133770}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{1DBBC150-F085-43EF-B41D-27C72D133770}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{1DBBC150-F085-43EF-B41D-27C72D133770}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
@@ -1602,9 +1574,7 @@ Global
 		{9DE0AA56-0DE7-4ADC-BAAC-CD38B7139EBC} = {A7ED9F01-7D78-4381-90A6-D50E51C17250}
 		{570B0FF9-246F-4C6C-8384-F6BE1887A4A9} = {A7ED9F01-7D78-4381-90A6-D50E51C17250}
 		{7CA99C7B-E3A2-4DE6-9D6C-314AE39BBBB7} = {A7ED9F01-7D78-4381-90A6-D50E51C17250}
-		{75D71310-ECF7-4592-9E35-3FE540040982} = {1AE1AC60-5D2F-4CA7-AE20-888F44551185}
 		{F32FDA80-B6DD-47A8-8681-437E2C0D3F31} = {4C142567-C42B-40F5-B092-798882190209}
-		{B84ECB15-5E3F-4BD1-AB87-333BAE9B70F9} = {A7ED9F01-7D78-4381-90A6-D50E51C17250}
 		{1DBBC150-F085-43EF-B41D-27C72D133770} = {4C142567-C42B-40F5-B092-798882190209}
 		{370ADF53-DFFA-461E-B72A-1302C0A0DE00} = {A47B451D-3162-410F-BAF7-C650C4B7A4B0}
 		{33CC6216-3F30-4B5A-BB29-C5B47EFFA713} = {A7ED9F01-7D78-4381-90A6-D50E51C17250}

nuget.config

@@ -2,16 +2,23 @@
 <configuration>
   <packageSources>
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
-    <!-- Uncomment if switching back to Sdk="Stride.Build.Sdk" project style (see build/docs/SDK-GUIDE.md). -->
-    <!-- <add key="stride-sdks" value="build/packages" /> -->
+    <!-- Auto-pack output. Stride.* dev versions shadow public ones during restore. -->
+    <add key="stride-local" value="bin/packages" />
   </packageSources>
   <packageSourceMapping>
     <packageSource key="nuget.org">
       <package pattern="*" />
+      <!-- 3rd-party packages with a Stride prefix still resolve from nuget.org via longer-pattern wins. -->
+      <package pattern="Stride.GNU.*" />
+      <package pattern="Stride.Mono.*" />
+      <package pattern="Stride.Dependencies.*" />
+      <package pattern="Stride.GraphX.*" />
+      <package pattern="Stride.Metrics" />
+      <package pattern="Stride.QuickGraph" />
+    </packageSource>
+    <packageSource key="stride-local">
+      <package pattern="Stride" />
+      <package pattern="Stride.*" />
     </packageSource>
-    <!-- Uncomment if switching back to Sdk="Stride.Build.Sdk" project style (see build/docs/SDK-GUIDE.md). -->
-    <!-- <packageSource key="stride-sdks">
-      <package pattern="Stride.Build.Sdk*" />
-    </packageSource> -->
   </packageSourceMapping>
 </configuration>