AutoTester: Docker-based in-game integration test framework

.github/workflows/autotester-tests.yml

@@ -0,0 +1,16 @@
+name: step.autotester-tests
+
+# Fast, Docker-free tests for the autotester step engine (selectors, conditions,
+# templating, the executor, and the shipped scenarios run through a fake bridge).
+on: [workflow_call]
+
+jobs:
+  autotester-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - name: Run engine tests
+        working-directory: autotester
+        run: uv run --group dev pytest

.github/workflows/build.yml

@@ -12,6 +12,11 @@ on:
         type: string
         required: false
         default: ''
+      autotest:
+        description: bundle the autotester instrumentation into the jars (never for releases)
+        type: boolean
+        required: false
+        default: false
 
 jobs:
   build:
@@ -31,13 +36,17 @@ jobs:
       - name: Execute Gradle build
         run: |
           chmod +x gradlew
+          flags=""
+          if [ "${{ inputs.autotest }}" = "true" ]; then
+              flags="-Pautomodpack.autotest"
+          fi
           if [ -z "${{ inputs.target_subproject }}" ]; then
               echo "Building all subprojects"
-              ./gradlew build
+              ./gradlew build $flags
           else
               args=$(echo "${{ inputs.target_subproject }}" | tr ',' '\n' | sed 's/$/:build/' | paste -sd ' ')
               echo "Building with arguments=$args"
-              ./gradlew $args
+              ./gradlew $args $flags
           fi
         env:
           BUILD_ID: ${{ github.run_number }}

.github/workflows/gradle.yml

@@ -6,6 +6,9 @@ jobs:
   tests: # runs on linux and windows
     uses: ./.github/workflows/tests.yml
     secrets: inherit
+  autotester-tests: # autotester engine unit tests (no Docker)
+    uses: ./.github/workflows/autotester-tests.yml
+    secrets: inherit
   build: # runs on linux
     uses: ./.github/workflows/build.yml
     secrets: inherit

.github/workflows/ingame-tests.yml

@@ -0,0 +1,142 @@
+name: In-game Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      scenario:
+        description: "Autotest scenario"
+        default: "sync"
+      target:
+        description: "Target ID (or 'all')"
+        default: "all"
+      jobs:
+        description: "Parallel runs"
+        default: "1"
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      scenario: ${{ steps.set-scenario.outputs.scenario }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - id: set-matrix
+        run: |
+          TARGET="${{ github.event.inputs.target || 'all' }}"
+          ALL=$(uv run --project autotester python -c "import json,yaml; ids=[t['id'] for t in yaml.safe_load(open('autotester/targets.yaml')).get('targets',[])]; print(json.dumps(ids))")
+          if [ "$TARGET" = "all" ]; then
+            echo "matrix={\"target\":$ALL}" >> $GITHUB_OUTPUT
+          else
+            echo "matrix={\"target\":[\"$TARGET\"]}" >> $GITHUB_OUTPUT
+          fi
+      - id: set-scenario
+        run: |
+          echo "scenario=${{ github.event.inputs.scenario || 'sync' }}" >> $GITHUB_OUTPUT
+
+  unit:
+    uses: ./.github/workflows/autotester-tests.yml
+    secrets: inherit
+
+  build:
+    needs: [prepare, unit]
+    uses: ./.github/workflows/build.yml
+    with:
+      autotest: true
+    secrets: inherit
+
+  ingame:
+    needs: [build, prepare]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - uses: actions/download-artifact@v6
+        with:
+          name: build-artifacts
+          path: merged/
+      - name: Build autotest client image
+        run: uv --project autotester run autotester build-images
+      - name: Run autotest
+        run: >
+          uv --project autotester run autotester run
+          --target "${{ matrix.target }}"
+          --scenario "${{ needs.prepare.outputs.scenario }}"
+      - uses: actions/upload-artifact@v6
+        if: always()
+        with:
+          name: autotester-${{ matrix.target }}
+          path: autotester/out
+
+  report:
+    needs: [ingame]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+      - uses: actions/download-artifact@v6
+        with:
+          pattern: autotester-*
+      - name: Aggregate results
+        id: aggregate
+        run: |
+          cat > /tmp/aggregate.py << 'PYEOF'
+          import json, glob, os
+          results = []
+          for f in sorted(glob.glob("**/results.json", recursive=True)):
+              try:
+                  data = json.load(open(f))
+                  results.extend(data.get("results", []))
+              except Exception:
+                  name = f.split(os.sep)[0] if os.sep in f else "unknown"
+                  results.append({"target": name, "ok": False, "error": "results.json missing or invalid"})
+
+          total = len(results)
+          passed = sum(1 for r in results if r.get("ok"))
+          any_ok = passed == total
+
+          for r in results:
+              if not r.get("ok"):
+                  err = r.get("error", "unknown error")
+                  print(f"::error title={r['target']}::{err}")
+
+          summary = {"ok": any_ok, "total": total, "passed": passed, "failed": total - passed, "results": results}
+          with open("aggregated.json", "w") as f:
+              json.dump(summary, f, indent=2)
+
+          md = "## In-game Test Results\n\n"
+          md += f"**{'PASS' if any_ok else 'FAIL'}** — {passed}/{total} passed, {total - passed} failed\n\n"
+          md += "| Status | Target | Duration | Error |\n"
+          md += "|--------|--------|----------|-------|\n"
+          for r in results:
+              status = "✅" if r.get("ok") else "❌"
+              dur = f"{r.get('duration', 0):.1f}s" if "duration" in r else "-"
+              err = r.get("error", "") if not r.get("ok") else ""
+              md += f"| {status} | {r['target']} | {dur} | {err} |\n"
+
+          with open("summary.md", "w") as f:
+              f.write(md)
+
+          print(f"ok={str(any_ok).lower()}", file=open(os.environ["GITHUB_OUTPUT"], "a"))
+          PYEOF
+          python3 /tmp/aggregate.py
+      - uses: actions/upload-artifact@v6
+        with:
+          name: aggregated-results
+          path: |
+            aggregated.json
+            summary.md
+      - name: Post job summary
+        run: cat summary.md >> $GITHUB_STEP_SUMMARY
+      - name: Check overall result
+        if: steps.aggregate.outputs.ok != 'true'
+        run: |
+          echo "Some tests failed"
+          exit 1

.gitignore

@@ -40,4 +40,11 @@ output/
 /merged/
 /core/automodpack/
 
-dupes
+dupes
+
+# python
+
+__pycache__/
+*.egg-info/
+autotester/out/
+autotester/.hmc-cache/