Translate smoketests from Python to Rust

.cargo/config.toml

@@ -5,6 +5,8 @@ rustflags = ["--cfg", "tokio_unstable"]
 bump-versions = "run -p upgrade-version --"
 llm = "run --package xtask-llm-benchmark --bin llm_benchmark --"
 ci = "run -p ci --"
+smoketest = "run -p xtask-smoketest -- smoketest"
+smoketests = "smoketest"
 
 [target.x86_64-pc-windows-msvc]
 # Use a different linker. Otherwise, the build fails with some obscure linker error that

.github/workflows/ci.yml

@@ -18,9 +18,107 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  docker_smoketests:
-    needs: [lints]
+  smoketests:
     name: Smoketests
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [spacetimedb-new-runner, windows-latest]
+        include:
+          - runner: spacetimedb-new-runner
+            container:
+              image: localhost:5000/spacetimedb-ci:latest
+              options: --privileged
+          - runner: windows-latest
+            container: null
+    runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container }}
+    timeout-minutes: 120
+    env:
+      CARGO_TARGET_DIR: ${{ github.workspace }}/target
+    steps:
+      - name: Find Git ref
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          PR_NUMBER="${{ github.event.inputs.pr_number || null }}"
+          if test -n "${PR_NUMBER}"; then
+            GIT_REF="$( gh pr view --repo clockworklabs/SpacetimeDB $PR_NUMBER --json headRefName --jq .headRefName )"
+          else
+            GIT_REF="${{ github.ref }}"
+          fi
+          echo "GIT_REF=${GIT_REF}" >>"$GITHUB_ENV"
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.GIT_REF }}
+      - uses: dsherret/rust-toolchain-file@v1
+      - name: Cache Rust dependencies
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ${{ github.workspace }}
+          shared-key: spacetimedb
+          cache-on-failure: false
+          cache-all-crates: true
+          cache-workspace-crates: true
+          prefix-key: v1
+
+      - uses: actions/setup-dotnet@v4
+        with:
+          global-json-file: global.json
+
+      # nodejs and pnpm are required for the typescript quickstart smoketest
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        with:
+          run_install: true
+
+      - name: Install psql (Windows)
+        if: runner.os == 'Windows'
+        run: choco install psql -y --no-progress
+        shell: powershell
+
+      - name: Update dotnet workloads
+        if: runner.os == 'Windows'
+        run: |
+          # Fail properly if any individual command fails
+          $ErrorActionPreference = 'Stop'
+          $PSNativeCommandUseErrorActionPreference = $true
+
+          cd modules
+          # the sdk-manifests on windows-latest are messed up, so we need to update them
+          dotnet workload config --update-mode manifests
+          dotnet workload update
+
+      # This step shouldn't be needed, but somehow we end up with caches that are missing librusty_v8.a.
+      # ChatGPT suspects that this could be due to different build invocations using the same target dir,
+      # and this makes sense to me because we only see it in this job where we mix `cargo build -p` with
+      # `cargo build --manifest-path` (which apparently build different dependency trees).
+      # However, we've been unable to fix it so... /shrug
+      - name: Check v8 outputs
+        shell: bash
+        run: |
+          find "${CARGO_TARGET_DIR}"/ -type f | grep '[/_]v8' || true
+          if ! [ -f "${CARGO_TARGET_DIR}"/debug/gn_out/obj/librusty_v8.a ]; then
+            echo "Could not find v8 output file librusty_v8.a; rebuilding manually."
+            cargo clean -p v8 || true
+            cargo build -p v8
+          fi
+
+      - name: Install cargo-nextest
+        uses: taiki-e/install-action@nextest
+
+      - name: Run smoketests
+        run: cargo ci smoketests
+
+  smoketests-python:
+    needs: [lints]
+    name: Smoketests (Python Legacy)
     strategy:
       matrix:
         runner: [spacetimedb-new-runner, windows-latest]
@@ -92,8 +190,10 @@ jobs:
         if: runner.os == 'Windows'
         run: choco install psql -y --no-progress
         shell: powershell
+
       - name: Build crates
         run: cargo build -p spacetimedb-cli -p spacetimedb-standalone -p spacetimedb-update
+
       - name: Start Docker daemon
         if: runner.os == 'Linux'
         run: /usr/local/bin/start-docker.sh
@@ -104,6 +204,7 @@ jobs:
           # Our .dockerignore omits `target`, which our CI Dockerfile needs.
           rm .dockerignore
           docker compose -f .github/docker-compose.yml up -d
+
       - name: Build and start database (Windows)
         if: runner.os == 'Windows'
         run: |
@@ -116,14 +217,18 @@ jobs:
           # the sdk-manifests on windows-latest are messed up, so we need to update them
           dotnet workload config --update-mode manifests
           dotnet workload update
+
       - uses: actions/setup-python@v5
         with: { python-version: "3.12" }
         if: runner.os == 'Windows'
+
       - name: Install python deps
         run: python -m pip install -r smoketests/requirements.txt
-      - name: Run smoketests
+
+      - name: Run Python smoketests
         # Note: clear_database and replication only work in private
-        run: cargo ci smoketests -- ${{ matrix.smoketest_args }} -x clear_database replication teams
+        run: python -m smoketests ${{ matrix.smoketest_args }} -x clear_database replication teams
+
       - name: Stop containers (Linux)
         if: always() && runner.os == 'Linux'
         run: docker compose -f .github/docker-compose.yml down
@@ -943,3 +1048,32 @@ jobs:
 
       - name: Check global.json policy
         run: cargo ci global-json-policy
+
+  warn-python-smoketests:
+    name: Check for Python smoketest edits
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Fail if Python smoketests were modified
+        run: |
+          PYTHON_SMOKETEST_CHANGES=$(git diff --name-only origin/${{ github.base_ref }} HEAD -- 'smoketests/**.py')
+
+          if [ -n "$PYTHON_SMOKETEST_CHANGES" ]; then
+            echo "::error::This PR modifies legacy Python smoketests. Please add new tests to the Rust smoketests in crates/smoketests/ instead."
+            echo ""
+            echo "Changed files:"
+            echo "$PYTHON_SMOKETEST_CHANGES"
+            echo ""
+            echo "The Python smoketests are being replaced by Rust smoketests."
+            echo "See crates/smoketests/DEVELOP.md for instructions on adding Rust smoketests."
+            exit 1
+          fi
+
+          echo "No Python smoketest changes detected."

.gitignore

@@ -211,6 +211,10 @@ crates/bench/spacetime.svg
 crates/bench/sqlite.svg
 .vs/
 
+# .NET build artifacts
+**/obj/
+**/bin/
+
 # benchmark files
 out.json
 old.json

Cargo.toml

@@ -1,4 +1,5 @@
 [workspace]
+exclude = ["crates/smoketests/modules"]
 members = [
   "crates/auth",
   "crates/bench",
@@ -26,6 +27,7 @@ members = [
   "crates/query",
   "crates/sats",
   "crates/schema",
+  "crates/smoketests",
   "sdks/rust",
   "sdks/unreal",
   "crates/snapshot",
@@ -41,6 +43,7 @@ members = [
   "modules/keynote-benchmarks",
   "modules/perf-test",
   "modules/module-test",
+  "templates/basic-rs/spacetimedb",
   "templates/chat-console-rs/spacetimedb",
   "modules/sdk-test",
   "modules/sdk-test-connect-disconnect",
@@ -58,6 +61,7 @@ members = [
   "tools/generate-client-api",
   "tools/gen-bindings",
   "tools/xtask-llm-benchmark",
+  "tools/xtask-smoketest",
   "crates/bindings-typescript/test-app/server",
   "crates/bindings-typescript/test-react-router-app/server",
   "crates/query-builder",

crates/cli/Cargo.toml

@@ -87,9 +87,6 @@ notify.workspace = true
 [dev-dependencies]
 pretty_assertions.workspace = true
 fs_extra.workspace = true
-assert_cmd = "2"
-predicates = "3"
-spacetimedb-guard.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = { workspace = true }

crates/cli/src/subcommands/describe.rs

@@ -88,6 +88,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error
             None => sats_to_json(&module_def)?,
         };
 
+        // TODO: validate the JSON output
         println!("{json}");
     } else {
         // TODO: human-readable API