Fix for Issue #232: generative + invariant-driven DSL testing (Phases 1-6)

.github/workflows/ci.yml

@@ -60,6 +60,68 @@ jobs:
       - name: Check formatting
         run: cargo fmt --package pg_durable -- --check
 
+  generated-matrix:
+    # Phase 2 + Phase 4 + Phase 3 (#232): the combinator-nesting matrix
+    # generator, the Phase 4 metamorphic-relations registry, and the Phase 3
+    # model-level proptest properties all live in one standalone std-only crate,
+    # so their lint, determinism, unit, and property gates need no PostgreSQL
+    # build and run fast as their own blocking gate. The live/quarantine SQL
+    # (incl. the meta-*.sql relations) is exercised inside the `test` job, which
+    # already has the extension installed.
+    name: Generated Matrix (lint + determinism + unit + proptest)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # std-only stable crate — uses the runner's pre-installed stable toolchain
+      # (no nightly needed, unlike the extension's nightly-rustfmt format job).
+
+      # Lints the whole generator crate INCLUDING the #[cfg(test)] Phase 3
+      # proptest module (--all-targets), so prop.rs stays clippy-clean alongside
+      # the generation binary.
+      - name: Generator clippy (-D warnings, lints the proptest module)
+        run: cargo clippy --manifest-path tests/e2e/generated/generator/Cargo.toml --all-targets -- -D warnings
+
+      # `cargo test` runs the std unit tests AND the Phase 3 proptest properties.
+      # proptest ALWAYS replays the committed proptest-regressions/ corpus first
+      # (the deterministic regression guard for every past counterexample), then
+      # explores PROPTEST_CASES fresh random trees per property within the PR
+      # time budget. The nightly step below widens that budget for deeper search.
+      - name: Generator unit + property tests (classifier, metamorphic, proptest)
+        env:
+          PROPTEST_CASES: 256
+        run: cargo test --manifest-path tests/e2e/generated/generator/Cargo.toml
+
+      # Regenerates the manifests in memory and diffs them against the committed
+      # tests/e2e/generated/{manifest,meta-manifest}.json. Fails if generation
+      # drifted or a committed golden is stale — keeping generation deterministic.
+      # (proptest is a dev-dependency only, so it never perturbs these goldens.)
+      - name: Check manifest determinism
+        run: cargo run --manifest-path tests/e2e/generated/generator/Cargo.toml -- --check
+
+      # Nightly / on-demand deep proptest exploration: a far larger case budget
+      # and a fresh random seed each run, to hunt for counterexamples the 256-case
+      # PR gate misses. Non-blocking (mirrors the quarantine nightly) — any NEW
+      # minimal counterexample is printed AND written to proptest-regressions/;
+      # commit that seed to turn it into a blocking regression guard.
+      - name: Deep proptest exploration (non-blocking, nightly)
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        continue-on-error: true
+        env:
+          PROPTEST_CASES: 8192
+        run: cargo test --manifest-path tests/e2e/generated/generator/Cargo.toml
+
+      - name: Surface nightly proptest counterexamples
+        if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        run: |
+          corpus=tests/e2e/generated/generator/proptest-regressions/prop.txt
+          if [ -f "$corpus" ] && grep -qE '^cc ' "$corpus"; then
+            echo "::warning::Phase 3 proptest corpus has saved counterexample seed(s) — review and commit any new ones:"
+            grep -E '^cc ' "$corpus"
+          else
+            echo "No proptest counterexample seeds present — nightly found nothing beyond the committed corpus."
+          fi
+
   prepare:
     name: Prepare Matrix
     runs-on: ubuntu-latest
@@ -83,9 +145,13 @@ jobs:
   test:
     name: Clippy & Tests (PG${{ matrix.pg_version }})
     runs-on: ubuntu-latest
-    needs: [azure_example_smoke, format, prepare]
+    needs: [azure_example_smoke, format, generated-matrix, prepare]
     permissions:
       contents: read
+    # Generous bound: this job builds the extension and runs the full E2E suite
+    # (incl. the live generated matrix) plus upgrade tests. Caps a hung backend
+    # worker / stuck instance instead of burning the 6h GitHub default.
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
@@ -195,9 +261,60 @@ jobs:
       - name: Run unit tests
         run: cargo pgrx test pg${{ matrix.pg_version }} --features http-allow-test-domains
 
-      - name: Run E2E tests (default-build phases)
+      # Phase 2 + Phase 4 (#232): render the combinator matrix and the
+      # metamorphic relations into tests/e2e/generated/{sql,quarantine}/*.sql
+      # before the E2E run. The crate is std-only and fast; --include-generated
+      # below runs the live (sql/) set, which holds both gen-*.sql and meta-*.sql.
+      - name: Generate Phase 2 + Phase 4 matrix
+        run: cargo run --manifest-path tests/e2e/generated/generator/Cargo.toml
+
+      - name: Run E2E tests (default-build phases + generated matrix)
         id: e2e_default_build_phases
-        run: ./scripts/test-e2e-local.sh --clean --default-build-phases --pg-version ${{ matrix.pg_version }}
+        run: ./scripts/test-e2e-local.sh --clean --default-build-phases --include-generated --pg-version ${{ matrix.pg_version }}
+
+      # Quarantined shapes (xfail) reproduce known, filed loop-nesting bugs
+      # (#227/#230/#233): each asserts the *correct* behavior, so it fails today.
+      # Non-blocking and gated to nightly/manual so PRs stay fast — its purpose is
+      # to keep those bugs continuously documented, not to gate merges.
+      # The run collects the base tests/e2e/sql suite plus quarantine/ (NOT the
+      # live generated sql/ set), so a `gen-* ... PASS` line can only be a
+      # quarantined shape that unexpectedly passed.
+      - name: Run quarantined matrix (non-blocking, nightly)
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        continue-on-error: true
+        run: |
+          ./scripts/test-e2e-local.sh --clean --include-generated-quarantine --pg-version ${{ matrix.pg_version }} \
+            2>&1 | tee quarantine-pg${{ matrix.pg_version }}.log
+
+      # A quarantined shape asserts the CORRECT behavior for an open bug, so it is
+      # expected to FAIL. If one unexpectedly PASSES, the underlying loop-nesting
+      # bug (#227/#230/#233) may be fixed — surface it loudly so the shape can be
+      # promoted out of quarantine/ into the blocking sql/ set. Never fails the job.
+      - name: Flag unexpectedly-passing quarantined shapes
+        if: always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        run: |
+          log="quarantine-pg${{ matrix.pg_version }}.log"
+          if [ ! -f "$log" ]; then
+            echo "No quarantine log ($log); nothing to inspect."
+            exit 0
+          fi
+          # Strip ANSI colour codes, then count quarantined (gen-*) PASS lines.
+          stripped=$(sed -r 's/\x1b\[[0-9;]*m//g' "$log")
+          passes=$(printf '%s\n' "$stripped" | grep -cE 'gen-[0-9]+ +\.\.\. +PASS$' || true)
+          if [ "$passes" -gt 0 ]; then
+            msg="$passes quarantined shape(s) unexpectedly PASSED on PG${{ matrix.pg_version }} — a known loop-nesting bug may be fixed; review for promotion out of quarantine/."
+            echo "::warning::$msg"
+            {
+              echo "### ⚠️ Quarantine watch (PG${{ matrix.pg_version }})"
+              echo "$msg"
+              echo ""
+              echo '```'
+              printf '%s\n' "$stripped" | grep -E 'gen-[0-9]+ +\.\.\. +PASS$' || true
+              echo '```'
+            } >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "All quarantined shapes failed as expected (0 unexpected passes)."
+          fi
 
       - name: Run upgrade tests
         id: upgrade_tests

.gitignore

@@ -35,3 +35,11 @@ results/
 tmp_check/
 tmp_check_iso/
 log/
+
+# Phase 2 + Phase 4 generated matrix tests — regenerated by the generator on demand.
+# The golden manifest.json + meta-manifest.json ARE committed; the per-shape and
+# per-relation .sql files (gen-*.sql, meta-*.sql) are not.
+/tests/e2e/generated/sql/
+/tests/e2e/generated/quarantine/
+# Generator crate build output (nested crate, not caught by /target/).
+/tests/e2e/generated/generator/target/

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pg_durable"
-version = "0.2.3"
+version = "0.2.4"
 edition = "2021"
 license = "PostgreSQL"
 repository = "https://github.com/microsoft/pg_durable"

Makefile

@@ -8,7 +8,7 @@ PG_VERSION ?= pg17
 ACR_REGISTRY ?= myregistry.azurecr.io
 ACR_IMAGE ?= pg_durable
 
-.PHONY: build test test-unit test-e2e test-regress pg-clean docker-build docker-push pg-install
+.PHONY: build test test-unit test-e2e test-regress pg-clean docker-build docker-push pg-install generate-matrix proptest
 
 # Default target
 all: build
@@ -29,6 +29,20 @@ test-unit:
 test-e2e:
 	./scripts/test.sh --e2e
 
+# Generate the Phase 2 combinator-nesting + Phase 4 metamorphic E2E matrix.
+# Writes tests/e2e/generated/sql/*.sql (gitignored: gen-*.sql + meta-*.sql) and
+# refreshes manifest.json + meta-manifest.json.
+generate-matrix:
+	cargo run --manifest-path tests/e2e/generated/generator/Cargo.toml
+
+# Phase 3 (#232): run the model-level proptest properties (random labeled-leaf
+# trees over the Phase 4 reference interpreter + renderer, with shrinking). The
+# committed proptest-regressions/ corpus replays first; PROPTEST_CASES sets the
+# fresh-exploration budget (override for a deeper local hunt, e.g.
+# `PROPTEST_CASES=8192 make proptest`).
+proptest:
+	PROPTEST_CASES=$${PROPTEST_CASES:-1024} cargo test --manifest-path tests/e2e/generated/generator/Cargo.toml
+
 # Build Docker image
 docker-build:
 	docker build --platform linux/amd64 -t pg_durable:latest .
@@ -70,6 +84,8 @@ help:
 	@echo "  test          - Run all tests (unit + E2E)"
 	@echo "  test-unit     - Run pgrx unit tests only"
 	@echo "  test-e2e      - Run E2E tests only (Docker)"
+	@echo "  generate-matrix - Generate the Phase 2 + Phase 4 generated E2E matrix"
+	@echo "  proptest      - Run the Phase 3 model-level proptest properties (shrinking)"
 	@echo "  test-regress  - Run pg_regress tests (resets and starts PostgreSQL)"
 	@echo "  installcheck  - Run pg_regress tests (requires PostgreSQL running, via PGXS)"
 	@echo "  docker-build  - Build Docker image"

USER_GUIDE.md

@@ -186,6 +186,7 @@ df.sql('SELECT 1') ~> df.sql('SELECT 2')
 | `df.status(id)` | Get status | `df.status('a1b2c3d4')` |
 | `df.result(id)` | Get result | `df.result('a1b2c3d4')` |
 | `df.explain(input)` | Visualize graph | `df.explain('a1b2c3d4')` |
+| `df.assert_structural_invariants(id, fail_on_violation)` | Validate a completed instance's structure (`STRICT`: a NULL arg returns no rows, so pass a non-NULL boolean) | `df.assert_structural_invariants('a1b2c3d4', true)` |
 | `df.setvar(name, value)` | Set durable function variable | `df.setvar('api_url', 'https://...')` |
 | `df.getvar(name)` | Get durable function variable | `df.getvar('api_url')` |
 | `df.unsetvar(name)` | Remove durable function variable | `df.unsetvar('api_url')` |
@@ -1504,6 +1505,47 @@ SELECT started_at, last_seen_at,
 
 The background worker updates `last_seen_at` every ~5 seconds as part of its normal operation.
 
+### Validating Structural Invariants
+
+`df.assert_structural_invariants(instance_id, fail_on_violation => false)` checks a
+**completed** instance's node graph against pg_durable's operational-semantics contract
+(see `docs/dsl-semantics.md`). It is a read-only diagnostic — handy in tests and CI to catch
+regressions in branching, parallelism, and reachability.
+
+```sql
+-- Inspect every invariant (one row each; passed = true when it holds)
+SELECT * FROM df.assert_structural_invariants('a1b2c3d4');
+
+-- Assertion form: raises an error listing any violations (one-line test check)
+SELECT * FROM df.assert_structural_invariants('a1b2c3d4', true);
+```
+
+**Columns:** `invariant`, `passed`, `node_id`, `detail`. When an invariant holds you get a
+single `passed = true` row for it; when it is violated you get one `passed = false` row per
+offending node, with `detail` explaining why.
+
+**Invariants checked** (all derived purely from the `df.nodes` snapshot):
+
+- `every_reachable_node_completed` — every node on a taken path is `completed`/`failed`.
+- `join_all_branches_completed` — every branch of a `JOIN` completed.
+- `race_at_least_one_branch_completed` — a resolved `RACE` has a completed branch.
+- `untaken_if_branch_pending` — the untaken `IF` branch stays `pending`.
+- `join_branch_result_name_disjoint` — parallel `JOIN` branches don't bind the same result name.
+- `query_json_well_formed` — internal query JSON (IF/LOOP condition wiring) parses.
+
+**Caveats:**
+
+- **Terminal only.** It evaluates a *snapshot*, so it skips instances that are still running
+  (returning a single `passed = true` `instance_terminal` row). Run it after the instance
+  reaches `completed`/`failed`/`cancelled` — e.g. after `df.wait_for_completion()`.
+- **Relaxed inside loops.** Because `df.nodes` is current state with no per-iteration history,
+  completeness checks are deliberately relaxed for nodes inside a `LOOP` body to avoid false
+  positives (a `break` can abandon an in-flight sibling; an `IF` can leave both branches
+  `completed` across iterations). It never reports a false violation, but catches fewer issues
+  inside loop bodies. It cannot check execution/iteration counts at all.
+- **RLS-scoped.** It only sees instances visible to the calling role; an unknown or
+  not-visible id returns a `passed = false` `instance_found` row.
+
 ---
 
 ## User Isolation & Privileges