Implement resilience test plan: Phase 2–5 E2E tests

tests/e2e/sql/51_wide_parallel.sql

@@ -0,0 +1,41 @@
+-- Test: Wide parallel graph — 9 concurrent branches via nested join3 (A3)
+-- Demonstrates: duroxide runtime handles many simultaneous parallel sub-orchestrations
+-- Expected: All branches complete; instance ends in completed state
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+BEGIN
+    -- Build 9 parallel branches by nesting join3 calls
+    inst_id := df.start(
+        df.join3(
+            df.join3(
+                df.sql('SELECT 1 AS branch'),
+                df.sql('SELECT 2 AS branch'),
+                df.sql('SELECT 3 AS branch')
+            ),
+            df.join3(
+                df.sql('SELECT 4 AS branch'),
+                df.sql('SELECT 5 AS branch'),
+                df.sql('SELECT 6 AS branch')
+            ),
+            df.join3(
+                df.sql('SELECT 7 AS branch'),
+                df.sql('SELECT 8 AS branch'),
+                df.sql('SELECT 9 AS branch')
+            )
+        ),
+        'test-wide-parallel-9'
+    );
+
+    SELECT df.wait_for_completion(inst_id, 60) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [A3]: 9-branch parallel graph expected completed, got %', status;
+    END IF;
+
+    RAISE NOTICE 'PASSED [A3]: 9-branch parallel graph completed successfully';
+END $$;
+
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/52_long_loop_history.sql

@@ -0,0 +1,41 @@
+-- Test: Long loop execution history — 100 iterations (A4)
+-- Demonstrates: df.loop() with a finite counter condition running 100 iterations
+--               does not OOM, stack-overflow, or time out unreasonably.
+-- Expected: Instance completes in completed state; loop table has exactly 100 rows.
+
+DROP TABLE IF EXISTS test_long_loop_log;
+CREATE TABLE test_long_loop_log (id SERIAL);
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    row_count INT;
+BEGIN
+    -- Loop body: insert a row. Condition: stop after 100 rows.
+    inst_id := df.start(
+        df.loop(
+            'INSERT INTO test_long_loop_log DEFAULT VALUES',
+            'SELECT COUNT(*) < 100 FROM test_long_loop_log'
+        ),
+        'test-long-loop-100'
+    );
+
+    -- Allow up to 120 seconds; 100 iterations should complete well under that
+    SELECT df.wait_for_completion(inst_id, 120) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [A4]: long loop expected completed, got %', status;
+    END IF;
+
+    SELECT COUNT(*) INTO row_count FROM test_long_loop_log;
+
+    IF row_count != 100 THEN
+        RAISE EXCEPTION 'TEST FAILED [A4]: expected 100 rows, got %', row_count;
+    END IF;
+
+    RAISE NOTICE 'PASSED [A4]: loop ran 100 iterations and completed successfully';
+END $$;
+
+DROP TABLE test_long_loop_log;
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/53_large_result_sets.sql

@@ -0,0 +1,26 @@
+-- Test: Large SQL result set — query returning 10,000 rows (A5)
+-- Demonstrates: execute_sql activity (which uses fetch_all()) can handle
+--               a large result set without OOM or timeout.
+-- Expected: Instance completes successfully.
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+BEGIN
+    -- Generate a 10,000-row result by cross-joining small sets
+    inst_id := df.start(
+        df.sql('SELECT g1.n AS a, g2.n AS b FROM generate_series(1, 100) g1(n) CROSS JOIN generate_series(1, 100) g2(n)'),
+        'test-large-result-10k'
+    );
+
+    SELECT df.wait_for_completion(inst_id, 60) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [A5]: large result set expected completed, got %', status;
+    END IF;
+
+    RAISE NOTICE 'PASSED [A5]: 10,000-row result set handled without error';
+END $$;
+
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/54_large_query_text.sql

@@ -0,0 +1,34 @@
+-- Test: Very large SQL query text (A6)
+-- Demonstrates: execute_sql activity can handle a very long query string
+--               without truncation or serialization errors.
+-- Expected: Instance completes successfully.
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    long_sql TEXT;
+    i INT;
+BEGIN
+    -- Build a SELECT with a 500-element VALUES list, producing a ~10KB query
+    long_sql := 'SELECT v FROM (VALUES ';
+    FOR i IN 1..500 LOOP
+        IF i > 1 THEN
+            long_sql := long_sql || ',';
+        END IF;
+        long_sql := long_sql || format('(%s)', i);
+    END LOOP;
+    long_sql := long_sql || ') t(v) WHERE v = 1';
+
+    inst_id := df.start(df.sql(long_sql), 'test-large-query-text');
+
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [A6]: large query text expected completed, got %', status;
+    END IF;
+
+    RAISE NOTICE 'PASSED [A6]: ~10KB query text executed without error';
+END $$;
+
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/55_large_variables.sql

@@ -0,0 +1,38 @@
+-- Test: Large variable payloads (A8)
+-- Demonstrates: df.setvar() and {var} substitution handles large string values
+--               without truncation or serialization errors.
+-- Expected: Instance completes; the large value is preserved in the workflow.
+
+SELECT df.clearvars();
+
+-- Set a ~5KB variable value (a repeated pattern string)
+SELECT df.setvar('big_val', repeat('abcdefghij', 500));  -- 5000 chars
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    result_text TEXT;
+BEGIN
+    -- Pass the big variable through a workflow step
+    inst_id := df.start(
+        'SELECT length(''{big_val}'') AS len' |=> 'result',
+        'test-large-variable'
+    );
+
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [A8]: large variable expected completed, got %', status;
+    END IF;
+
+    SELECT r INTO result_text FROM df.result(inst_id) r;
+    IF result_text IS NULL OR result_text NOT LIKE '%5000%' THEN
+        RAISE EXCEPTION 'TEST FAILED [A8]: expected length 5000 in result, got %', result_text;
+    END IF;
+
+    RAISE NOTICE 'PASSED [A8]: 5KB variable payload handled without error';
+END $$;
+
+SELECT df.clearvars();
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/56_rapid_status_poll.sql

@@ -0,0 +1,31 @@
+-- Test: Rapid df.status() polling stress test (C5)
+-- Demonstrates: Many rapid status() calls do not exhaust resources or deadlock.
+-- Each df.status() call creates a fresh tokio runtime and duroxide provider —
+-- this test verifies the system remains stable under polling pressure.
+-- Expected: No errors, no hangs; instance completes normally.
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    poll_count INT := 0;
+BEGIN
+    -- Start a slow instance so we can poll it many times before it finishes
+    inst_id := df.start(df.sleep(3), 'test-rapid-poll');
+
+    -- Tight-loop poll until completed, counting iterations
+    LOOP
+        SELECT s INTO status FROM df.status(inst_id) s;
+        poll_count := poll_count + 1;
+        EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled');
+        EXIT WHEN poll_count > 1000;  -- safety limit
+    END LOOP;
+
+    IF lower(status) != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [C5]: expected completed, got % after % polls', status, poll_count;
+    END IF;
+
+    RAISE NOTICE 'PASSED [C5]: rapid polling ran % times without resource errors', poll_count;
+END $$;
+
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/57_variable_name_conflict.sql

@@ -0,0 +1,81 @@
+-- Test: Variable substitution edge cases — name conflicts and overwrites (B4 / B14)
+-- B4: A result binding that shadows a user variable of the same name
+-- B14: Two sequential steps binding the same result name (the second overwrites the first)
+-- Expected: System handles shadowing/overwriting gracefully without error
+
+SELECT df.clearvars();
+
+-- ============================================================================
+-- B14: Two steps bound to the same result name — second value wins
+-- ============================================================================
+DROP TABLE IF EXISTS test_var_conflict_log;
+CREATE TABLE test_var_conflict_log (id SERIAL, val TEXT);
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    log_val TEXT;
+BEGIN
+    -- Step 1 binds result to "x" (value: 'first')
+    -- Step 2 binds result to "x" (value: 'second')  — overwrites
+    -- Step 3 uses $x — should see 'second'
+    inst_id := df.start(
+        'SELECT ''first'' AS val' |=> 'x'
+        ~> ('SELECT ''second'' AS val' |=> 'x')
+        ~> 'INSERT INTO test_var_conflict_log (val) VALUES ($x) RETURNING val',
+        'test-var-name-conflict-b14'
+    );
+
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B14]: expected completed, got %', status;
+    END IF;
+
+    SELECT val INTO log_val FROM test_var_conflict_log ORDER BY id DESC LIMIT 1;
+    IF log_val IS NULL OR log_val NOT LIKE '%second%' THEN
+        RAISE EXCEPTION 'TEST FAILED [B14]: expected second to win name conflict, got %', log_val;
+    END IF;
+
+    RAISE NOTICE 'PASSED [B14]: second binding of same name correctly overwrites first';
+END $$;
+
+-- ============================================================================
+-- B4: User variable shadowed by a step result of the same name
+-- ============================================================================
+SELECT df.clearvars();
+SELECT df.setvar('user_val', 'from_user');
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    log_val TEXT;
+BEGIN
+    -- {user_val} is substituted at graph-build time as 'from_user'.
+    -- The step then binds its result to user_val as well.
+    -- A subsequent step that references $user_val gets the step result, not the original.
+    inst_id := df.start(
+        'SELECT ''from_step'' AS val' |=> 'user_val'
+        ~> 'INSERT INTO test_var_conflict_log (val) VALUES ($user_val) RETURNING val',
+        'test-var-shadow-b4'
+    );
+
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B4]: expected completed, got %', status;
+    END IF;
+
+    SELECT val INTO log_val FROM test_var_conflict_log ORDER BY id DESC LIMIT 1;
+    IF log_val IS NULL OR log_val NOT LIKE '%from_step%' THEN
+        RAISE EXCEPTION 'TEST FAILED [B4]: expected step result to shadow user var, got %', log_val;
+    END IF;
+
+    RAISE NOTICE 'PASSED [B4]: step result correctly shadows user-defined var of same name';
+END $$;
+
+SELECT df.clearvars();
+DROP TABLE test_var_conflict_log;
+SELECT 'TEST PASSED' AS result;