[WIP] Add resilience testing plan

tests/e2e/sql/38_infinite_loop.sql

@@ -0,0 +1,79 @@
+-- Test: Infinite loop cancellation (B1 / B2)
+-- Demonstrates: df.loop() with always-true condition and unconditional loop
+-- Expected:
+--   - Loops run indefinitely; df.cancel() successfully stops them
+--   - Instance ends in canceled/failed state, not stuck in running
+
+DROP TABLE IF EXISTS test_infinite_log;
+CREATE TABLE test_infinite_log (id SERIAL, variant TEXT, ts TIMESTAMP DEFAULT now());
+
+CREATE TEMP TABLE _inf_state (instance_id TEXT, variant TEXT);
+
+-- B1: always-true while-condition loop
+INSERT INTO _inf_state
+SELECT df.start(
+    df.loop(
+        'INSERT INTO test_infinite_log (variant) VALUES (''while_true'')',
+        'SELECT true'   -- condition never becomes false
+    ),
+    'test-infinite-while-true'
+), 'while_true';
+
+-- B2: unconditional loop (no condition argument)
+INSERT INTO _inf_state
+SELECT df.start(
+    df.loop(
+        'INSERT INTO test_infinite_log (variant) VALUES (''unconditional'')'
+    ),
+    'test-infinite-unconditional'
+), 'unconditional';
+
+DO $$
+DECLARE
+    rec RECORD;
+    cnt INT;
+    status TEXT;
+    attempts INT;
+BEGIN
+    FOR rec IN SELECT instance_id, variant FROM _inf_state LOOP
+        RAISE NOTICE 'Testing infinite loop [%]: %', rec.variant, rec.instance_id;
+
+        -- Wait for at least 2 iterations to prove the loop is actually running
+        attempts := 0;
+        LOOP
+            SELECT COUNT(*) INTO cnt FROM test_infinite_log WHERE variant = rec.variant;
+            EXIT WHEN cnt >= 2 OR attempts > 200;
+            PERFORM pg_sleep(0.1);
+            attempts := attempts + 1;
+        END LOOP;
+
+        IF cnt < 2 THEN
+            RAISE EXCEPTION 'TEST FAILED [%]: expected >= 2 iterations before cancel, got %',
+                rec.variant, cnt;
+        END IF;
+
+        -- Cancel the running loop
+        PERFORM df.cancel(rec.instance_id, 'test-cancel');
+
+        -- Wait for cancellation to take effect
+        attempts := 0;
+        LOOP
+            SELECT s INTO status FROM df.status(rec.instance_id) s;
+            EXIT WHEN lower(status) IN ('canceled', 'cancelled', 'failed') OR attempts > 100;
+            PERFORM pg_sleep(0.2);
+            attempts := attempts + 1;
+        END LOOP;
+
+        IF lower(status) NOT IN ('canceled', 'cancelled', 'failed') THEN
+            RAISE EXCEPTION 'TEST FAILED [%]: expected canceled/failed after cancel, got %',
+                rec.variant, status;
+        END IF;
+
+        RAISE NOTICE 'PASSED [%]: ran % iterations, then canceled (status=%)',
+            rec.variant, cnt, status;
+    END LOOP;
+END $$;
+
+DROP TABLE _inf_state;
+DROP TABLE test_infinite_log;
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/39_truthiness_edge_cases.sql

@@ -0,0 +1,121 @@
+-- Test: Loop condition truthiness edge cases (B3)
+-- Demonstrates: evaluate_condition / is_truthy behavior for ambiguous values
+-- Expected: Documents and verifies the actual truthiness semantics for:
+--   NULL, integer 0, float 0.0, empty string, string "false", string "no",
+--   empty JSON array, empty JSON object
+
+-- Each sub-test starts a df.loop(body, condition) and checks whether the loop
+-- stops (condition is falsy) or runs at least 2 iterations before cancel
+-- (condition is truthy).
+
+DROP TABLE IF EXISTS test_truth_log;
+CREATE TABLE test_truth_log (id SERIAL, variant TEXT, ts TIMESTAMP DEFAULT now());
+
+-- Helper: run a loop with the given condition SQL, return 'truthy' or 'falsy'
+-- based on whether the loop keeps running (truthy) or stops on its own.
+CREATE OR REPLACE FUNCTION _run_truth_test(
+    p_variant TEXT,
+    p_condition_sql TEXT
+) RETURNS TEXT
+LANGUAGE plpgsql AS $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    cnt INT;
+    attempts INT := 0;
+BEGIN
+    inst_id := df.start(
+        df.loop(
+            format('INSERT INTO test_truth_log (variant) VALUES (%L)', p_variant),
+            p_condition_sql
+        ),
+        format('truth-%s', p_variant)
+    );
+
+    -- Wait up to 10s for the loop to either stop on its own or run 2 iterations
+    LOOP
+        SELECT s INTO status FROM df.status(inst_id) s;
+        SELECT COUNT(*) INTO cnt FROM test_truth_log WHERE variant = p_variant;
+        EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled', 'cancelled')
+               OR cnt >= 2
+               OR attempts > 100;
+        PERFORM pg_sleep(0.1);
+        attempts := attempts + 1;
+    END LOOP;
+
+    IF lower(status) IN ('completed', 'failed', 'canceled', 'cancelled') THEN
+        -- Loop stopped by itself → condition was falsy
+        RETURN 'falsy';
+    ELSIF cnt >= 2 THEN
+        -- Loop kept running beyond 1 iteration → condition is truthy; cancel it
+        PERFORM df.cancel(inst_id, 'truth-test-done');
+        -- Wait for cancel to land
+        attempts := 0;
+        LOOP
+            SELECT s INTO status FROM df.status(inst_id) s;
+            EXIT WHEN lower(status) IN ('completed', 'failed', 'canceled', 'cancelled')
+                   OR attempts > 50;
+            PERFORM pg_sleep(0.1);
+            attempts := attempts + 1;
+        END LOOP;
+        RETURN 'truthy';
+    ELSE
+        -- Timeout: instance did not start within 10s (worker busy or dead)
+        PERFORM df.cancel(inst_id, 'truth-test-timeout');
+        RAISE EXCEPTION 'Timeout waiting for truth test [%] (status=%, cnt=%)', p_variant, status, cnt;
+    END IF;
+END $$;
+
+-- NOTE on known behavior quirks:
+-- String "false" and "no" are treated as TRUTHY by is_truthy() because they are
+-- non-empty strings that don't parse as integers. A user writing
+-- `df.loop(..., 'SELECT ''false''')` may expect the loop to stop but it will not.
+-- The correct way to return a falsy condition is `SELECT false` (boolean) or `SELECT 0`.
+
+DO $$
+DECLARE
+    -- Each entry: (variant, condition_sql, expected_actual_result)
+    -- expected values reflect the CURRENT implementation behavior.
+    -- Entries marked with [KNOWN QUIRK] behave differently than users may expect.
+    cases TEXT[][] := ARRAY[
+        ARRAY['null_val',    'SELECT NULL',          'falsy'],
+        ARRAY['int_zero',    'SELECT 0',             'falsy'],
+        ARRAY['int_one',     'SELECT 1',             'truthy'],
+        ARRAY['bool_false',  'SELECT false',         'falsy'],
+        ARRAY['bool_true',   'SELECT true',          'truthy'],
+        -- [KNOWN QUIRK] Non-empty strings that are not "true"/"t"/"yes"/"1" and
+        -- not parseable as non-zero integers fall through to !s.is_empty() = true.
+        ARRAY['str_false',   'SELECT ''false''',     'truthy'],
+        ARRAY['str_no',      'SELECT ''no''',        'truthy'],
+        ARRAY['empty_str',   'SELECT ''''',          'falsy'],
+        ARRAY['float_zero',  'SELECT 0.0',           'falsy'],
+        ARRAY['empty_array', 'SELECT ''[]''::jsonb', 'falsy'],
+        ARRAY['empty_obj',   'SELECT ''{}''::jsonb', 'falsy']
+    ];
+    rec TEXT[];
+    got TEXT;
+    expected TEXT;
+    failures INT := 0;
+BEGIN
+    FOREACH rec SLICE 1 IN ARRAY cases LOOP
+        got := _run_truth_test(rec[1], rec[2]);
+        expected := rec[3];
+        RAISE NOTICE 'Truthiness [%]: condition=% → %', rec[1], rec[2], got;
+        IF got != expected THEN
+            RAISE WARNING 'REGRESSION [%]: got % expected %', rec[1], got, expected;
+            failures := failures + 1;
+        END IF;
+    END LOOP;
+
+    -- Emit a clear notice about the known quirks so they are visible in test output
+    RAISE NOTICE 'KNOWN QUIRK: SELECT ''false'' and SELECT ''no'' are truthy in loop conditions. '
+        'Use SELECT false (boolean) or SELECT 0 to stop a loop.';
+
+    IF failures > 0 THEN
+        RAISE EXCEPTION 'TEST FAILED: % truthiness regression(s) — see WARNINGs above', failures;
+    END IF;
+END $$;
+
+DROP FUNCTION _run_truth_test(TEXT, TEXT);
+DROP TABLE test_truth_log;
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/40_empty_dml_results.sql

@@ -0,0 +1,82 @@
+-- Test: SQL nodes returning 0 rows or DML without RETURNING (B5 / B6)
+-- Demonstrates: How empty result sets and DML results flow through |=> and $var
+-- Expected: Both patterns complete successfully; documents the JSON result shape
+
+DROP TABLE IF EXISTS test_dml_target;
+CREATE TABLE test_dml_target (id SERIAL, val TEXT);
+
+-- ============================================================================
+-- B5: SQL node that returns 0 rows, result used in next node
+-- ============================================================================
+CREATE TEMP TABLE _b5_state AS
+SELECT df.start(
+    'SELECT 1 WHERE false' |=> 'empty_result'
+    ~> 'SELECT $empty_result',   -- uses the empty result JSON
+    'test-empty-result'
+) AS instance_id;
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    res TEXT;
+BEGIN
+    SELECT instance_id INTO inst_id FROM _b5_state;
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B5]: expected Completed, got %', status;
+    END IF;
+
+    SELECT r INTO res FROM df.result(inst_id) r;
+    RAISE NOTICE 'B5 result (empty result passed as $var): %', res;
+    RAISE NOTICE 'PASSED [B5]: zero-row SQL result flows through |=> correctly';
+END $$;
+
+DROP TABLE _b5_state;
+
+-- ============================================================================
+-- B6: DML node without RETURNING, result used in next node
+-- ============================================================================
+INSERT INTO test_dml_target (val) VALUES ('initial');
+
+CREATE TEMP TABLE _b6_state AS
+SELECT df.start(
+    'UPDATE test_dml_target SET val = ''updated''' |=> 'update_result'
+    ~> 'SELECT $update_result',  -- uses the DML result JSON (0 rows, row_count > 0)
+    'test-dml-result'
+) AS instance_id;
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    res TEXT;
+    updated_val TEXT;
+BEGIN
+    SELECT instance_id INTO inst_id FROM _b6_state;
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B6]: expected Completed, got %', status;
+    END IF;
+
+    SELECT r INTO res FROM df.result(inst_id) r;
+    RAISE NOTICE 'B6 result (DML result passed as $var): %', res;
+
+    -- Verify the DML actually ran
+    SELECT val INTO updated_val FROM test_dml_target LIMIT 1;
+    IF updated_val != 'updated' THEN
+        RAISE EXCEPTION 'TEST FAILED [B6]: DML did not execute, val = %', updated_val;
+    END IF;
+
+    RAISE NOTICE 'PASSED [B6]: DML result flows through |=> correctly';
+END $$;
+
+DROP TABLE _b6_state;
+
+-- ============================================================================
+-- Cleanup
+-- ============================================================================
+DROP TABLE test_dml_target;
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/41_break_outside_loop.sql

@@ -0,0 +1,35 @@
+-- Test: df.break() used at the top level outside any loop (B10)
+-- Demonstrates: Break sentinel propagated as final instance result
+-- Expected: Instance completes (the break sentinel becomes the result),
+--           does NOT hang or crash.
+
+CREATE TEMP TABLE _b10_state AS
+SELECT df.start(
+    df.break('{"reason": "top-level-break"}'),
+    'test-break-outside-loop'
+) AS instance_id;
+
+DO $$
+DECLARE
+    inst_id TEXT;
+    status TEXT;
+    res TEXT;
+BEGIN
+    SELECT instance_id INTO inst_id FROM _b10_state;
+
+    -- A top-level break has no enclosing loop to consume it, so the break
+    -- sentinel propagates as the final result.  The instance should complete
+    -- rather than hang or fail with an error.
+    SELECT df.wait_for_completion(inst_id, 30) INTO status;
+
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B10]: expected Completed for top-level break, got %', status;
+    END IF;
+
+    SELECT r INTO res FROM df.result(inst_id) r;
+    RAISE NOTICE 'B10 result (top-level break value): %', res;
+    RAISE NOTICE 'PASSED [B10]: df.break() at top level completes gracefully';
+END $$;
+
+DROP TABLE _b10_state;
+SELECT 'TEST PASSED' AS result;

tests/e2e/sql/42_recursive_start.sql

@@ -0,0 +1,52 @@
+-- Test: Calling df.start() from inside a workflow SQL node (B11)
+-- Demonstrates: df.start() is not guarded by is_in_workflow_context().
+--               A SQL node can spawn child instances, which the background
+--               worker picks up independently.
+-- Expected: Outer instance completes; child instance is created and completes.
+
+DROP TABLE IF EXISTS test_recursive_log;
+CREATE TABLE test_recursive_log (id SERIAL, spawned_id TEXT, ts TIMESTAMP DEFAULT now());
+
+CREATE TEMP TABLE _b11_outer AS
+SELECT df.start(
+    -- This SQL node calls df.start() to spawn a child instance and records the ID.
+    'INSERT INTO test_recursive_log (spawned_id)
+     SELECT df.start(df.sql(''SELECT 1''), ''child-from-workflow'')',
+    'test-recursive-start-outer'
+) AS instance_id;
+
+DO $$
+DECLARE
+    outer_id TEXT;
+    child_id TEXT;
+    status TEXT;
+BEGIN
+    SELECT instance_id INTO outer_id FROM _b11_outer;
+    RAISE NOTICE 'Outer instance: %', outer_id;
+
+    -- Wait for the outer instance to complete
+    SELECT df.wait_for_completion(outer_id, 30) INTO status;
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B11]: outer instance expected Completed, got %', status;
+    END IF;
+
+    -- Verify that a child instance was spawned
+    SELECT spawned_id INTO child_id FROM test_recursive_log LIMIT 1;
+    IF child_id IS NULL THEN
+        RAISE EXCEPTION 'TEST FAILED [B11]: expected a child instance to be spawned';
+    END IF;
+    RAISE NOTICE 'Child instance spawned: %', child_id;
+
+    -- Wait for the child instance to complete
+    SELECT df.wait_for_completion(child_id, 30) INTO status;
+    IF status != 'completed' THEN
+        RAISE EXCEPTION 'TEST FAILED [B11]: child instance expected Completed, got %', status;
+    END IF;
+
+    RAISE NOTICE 'PASSED [B11]: df.start() inside a workflow spawns a running child instance';
+    RAISE NOTICE 'NOTE: No recursion guard exists — unbounded spawning is possible if used carelessly';
+END $$;
+
+DROP TABLE _b11_outer;
+DROP TABLE test_recursive_log;
+SELECT 'TEST PASSED' AS result;