test(scripts): stress tests for pre-gate idempotency claim

Four scenarios that the unit-test stubs and the cold-gate burst (04)
don't exercise. All green against a live webapp with the claim system
wired in.

16 — claimant-crash recovery. Planted "pending" claim externally, fired
5 same-key triggers (all polling), DEL'd the claim mid-poll. Verifies
the retry-SETNX path: 1 waiter wins, 4 polling losers resolve to the
same runId.

17 — stale-runId recovery. Claim resolves to a runId that exists in
neither PG nor the buffer. IdempotencyKeyConcern logs a warn and falls
through; the trigger creates a fresh run. Validates the
"resolved-but-not-findable" branch.

18 — claim safety-net timeout. Long-lived "pending" claim with no
publisher; same-key trigger polls until safetyNetMs elapses, returns
503. Validates the wait/poll budget caps.

19 — burst → drain → re-burst with the same key. First burst converges
via the claim (drainer ON, materialises post-burst); second burst
resolves via PG-findFirst (existing IdempotencyKeyConcern behaviour),
bypassing the claim entirely. Validates that the new claim path
doesn't break the existing PG-cache resolution that takes over once
the run is in PG.

Author: d-cs

scripts/mollifier-challenge/16-claimant-crash-recovery.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# 16 — claimant-crash recovery. The trigger pipeline's try/catch must
+# release the claim so polling waiters can retry. We simulate by
+# planting a "pending" claim externally, firing N same-key triggers
+# (all polling), DEL-ing the claim mid-poll to simulate a release,
+# and verifying one of the waiters re-claims + succeeds.
+#
+# Required: drainer OFF + redis-cli.
+
+source "$(dirname "$0")/00-lib.sh"
+
+header "Claimant-crash recovery: release → waiter re-claim"
+
+if [[ -z "${REDIS_CLI:-}" ]]; then
+  if command -v redis-cli >/dev/null 2>&1; then REDIS_CLI=(redis-cli)
+  elif docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^redis$'; then
+    REDIS_CLI=(docker exec -i redis redis-cli)
+  else fail "no redis-cli; set REDIS_CLI"; summary; fi
+else read -ra REDIS_CLI <<< "$REDIS_CLI"
+fi
+
+KEY="challenge-crash-$(date +%s)-$RANDOM"
+CLAIM_KEY="mollifier:claim:${ENV_ID:?ENV_ID required}:$TASK_ID:$KEY"
+
+# Pre-plant a "pending" claim so all incoming triggers will poll.
+"${REDIS_CLI[@]}" SET "$CLAIM_KEY" "pending" EX 60 >/dev/null
+info "planted pending claim at $CLAIM_KEY"
+
+# Fire 5 same-key triggers in parallel — all should enter poll mode.
+WAITERS=$WORK/w
+mkdir -p "$WAITERS"
+for i in $(seq 1 5); do
+  curl -s -o "$WAITERS/$i.json" -X POST \
+    -H "Authorization: Bearer $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d "{\"payload\":{\"i\":$i},\"options\":{\"idempotencyKey\":\"$KEY\"}}" \
+    "$API_BASE/api/v1/tasks/$TASK_ID/trigger" &
+done
+
+# After 1 second, simulate the claimant's release by DEL-ing the claim
+# key. Polling waiters should detect the absent key, retry SETNX, and
+# one of them should win + proceed.
+sleep 1
+"${REDIS_CLI[@]}" DEL "$CLAIM_KEY" >/dev/null
+info "released pending claim (DEL fired)"
+
+wait
+
+# Collect runIds.
+declare -a IDS=()
+for f in "$WAITERS"/*.json; do
+  id=$(jq -r '.id // empty' "$f")
+  if [[ -n "$id" ]]; then IDS+=( "$id" ); fi
+done
+UNIQUE=$(printf "%s\n" "${IDS[@]}" | sort -u)
+n=$(echo "$UNIQUE" | wc -l | tr -d ' ')
+
+info "responses: ${#IDS[@]}, unique runIds: $n"
+echo "$UNIQUE" | head -3 | while read -r id; do info "  $id"; done
+
+if [[ "$n" == "1" ]]; then
+  pass "all 5 waiters resolved to one runId after release"
+else
+  fail "expected 1 unique runId, got $n — retry path broken?"
+fi
+
+NOT_CACHED=$(jq -s 'map(select(.isCached == false)) | length' "$WAITERS"/*.json)
+if [[ "$NOT_CACHED" == "1" ]]; then
+  pass "exactly one waiter became the new claimant (isCached:false)"
+else
+  fail "expected 1 isCached:false response, got $NOT_CACHED"
+fi
+
+summary

scripts/mollifier-challenge/17-stale-runid-recovery.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# 17 — stale-runId recovery. The claim resolves to a runId that exists
+# in neither PG nor the buffer (e.g., claimant errored after publish, or
+# both stores expired). IdempotencyKeyConcern should detect this, log a
+# warn, and fall through to a fresh trigger rather than echoing the
+# dead runId.
+#
+# Required: drainer OFF + redis-cli.
+
+source "$(dirname "$0")/00-lib.sh"
+
+header "Stale-runId recovery: claim points at a ghost"
+
+if [[ -z "${REDIS_CLI:-}" ]]; then
+  if command -v redis-cli >/dev/null 2>&1; then REDIS_CLI=(redis-cli)
+  elif docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^redis$'; then
+    REDIS_CLI=(docker exec -i redis redis-cli)
+  else fail "no redis-cli; set REDIS_CLI"; summary; fi
+else read -ra REDIS_CLI <<< "$REDIS_CLI"
+fi
+
+KEY="challenge-stale-$(date +%s)-$RANDOM"
+CLAIM_KEY="mollifier:claim:${ENV_ID:?ENV_ID required}:$TASK_ID:$KEY"
+GHOST_ID="run_doesnotexist_$(date +%s)"
+
+# Plant a claim that points at a non-existent runId.
+"${REDIS_CLI[@]}" SET "$CLAIM_KEY" "$GHOST_ID" EX 60 >/dev/null
+info "planted stale claim: $CLAIM_KEY -> $GHOST_ID"
+
+# Fire a same-key trigger. IdempotencyKeyConcern's flow:
+#   1. claimOrAwait → returns { resolved, runId: ghost }
+#   2. PG findFirst(idempotencyKey=K) → miss (no row)
+#   3. findBufferedRunWithIdempotency → miss
+#   4. Log warn ("claim resolved but runId not findable"), fall through
+#   5. The trigger proceeds normally and SHOULD create a fresh new run
+api POST "/api/v1/tasks/$TASK_ID/trigger" \
+  "{\"payload\":{\"x\":1},\"options\":{\"idempotencyKey\":\"$KEY\"}}"
+if ! last_status_ok; then
+  fail "trigger returned $(cat "$WORK/last.status") body=$(last_body | head -c 200)"
+  summary
+fi
+NEW_ID=$(last_body | jq -r '.id')
+NEW_CACHED=$(last_body | jq -r '.isCached')
+
+if [[ "$NEW_ID" == "$GHOST_ID" ]]; then
+  fail "trigger returned the ghost runId — fall-through broken"
+elif [[ "$NEW_CACHED" == "true" ]]; then
+  fail "trigger returned isCached:true (id=$NEW_ID) — should be fresh"
+else
+  pass "fresh runId returned: $NEW_ID (isCached:false)"
+fi
+
+# Verify the new run is actually resolvable (not another ghost).
+api GET "/api/v3/runs/$NEW_ID"
+if last_status_ok; then
+  pass "new runId is resolvable"
+else
+  fail "new runId $(cat "$WORK/last.status")"
+fi
+
+summary

scripts/mollifier-challenge/18-claim-ttl-expiry.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# 18 — claim safety-net timeout. Plant a "pending" claim with a TTL
+# longer than the wait safety net (default 5s); fire a same-key trigger;
+# verify it polls for the safetyNet and returns 503 (not 200, not 5xx,
+# not a fresh trigger).
+#
+# Required: drainer OFF + redis-cli.
+
+source "$(dirname "$0")/00-lib.sh"
+
+header "Claim safety-net timeout"
+
+if [[ -z "${REDIS_CLI:-}" ]]; then
+  if command -v redis-cli >/dev/null 2>&1; then REDIS_CLI=(redis-cli)
+  elif docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^redis$'; then
+    REDIS_CLI=(docker exec -i redis redis-cli)
+  else fail "no redis-cli; set REDIS_CLI"; summary; fi
+else read -ra REDIS_CLI <<< "$REDIS_CLI"
+fi
+
+KEY="challenge-ttl-$(date +%s)-$RANDOM"
+CLAIM_KEY="mollifier:claim:${ENV_ID:?ENV_ID required}:$TASK_ID:$KEY"
+
+# Plant "pending" with TTL=20s — comfortably outlives the 5s safety net.
+"${REDIS_CLI[@]}" SET "$CLAIM_KEY" "pending" EX 20 >/dev/null
+info "planted long-lived pending claim ($CLAIM_KEY, TTL=20s)"
+
+# Fire a same-key trigger. Time the response.
+t0=$(date +%s)
+api POST "/api/v1/tasks/$TASK_ID/trigger" \
+  "{\"payload\":{\"x\":1},\"options\":{\"idempotencyKey\":\"$KEY\"}}"
+t1=$(date +%s)
+elapsed=$((t1 - t0))
+status=$(cat "$WORK/last.status")
+
+info "response status=$status, elapsed=${elapsed}s"
+info "body: $(last_body | head -c 200)"
+
+if [[ "$status" == "503" ]]; then
+  pass "returned 503 (safety net hit)"
+else
+  fail "expected 503, got $status"
+fi
+
+# Wait should be ~5s (safetyNetMs default). Accept [4, 8] to absorb
+# polling jitter and webapp overhead.
+if (( elapsed >= 4 && elapsed <= 8 )); then
+  pass "wait time ${elapsed}s ≈ safetyNetMs (5s)"
+else
+  fail "wait time ${elapsed}s outside [4, 8]s — safetyNet misconfigured?"
+fi
+
+# Cleanup so other tests don't see stale pending.
+"${REDIS_CLI[@]}" DEL "$CLAIM_KEY" >/dev/null
+
+summary

scripts/mollifier-challenge/19-burst-drain-reburst.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# 19 — burst → drain → re-burst with the same idempotency key.
+# Verifies the new claim system doesn't *break* the existing
+# post-materialisation cached-hit path: once the buffered (or PG) winner
+# of the first burst is materialised into PG, the second burst's
+# triggers should resolve via IdempotencyKeyConcern's PG-findFirst
+# (existing behaviour), bypassing the claim entirely.
+#
+# Required: drainer ON.
+
+source "$(dirname "$0")/00-lib.sh"
+
+header "Burst → drain → re-burst (cross-store cached resolve)"
+
+KEY="challenge-reburst-$(date +%s)-$RANDOM"
+info "shared idempotencyKey=$KEY"
+
+# Burst 1 — cold gate, same-key triggers serialise through the claim.
+info "burst 1 — 20 same-key triggers"
+B1=$WORK/burst1
+mkdir -p "$B1"
+for i in $(seq 1 20); do
+  curl -s -o "$B1/$i.json" -X POST \
+    -H "Authorization: Bearer $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d "{\"payload\":{\"i\":$i},\"options\":{\"idempotencyKey\":\"$KEY\"}}" \
+    "$API_BASE/api/v1/tasks/$TASK_ID/trigger" &
+done
+wait
+
+declare -a IDS1=()
+for f in "$B1"/*.json; do
+  id=$(jq -r '.id // empty' "$f")
+  if [[ -n "$id" ]]; then IDS1+=( "$id" ); fi
+done
+U1=$(printf "%s\n" "${IDS1[@]}" | sort -u)
+n1=$(echo "$U1" | wc -l | tr -d ' ')
+info "burst 1: ${#IDS1[@]} responses, $n1 unique runId(s)"
+if [[ "$n1" == "1" ]]; then
+  pass "burst 1 converged on one runId via the claim"
+  WINNER=$(echo "$U1" | head -1)
+  info "winner runId: $WINNER"
+else
+  fail "burst 1 produced $n1 unique runIds — claim path broken"
+  summary
+fi
+
+# Wait for the winner to materialise into PG (drainer must be ON).
+info "polling for materialisation (drainer must be ON)"
+deadline=$(($(date +%s) + 60))
+materialised=""
+while (( $(date +%s) < deadline )); do
+  api GET "/api/v3/runs/$WINNER" >/dev/null
+  if last_body | jq -e '.attempts // [] | length > 0' >/dev/null 2>&1; then
+    materialised="yes"
+    break
+  fi
+  status=$(last_body | jq -r '.status // empty')
+  if [[ "$status" != "" && "$status" != "PENDING" && "$status" != "QUEUED" && "$status" != "DELAYED" ]]; then
+    materialised="yes"
+    break
+  fi
+  sleep 1
+done
+if [[ -z "$materialised" ]]; then
+  fail "winner did not materialise within 60s — drainer not on?"
+  summary
+fi
+pass "winner $WINNER materialised into PG"
+
+# Burst 2 — same key. Should ALL resolve via PG-findFirst (existing
+# IdempotencyKeyConcern behaviour) without ever reaching the claim path.
+info "burst 2 — 20 same-key triggers (post-materialisation)"
+B2=$WORK/burst2
+mkdir -p "$B2"
+for i in $(seq 1 20); do
+  curl -s -o "$B2/$i.json" -X POST \
+    -H "Authorization: Bearer $API_KEY" \
+    -H "Content-Type: application/json" \
+    -d "{\"payload\":{\"i\":$i,\"phase\":2},\"options\":{\"idempotencyKey\":\"$KEY\"}}" \
+    "$API_BASE/api/v1/tasks/$TASK_ID/trigger" &
+done
+wait
+
+declare -a IDS2=()
+for f in "$B2"/*.json; do
+  id=$(jq -r '.id // empty' "$f")
+  if [[ -n "$id" ]]; then IDS2+=( "$id" ); fi
+done
+U2=$(printf "%s\n" "${IDS2[@]}" | sort -u)
+n2=$(echo "$U2" | wc -l | tr -d ' ')
+info "burst 2: ${#IDS2[@]} responses, $n2 unique runId(s)"
+
+if [[ "$n2" == "1" ]]; then
+  pass "burst 2 converged on one runId"
+else
+  fail "burst 2 produced $n2 unique runIds — PG-cache resolution broken"
+fi
+
+SHARED=$(echo "$U2" | head -1)
+if [[ "$SHARED" == "$WINNER" ]]; then
+  pass "burst 2's runId matches burst 1's winner — cross-store dedup intact"
+else
+  fail "burst 2 runId=$SHARED, burst 1 winner=$WINNER — they should match"
+fi
+
+# Burst 2 should be ALL isCached:true (PG-findFirst hit).
+CACHED2=$(jq -s 'map(select(.isCached == true)) | length' "$B2"/*.json)
+if [[ "$CACHED2" == "20" ]]; then
+  pass "all 20 burst-2 responses are isCached:true (PG cache hit, not claim)"
+else
+  fail "burst 2 had $CACHED2/20 isCached:true responses"
+fi
+
+summary