fix(wake): harden test cleanup and wake-queue locking against tmp leaks and filesystem failures

bin/fm-afk-return.sh

@@ -205,7 +205,7 @@ main() {
   . "$SCRIPT_DIR/fm-classify-lib.sh"
 
   mkdir -p "$STATE" || return 1
-  fm_lock_acquire_wait "$LOCK"
+  fm_lock_acquire_wait "$LOCK" || return "$?"
   trap 'fm_lock_release "$LOCK"' EXIT
   write_pending_seed || { fm_lock_release "$LOCK"; trap - EXIT; return 1; }
   return_reconcile

bin/fm-guard.sh

@@ -63,7 +63,7 @@ fm_guard_claim_stale_banner() {
   local state=$1 key=$2
   local marker="$state/.guard-watcher-stale-banner"
   local lock="$state/.guard-watcher-stale-banner.lock"
-  local seen i
+  local seen i rc
 
   seen=$(cat "$marker" 2>/dev/null || true)
   # Strip a single trailing newline so key comparison is line-content based.
@@ -85,7 +85,12 @@ fm_guard_claim_stale_banner() {
       printf '%s\n' "$key" > "$marker" || true
       fm_lock_release "$lock" 2>/dev/null || true
       return 0
+    else
+      rc=$?
     fi
+    # A filesystem failure is already reported by the lock helper. Keep the
+    # operator-facing alarm loud without retrying the broken operation 50 times.
+    [ "$rc" -eq 2 ] && return 0
     seen=$(cat "$marker" 2>/dev/null || true)
     seen=${seen%$'\n'}
     if [ "$seen" = "$key" ]; then

bin/fm-pr-check-migrate.sh

@@ -292,6 +292,12 @@ while [ "$i" -lt 100 ]; do
   if fm_lock_try_acquire "$WATCH_LOCK"; then
     lock_held=1
     break
+  else
+    lock_rc=$?
+  fi
+  if [ "$lock_rc" -eq 2 ]; then
+    echo "PR_CHECK_MIGRATION: watcher exclusion lock could not be created" >&2
+    exit 1
   fi
   # A concurrent migration may have completed while this process waited.
   # Its validated marker proves the old watcher crossed the boundary, so this

bin/fm-supervise-daemon.sh

@@ -1264,7 +1264,13 @@ fm_super_main() {
   [ -x "$WATCH" ] || { echo "error: watcher not found or not executable: $WATCH" >&2; exit 1; }
 
   # --- single instance (portable lock, no flock dependency) ------------------
-  if ! fm_lock_try_acquire "$LOCK"; then
+  local lock_rc=0
+  fm_lock_try_acquire "$LOCK" || lock_rc=$?
+  if [ "$lock_rc" -ne 0 ]; then
+    if [ "$lock_rc" -eq 2 ]; then
+      echo "error: supervise-daemon lock acquisition failed for $LOCK" >&2
+      exit 1
+    fi
     if [ -n "${FM_LOCK_HELD_PID:-}" ]; then
       echo "error: another fm-supervise-daemon is already running (pid $FM_LOCK_HELD_PID, lock $LOCK held)" >&2
     else

bin/fm-wake-drain.sh

@@ -40,7 +40,7 @@ trap cleanup EXIT
 trap 'exit 130' INT
 trap 'exit 143' TERM
 
-fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || exit "$?"
 DRAIN_LOCK_HELD=true
 
 if [ ! -s "$FM_WAKE_QUEUE" ]; then

bin/fm-wake-lib.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 # Shared durable wake queue and portable lock helpers.
+# Lock acquisition returns 0 when acquired, 1 for genuine contention, and 2 for
+# an operational filesystem failure; FM_LOCK_ERROR describes the latter.
 
 FM_WAKE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 FM_WAKE_DEFAULT_ROOT="$(cd "$FM_WAKE_LIB_DIR/.." && pwd)"
@@ -9,6 +11,7 @@ STATE="${FM_STATE_OVERRIDE:-${STATE:-$FM_HOME/state}}"
 FM_WAKE_QUEUE="${FM_WAKE_QUEUE:-$STATE/.wake-queue}"
 FM_WAKE_QUEUE_LOCK="${FM_WAKE_QUEUE_LOCK:-$STATE/.wake-queue.lock}"
 FM_LOCK_STALE_AFTER="${FM_LOCK_STALE_AFTER:-2}"
+FM_LOCK_STEAL_MAX_DEPTH="${FM_LOCK_STEAL_MAX_DEPTH:-8}"
 mkdir -p "$STATE"
 
 fm_current_pid() {
@@ -157,12 +160,12 @@ fm_lock_claim() {
   mypid=${BASHPID:-$$}
   if ! { printf '%s\n' "$mypid" > "$ownerdir/pid"; } 2>/dev/null; then
     fm_lock_discard_owner "$ownerdir"
-    return 1
+    return 2
   fi
   back=$(cat "$ownerdir/pid" 2>/dev/null || true)
   if [ "$back" != "$mypid" ]; then
     fm_lock_discard_owner "$ownerdir"
-    return 1
+    return 2
   fi
   if ! fm_lock_points_to_owner "$lockdir" "$ownerdir"; then
     fm_lock_discard_owner "$ownerdir"
@@ -179,29 +182,49 @@ fm_lock_claim() {
 }
 
 fm_lock_try_create() {
-  local lockdir=$1 allowed_steal_owner=${2:-} ownerdir
+  local lockdir=$1 allowed_steal_owner=${2:-} ownerdir claim_rc
   FM_LOCK_OWNER_DIR=
-  ownerdir=$(fm_lock_owner_dir "$lockdir") || return 1
+  if ! ownerdir=$(fm_lock_owner_dir "$lockdir"); then
+    FM_LOCK_ERROR="could not create owner directory for $lockdir"
+    return 2
+  fi
   if [ -e "$lockdir" ] || [ -L "$lockdir" ]; then
     fm_lock_discard_owner "$ownerdir"
     return 1
   fi
   if ! fm_lock_prepare_owner "$ownerdir"; then
     fm_lock_discard_owner "$ownerdir"
-    return 1
+    FM_LOCK_ERROR="could not prepare owner directory for $lockdir"
+    return 2
   fi
-  if ln -s "$ownerdir" "$lockdir" 2>/dev/null && fm_lock_points_to_owner "$lockdir" "$ownerdir"; then
-    if fm_lock_claim "$lockdir" "$ownerdir" "$allowed_steal_owner"; then
-      FM_LOCK_OWNER_DIR=$ownerdir
-      return 0
-    fi
-    if fm_lock_points_to_owner "$lockdir" "$ownerdir"; then
-      rm -f "$lockdir" 2>/dev/null || true
+  if ! ln -s "$ownerdir" "$lockdir" 2>/dev/null; then
+    fm_lock_remove_stray_owner_link "$lockdir" "$ownerdir"
+    fm_lock_discard_owner "$ownerdir"
+    if [ -e "$lockdir" ] || [ -L "$lockdir" ]; then
+      return 1
     fi
-  else
+    FM_LOCK_ERROR="could not publish lock $lockdir"
+    return 2
+  fi
+  if ! fm_lock_points_to_owner "$lockdir" "$ownerdir"; then
     fm_lock_remove_stray_owner_link "$lockdir" "$ownerdir"
+    fm_lock_discard_owner "$ownerdir"
+    return 1
+  fi
+  if fm_lock_claim "$lockdir" "$ownerdir" "$allowed_steal_owner"; then
+    FM_LOCK_OWNER_DIR=$ownerdir
+    return 0
+  else
+    claim_rc=$?
+  fi
+  if fm_lock_points_to_owner "$lockdir" "$ownerdir"; then
+    rm -f "$lockdir" 2>/dev/null || true
   fi
   fm_lock_discard_owner "$ownerdir"
+  if [ "$claim_rc" -eq 2 ]; then
+    FM_LOCK_ERROR="could not record owner for $lockdir"
+    return 2
+  fi
   return 1
 }
 
@@ -248,13 +271,18 @@ fm_lock_recheck_stale_owner() {
   return 0
 }
 
-fm_lock_try_acquire() {
-  local lockdir=$1 pid steal cur rc steal_owner primary_owner
+fm_lock_try_acquire_inner() {
+  local lockdir=$1 depth=$2 pid steal cur rc steal_owner primary_owner max_depth create_rc
   FM_LOCK_HELD_PID=
   FM_LOCK_OWNER_DIR=
 
   if fm_lock_try_create "$lockdir"; then
     return 0
+  else
+    create_rc=$?
+  fi
+  if [ "$create_rc" -eq 2 ]; then
+    return 2
   fi
 
   pid=$(cat "$lockdir/pid" 2>/dev/null || true)
@@ -267,11 +295,23 @@ fm_lock_try_acquire() {
     return 1
   fi
 
+  max_depth=$FM_LOCK_STEAL_MAX_DEPTH
+  case "$max_depth" in
+    ''|*[!0-9]*) max_depth=8 ;;
+  esac
+  if [ "$depth" -ge "$max_depth" ]; then
+    FM_LOCK_ERROR="steal depth exceeded $max_depth while acquiring $lockdir"
+    return 2
+  fi
+
   steal="$lockdir.steal"
-  if ! fm_lock_try_acquire "$steal"; then
+  if fm_lock_try_acquire_inner "$steal" "$((depth + 1))"; then
+    :
+  else
+    rc=$?
     FM_LOCK_HELD_PID=$(cat "$lockdir/pid" 2>/dev/null || true)
     FM_LOCK_OWNER_DIR=
-    return 1
+    return "$rc"
   fi
   steal_owner=${FM_LOCK_OWNER_DIR:-}
 
@@ -311,6 +351,8 @@ fm_lock_try_acquire() {
   rc=1
   if fm_lock_try_create "$lockdir" "$steal_owner"; then
     rc=0
+  else
+    rc=$?
   fi
   if [ "$rc" -ne 0 ]; then
     # shellcheck disable=SC2034 # Read by callers after fm_lock_try_acquire returns.
@@ -321,9 +363,29 @@ fm_lock_try_acquire() {
   return "$rc"
 }
 
+fm_lock_try_acquire() {
+  local lockdir=$1 rc
+  FM_LOCK_ERROR=
+  if fm_lock_try_acquire_inner "$lockdir" 0; then
+    return 0
+  else
+    rc=$?
+  fi
+  if [ "$rc" -eq 2 ]; then
+    printf 'fm_lock_try_acquire: %s\n' "${FM_LOCK_ERROR:-filesystem operation failed for $lockdir}" >&2
+  fi
+  return "$rc"
+}
+
 fm_lock_acquire_wait() {
-  local lockdir=$1
-  while ! fm_lock_try_acquire "$lockdir"; do
+  local lockdir=$1 rc
+  while :; do
+    if fm_lock_try_acquire "$lockdir"; then
+      return 0
+    else
+      rc=$?
+    fi
+    [ "$rc" -eq 1 ] || return "$rc"
     sleep 0.1
   done
 }
@@ -364,7 +426,7 @@ fm_wake_append() {
   seq_file="$STATE/.wake-queue.seq"
   status=0
 
-  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK"
+  fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return "$?"
   seq=$(cat "$seq_file" 2>/dev/null || echo 0)
   case "$seq" in
     ''|*[!0-9]*) seq=0 ;;

bin/fm-watch.sh

@@ -744,7 +744,13 @@ fi
   exit 1
 }
 
-if ! fm_lock_try_acquire "$WATCH_LOCK"; then
+lock_rc=0
+fm_lock_try_acquire "$WATCH_LOCK" || lock_rc=$?
+if [ "$lock_rc" -ne 0 ]; then
+  if [ "$lock_rc" -eq 2 ]; then
+    echo "watcher: lock acquisition failed for $WATCH_LOCK" >&2
+    exit 1
+  fi
   BEAT="$STATE/.last-watcher-beat"
   if [ -n "${FM_LOCK_HELD_PID:-}" ]; then
     if [ -e "$BEAT" ]; then

docs/configuration.md

@@ -424,6 +424,7 @@ FMX_X_THREAD_MAX=25     # maximum messages in one auto-split reply thread
 FMX_FOLLOWUP_MAX_AGE_SECS=604800   # local window for posting X-mode completion follow-ups (7 days)
 FMX_FOLLOWUP_MAX_COUNT=3   # local cap on X-mode completion follow-ups per linked mention
 FM_LOCK_STALE_AFTER=2   # seconds before dead-pid lock records can be reclaimed; mid-acquire locks keep at least 2s grace
+FM_LOCK_STEAL_MAX_DEPTH=8   # hard cap on nested stale-lock steal recursion; acquisition fails loudly (rc 2) past this depth instead of recursing unbounded
 FM_GUARD_GRACE=300      # seconds before guard warnings, arm health checks, and the primary turn-end guard treat a watcher beacon as stale
 FM_ARM_CONFIRM_TIMEOUT=10   # seconds fm-watch-arm waits to confirm a fresh watcher before reporting FAILED
 FM_ARM_ATTACH_POLL=0.5  # seconds between checks while fm-watch-arm is attached to an existing healthy watcher cycle

tests/fm-afk-pi-herdr-return-e2e.test.sh

@@ -31,7 +31,8 @@ done
 
 LAB_HELPER=${HERDR_LAB_HELPER:-$ROOT/bin/fm-herdr-lab.sh}
 SESSION=$("$LAB_HELPER" name fm-afk-pi-return-e2e)
-TMP_ROOT=$(fm_test_tmproot fm-afk-pi-return-e2e)
+fm_test_tmproot TMP_ROOT fm-afk-pi-return-e2e
+
 HOME_DIR="$TMP_ROOT/home"
 STATE="$HOME_DIR/state"
 PROJECT="$TMP_ROOT/project"

tests/fm-afk-return.test.sh

@@ -11,7 +11,7 @@ set -u
 # shellcheck source=tests/lib.sh
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-TMP_ROOT=$(fm_test_tmproot fm-afk-return-tests)
+fm_test_tmproot TMP_ROOT fm-afk-return-tests
 
 install_runner() {  # <case-dir>
   local dir=$1

tests/fm-arm-pretool-check.test.sh

@@ -142,8 +142,8 @@ matrix_case E16 allow '~/firstmate/bin/fm-watch-checkpoint.sh --seconds 180'
 matrix_case E17 allow 'for f in 1; do echo fm-watch; done'
 
 MATRIX_TMP=$(mktemp -d "${TMPDIR:-/tmp}/fm-arm-policy-matrix.XXXXXX")
+fm_test_install_cleanup_traps
 FM_TEST_CLEANUP_DIRS+=("$MATRIX_TMP")
-trap fm_test_cleanup EXIT
 
 run_matrix_entry() {
   local id=$1 expected=$2 entry=$3 cmd=$4 payload out_file err_file rc
@@ -371,7 +371,7 @@ test_failopen_garbage_stdin() {
 
 test_failopen_missing_jq() {
   local dir fakebin rc real
-  dir=$(fm_test_tmproot fm-arm-pretool-check)
+  fm_test_tmproot dir fm-arm-pretool-check
   fakebin="$dir/fakebin"
   mkdir -p "$fakebin"
   local tool
@@ -387,7 +387,7 @@ test_failopen_missing_jq() {
 
 test_failopen_missing_node() {
   local dir fakebin rc real tool
-  dir=$(fm_test_tmproot fm-arm-pretool-node)
+  fm_test_tmproot dir fm-arm-pretool-node
   fakebin="$dir/fakebin"
   mkdir -p "$fakebin"
   for tool in bash dirname; do

tests/fm-axi-suite.test.sh

@@ -5,7 +5,8 @@ set -u
 # shellcheck source=tests/lib.sh disable=SC1091
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-TMP_ROOT=$(fm_test_tmproot fm-axi-suite-tests)
+fm_test_tmproot TMP_ROOT fm-axi-suite-tests
+
 BASE_PATH=${FM_TEST_BASE_PATH:-/usr/bin:/bin:/usr/sbin:/sbin}
 
 make_tool() {

tests/fm-backend-cmux.test.sh

@@ -15,7 +15,7 @@ set -u
 
 command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the cmux adapter)"; exit 0; }
 
-TMP_ROOT=$(fm_test_tmproot fm-backend-cmux-tests)
+fm_test_tmproot TMP_ROOT fm-backend-cmux-tests
 
 # make_cmux_fakebin: a `cmux` stub that logs every invocation (one line,
 # unit-separated args, to $FM_CMUX_LOG) and returns the canned response for

tests/fm-backend-herdr.test.sh

@@ -15,7 +15,8 @@ set -u
 
 command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the herdr adapter)"; exit 0; }
 
-TMP_ROOT=$(fm_test_tmproot fm-backend-herdr-tests)
+fm_test_tmproot TMP_ROOT fm-backend-herdr-tests
+
 export FM_BACKEND_HERDR_SUBMIT_MIN_SLEEP=0
 
 # make_herdr_fakebin: a `herdr` stub that logs every invocation (one line,

tests/fm-backend-orca.test.sh

@@ -6,7 +6,7 @@ set -u
 # shellcheck source=tests/lib.sh
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-TMP_ROOT=$(fm_test_tmproot fm-backend-orca-tests)
+fm_test_tmproot TMP_ROOT fm-backend-orca-tests
 
 make_orca_fakebin() {  # <dir> -> echoes fakebin dir
   local fb="$1/fakebin"

tests/fm-backend-zellij.test.sh

@@ -14,7 +14,7 @@ set -u
 
 command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (required by the zellij adapter)"; exit 0; }
 
-TMP_ROOT=$(fm_test_tmproot fm-backend-zellij-tests)
+fm_test_tmproot TMP_ROOT fm-backend-zellij-tests
 
 # make_zellij_fakebin: a `zellij` stub that logs every invocation (one line,
 # unit-separated args, to $FM_ZELLIJ_LOG) and returns the canned response for

tests/fm-backend.test.sh

@@ -33,7 +33,7 @@ fm_git_identity fmtest fmtest@example.invalid
 # shellcheck source=bin/fm-backend.sh
 . "$ROOT/bin/fm-backend.sh"
 
-TMP_ROOT=$(fm_test_tmproot fm-backend-tests)
+fm_test_tmproot TMP_ROOT fm-backend-tests
 
 # fm_backend_detect's cmux fallback (bundle id + process ancestry,
 # docs/cmux-backend.md "Runtime auto-detection") consults uname, lsappinfo,

tests/fm-backlog-handoff.test.sh

@@ -13,7 +13,7 @@ set -u
 # binary. Skip cleanly when it is absent (matching the backend smoke suites).
 command -v tasks-axi >/dev/null 2>&1 || { echo "skip: tasks-axi not found (required by the delegated handoff path)"; exit 0; }
 
-TMP_ROOT=$(fm_test_tmproot fm-backlog-handoff)
+fm_test_tmproot TMP_ROOT fm-backlog-handoff
 
 setup_homes() {
   local home=$1 subhome=$2 id=${3:-design}