test: retry real-herdr smoke tests on treehouse contention

CONTRIBUTING.md

@@ -112,8 +112,8 @@ tests/fm-backend.test.sh                  # runtime-backend abstraction: fm-back
 tests/fm-backend-tmux-smoke.test.sh       # real (private-socket) tmux smoke test for the tmux adapter: create/duplicate-refuse, send text + Enter, send literal + key, bounded capture, live-window resolve, kill
 tests/fm-backend-herdr.test.sh            # fake herdr CLI unit tests for the experimental herdr adapter, including version/tool gates, target parsing, send/capture, structural composer-state verification, slash-submit retry regression coverage, native busy state, per-home workspace-label resolution, default-tab prune safety, restored-layout husk replacement, and verified CLI bug workarounds
 tests/fm-backend-herdr-smoke.test.sh      # real herdr adapter smoke test, skipped when herdr or jq is unavailable, using an isolated throwaway HERDR_SESSION and guarded session cleanup, including live-agent duplicate refusal and no-agent husk replacement
-tests/fm-backend-autodetect-smoke.test.sh # real herdr auto-detection smoke test, skipped when herdr, jq, or treehouse is unavailable, using the same guarded session cleanup
-tests/fm-backend-herdr-workspace-per-home-e2e.test.sh # mandatory isolated E2E for workspace-per-home: primary and secondmate-shaped homes, a crewmate spawned from a secondmate home, teardown, list-live recovery
+tests/fm-backend-autodetect-smoke.test.sh # real herdr auto-detection smoke test, skipped when herdr, jq, or treehouse is unavailable, using the same guarded session cleanup; whole-script retry (tests/real-herdr-smoke-retry.sh) on fm-spawn.sh's treehouse-lease-contention timeout signature
+tests/fm-backend-herdr-workspace-per-home-e2e.test.sh # mandatory isolated E2E for workspace-per-home: primary and secondmate-shaped homes, a crewmate spawned from a secondmate home, teardown, list-live recovery; whole-script retry (tests/real-herdr-smoke-retry.sh) on fm-spawn.sh's treehouse-lease-contention timeout signature
 tests/fm-backend-herdr-prune-safety-e2e.test.sh # isolated real-herdr E2E for the default-tab prune self-kill regression: adopted label-collision workspaces are never pruned, while freshly created workspaces still prune their seeded default tab
 tests/fm-backend-herdr-respawn-idem-e2e.test.sh # isolated real-herdr E2E for restored-layout husk respawn idempotency across a real session restart, covering crewmate/scout and secondmate-shaped tabs plus live-agent duplicate refusal
 tests/fm-backend-zellij.test.sh           # fake zellij CLI unit tests for the experimental zellij adapter, including version/tool gates, target parsing, home-scoped title creation, legacy-title fallback, send/capture, current-path probing, label-checked target safety, secondmate child cleanup, and tab cleanup

docs/herdr-backend.md

@@ -389,3 +389,6 @@ This is a test-harness-only concern - `fm_backend_herdr_composer_state` and `fm_
 - **RESOLVED: a restart's restored-layout husk no longer needs a manual pane close before respawn.** See "Respawn idempotency: a restored task tab is a husk, not a duplicate" above for the fix (`fm_backend_herdr_pane_agent_state`, `fm_backend_herdr_create_task`'s close-and-replace).
   Left over from that fix: the `dead` (`pane_not_found`) husk classification is exercised only at the unit level, never against the real binary - killing a pane's process on a live server was observed to make herdr reap the whole tab immediately (never leaving a dead-but-still-listed pane for the duplicate check to find), and a real session restart was never observed to produce one either.
   It remains a conservative, defensively-coded path for a herdr failure mode (e.g. a restored process that fails to start) nobody has reproduced against the real binary yet.
+- **RESOLVED: intermittent `tests/fm-backend-autodetect-smoke.test.sh` / `tests/fm-backend-herdr-workspace-per-home-e2e.test.sh` flakes under concurrent fleet load.** Root-caused as environmental treehouse lease contention under concurrent `treehouse get` calls from other live fleet activity, not a production `bin/` bug - `bin/fm-spawn.sh`'s own worktree-acquisition behavior is unchanged.
+  A test-only lock was rejected because it would only serialize these two tests against each other and cannot be honored by unrelated live fleet treehouse operations.
+  Fix: `tests/real-herdr-smoke-retry.sh` gives both tests a bounded whole-script retry (default 3 attempts) that triggers ONLY on `fm-spawn.sh`'s exact timeout signature (`error: treehouse get did not enter a worktree within 60s; inspect window ...`); any other failure, and a signature match past the retry budget, still fails immediately.

tests/fm-backend-autodetect-smoke.test.sh

@@ -29,6 +29,10 @@ set -u
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
+# shellcheck source=tests/real-herdr-smoke-retry.sh
+. "$ROOT/tests/real-herdr-smoke-retry.sh"
+fm_real_herdr_smoke_retry_on_treehouse_contention "$@"
+
 fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; }
 pass() { printf 'ok - %s\n' "$1"; }
 assert_contains_local() {  # <haystack> <needle> <msg>

tests/fm-backend-herdr-workspace-per-home-e2e.test.sh

@@ -32,6 +32,10 @@ set -u
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
+# shellcheck source=tests/real-herdr-smoke-retry.sh
+. "$ROOT/tests/real-herdr-smoke-retry.sh"
+fm_real_herdr_smoke_retry_on_treehouse_contention "$@"
+
 fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; }
 pass() { printf 'ok - %s\n' "$1"; }
 assert_contains_local() {  # <haystack> <needle> <msg>

tests/real-herdr-smoke-retry.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+# Shared whole-script retry for real-herdr smoke tests that exercise
+# bin/fm-spawn.sh's real treehouse acquisition path.
+#
+# These tests are intentionally end-to-end, so the least invasive flake
+# handling is at the test-process boundary: retry only when fm-spawn.sh reports
+# the exact environmental signature where `treehouse get` never moved the pane
+# into a worktree. All other failures surface immediately, and deterministic
+# regressions still fail after the bounded retry count.
+set -u
+
+fm_real_herdr_smoke_retry_on_treehouse_contention() {
+  [ -z "${FM_REAL_HERDR_SMOKE_ATTEMPT:-}" ] || return 0
+
+  local attempts=${FM_REAL_HERDR_SMOKE_RETRIES:-3}
+  local tmp attempt rc combined
+  tmp=$(mktemp -d "$(cd "${TMPDIR:-/tmp}" && pwd -P)/fm-real-herdr-smoke-retry.XXXXXX") || exit 1
+
+  attempt=1
+  while [ "$attempt" -le "$attempts" ]; do
+    FM_REAL_HERDR_SMOKE_ATTEMPT=$attempt "$BASH" "$0" "$@" >"$tmp/out" 2>"$tmp/err"
+    rc=$?
+    cat "$tmp/out"
+    cat "$tmp/err" >&2
+    [ "$rc" -eq 0 ] && { rm -rf "$tmp"; exit 0; }
+
+    combined=$(cat "$tmp/out" "$tmp/err")
+    case "$combined" in
+      *"error: treehouse get did not enter a worktree within 60s; inspect window "*)
+        if [ "$attempt" -lt "$attempts" ]; then
+          printf 'retry: real-herdr smoke hit treehouse lease contention signature (attempt %s/%s); retrying\n' "$attempt" "$attempts" >&2
+          attempt=$((attempt + 1))
+          sleep 2
+          continue
+        fi
+        ;;
+    esac
+
+    rm -rf "$tmp"
+    exit "$rc"
+  done
+}