From 0fdba614738eb4bcf52a59d7c5c3d9f6caf3a9d0 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:46:24 -0700 Subject: [PATCH 1/2] test: retry real-herdr smoke on treehouse contention The real-herdr smoke tests fail under fleet load when fm-spawn.sh waits for treehouse get and the pane never moves into a worktree. A shared test-only lock would serialize these two scripts with each other, but live fleet treehouse operations would not take that lock. A bounded whole-script retry fits the existing end-to-end test shape better: it only retries the exact treehouse-get timeout signature, while unrelated failures still surface immediately and deterministic regressions still fail after the retry budget. --- tests/fm-backend-autodetect-smoke.test.sh | 4 ++ ...ckend-herdr-workspace-per-home-e2e.test.sh | 4 ++ tests/real-herdr-smoke-retry.sh | 42 +++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 tests/real-herdr-smoke-retry.sh diff --git a/tests/fm-backend-autodetect-smoke.test.sh b/tests/fm-backend-autodetect-smoke.test.sh index 2d7aa0c14..6a7ff2df2 100755 --- a/tests/fm-backend-autodetect-smoke.test.sh +++ b/tests/fm-backend-autodetect-smoke.test.sh @@ -29,6 +29,10 @@ set -u ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=tests/real-herdr-smoke-retry.sh +. "$ROOT/tests/real-herdr-smoke-retry.sh" +fm_real_herdr_smoke_retry_on_treehouse_contention "$@" + fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } pass() { printf 'ok - %s\n' "$1"; } assert_contains_local() { # diff --git a/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh b/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh index f30857292..e4a9a19df 100755 --- a/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +++ b/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh @@ -32,6 +32,10 @@ set -u ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=tests/real-herdr-smoke-retry.sh +. "$ROOT/tests/real-herdr-smoke-retry.sh" +fm_real_herdr_smoke_retry_on_treehouse_contention "$@" + fail() { printf 'not ok - %s\n' "$1" >&2; cleanup_all; exit 1; } pass() { printf 'ok - %s\n' "$1"; } assert_contains_local() { # diff --git a/tests/real-herdr-smoke-retry.sh b/tests/real-herdr-smoke-retry.sh new file mode 100644 index 000000000..405eee1bf --- /dev/null +++ b/tests/real-herdr-smoke-retry.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Shared whole-script retry for real-herdr smoke tests that exercise +# bin/fm-spawn.sh's real treehouse acquisition path. +# +# These tests are intentionally end-to-end, so the least invasive flake +# handling is at the test-process boundary: retry only when fm-spawn.sh reports +# the exact environmental signature where `treehouse get` never moved the pane +# into a worktree. All other failures surface immediately, and deterministic +# regressions still fail after the bounded retry count. +set -u + +fm_real_herdr_smoke_retry_on_treehouse_contention() { + [ -z "${FM_REAL_HERDR_SMOKE_ATTEMPT:-}" ] || return 0 + + local attempts=${FM_REAL_HERDR_SMOKE_RETRIES:-3} + local tmp attempt rc combined + tmp=$(mktemp -d "$(cd "${TMPDIR:-/tmp}" && pwd -P)/fm-real-herdr-smoke-retry.XXXXXX") || exit 1 + + attempt=1 + while [ "$attempt" -le "$attempts" ]; do + FM_REAL_HERDR_SMOKE_ATTEMPT=$attempt "$BASH" "$0" "$@" >"$tmp/out" 2>"$tmp/err" + rc=$? + cat "$tmp/out" + cat "$tmp/err" >&2 + [ "$rc" -eq 0 ] && { rm -rf "$tmp"; exit 0; } + + combined=$(cat "$tmp/out" "$tmp/err") + case "$combined" in + *"error: treehouse get did not enter a worktree within 60s; inspect window "*) + if [ "$attempt" -lt "$attempts" ]; then + printf 'retry: real-herdr smoke hit treehouse lease contention signature (attempt %s/%s); retrying\n' "$attempt" "$attempts" >&2 + attempt=$((attempt + 1)) + sleep 2 + continue + fi + ;; + esac + + rm -rf "$tmp" + exit "$rc" + done +} From eecb997194ef353e7f1871bd508e056f3d924d01 Mon Sep 17 00:00:00 2001 From: mielyemitchell <249051873+mielyemitchell@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:07:50 -0700 Subject: [PATCH 2/2] no-mistakes(document): docs: note treehouse-contention retry on real-herdr smoke tests --- CONTRIBUTING.md | 4 ++-- docs/herdr-backend.md | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 77a674f77..e27af3e1e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,8 +112,8 @@ tests/fm-backend.test.sh # runtime-backend abstraction: fm-back tests/fm-backend-tmux-smoke.test.sh # real (private-socket) tmux smoke test for the tmux adapter: create/duplicate-refuse, send text + Enter, send literal + key, bounded capture, live-window resolve, kill tests/fm-backend-herdr.test.sh # fake herdr CLI unit tests for the experimental herdr adapter, including version/tool gates, target parsing, send/capture, structural composer-state verification, slash-submit retry regression coverage, native busy state, per-home workspace-label resolution, default-tab prune safety, restored-layout husk replacement, and verified CLI bug workarounds tests/fm-backend-herdr-smoke.test.sh # real herdr adapter smoke test, skipped when herdr or jq is unavailable, using an isolated throwaway HERDR_SESSION and guarded session cleanup, including live-agent duplicate refusal and no-agent husk replacement -tests/fm-backend-autodetect-smoke.test.sh # real herdr auto-detection smoke test, skipped when herdr, jq, or treehouse is unavailable, using the same guarded session cleanup -tests/fm-backend-herdr-workspace-per-home-e2e.test.sh # mandatory isolated E2E for workspace-per-home: primary and secondmate-shaped homes, a crewmate spawned from a secondmate home, teardown, list-live recovery +tests/fm-backend-autodetect-smoke.test.sh # real herdr auto-detection smoke test, skipped when herdr, jq, or treehouse is unavailable, using the same guarded session cleanup; whole-script retry (tests/real-herdr-smoke-retry.sh) on fm-spawn.sh's treehouse-lease-contention timeout signature +tests/fm-backend-herdr-workspace-per-home-e2e.test.sh # mandatory isolated E2E for workspace-per-home: primary and secondmate-shaped homes, a crewmate spawned from a secondmate home, teardown, list-live recovery; whole-script retry (tests/real-herdr-smoke-retry.sh) on fm-spawn.sh's treehouse-lease-contention timeout signature tests/fm-backend-herdr-prune-safety-e2e.test.sh # isolated real-herdr E2E for the default-tab prune self-kill regression: adopted label-collision workspaces are never pruned, while freshly created workspaces still prune their seeded default tab tests/fm-backend-herdr-respawn-idem-e2e.test.sh # isolated real-herdr E2E for restored-layout husk respawn idempotency across a real session restart, covering crewmate/scout and secondmate-shaped tabs plus live-agent duplicate refusal tests/fm-backend-zellij.test.sh # fake zellij CLI unit tests for the experimental zellij adapter, including version/tool gates, target parsing, home-scoped title creation, legacy-title fallback, send/capture, current-path probing, label-checked target safety, secondmate child cleanup, and tab cleanup diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 055cc11af..0e6f79661 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -389,3 +389,6 @@ This is a test-harness-only concern - `fm_backend_herdr_composer_state` and `fm_ - **RESOLVED: a restart's restored-layout husk no longer needs a manual pane close before respawn.** See "Respawn idempotency: a restored task tab is a husk, not a duplicate" above for the fix (`fm_backend_herdr_pane_agent_state`, `fm_backend_herdr_create_task`'s close-and-replace). Left over from that fix: the `dead` (`pane_not_found`) husk classification is exercised only at the unit level, never against the real binary - killing a pane's process on a live server was observed to make herdr reap the whole tab immediately (never leaving a dead-but-still-listed pane for the duplicate check to find), and a real session restart was never observed to produce one either. It remains a conservative, defensively-coded path for a herdr failure mode (e.g. a restored process that fails to start) nobody has reproduced against the real binary yet. +- **RESOLVED: intermittent `tests/fm-backend-autodetect-smoke.test.sh` / `tests/fm-backend-herdr-workspace-per-home-e2e.test.sh` flakes under concurrent fleet load.** Root-caused as environmental treehouse lease contention under concurrent `treehouse get` calls from other live fleet activity, not a production `bin/` bug - `bin/fm-spawn.sh`'s own worktree-acquisition behavior is unchanged. + A test-only lock was rejected because it would only serialize these two tests against each other and cannot be honored by unrelated live fleet treehouse operations. + Fix: `tests/real-herdr-smoke-retry.sh` gives both tests a bounded whole-script retry (default 3 attempts) that triggers ONLY on `fm-spawn.sh`'s exact timeout signature (`error: treehouse get did not enter a worktree within 60s; inspect window ...`); any other failure, and a signature match past the retry budget, still fails immediately.