diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index ed40467c19..e13ac018e2 100644 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -1297,7 +1297,10 @@ FM_BACKEND_HERDR_IDLE_RE=${FM_BACKEND_HERDR_IDLE_RE:-'^Type a message\.\.\.$'} # Known bare (unbordered) prompt glyphs a composer row may start with: ❯ # (claude) and › (codex) only. Generic shell-style glyphs > $ % # are still # recognized after a bordered composer row has already been structurally found. -FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^[❯›]'} +# Written as an anchored alternation, not a bracket class: under a C/byte +# locale a bracket class matches these glyphs' individual UTF-8 bytes, so box +# corners like ╰ (shared 0xE2 lead byte) would misread as a bare prompt row. +FM_BACKEND_HERDR_BARE_PROMPT_RE=${FM_BACKEND_HERDR_BARE_PROMPT_RE:-'^(❯|›)'} # Pi allows a multi-line composer between its horizontal separators. Bound the # structural candidate so two unrelated transcript rules with an arbitrarily # large region between them can never be promoted into a composer. @@ -1461,7 +1464,7 @@ EOF fi # Delegate the empty/pending/unknown decision to the shared owner. The bare # shape only ever starts with an AGENT glyph (FM_BACKEND_HERDR_BARE_PROMPT_RE - # is '^[❯›]'), so a bare shell prompt never reaches here - it stays 'unknown' + # is '^(❯|›)'), so a bare shell prompt never reaches here - it stays 'unknown' # via the no-composer-row path above, exactly as before. fm_composer_classify_content "$bordered" "$stripped" "$FM_BACKEND_HERDR_IDLE_RE" } diff --git a/bin/fm-classify-lib.sh b/bin/fm-classify-lib.sh index d80840f6a1..991f1f6fa7 100755 --- a/bin/fm-classify-lib.sh +++ b/bin/fm-classify-lib.sh @@ -318,13 +318,17 @@ signal_reason_is_actionable() { # ... # Classify WHY an idle/stale crew MIGHT be safely absorbed instead of surfaced, # from bin/fm-crew-state.sh's one authoritative current-state line # ("state: · source: · "). Prints exactly one token: -# working - an actively-running no-mistakes step (running/fixing/ci) or a busy -# pane; the crew is legitimately mid-work on a static-looking pane -# (e.g. waiting on CI); -# paused - the crew's authoritative current state is a declared external-wait -# pause (paused:), which is EXPECTED to idle; -# none - neither, so the wake must surface (a stopped/finished/parked/failed/ -# torn-down/unknown crew, or an unreadable verdict). +# working - an actively-running no-mistakes step (running/fixing/ci) or a busy +# pane; the crew is legitimately mid-work on a static-looking pane +# (e.g. waiting on CI); +# paused - the crew's authoritative current state is a declared external-wait +# pause (paused:), which is EXPECTED to idle; +# none - neither, so the wake must surface (a stopped/finished/parked/ +# failed/torn-down/unknown crew); +# unreadable - the fm-crew-state.sh read itself failed or produced no parseable +# state line: NOT a verdict about the crew. Consumers must treat it +# like none for surfacing (conservative) but never durably cache it +# as a negative verdict, so the next poll re-reads. # One fm-crew-state.sh read serves BOTH absorb reasons at once. Reading the state # authoritatively (not the status log) is what keeps run-step precedence: a crew # that appended paused: but then STARTED a run reports working, never paused. @@ -334,8 +338,8 @@ signal_reason_is_actionable() { # ... crew_absorb_class() { # local id=$1 line state src [ -n "$id" ] || { printf 'none'; return; } - line=$("$FM_CREW_STATE_BIN" "$id" 2>/dev/null) || true - case "$line" in state:*) ;; *) printf 'none'; return ;; esac + line=$("$FM_CREW_STATE_BIN" "$id" 2>/dev/null) || { printf 'unreadable'; return; } + case "$line" in state:*) ;; *) printf 'unreadable'; return ;; esac state=${line#state: }; state=${state%% *} if [ "$state" = paused ]; then printf 'paused'; return; fi if [ "$state" = working ]; then @@ -351,7 +355,8 @@ crew_absorb_class() { # # ONLY when this returns 0, and SURFACED otherwise (the crew may be done, waiting # on a decision, or wedged). For stale panes it is checked before trusting the # status log so a pre-validation captain-relevant line does not override an active -# run. See crew_absorb_class for the exact working/paused/none decision. +# run. See crew_absorb_class for the exact working/paused/none/unreadable +# decision (unreadable counts as not provably working here). crew_is_provably_working() { # [ "$(crew_absorb_class "$1")" = working ] } diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 4d09ea6fe0..6b5402c3d5 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -23,7 +23,20 @@ # re-surface cadence, never as a wedge. Only when neither # absorb class applies does the log's last line decide: # terminal (captain-relevant) or non-terminal (no verb), -# both surfaced at once. A provably-working stale past the +# both surfaced at once - and both surfaced ONCE PER +# SITUATION (.stale-surfaced-*, keyed on the last status +# line), not once per pane hash: a live idle agent's +# ticking status bar churns the hash every minute, so a +# repeat of an already-surfaced situation is absorbed +# until positive working evidence (a busy pane, an +# active run) or a new status line re-arms surfacing. +# A paused: log line found superseded by the +# authoritative crew state is cached against the status +# file's signature (.paused-none-*) so it is neither +# re-read nor re-surfaced every poll; a FAILED crew-state +# read is never cached, so a transient misread cannot +# disarm a genuine pause's recheck cadence. +# A provably-working stale past the # wedge threshold also surfaces, with an "escalation N" # count in the reason; at FM_WEDGE_DEMAND_INSPECT_COUNT # consecutive escalations on the SAME pane, the reason @@ -350,12 +363,22 @@ clear_pause_tracking() { # key=${key//\//_} key=${key//./_} clear_pause_state "$win" - rm -f "$STATE/.stale-$key" "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key" + rm -f "$STATE/.stale-$key" "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key" "$STATE/.paused-none-$key" } # Reconcile a declared pause or captain-held status with authoritative crew state. # Only a confidently dead ordinary crew may recover paused classification after # fm-crew-state has fallen back to stopped or unknown. +# Prints working/paused/none/unreadable as crew_absorb_class does, plus one token +# of its own: +# hold - the declared pause or captain hold is NOT disproven (either the read +# was skipped inside the bounded recheck window, or it still reported +# paused) while the agent process is not confidently dead. Not a verdict +# that the crew stopped: a fresh stale surfaces it once, but a repeat on +# an unchanged pane keeps the bounded pause cadence instead. +# Keeping hold distinct from none/unreadable is what lets a live external-decision +# gate retain that cadence while a pause the authoritative state SUPERSEDES (none) +# is surfaced once and cached, and an unreadable read is surfaced without caching. pause_state_class() { # local win=$1 task=$2 key last recheck_file class agent_alive key=${win//:/_} @@ -372,8 +395,10 @@ pause_state_class() { # if [ "$(window_kind "$win")" != secondmate ]; then agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown if [ "$agent_alive" != dead ]; then - rm -f "$recheck_file" - printf 'none' + # A hold keeps the recheck stamp: like paused, it is still on the bounded + # cadence, so dropping the stamp would re-run the costly crew-state read + # on every poll of an unchanged pane. + printf 'hold' return fi fi @@ -389,8 +414,15 @@ pause_state_class() { # if [ "$(window_kind "$win")" != secondmate ]; then agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown if [ "$agent_alive" != dead ]; then - rm -f "$recheck_file" - printf 'none' + # A still-paused authoritative read on a live agent is a hold, not a + # verdict that the crew stopped; none/unreadable stay themselves so the + # superseded-pause cache and the no-verdict path can tell them apart. + # A hold stamps the recheck like paused does, so the bounded cadence - + # not every poll - owns the next costly read. + case "$class" in + paused) date +%s > "$recheck_file"; printf 'hold' ;; + *) rm -f "$recheck_file"; printf '%s' "$class" ;; + esac return fi fi @@ -402,13 +434,34 @@ pause_state_class() { # printf '%s' "$class" } +# Surface a stopped crew's stale pane - once per distinct SITUATION, not once +# per pane hash. A surfaced stale records "stopped|" in +# .stale-surfaced-; a later stale whose situation matches that record is +# absorbed instead of re-surfaced. The pane hash alone is not a stable dedup +# key: a live idle agent's ticking status bar changes the hash every minute, +# and a watcher restart can re-see a drifted hash, so hash-keyed dedup alone +# re-surfaced the same finished merge-wait crew on effectively every poll +# (the 2026-07-11 fix-sync-ssh-m4 incident). The record is cleared only on +# POSITIVE working evidence (a busy pane, an active run), so a crew that +# visibly resumes and stops again - or writes a new status line - still +# surfaces; the first stop of any situation always surfaces, preserving +# absorb-only-when-provably-working. surface_nonterminal_stale() { # - local win=$1 h=$2 key task last + local win=$1 h=$2 key task last sit sitf key=$(printf '%s' "$win" | tr ':/.' '___') + task=$(window_to_task "$win" "$STATE") + sit="stopped|$(last_status_line "$STATE/$task.status")" + sitf="$STATE/.stale-surfaced-$key" + if [ "$sit" = "$(cat "$sitf" 2>/dev/null)" ]; then + printf '%s' "$h" > "$STATE/.stale-$key" + rm -f "$STATE/.stale-since-$key" "$STATE/.paused-$key" "$STATE/.paused-rechecked-$key" "$STATE/.paused-resurfaced-$key" + triage_log "absorbed stale (stopped situation already surfaced): $win" + return 0 + fi fm_wake_append stale "$win" "stale: $win" || exit 1 printf '%s' "$h" > "$STATE/.stale-$key" + printf '%s' "$sit" > "$sitf" rm -f "$STATE/.stale-since-$key" - task=$(window_to_task "$win" "$STATE") last=$(last_status_line "$STATE/$task.status") if status_is_paused_or_captain_held "$last"; then : > "$STATE/.paused-$key" @@ -893,6 +946,8 @@ EOF ssf="$STATE/.stale-since-$key" ewf="$STATE/.wedge-escalations-$key" pf="$STATE/.paused-$key" # flag: this key's stale is using the bounded pause cadence + sitf="$STATE/.stale-surfaced-$key" # situation record of the last SURFACED stale (see surface_nonterminal_stale) + pnf="$STATE/.paused-none-$key" # status-file signature at which a paused: log line was last found superseded prev=$(cat "$hf" 2>/dev/null || true) if [ "$h" = "$prev" ]; then n=$(( $(cat "$cf" 2>/dev/null || echo 0) + 1 )) @@ -933,12 +988,22 @@ EOF # over the log) a chance to override before trusting the log. if [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then if crew_is_provably_working "$(window_to_task "$w" "$STATE")"; then + rm -f "$sitf" printf '%s' "$h" > "$sf" date +%s > "$ssf" triage_log "absorbed stale (provably working, overriding a stale captain-relevant status): $w" + elif [ "terminal|$(last_status_line "$STATE/$task.status")" = "$(cat "$sitf" 2>/dev/null)" ]; then + # Already surfaced this exact terminal situation: only the pane + # hash moved (a live idle agent's ticking status bar, or a + # watcher restart re-seeing a drifted pane). Advance the + # suppressor quietly instead of waking once per tick. + printf '%s' "$h" > "$sf" + rm -f "$ssf" + triage_log "absorbed stale (terminal situation already surfaced): $w" else fm_wake_append stale "$w" "stale: $w" || exit 1 printf '%s' "$h" > "$sf" + printf 'terminal|%s' "$(last_status_line "$STATE/$task.status")" > "$sitf" rm -f "$ssf" mark_surfaced "$STATE/$(window_to_task "$w" "$STATE").status" wake "stale: $w" @@ -967,12 +1032,16 @@ EOF # pause - the crew has STOPPED. Surface immediately so firstmate peeks # (it may be done via an interactive menu that wrote no done: status, # waiting on a decision, or wedged) instead of leaving the finish to - # wait out the timer. + # wait out the timer; + # - hold (or an unreadable read): no verdict either way, so a FRESH + # stale surfaces here exactly as none does; only the unchanged-hash + # re-entry below distinguishes them. if [ "$(cat "$sf" 2>/dev/null || true)" != "$h" ]; then task=$(window_to_task "$w" "$STATE") case "$(pause_state_class "$w" "$task")" in working) clear_pause_tracking "$w" + rm -f "$sitf" printf '%s' "$h" > "$sf" date +%s > "$ssf" triage_log "absorbed non-terminal stale (provably working): $w" @@ -986,16 +1055,44 @@ EOF esac else task=$(window_to_task "$w" "$STATE") - if [ -e "$pf" ] || status_is_paused_or_captain_held "$(last_status_line "$STATE/$task.status")"; then + statusf="$STATE/$task.status" + # The paused-looking re-entry is gated by a negative-verdict cache: + # once pause_state_class found the paused: log line superseded + # (class none, e.g. run-step precedence reporting done) at this + # exact status-file signature, re-running the costly read every + # poll would only re-decide the same thing - and, before this + # cache, re-SURFACE the same stopped crew on every poll through + # this branch (the 2026-07-11 merge-wait incident). A new status + # write changes the signature and re-evaluates. Only a DEFINITIVE + # none verdict is cached: an unreadable crew-state read is no + # verdict at all, so it surfaces conservatively but leaves the + # recheck cadence armed for the next poll instead of freezing a + # possibly-genuine pause behind a transient read failure. + # A hold - the pause still undisproven on a not-confidently-dead + # agent - is likewise no verdict, but here it keeps the bounded + # PAUSE_RESURFACE_SECS cadence rather than re-surfacing, so a live + # external-decision gate that already surfaced once stays parked. + if [ -e "$pf" ] || { status_is_paused_or_captain_held "$(last_status_line "$statusf")" \ + && [ "$(stat_sig "$statusf")" != "$(cat "$pnf" 2>/dev/null)" ]; }; then case "$(pause_state_class "$w" "$task")" in - paused) handle_paused_stale "$w" "$task" "$h" ;; - working) clear_pause_state "$w" + paused) rm -f "$pnf" + handle_paused_stale "$w" "$task" "$h" ;; + working) rm -f "$pnf" "$sitf" + clear_pause_state "$w" printf '%s' "$h" > "$sf" wedge_timer_check "$w" "$ssf" "non-terminal stale (provably working after a declared pause)" "$ewf" triage_log "absorbed non-terminal stale (provably working): $w" ;; - *) handle_paused_stale "$w" "$task" "$h" ;; + none) stat_sig "$statusf" > "$pnf" || : > "$pnf" + surface_nonterminal_stale "$w" "$h" ;; + hold) rm -f "$pnf" + handle_paused_stale "$w" "$task" "$h" ;; + *) surface_nonterminal_stale "$w" "$h" ;; esac - else + elif [ -e "$ssf" ] || [ ! -e "$sitf" ]; then + # Wedge timing belongs to provably-working absorbs (and heals a + # timer lost to a crash between the hash and timer writes); an + # already-SURFACED stopped stale stays quiet instead of being + # re-nagged as a possible wedge every STALE_ESCALATE_SECS. wedge_timer_check "$w" "$ssf" "non-terminal stale" "$ewf" fi fi @@ -1003,6 +1100,10 @@ EOF else # Pane busy or not yet stably stale: reset pending escalation bookkeeping. rm -f "$ssf" "$ewf" + # Reaching here with n >= 2 means window_is_busy returned busy: positive + # working evidence, so clear the surfaced-situation record - the next + # stop is a genuinely new event and must surface again. + [ "$n" -ge 2 ] && rm -f "$sitf" if [ -e "$pf" ] && { [ "$n" -ge 2 ] || ! status_is_paused_or_captain_held "$(last_status_line "$STATE/$(window_to_task "$w" "$STATE").status")"; }; then clear_pause_tracking "$w" fi @@ -1012,6 +1113,7 @@ EOF echo 0 > "$cf" rm -f "$ssf" "$ewf" task=$(window_to_task "$w" "$STATE") + [ -e "$sitf" ] && window_is_busy "$w" "$tail40" && rm -f "$sitf" if ! afk_present && status_is_paused_or_captain_held "$(last_status_line "$STATE/$task.status")" && ! window_is_busy "$w" "$tail40"; then case "$(pause_state_class "$w" "$task")" in paused) handle_paused_stale "$w" "$task" "$h" ;; diff --git a/docs/architecture.md b/docs/architecture.md index 82960b1ff1..df9e3b78d2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ firstmate's always-loaded operating contract and routing index for conditional p ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. -Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. +Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal - each surfaced once per distinct situation, never once per poll or per pane-hash tick - provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated provably-working stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so a missed process exit can be recovered by draining the queue. No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step attributed to that crew's current code or a backend busy signature. @@ -18,6 +18,14 @@ For an ordinary crew that has stopped, the normal-mode watcher first surfaces on Live or inconclusive liveness remains fail-open at that initial surface, and the secondmate idle-endpoint exemption is unchanged. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. Fresh stale panes use the same current-state read before trusting the status log, so an active run or busy pane outranks an old captain-relevant status-log line left behind before validation. +A surfaced stale also records its situation - the branch taken plus the crew's last status line - in `state/.stale-surfaced-*`, and an identical situation is absorbed on later polls, on pane-hash ticks from a live idle agent's status bar, and across watcher restarts, so a finished crew awaiting a merge wakes the first mate once instead of once per poll. +Positive working evidence - a busy pane or an actively running step - clears that record, and a new status line changes the situation, so a crew that visibly resumes and stops again, or reports anything new, still surfaces. +An already-surfaced stopped stale also stays off the wedge timer, which remains reserved for provably-working absorbs. +A `paused:` log line that the authoritative crew state supersedes - run-step precedence, such as a finished run behind a declared merge wait - surfaces once as a stopped crew, then is cached against the status file's signature in `state/.paused-none-*` so it is neither re-read nor re-surfaced on every poll. +Only a definitive superseded verdict is cached: a failed or unparseable crew-state read still surfaces conservatively but writes no cache, so one transient misread cannot permanently disarm a genuine pause's recheck cadence on a static pane. +A pause the authoritative read leaves standing while the backend does not confidently report the agent dead is no verdict either: it surfaces once on a fresh stale pane, then holds the bounded pause cadence on the unchanged pane instead of re-surfacing, so a live external-decision gate keeps that cadence rather than being reclassified as a stopped crew. +Evidence for the once-per-situation rule: on 2026-07-11 the task fix-sync-ssh-m4, parked at `done: PR ... checks green` awaiting the captain's merge, was re-surfaced as `stale:` on effectively every poll (wake records 31-38 inside ~7 minutes) - first once per pane-hash tick from the live agent's ticking status bar, then once per poll through the paused re-entry branch after run-step precedence reported the declared pause superseded. +The mechanism is reproduced by `bash tests/fm-watch-triage.test.sh` (test_terminal_stale_not_resurfaced_per_pane_hash_tick, test_paused_log_done_crew_not_resurfaced_per_poll, test_surfaced_stopped_stale_not_wedge_nagged, test_busy_pane_rearms_stale_surfacing, test_streaming_busy_pane_rearms_stale_surfacing), which failed against the pre-fix watcher and pass with the situation-keyed markers. No-change heartbeats are also benign. Absorbed wakes advance their suppression markers, log to `state/.watch-triage.log`, and keep the watcher blocking without a queue record or LLM turn. After each drain, `fm-wake-drain.sh` runs the same liveness guard as the supervision scripts, so a lapsed watcher chain surfaces even on a turn that only drains and handles queued wakes. diff --git a/docs/configuration.md b/docs/configuration.md index 61838643f0..ae58a336ac 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -355,7 +355,7 @@ FM_BACKEND= # optional runtime backend override for new spawns; tmux HERDR_SESSION=default # herdr-only: named session for normal backend ops; not enough for destructive cleanup (docs/herdr-backend.md) FM_BACKEND_HERDR_COMPOSER_LINES=20 # herdr-only: tail lines scanned by composer-state guard/fallback paths; idle-baseline submit confirmation uses agent-state FM_BACKEND_HERDR_IDLE_RE='^Type a message\.\.\.$' # herdr-only: empty-composer placeholder regex after shared ghost extraction plus border and prompt stripping -FM_BACKEND_HERDR_BARE_PROMPT_RE='^[❯›]' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)") +FM_BACKEND_HERDR_BARE_PROMPT_RE='^(❯|›)' # herdr-only: verified agent glyphs recognized as an UNBORDERED (bare) composer row, e.g. claude's ❯ or codex's ›; an anchored alternation rather than a bracket class, because under a C/byte locale a bracket class matches the glyphs' individual UTF-8 bytes and misreads box corners such as ╰ (shared 0xE2 lead byte) as a bare prompt row; shell glyphs remain unknown rather than empty, and de-emphasised ghost/placeholder text (dim or dark-truecolor) after an agent prompt reads empty via the shared fm_composer_strip_ghost (docs/herdr-backend.md "Incident (2026-07-08)", "Incident (2026-07-10)") FM_BACKEND_HERDR_PI_COMPOSER_MAX_LINES=8 # herdr-only: maximum rows admitted between Pi's native-identity-corroborated separator pair; taller or ambiguous candidates stay unknown (docs/herdr-backend.md "Incident (2026-07-14)") FM_BACKEND_HERDR_SUBMIT_POLLS=6 # herdr-only: agent-state samples spread across each Enter attempt's budget when confirming a submit (docs/herdr-backend.md "Native agent-state submit confirmation") FM_BACKEND_HERDR_SUBMIT_MIN_SLEEP=0.6 # herdr-only: minimum per-Enter confirmation budget before polling agent-state after an idle baseline @@ -403,7 +403,7 @@ FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watche FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # captain-relevant status regex; nonterminal progress verbs remain excluded even when their prose matches FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked -FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb +FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stale pane escalates; stale panes whose crew is not provably working surface immediately unless they declare the pause verb, but only once per distinct situation (state/.stale-surfaced-*; docs/architecture.md), never per poll or pane-hash tick FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's absorbed-wake debug log diff --git a/tests/fm-bootstrap.test.sh b/tests/fm-bootstrap.test.sh index 851ba66ec0..03724fefcd 100755 --- a/tests/fm-bootstrap.test.sh +++ b/tests/fm-bootstrap.test.sh @@ -170,9 +170,9 @@ run_bootstrap_timeout_case() { sleep() { local inc=${1:-1} SECONDS=$((SECONDS + inc)) - # Advance fake time quickly, but yield on every tick so the background - # fleet-sync process can deterministically write its partial output before - # the simulated timeout kills it, even on a busy full-suite runner. + # Yield real time on every poll so the backgrounded fake fleet-sync is + # always scheduled (and its partial output written) before the virtual + # timeout expires; a capped yield count raced the child on cold runs. command sleep 0.01 } # shellcheck disable=SC2317,SC2329 # Exported and invoked by the bootstrap subprocess. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 3927a423a8..6beb58eb15 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -271,8 +271,22 @@ test_crew_absorb_class_classifier() { [ "$(crew_absorb_class a)" = none ] || fail "unknown crew classed absorbable" ! crew_is_paused a || fail "unknown crew classed paused" [ "$(crew_absorb_class "")" = none ] || fail "empty id not classed none" + # A failed or unparseable read is NOT a verdict: it must be distinguishable + # from a definitive none so consumers never durably cache it. + FM_FAKE_CREW_STATE='error: cannot resolve the task worktree' + [ "$(crew_absorb_class a)" = unreadable ] || fail "unparseable crew-state output not classed unreadable" + ! crew_is_paused a || fail "an unreadable verdict classed paused" + ! crew_is_provably_working a || fail "an unreadable verdict treated as provably working" + cat > "$fakebin/fm-crew-state-fail.sh" <<'SH' +#!/usr/bin/env bash +exit 1 +SH + chmod +x "$fakebin/fm-crew-state-fail.sh" + FM_CREW_STATE_BIN="$fakebin/fm-crew-state-fail.sh" + [ "$(crew_absorb_class a)" = unreadable ] || fail "a failing crew-state read not classed unreadable" + export FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" unset FM_FAKE_CREW_STATE - pass "crew_absorb_class: working/paused/none from one read; crew_is_paused and crew_is_provably_working agree" + pass "crew_absorb_class: working/paused/none from one read, unreadable on a failed read; crew_is_paused and crew_is_provably_working agree" } # signal_crew_provably_working: a no-verb "signal:" wake is benign ONLY when EVERY @@ -1096,6 +1110,326 @@ test_nonterminal_stale_repairs_missing_or_corrupt_timer() { pass "matching non-terminal stale suppressors repair missing or corrupt stale-since timers" } +# --- a finished crew awaiting merge surfaces once per situation --------------- +# Regression for the 2026-07-11 merge-wait incident (task fix-sync-ssh-m4): a +# ship task whose run finished (done via run-step, PR open awaiting the captain's +# merge) was re-surfaced as stale on effectively every poll. Two mechanisms: a +# live idle agent's ticking status bar changes the pane hash every minute, so the +# hash-keyed .stale-* suppressor never held; and a paused: log line superseded by +# run-step precedence re-entered classification (and surfacing) on every poll +# through the status_is_paused re-entry branch. The fix keys surfaced stales on +# the SITUATION (branch + last status line, .stale-surfaced-*): an identical +# situation is absorbed across polls, hash ticks, and watcher restarts, while +# positive working evidence (busy pane, active run) re-arms surfacing. + +test_terminal_stale_not_resurfaced_per_pane_hash_tick() { + local dir state fakebin out capture_file window key sig pid + dir=$(make_case terminal-stale-tick); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-merge-wait" + printf 'composer idle | 11:52' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/merge-wait.meta" + printf 'done: PR https://example.test/pr/9 checks green\n' > "$state/merge-wait.status" + sig=$(seen_sig "$state/merge-wait.status"); printf '%s' "$sig" > "$state/.seen-merge-wait_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'composer idle | 11:52')" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # The run finished (checks green, awaiting the captain's merge): done via + # run-step, NOT provably working, so the first stale must surface - only once. + export FM_FAKE_CREW_STATE='state: done · source: run-step · checks green, awaiting merge' + + # Round 1: the finished crew's first terminal stale surfaces. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "the finished crew's first terminal stale did not surface" + grep -Fx "stale: $window" "$out" >/dev/null || fail "round 1 did not print the terminal stale wake" + + # Round 2: only the ticking status bar changed (a new pane hash, same finished + # run, same terminal status line). Pre-fix, this re-surfaced once per tick. + printf 'composer idle | 11:53' > "$capture_file" + : > "$out"; : > "$state/.wake-queue" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "a pane-hash tick re-surfaced an already-surfaced terminal situation: $(cat "$out")" + fi + [ ! -s "$out" ] || { reap "$pid"; fail "absorbed hash tick printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "absorbed hash tick enqueued a wake"; } + [ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$(hash_text 'composer idle | 11:53')" ] \ + || { reap "$pid"; fail "absorbed hash tick did not advance the stale suppressor"; } + reap "$pid" + + # Round 3: a genuinely NEW terminal situation (a new captain-relevant status + # line) on yet another hash must still surface. + printf 'failed: merge conflict while rebasing\n' >> "$state/merge-wait.status" + sig=$(seen_sig "$state/merge-wait.status"); printf '%s' "$sig" > "$state/.seen-merge-wait_status" + printf 'composer idle | 11:54' > "$capture_file" + : > "$out"; : > "$state/.wake-queue" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a new terminal status line did not re-surface" + grep -Fx "stale: $window" "$out" >/dev/null || fail "round 3 did not print the new-situation stale wake" + + # Round 4: the crew visibly resumes (an active run, fresh hash): absorbed, and + # the surfaced-situation record is cleared so the next stop counts as new. + printf 'composer idle | 11:55' > "$capture_file" + : > "$out" + FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "an active run's stale was surfaced instead of absorbed: $(cat "$out")" + fi + [ ! -e "$state/.stale-surfaced-$key" ] || { reap "$pid"; fail "positive working evidence did not clear the surfaced-situation record"; } + reap "$pid" + + # Round 5: it stops again with the SAME last status line - a new stop after a + # working interlude, so it surfaces again. + printf 'composer idle | 11:56' > "$capture_file" + : > "$out"; : > "$state/.wake-queue" + FM_FAKE_CREW_STATE='state: done · source: run-step · checks green, awaiting merge' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "a stop after a working interlude did not re-surface" + grep -Fx "stale: $window" "$out" >/dev/null || fail "round 5 did not print the post-interlude stale wake" + unset FM_FAKE_CREW_STATE + pass "a finished crew awaiting merge surfaces once per terminal situation, not once per pane-hash tick" +} + +test_paused_log_done_crew_not_resurfaced_per_poll() { + local dir state fakebin out capture_file window key sig pid calls + dir=$(make_case paused-done-per-poll); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; calls="$dir/crew-state-calls" + window="test:fm-pause-done" + printf 'composer idle at prompt' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/pause-done.meta" + # The crew declared a pause for the merge wait, but the authoritative run-step + # reports done (run-step precedence), so the pause never holds. + printf 'paused: awaiting captain merge of the open PR\n' > "$state/pause-done.status" + sig=$(seen_sig "$state/pause-done.status"); printf '%s' "$sig" > "$state/.seen-pause-done_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'composer idle at prompt')" > "$state/.hash-$key" + printf '%s' "$(hash_text 'composer idle at prompt')" > "$state/.stale-$key" + printf '1\n' > "$state/.count-$key" + # Counting fake: every read appends a line, so the test can assert the costly + # crew-state read is not repeated on every poll. + cat > "$fakebin/fm-crew-state.sh" <> "$calls" +printf '%s\n' "\${FM_FAKE_CREW_STATE:-state: unknown · source: none · fake default}" +exit 0 +SH + chmod +x "$fakebin/fm-crew-state.sh" + export FM_FAKE_CREW_STATE='state: done · source: run-step · checks green, awaiting merge' + + # Round 1: the pause recheck discovers the crew actually stopped: surface once. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "the superseded pause's stopped crew did not surface once" + grep -Fx "stale: $window" "$out" >/dev/null || fail "round 1 did not print the stopped-crew stale wake" + + # Round 2: identical situation on an unchanged hash. Pre-fix, the paused + # re-entry branch re-classified and re-surfaced this on EVERY poll. + : > "$out"; : > "$state/.wake-queue"; : > "$calls" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "an unchanged superseded-pause situation was re-surfaced per poll: $(cat "$out")" + fi + [ ! -s "$out" ] || { reap "$pid"; fail "absorbed superseded pause printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "absorbed superseded pause enqueued a wake"; } + [ ! -s "$calls" ] || { reap "$pid"; fail "the superseded pause re-ran the costly crew-state read every poll ($(wc -l < "$calls" | tr -d '[:space:]') reads)"; } + reap "$pid" + unset FM_FAKE_CREW_STATE + pass "a done crew behind a superseded paused: line surfaces once, then stays quiet without per-poll reads" +} + +# A genuinely paused crew on a fully static pane whose recheck's crew-state read +# transiently FAILS: the failed read surfaces conservatively but must never be +# cached in .paused-none-* as a definitive superseded-pause verdict, or the +# FM_PAUSE_RESURFACE_SECS recheck cadence stays disarmed until the crew writes a +# new status line or the pane changes, letting the declared wait rot invisibly. +test_paused_crew_transient_read_failure_not_cached() { + local dir state fakebin out capture_file window key sig pid + dir=$(make_case paused-read-failure); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-pause-misread" + printf 'composer idle at prompt' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/pause-misread.meta" + printf 'paused: awaiting the upstream release\n' > "$state/pause-misread.status" + sig=$(seen_sig "$state/pause-misread.status"); printf '%s' "$sig" > "$state/.seen-pause-misread_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'composer idle at prompt')" > "$state/.hash-$key" + printf '%s' "$(hash_text 'composer idle at prompt')" > "$state/.stale-$key" + printf '1\n' > "$state/.count-$key" + : > "$state/.paused-$key" + # Fake whose read can be made to FAIL (exit 1, no verdict line) via + # FM_FAKE_CREW_STATE_FAIL, simulating a transient fm-crew-state.sh misread. + cat > "$fakebin/fm-crew-state.sh" <<'SH' +#!/usr/bin/env bash +set -u +[ "${FM_FAKE_CREW_STATE_FAIL:-0}" = 1 ] && exit 1 +printf '%s\n' "${FM_FAKE_CREW_STATE:-state: unknown · source: none · fake default}" +exit 0 +SH + chmod +x "$fakebin/fm-crew-state.sh" + + # Round 1: the pause recheck's crew-state read fails. Surface conservatively + # (an unreadable crew might really have stopped), but write NO negative cache. + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" FM_FAKE_CREW_STATE_FAIL=1 \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + wait_for_exit "$pid" 40 || fail "an unreadable paused recheck did not surface conservatively" + grep -Fx "stale: $window" "$out" >/dev/null || fail "round 1 did not print the conservative stale wake" + [ ! -e "$state/.paused-none-$key" ] || fail "a failed crew-state read was cached as a superseded-pause verdict" + + # Round 2: the read succeeds and confirms the declared pause on the same + # static pane and status line. Pre-fix, the round-1 cache matched the status + # signature and skipped this recheck forever; post-fix the cadence is still + # armed, so the pause is re-absorbed and tracked again. + : > "$out"; : > "$state/.wake-queue" + export FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting the upstream release' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 30; then + reap "$pid"; fail "a confirmed pause after a misread was surfaced instead of re-absorbed: $(cat "$out")" + fi + [ -e "$state/.paused-$key" ] || { reap "$pid"; fail "the recheck cadence stayed disarmed after the misread (pause not re-absorbed)"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the re-absorbed pause enqueued a wake"; } + reap "$pid" + unset FM_FAKE_CREW_STATE + pass "a transient crew-state misread surfaces once without caching, leaving the pause recheck cadence armed" +} + +test_surfaced_stopped_stale_not_wedge_nagged() { + local dir state fakebin out capture_file window key sig pid + dir=$(make_case surfaced-stale-quiet); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stopped-quiet" + printf 'stopped at the prompt' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stopped-quiet.meta" + printf 'working: implementing\n' > "$state/stopped-quiet.status" + sig=$(seen_sig "$state/stopped-quiet.status"); printf '%s' "$sig" > "$state/.seen-stopped-quiet_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'stopped at the prompt')" > "$state/.hash-$key" + printf '%s' "$(hash_text 'stopped at the prompt')" > "$state/.stale-$key" + printf '1\n' > "$state/.count-$key" + # This stale was already SURFACED (situation recorded, no wedge timer). Later + # identical polls must stay quiet instead of re-arming the wedge timer and + # nagging "possible wedge" every FM_STALE_ESCALATE_SECS. + printf 'stopped|working: implementing' > "$state/.stale-surfaced-$key" + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=1 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + if ! wait_live "$pid" 40; then + reap "$pid"; fail "an already-surfaced stopped stale was wedge-nagged: $(cat "$out")" + fi + [ ! -s "$out" ] || { reap "$pid"; fail "already-surfaced stopped stale printed a wake reason: $(cat "$out")"; } + [ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "already-surfaced stopped stale enqueued a wake"; } + [ ! -e "$state/.stale-since-$key" ] || { reap "$pid"; fail "already-surfaced stopped stale re-armed the wedge timer"; } + reap "$pid" + pass "an already-surfaced stopped stale stays quiet; wedge timing remains reserved for provably-working absorbs" +} + +test_busy_pane_rearms_stale_surfacing() { + local dir state fakebin out capture_file window key sig pid i + dir=$(make_case busy-pane-rearm); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-busy-rearm" + printf 'building... esc to interrupt' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/busy-rearm.meta" + printf 'working: implementing\n' > "$state/busy-rearm.status" + sig=$(seen_sig "$state/busy-rearm.status"); printf '%s' "$sig" > "$state/.seen-busy-rearm_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + printf '%s' "$(hash_text 'building... esc to interrupt')" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + # A previously surfaced situation is on record; the pane then shows a busy + # signature (positive evidence), which must clear that record so the NEXT stop + # surfaces even with an unchanged status line. + printf 'stopped|working: implementing' > "$state/.stale-surfaced-$key" + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + i=0 + while [ "$i" -lt 40 ] && [ -e "$state/.stale-surfaced-$key" ]; do + sleep 0.1 + i=$((i + 1)) + done + [ ! -e "$state/.stale-surfaced-$key" ] || { reap "$pid"; fail "a busy pane did not clear the surfaced-situation record"; } + # The crew stops again: same status line, new idle hash - must surface. + printf 'stopped at the prompt' > "$capture_file" + wait_for_exit "$pid" 80 || fail "a stop after visible busy activity did not re-surface" + grep -Fx "stale: $window" "$out" >/dev/null || fail "post-busy stop did not print a stale wake" + unset FM_FAKE_CREW_STATE + pass "a busy pane re-arms stale surfacing so a later stop with the same status line surfaces" +} + +test_streaming_busy_pane_rearms_stale_surfacing() { + local dir state fakebin out capture_file window key sig pid writer i + dir=$(make_case streaming-busy-rearm); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt" + window="test:fm-stream-rearm" + printf 'building step 0... esc to interrupt' > "$capture_file" + printf 'window=%s\nkind=ship\n' "$window" > "$state/stream-rearm.meta" + printf 'working: implementing\n' > "$state/stream-rearm.status" + sig=$(seen_sig "$state/stream-rearm.status"); printf '%s' "$sig" > "$state/.seen-stream-rearm_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + # An actively STREAMING crew changes its pane content faster than the poll + # cadence, so every poll sees a new hash and the same-hash busy clear (which + # needs two identical busy polls) never runs. The surfaced-situation record + # must still be cleared by that busy evidence on the hash-changed path, or a + # later stop with the same status line would be absorbed as already-surfaced. + printf 'stopped|working: implementing' > "$state/.stale-surfaced-$key" + : > "$dir/.streaming" + ( i=1 + while [ -e "$dir/.streaming" ]; do + printf 'building step %d... esc to interrupt' "$i" > "$capture_file" + i=$((i + 1)) + sleep 0.2 + done ) & + writer=$! + export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available' + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & + pid=$! + i=0 + while [ "$i" -lt 60 ] && [ -e "$state/.stale-surfaced-$key" ]; do + sleep 0.1 + i=$((i + 1)) + done + rm -f "$dir/.streaming" + wait "$writer" 2>/dev/null || true + [ ! -e "$state/.stale-surfaced-$key" ] || { reap "$pid"; fail "a streaming busy pane did not clear the surfaced-situation record"; } + # The crew stops again: same status line, new idle hash - must surface. + printf 'stopped at the prompt' > "$capture_file" + wait_for_exit "$pid" 80 || fail "a stop after streaming busy activity did not re-surface" + grep -Fx "stale: $window" "$out" >/dev/null || fail "post-streaming stop did not print a stale wake" + unset FM_FAKE_CREW_STATE + pass "a streaming busy pane (hash changing every poll) re-arms stale surfacing" +} + # --- triage debug log stays size capped ------------------------------------- test_triage_log_size_cap_accepts_spaced_wc_counts() { @@ -1298,6 +1632,12 @@ test_nonterminal_stale_pause_transitions_reclassify_unchanged_hash test_nonterminal_paused_rechecks_authoritative_state test_paused_authoritative_working_preserves_wedge_timer test_nonterminal_stale_repairs_missing_or_corrupt_timer +test_terminal_stale_not_resurfaced_per_pane_hash_tick +test_paused_log_done_crew_not_resurfaced_per_poll +test_paused_crew_transient_read_failure_not_cached +test_surfaced_stopped_stale_not_wedge_nagged +test_busy_pane_rearms_stale_surfacing +test_streaming_busy_pane_rearms_stale_surfacing test_triage_log_size_cap_accepts_spaced_wc_counts test_heartbeat_no_change_absorbed test_heartbeat_backstop_surfaces_unsurfaced_status