diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 56f3008d3..d4921054a 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -196,12 +196,12 @@ HERDR_SECTION=$(printf '%s\n' \ '# Herdr isolation - HARD SAFETY CONTRACT' \ 'This brief was explicitly scaffolded with `--herdr-lab` because the task will drive Herdr lifecycle behavior.' \ 'On Herdr 0.7.3 the API socket is not relocatable by `HERDR_CONFIG_PATH`, `XDG_CONFIG_HOME`, or `HOME`.' \ -'A named non-`default` session plus a trailing `--session ` on every call is the only viable local isolation.' \ +'A named non-`default` session plus an explicit Herdr-global `--session ` on every call is the only viable local isolation.' \ '' \ '1. Set `HERDR_LAB_HELPER='"$HERDR_LAB_HELPER"'` and generate the session name with `HERDR_LAB_SESSION=$("$HERDR_LAB_HELPER" name '"$ID"')`.' \ ' Install `trap '\''"$HERDR_LAB_HELPER" teardown "$HERDR_LAB_SESSION"'\'' EXIT` before provisioning, then provision only with `"$HERDR_LAB_HELPER" provision "$HERDR_LAB_SESSION"`.' \ '2. Run every task-specific non-lifecycle Herdr command through `"$HERDR_LAB_HELPER" run "$HERDR_LAB_SESSION" `.' \ -' The helper appends the required trailing `--session "$HERDR_LAB_SESSION"`; `HERDR_SESSION` alone is never accepted as isolation.' \ +' The helper appends `--session "$HERDR_LAB_SESSION"` for ordinary calls and places it immediately before any `--` command-argv delimiter; `HERDR_SESSION` alone is never accepted as isolation.' \ '3. Teardown only through `"$HERDR_LAB_HELPER" teardown "$HERDR_LAB_SESSION"`.' \ ' It re-checks refuse-default immediately before stop and again immediately before delete, and fails closed on ambiguity.' \ '4. If an experiment requires a deliberate mid-run session stop, use only `"$HERDR_LAB_HELPER" stop "$HERDR_LAB_SESSION"`; it performs the same immediate refuse-default check.' \ diff --git a/bin/fm-herdr-lab.sh b/bin/fm-herdr-lab.sh index f8ea014c6..ca68aa745 100755 --- a/bin/fm-herdr-lab.sh +++ b/bin/fm-herdr-lab.sh @@ -13,7 +13,9 @@ # Session names must begin with "fm-lab-" and can never be "default". # The name command sanitizes the label, caps it at 16 characters, and appends # process/random suffixes to keep generated socket paths short. -# Every Herdr call made here carries a trailing --session . +# Every Herdr call made here carries --session as a Herdr global +# option: trailing for ordinary calls, or immediately before a `--` command-argv +# delimiter so the child command cannot consume the isolation flag. # The run command rejects caller-supplied --session flags, any leading option # before the subcommand, all session lifecycle operations, and every server # operation. @@ -49,9 +51,21 @@ fm_herdr_lab_tripwire_path() { # } fm_herdr_lab_raw() { # - local name=$1 + local name=$1 arg found_delimiter=0 + local herdr_args=() shift - HERDR_SESSION="$name" herdr "$@" --session "$name" + for arg in "$@"; do + if [ "$found_delimiter" -eq 0 ] && [ "$arg" = -- ]; then + herdr_args+=(--session "$name" --) + found_delimiter=1 + else + herdr_args+=("$arg") + fi + done + if [ "$found_delimiter" -eq 0 ]; then + herdr_args+=(--session "$name") + fi + HERDR_SESSION="$name" herdr "${herdr_args[@]}" } fm_herdr_lab_session_list() { # diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 4ea6da70a..b7ec38657 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -155,6 +155,146 @@ _event_cap_key="" _event_cap_ok=0 _event_cap_fails=0 +# Herdr's agent lifecycle state must stay truthful: an idle primary remains idle +# while this watcher blocks. Display-only pane metadata provides the distinct +# captain-visible state instead. A watcher publishes `idle · supervised` only +# after it owns the singleton, refreshes a bounded TTL while it stays live, and +# clears its own unique source on exit. The TTL lets a SIGKILL or host crash age +# out rather than leaving a false permanent indicator. Other runtime backends +# have no equivalent shared display surface and remain unchanged. +FM_HERDR_SUPERVISION_STATUS_TTL_MS=${FM_HERDR_SUPERVISION_STATUS_TTL_MS:-360000} +FM_SUPERVISION_VISIBILITY_PUBLISHED=0 +FM_SUPERVISION_VISIBILITY_SOURCE= +FM_SUPERVISION_VISIBILITY_SESSION= +FM_SUPERVISION_VISIBILITY_PANE= +FM_SUPERVISION_VISIBILITY_REFRESH_PID= +FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY= + +fm_supervision_visibility_resolve_herdr() { + local target backend=${FM_SUPERVISOR_BACKEND:-} + FM_SUPERVISION_VISIBILITY_SESSION= + FM_SUPERVISION_VISIBILITY_PANE= + case "$backend" in + herdr) + target=${FM_SUPERVISOR_TARGET:-} + [ -n "$target" ] || return 1 + FM_SUPERVISION_VISIBILITY_SESSION=${target%%:*} + FM_SUPERVISION_VISIBILITY_PANE=${target#*:} + [ "$FM_SUPERVISION_VISIBILITY_PANE" != "$target" ] || return 1 + ;; + '') + [ "${HERDR_ENV:-}" = 1 ] || return 1 + [ -z "${TMUX:-}" ] || return 1 + [ -n "${HERDR_PANE_ID:-}" ] || return 1 + FM_SUPERVISION_VISIBILITY_SESSION=${HERDR_SESSION:-default} + FM_SUPERVISION_VISIBILITY_PANE=$HERDR_PANE_ID + ;; + *) return 1 ;; + esac + [ -n "$FM_SUPERVISION_VISIBILITY_SESSION" ] && [ -n "$FM_SUPERVISION_VISIBILITY_PANE" ] +} + +# Display metadata is optional observability and must never wedge the watcher. +# Use the same dual session targeting as fm_backend_herdr_cli, but put a hard +# three-second process bound around this cosmetic call. +fm_supervision_visibility_herdr_call() { # + local session=$1 + shift + command -v herdr >/dev/null 2>&1 || return 1 + if command -v timeout >/dev/null 2>&1; then + HERDR_SESSION="$session" timeout --kill-after=1 2 herdr "$@" --session "$session" + elif command -v gtimeout >/dev/null 2>&1; then + HERDR_SESSION="$session" gtimeout --kill-after=1 2 herdr "$@" --session "$session" + elif command -v perl >/dev/null 2>&1; then + HERDR_SESSION="$session" perl -e 'my $seconds = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV } my $stop = sub { $SIG{TERM} = $SIG{INT} = $SIG{HUP} = "IGNORE"; kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; waitpid $pid, 0; exit 124 }; local $SIG{ALRM} = $stop; local $SIG{TERM} = $stop; local $SIG{INT} = $stop; local $SIG{HUP} = $stop; alarm $seconds; waitpid $pid, 0; alarm 0; exit($? >> 8)' \ + 2 herdr "$@" --session "$session" + else + return 1 + fi +} + +fm_supervision_visibility_refresh() { + local ttl=$FM_HERDR_SUPERVISION_STATUS_TTL_MS + [ -n "$FM_SUPERVISION_VISIBILITY_SOURCE" ] || return 0 + fm_supervision_visibility_resolve_herdr || return 0 + case "$ttl" in ''|*[!0-9]*|0) ttl=360000 ;; esac + if fm_supervision_visibility_herdr_call "$FM_SUPERVISION_VISIBILITY_SESSION" \ + pane report-metadata "$FM_SUPERVISION_VISIBILITY_PANE" \ + --source "$FM_SUPERVISION_VISIBILITY_SOURCE" \ + --custom-status supervised \ + --state-label 'idle=idle · supervised' \ + --ttl-ms "$ttl" >/dev/null 2>&1; then + FM_SUPERVISION_VISIBILITY_PUBLISHED=1 + fi + return 0 +} + +fm_supervision_visibility_clear() { + [ -n "$FM_SUPERVISION_VISIBILITY_SOURCE" ] || return 0 + fm_supervision_visibility_resolve_herdr || return 0 + fm_supervision_visibility_herdr_call "$FM_SUPERVISION_VISIBILITY_SESSION" \ + pane report-metadata "$FM_SUPERVISION_VISIBILITY_PANE" \ + --source "$FM_SUPERVISION_VISIBILITY_SOURCE" \ + --clear-custom-status --clear-state-labels >/dev/null 2>&1 || true + # shellcheck disable=SC2034 # Read by deterministic tests after sourcing this script. + FM_SUPERVISION_VISIBILITY_PUBLISHED=0 +} + +fm_supervision_visibility_refresh_interval() { + local ttl=$FM_HERDR_SUPERVISION_STATUS_TTL_MS half seconds millis + case "$ttl" in ''|*[!0-9]*|0) ttl=360000 ;; esac + half=$((ttl / 2)) + [ "$half" -gt 0 ] || half=1 + seconds=$((half / 1000)) + millis=$((half % 1000)) + printf '%d.%03d\n' "$seconds" "$millis" +} + +fm_supervision_visibility_refresh_loop() { # + local owner_pid=$1 interval sleeper='' expected_identity='' current_identity='' + interval=$(fm_supervision_visibility_refresh_interval) + trap '[ -z "$sleeper" ] || kill -TERM "$sleeper" 2>/dev/null || true; exit 0' HUP INT TERM + while :; do + sleep "$interval" & + sleeper=$! + wait "$sleeper" 2>/dev/null || exit 0 + sleeper= + [ "$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)" = "$owner_pid" ] || exit 0 + expected_identity=$(cat "$WATCH_LOCK/pid-identity" 2>/dev/null || true) + current_identity=$(fm_pid_identity "$owner_pid" 2>/dev/null || true) + [ -n "$expected_identity" ] && [ "$current_identity" = "$expected_identity" ] || exit 0 + fm_supervision_visibility_refresh + done +} + +fm_supervision_visibility_start() { # + local owner_pid=$1 pid identity + fm_supervision_visibility_resolve_herdr || return 0 + fm_supervision_visibility_refresh + fm_supervision_visibility_refresh_loop "$owner_pid" & + pid=$! + identity=$(fm_pid_identity "$pid" 2>/dev/null || true) + if [ -z "$identity" ]; then + kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + return 0 + fi + FM_SUPERVISION_VISIBILITY_REFRESH_PID=$pid + FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY=$identity +} + +fm_supervision_visibility_stop() { + local pid=${FM_SUPERVISION_VISIBILITY_REFRESH_PID:-} identity=${FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY:-} current= + [ -n "$pid" ] || return 0 + current=$(fm_pid_identity "$pid" 2>/dev/null || true) + if [ -n "$identity" ] && [ "$current" = "$identity" ]; then + kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + FM_SUPERVISION_VISIBILITY_REFRESH_PID= + FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY= +} + # afk_present: 0 while the away-mode flag exists. When set, the daemon wraps this # watcher and owns triage, so the watcher must behave one-shot (enqueue + exit on # every wake) and let the daemon classify - never absorb here, or the daemon's @@ -686,6 +826,8 @@ if ! fm_lock_try_acquire "$WATCH_LOCK"; then exit 0 fi watcher_cleanup() { + fm_supervision_visibility_stop + fm_supervision_visibility_clear fm_active_check_stop || return 1 fm_check_output_cleanup fm_custom_check_snapshot_cleanup @@ -697,9 +839,11 @@ trap 'exit 1' HUP INT TERM # ${BASHPID:-$$} from this same main shell). Read directly, never via a command # substitution, so it matches the stored holder pid for the self-eviction check. WATCHER_PID=${BASHPID:-$$} +FM_SUPERVISION_VISIBILITY_SOURCE="firstmate-supervision:$WATCHER_PID" printf '%s\n' "$FM_HOME" > "$WATCH_LOCK/fm-home" || true printf '%s\n' "$WATCH_PATH" > "$WATCH_LOCK/watcher-path" || true fm_pid_identity "$WATCHER_PID" > "$WATCH_LOCK/pid-identity" 2>/dev/null || true +fm_supervision_visibility_start "$WATCHER_PID" [ -e "$STATE/.last-heartbeat" ] || touch "$STATE/.last-heartbeat" diff --git a/docs/architecture.md b/docs/architecture.md index 7bb809fe6..b484f51ff 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,6 +9,7 @@ firstmate's always-loaded operating contract and routing index for conditional p ## Event-driven supervision A zero-token bash watcher (`bin/fm-watch.sh`) sleeps on the fleet, classifies detected wakes in bash, and wakes the first mate only when something is actionable. +On Herdr, that live singleton also marks the idle primary as `idle · supervised` through display-only pane metadata without changing its real agent lifecycle state; [`herdr-backend.md`](herdr-backend.md#captain-visible-supervision-state-2026-07-19) owns the mechanism and verification evidence. Actionable wakes include captain-relevant status signals, no-verb signals whose crew is not provably working, authenticated check output such as PR merge polling or an X-mode mention, stale panes whose crew is not provably working whether their status log looks terminal or non-terminal, provably-working stale panes that persist past `FM_STALE_ESCALATE_SECS`, declared external waits that remain paused past `FM_PAUSE_RESURFACE_SECS`, and heartbeat backstop hits. Repeated provably-working stale escalations on the same unchanged pane add an escalation count to the wake reason and, at `FM_WEDGE_DEMAND_INSPECT_COUNT`, a `demand-deep-inspection` marker. Those actionable wakes are written to a durable local queue (`state/.wake-queue`) before detector state advances, so a missed process exit can be recovered by draining the queue. diff --git a/docs/configuration.md b/docs/configuration.md index 25560837d..20c169b53 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -394,6 +394,7 @@ FM_WATCH_REARM_RETRY_LIMIT=5 # Pi/OpenCode adapter launch-failure retries befo FM_WATCH_CYCLE_LOG_MAX_BYTES=262144 # size cap for the arm-owned watcher lifecycle ledger FM_WATCH_CYCLE_LOG_KEEP_LINES=1000 # newest complete lifecycle rows considered when the ledger is capped FM_WATCHER_STALE_GRACE=300 # defaults to FM_GUARD_GRACE; seconds a live watcher lock may have a stale beacon before re-arm errors +FM_HERDR_SUPERVISION_STATUS_TTL_MS=360000 # Herdr-only: display-metadata TTL for the live watcher's idle · supervised label; unset, blank, zero, or nonnumeric values use 360000 FM_SIGNAL_GRACE=30 # seconds to coalesce nearby status and turn-end signals into one wake FM_CAPTAIN_RE='done:|needs-decision:|blocked:|failed:|PR ready|checks green|ready in branch|merged' # status regex that makes watcher and daemon signal/stale/scan output captain-relevant FM_CLASSIFY_PAUSED_VERB=paused # leading status verb for a declared external wait; excluded from FM_CAPTAIN_RE and distinct from blocked diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 0bed40dd1..b23ff3975 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -193,6 +193,72 @@ Herdr tasks additionally record: | Workspace create / tab create (focus) | `herdr workspace create --no-focus`, `herdr tab create --no-focus` | Verified: neither focuses by default once a workspace already exists in the session, matching pre-P3 (flagless) behavior; `--no-focus` is passed anyway for defense in depth, since the very first workspace ever created in a brand-new session focuses regardless of the flag. `--focus` was separately verified to reliably focus, confirming the flag has real effect. | | Session targeting for DESTRUCTIVE calls | `herdr session stop --session --json`, then `herdr session delete --session --json`; never `herdr server stop` | Owned by `bin/fm-herdr-lab.sh` (which `tests/herdr-test-safety.sh` sources), re-querying `herdr session list --json` before every destructive call. See "Session targeting" below - `HERDR_SESSION` alone is not reliably honored once another herdr server is already running on the machine. | +## Captain-visible supervision state (2026-07-19) + +A Pi primary running `fm_watch_arm_pi` successfully started the attached watcher child, while Herdr continued to report `agent_status: idle` and its Agents row remained `firstmate idle · pi`. +Those are three separate facts: the arm action succeeded, Herdr correctly classified the model as idle while it waited, and the row had no independent supervision indicator. +Changing Herdr's lifecycle report to `working` would have hidden the symptom by inventing model activity, so that counterfactual was rejected. + +Herdr 0.7.3 already exposes the owner-aligned display-only surface `pane report-metadata`. +Its changelog describes `custom_status` as a compact status label and `state_labels` as visible state labels that do not take over integration-owned lifecycle state. +The smallest counterfactual applied `custom_status=supervised` and `state_labels.idle="idle · supervised"` to a synthetic idle Pi record. +`agent get` then retained `agent_status: idle` while exposing both display fields, and clearing that metadata restored the original idle record. +This disproved the idea that the Pi extension or Herdr's activity classifier needed to fake a busy turn. + +`bin/fm-watch.sh` now owns publication because the live singleton watcher is the backend-neutral truth that supervision is active. +After acquiring its singleton, a watcher publishes one unique, TTL-bounded Herdr metadata source, refreshes it while alive, and clears it on clean exit. +A process crash ages out through the TTL instead of leaving a permanent false claim. +Each cosmetic Herdr call sends TERM after two seconds and KILL no more than one second later, so a stalled display surface cannot wedge supervision beyond a three-second process bound. +A duplicate watcher stands down before publication, so the exactly-one-live-supervision-cycle contract is unchanged. +The visible state label is `idle · supervised`, while Herdr's real `agent_status` remains idle. + +This integration covers every supported primary harness that reaches the same watcher owner. +Claude and Grok run `fm-watch-arm.sh`, Pi and OpenCode launch that arm through their persistent adapters, and Codex runs `fm-watch.sh` through its foreground checkpoint. +Away mode uses its explicit Herdr supervisor target so metadata decorates the captain's pane rather than the daemon pane. +An idle secondmate with no live watcher remains plain idle, while a secondmate that arms supervision gets the same truthful label. +The runtime-backend axis is intentionally Herdr-only after inspection: tmux, zellij, Orca, and cmux do not render a Herdr Agents row, so they have no applicable display surface and no behavior change. +A tmux primary nested inside Herdr is also excluded so its outer pane cannot claim the inner runtime's supervision. + +Real verification used Herdr 0.7.3, protocol 16, a generated non-`default` session, `bin/fm-herdr-lab.sh`, and a synthetic idle Pi lifecycle record. +The deterministic test pins the exact argv emitted by the production refresh and clear functions. +The real smoke sends those same display arguments directly through the named-lab helper with a 15-second bound, avoiding any recursive CLI shim or agent launch. +The active read was: + +```json +{"agent":"pi","agent_status":"idle","custom_status":"supervised","state_labels":{"idle":"idle · supervised"}} +``` + +The inactive baseline, explicit-clear read, and short-TTL expiry read after the synthetic publisher was killed all retained `agent_status: idle` and omitted `custom_status` and `state_labels`. +Calling the active report twice produced the same effective record. +The exact real-smoke command run was: + +```sh +FM_SUPERVISION_VISIBILITY_HERDR_SMOKE=1 HERDR_LAB_HELPER="$PWD/bin/fm-herdr-lab.sh" bash tests/fm-supervision-visibility-herdr-smoke.test.sh +``` + +Its exact output was: + +```text +ok - real Herdr 0.7.3 keeps Pi idle with clean clearing and crash TTL cleanup +``` + +The exact deterministic test command for active, inactive, clean-exit after a transient initial publication failure, independent TTL refresh, duplicate-arm, non-Herdr, and nested-tmux coverage is: + +```sh +bash tests/fm-supervision-visibility.test.sh +``` + +Its exact output is: + +```text +ok - Herdr visibility publishes idle-supervised without faking work and clears on clean exit +ok - a hung optional Herdr metadata call times out without wedging supervision +ok - Herdr visibility refresh cadence is independent of watcher cycle duration +ok - clean exit clears metadata after a transient initial publication failure +ok - inactive, non-Herdr, and nested-tmux supervision do not claim a Herdr indicator +ok - duplicate arm preserves one live supervision cycle and one truthful Herdr indicator +``` + ## Incident (2026-07-13): the ASCII request separator erased the secondmate marker A routed request reached a Pi/Herdr secondmate without the visible `[fm-from-firstmate]` label, so the secondmate correctly treated it as direct captain conversation and returned nothing to the parent status path. @@ -425,11 +491,53 @@ The fix, verified against the real binary in an isolated session (both a genuine - For destructive session cleanup specifically, use `herdr session stop ` / `herdr session delete ` (the explicit-by-name forms - `` is a REQUIRED positional argument, so herdr cannot resolve it ambiguously; herdr's own help text requires literally typing `default` to affect the default session), never the ambient `herdr server stop`. `bin/fm-herdr-lab.sh` now owns this guard as the single source of truth: `fm_herdr_lab_teardown` does the stop-then-delete, gated by a read-only hard guard (`fm_herdr_lab_refuse_if_default`, re-querying `herdr session list --json` immediately before EVERY stop/delete call, refusing on a literal `default` name, a not-found name, or `default:true`) as a second, independent layer that fails closed on any ambiguity. `tests/herdr-test-safety.sh` now sources that helper, so its `herdr_safe_stop_and_delete`/`herdr_refuse_if_default` names are thin delegating wrappers over the same owner. The same guard is now a first-class production helper, `bin/fm-herdr-lab.sh`, not just test scaffolding. -It provisions an isolated never-`default` lab session (names must start with `fm-lab-`), runs every task command through `run ...` with a mandatory trailing `--session` appended, and refuses caller-supplied `--session`, any leading option before the subcommand, and every server or session-lifecycle subcommand. +It provisions an isolated never-`default` lab session (names must start with `fm-lab-`), runs every task command through `run ...` with a mandatory Herdr-global `--session`, and refuses caller-supplied `--session`, any leading option before the subcommand, and every server or session-lifecycle subcommand. +For ordinary calls the helper keeps the global flag trailing. +When the Herdr command contains a `--` child-command delimiter, the helper inserts its global session pair immediately before that delimiter so the child command cannot consume it. Destructive teardown goes only through `teardown ` (or a deliberate mid-run `stop `), each re-running the refuse-default check immediately before every stop and delete. It also adds a before/after fleet-state tripwire: `provision` records the live `default` session before creating the lab session, and `teardown` verifies that recorded state is byte-identical afterward before clearing it, treating any missing, stopped, or changed default session as a hard failure rather than a warning. Crewmate briefs for tasks that drive Herdr lifecycle get this exact contract embedded by scaffolding with `bin/fm-brief.sh --herdr-lab`; every crewmate brief scaffolded without the flag instead carries a loud not-enabled gate, because the scaffold cannot detect from the caller-supplied repo string whether the task will touch Herdr lifecycle. +### Incident (2026-07-19): a trailing lab session flag crossed the child-command delimiter + +`fm-herdr-lab.sh` formerly appended `--session ` after every argument without recognizing Herdr commands that use `-- `. +For `herdr agent start ... -- bash ...`, the option landed after the delimiter and became a child argument instead of a Herdr global option. +Herdr therefore used its ambient server and created the test agent in the running default session. +The operator removed that exact unexpected pane and confirmed the recorded primary pane was intact before work resumed. + +A benign fake-client reproduction used the named lab `fm-lab-benign-routing` and printed the helper's exact argv without invoking Herdr or starting an agent. +Before the fix it printed: + +```text + + + +<--> + + +<--session> + +``` + +The smallest fix is in the helper's single raw-call owner. +It now inserts `--session ` immediately before the first standalone `--`, while calls without a delimiter stay trailing as before. +The same benign probe after the fix printed: + +```text + + + +<--session> + +<--> + + +``` + +`tests/fm-herdr-lab.test.sh` now rejects a session pair after the child-command delimiter and accepts it only in Herdr's own option region. +The generated hard safety contract in `bin/fm-brief.sh` states the same delimiter rule. +No further real agent launch was used for this verification. + ## ID stability across a server restart The original design addendum flagged this as an open risk to verify. diff --git a/tests/fm-afk-inject-herdr-e2e.test.sh b/tests/fm-afk-inject-herdr-e2e.test.sh index 5e48ab43e..4570395eb 100755 --- a/tests/fm-afk-inject-herdr-e2e.test.sh +++ b/tests/fm-afk-inject-herdr-e2e.test.sh @@ -84,7 +84,7 @@ LOG_FILE="$STATE_DIR/submitted.log" CONTAINER_RAW=$(fm_backend_herdr_container_ensure /tmp) || fail "container_ensure failed" CONTAINER=${CONTAINER_RAW%%$'\t'*} SEEDED_TAB_ID=${CONTAINER_RAW#*$'\t'} -TASK_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "fm-afk-e2e-supervisor" /tmp "$SEEDED_TAB_ID") \ +TASK_IDS=$(fm_backend_herdr_create_task "$CONTAINER" "fm-afk-e2e-supervisor" "$STATE_DIR" "$SEEDED_TAB_ID") \ || fail "create_task for the scratch supervisor pane failed" read -r _TAB_ID PANE_ID <') break ;; + esac + sleep 0.1 + i=$((i + 1)) +done +case "$PANE_TAIL" in + *'❯'|*'$'|*'%'|*'#'|*'>') ;; + *) fail "scratch herdr pane did not reach a shell prompt" ;; +esac fm_backend_herdr_send_text_line "$SUPERVISOR_TARGET" "bash '$LOOP_SCRIPT' '$LOG_FILE'" \ || fail "could not start the supervisor-loop script in the scratch herdr pane" sleep 1 # let the loop start and settle diff --git a/tests/fm-afk-pi-herdr-return-e2e.test.sh b/tests/fm-afk-pi-herdr-return-e2e.test.sh index 12b421351..f19706af9 100755 --- a/tests/fm-afk-pi-herdr-return-e2e.test.sh +++ b/tests/fm-afk-pi-herdr-return-e2e.test.sh @@ -84,8 +84,9 @@ export default function (pi: ExtensionAPI) { } EOF -# Route production adapter invocations through the guarded helper too. The shim -# removes only the adapter's validated trailing pair, then the helper appends it. +# Route production adapter invocations through the guarded helper too. +# The shim removes only the adapter's validated trailing pair, then the helper +# restores the session pair in Herdr's option region. cat > "$FAKEBIN/herdr" <blocked ---------------- # report-agent is herdr's documented primitive for a non-built-in process to # report its own agent state (docs/herdr-backend.md); routed through the lab -# helper's guarded `run` so it carries the trailing --session. +# helper's guarded `run` so the session flag stays in Herdr's option region. fm_herdr_lab_cli "$SESSION" pane report-agent "$PANE_ID" --source fm-evwait-test --agent claude --state idle >/dev/null 2>&1 \ || fail "could not register the pane's agent as idle" diff --git a/tests/fm-backend-herdr-prune-safety-e2e.test.sh b/tests/fm-backend-herdr-prune-safety-e2e.test.sh index 1498e9869..a08cc71b0 100755 --- a/tests/fm-backend-herdr-prune-safety-e2e.test.sh +++ b/tests/fm-backend-herdr-prune-safety-e2e.test.sh @@ -76,9 +76,23 @@ pass "repro setup: a pre-existing workspace labeled 'firstmate' collides with th # Simulate a live long-running agent in that pane: a heartbeat loop that # appends to a marker file, so liveness is independently verifiable (not just # "the pane object still exists"). +PANE_TAIL= +i=0 +while [ "$i" -lt 80 ]; do + PANE_TAIL=$(fm_backend_herdr_capture "$SESSION:$LIVE_PANE_ID" 10 | tail -1) + case "$PANE_TAIL" in + *'❯'|*'$'|*'%'|*'#'|*'>') break ;; + esac + sleep 0.1 + i=$((i + 1)) +done +case "$PANE_TAIL" in + *'❯'|*'$'|*'%'|*'#'|*'>') ;; + *) fail "startup workspace pane did not reach a shell prompt" ;; +esac MARKER="$SCRATCH/heartbeat.log" fm_backend_herdr_cli "$SESSION" pane run "$LIVE_PANE_ID" \ - "sh -c 'while true; do date +%s >> $MARKER; sleep 1; done'" >/dev/null 2>&1 \ + "sh -c 'while true; do date +%s >> ../heartbeat.log; sleep 1; done'" >/dev/null 2>&1 \ || fail "could not start the live heartbeat process in the startup workspace's pane" sleep 2 [ -s "$MARKER" ] || fail "the live heartbeat process did not start writing its marker file" diff --git a/tests/fm-backend-tmux-smoke.test.sh b/tests/fm-backend-tmux-smoke.test.sh index 0d47b83ec..5989ddd9b 100755 --- a/tests/fm-backend-tmux-smoke.test.sh +++ b/tests/fm-backend-tmux-smoke.test.sh @@ -99,7 +99,12 @@ pass "real tmux: fm_backend_tmux_send_literal + fm_backend_tmux_send_key Enter s # far enough to still see the earliest line - the same -S -N bounding fm-peek.sh # and fm-watch.sh rely on for a bounded, cheap pane read. fm_backend_tmux_send_text_line "$TARGET" "for i in \$(seq 1 80); do echo tag-line-\$i; done" -sleep 0.6 +i=0 +while [ "$i" -lt 50 ]; do + fm_backend_tmux_capture "$TARGET" 3 | grep -q 'tag-line-80' && break + sleep 0.1 + i=$((i + 1)) +done small=$(fm_backend_tmux_capture "$TARGET" 3) || fail "fm_backend_tmux_capture (small window) failed" case "$small" in *tag-line-1$'\n'*) fail "a 3-line capture should not still see the very first numbered line"$'\n'"$small" ;; diff --git a/tests/fm-brief.test.sh b/tests/fm-brief.test.sh index 645e451c2..5f23bffbc 100755 --- a/tests/fm-brief.test.sh +++ b/tests/fm-brief.test.sh @@ -152,8 +152,8 @@ test_herdr_lab_contract_is_explicit_and_complete() { "Herdr lab brief missing helper-owned provisioning" assert_grep "\"\$HERDR_LAB_HELPER\" teardown \"\$HERDR_LAB_SESSION\"" "$brief" \ "Herdr lab brief missing helper-owned teardown" - assert_grep "required trailing \`--session \"\$HERDR_LAB_SESSION\"\`" "$brief" \ - "Herdr lab brief missing the per-call trailing session contract" + assert_grep "places it immediately before any \`--\` command-argv delimiter" "$brief" \ + "Herdr lab brief missing the command-argv-safe session contract" assert_grep "direct \`herdr server stop\`" "$brief" \ "Herdr lab brief missing the forbidden server-global command list" assert_grep "records the live default session before provisioning" "$brief" \ diff --git a/tests/fm-cd-pretool-check.test.sh b/tests/fm-cd-pretool-check.test.sh index d34bdda3a..f623430b1 100755 --- a/tests/fm-cd-pretool-check.test.sh +++ b/tests/fm-cd-pretool-check.test.sh @@ -435,6 +435,7 @@ test_pi_wiring() { } test_scripts_are_shellcheck_clean() { + command -v shellcheck >/dev/null 2>&1 || { pass "shellcheck not installed, skipping"; return; } shellcheck "$ROOT/bin/fm-cd-pretool-check.sh" >/dev/null 2>&1 \ || fail "bin/fm-cd-pretool-check.sh is not shellcheck-clean" pass "bin/fm-cd-pretool-check.sh is shellcheck-clean" diff --git a/tests/fm-herdr-lab.test.sh b/tests/fm-herdr-lab.test.sh index c004226aa..7c7fefa19 100755 --- a/tests/fm-herdr-lab.test.sh +++ b/tests/fm-herdr-lab.test.sh @@ -20,13 +20,23 @@ cat > "$FAKEBIN/herdr" <<'SH' set -eu printf '%s\n' "$*" >> "$FM_FAKE_HERDR_LOG" state=$FM_FAKE_HERDR_STATE -last= +session= +expect_session_value=0 +before_command_argv=1 for arg in "$@"; do - previous=$last - last=$arg + if [ "$before_command_argv" -eq 1 ] && [ "$expect_session_value" -eq 1 ]; then + session=$arg + expect_session_value=0 + continue + fi + if [ "$before_command_argv" -eq 1 ] && [ "$arg" = --session ]; then + expect_session_value=1 + continue + fi + [ "$arg" != -- ] || before_command_argv=0 done -[ "${previous:-}" = --session ] || { echo "fake herdr: missing trailing --session" >&2; exit 90; } -session=$last +[ -n "$session" ] && [ "$expect_session_value" -eq 0 ] \ + || { echo "fake herdr: missing --session before the command argv delimiter" >&2; exit 90; } default_socket=$(cat "$state/default-socket") lab_state=absent [ ! -f "$state/$session" ] || lab_state=$(cat "$state/$session") @@ -109,6 +119,13 @@ test_provision_run_and_guarded_teardown() { assert_present "$TRIPWIRES/$name.fleet-state.json" "provision did not record the fleet-state tripwire" run_with_fake fm_herdr_lab_cli "$name" workspace list >/dev/null || fail "safe run command failed" + run_with_fake fm_herdr_lab_cli "$name" agent start benign -- /usr/bin/printf ok >/dev/null \ + || fail "safe command-argv run failed" + grep -Fx "agent start benign --session $name -- /usr/bin/printf ok" "$FAKE_LOG" >/dev/null \ + || fail "lab session flag crossed the command argv delimiter" + if grep -Fx "agent start benign -- /usr/bin/printf ok --session $name" "$FAKE_LOG" >/dev/null; then + fail "lab session flag was passed to the child command instead of Herdr" + fi run_with_fake fm_herdr_lab_cli "$name" server >/dev/null 2>&1 || status=$? expect_code 1 "$status" "bare server start outside provision must be refused" status=0 @@ -139,8 +156,8 @@ test_provision_run_and_guarded_teardown() { while IFS= read -r line; do case "$line" in - *"--session $name") : ;; - *) fail "Herdr call lacks a trailing lab session: $line" ;; + *"--session $name"|*"--session $name --"*) : ;; + *) fail "Herdr call lacks a lab session before any command argv delimiter: $line" ;; esac done < "$FAKE_LOG" line_count=$(wc -l < "$FAKE_LOG" | tr -d ' ') diff --git a/tests/fm-send-secondmate-marker-herdr-e2e.test.sh b/tests/fm-send-secondmate-marker-herdr-e2e.test.sh index fa9eb151a..3fa06b31c 100755 --- a/tests/fm-send-secondmate-marker-herdr-e2e.test.sh +++ b/tests/fm-send-secondmate-marker-herdr-e2e.test.sh @@ -12,7 +12,7 @@ # Every Herdr call, including calls made inside the production backend adapter, # is routed through bin/fm-herdr-lab.sh. The PATH shim strips only the adapter's # already-validated trailing --session pair, then delegates to the lab helper, -# which appends its own required trailing --session before invoking real Herdr. +# which places its own required --session in Herdr's option region. set -u # shellcheck source=tests/lib.sh diff --git a/tests/fm-sessionstart-nudge.test.sh b/tests/fm-sessionstart-nudge.test.sh index 10992880f..2f367d062 100755 --- a/tests/fm-sessionstart-nudge.test.sh +++ b/tests/fm-sessionstart-nudge.test.sh @@ -22,7 +22,7 @@ make_primary() { run_nudge() { local root=$1 - FM_GATE_REFUSE_BYPASS=0 FM_ROOT_OVERRIDE="$root" FM_HOME="$root" "$NUDGE" + env -u NO_MISTAKES_GATE FM_GATE_REFUSE_BYPASS=0 FM_ROOT_OVERRIDE="$root" FM_HOME="$root" "$NUDGE" } expect_silent_zero() { diff --git a/tests/fm-supervision-visibility-herdr-smoke.test.sh b/tests/fm-supervision-visibility-herdr-smoke.test.sh new file mode 100755 index 000000000..3a759e0d1 --- /dev/null +++ b/tests/fm-supervision-visibility-herdr-smoke.test.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Opt-in real-Herdr proof for the display-only supervision state. +# It creates no agent process and uses only a synthetic idle lifecycle record in +# a guarded, named non-default lab session. +set -u + +if [ "${FM_SUPERVISION_VISIBILITY_HERDR_SMOKE:-0}" != 1 ]; then + echo "skip: set FM_SUPERVISION_VISIBILITY_HERDR_SMOKE=1 to run the isolated Herdr regression" + exit 0 +fi + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +HERDR_LAB_HELPER=${HERDR_LAB_HELPER:-$ROOT/bin/fm-herdr-lab.sh} +HERDR_LAB_SESSION=$("$HERDR_LAB_HELPER" name firstmate-visible-supervision) +LAB=$(mktemp -d "${TMPDIR:-/tmp}/fm-supervision-visibility-herdr.XXXXXX") +CRASH_PUBLISHER= + +cleanup() { + [ -z "$CRASH_PUBLISHER" ] || kill -KILL "$CRASH_PUBLISHER" 2>/dev/null || true + "$HERDR_LAB_HELPER" teardown "$HERDR_LAB_SESSION" + rm -rf "$LAB" + fm_test_cleanup +} +trap cleanup EXIT + +# Keep every task-specific real Herdr call both helper-scoped and independently +# bounded. The helper remains the only process that invokes the real CLI. +run_lab() { + perl -e 'my $timeout = shift; alarm $timeout; exec @ARGV' 15 \ + "$HERDR_LAB_HELPER" run "$HERDR_LAB_SESSION" "$@" +} + +mkdir -p "$LAB/home/state" "$LAB/home/config" +"$HERDR_LAB_HELPER" provision "$HERDR_LAB_SESSION" +out=$(run_lab workspace create --cwd "$ROOT" --label fm-visible-supervision --no-focus) +pane=$(printf '%s' "$out" | jq -r '.result.root_pane.pane_id // empty') +[ -n "$pane" ] || fail "lab workspace create did not return a pane" +run_lab pane report-agent "$pane" --source integration:pi --agent pi --state idle >/dev/null + +read_agent() { + run_lab agent get "$pane" +} + +baseline=$(read_agent) +printf '%s' "$baseline" | jq -e ' + .result.agent.agent_status == "idle" + and (.result.agent | has("custom_status") | not) + and (.result.agent | has("state_labels") | not) +' >/dev/null || fail "inactive baseline was not plain idle" + +# These are byte-for-byte the display arguments emitted by +# fm_supervision_visibility_refresh before its bounded Herdr call adds session +# targeting. The deterministic test separately pins that production argv. +run_lab pane report-metadata "$pane" \ + --source firstmate-supervision:smoke \ + --custom-status supervised \ + --state-label 'idle=idle · supervised' \ + --ttl-ms 360000 >/dev/null +run_lab pane report-metadata "$pane" \ + --source firstmate-supervision:smoke \ + --custom-status supervised \ + --state-label 'idle=idle · supervised' \ + --ttl-ms 360000 >/dev/null + +active=$(read_agent) +printf '%s' "$active" | jq -e ' + .result.agent.agent_status == "idle" + and .result.agent.custom_status == "supervised" + and .result.agent.state_labels.idle == "idle · supervised" +' >/dev/null || fail "active supervision metadata changed lifecycle state or was not visible: $active" + +run_lab pane report-metadata "$pane" \ + --source firstmate-supervision:smoke \ + --clear-custom-status --clear-state-labels >/dev/null + +cleared=$(read_agent) +printf '%s' "$cleared" | jq -e ' + .result.agent.agent_status == "idle" + and (.result.agent | has("custom_status") | not) + and (.result.agent | has("state_labels") | not) +' >/dev/null || fail "clean clear did not restore plain idle" + +crash_ready="$LAB/crash-ready" +( + run_lab pane report-metadata "$pane" \ + --source firstmate-supervision:crash \ + --custom-status supervised \ + --state-label 'idle=idle · supervised' \ + --ttl-ms 500 >/dev/null + touch "$crash_ready" + while :; do sleep 30; done +) & +crash_publisher=$! +CRASH_PUBLISHER=$crash_publisher +for _ in {1..100}; do + [ -e "$crash_ready" ] && break + sleep 0.05 +done +[ -e "$crash_ready" ] || fail "crash publisher did not report short-TTL metadata" +kill -KILL "$crash_publisher" 2>/dev/null || true +wait "$crash_publisher" 2>/dev/null || true +CRASH_PUBLISHER= + +expired= +for _ in {1..100}; do + crashed=$(read_agent) + if printf '%s' "$crashed" | jq -e ' + .result.agent.agent_status == "idle" + and (.result.agent | has("custom_status") | not) + and (.result.agent | has("state_labels") | not) + ' >/dev/null; then + expired=1 + break + fi + sleep 0.05 +done +[ "$expired" = 1 ] || fail "crashed publisher metadata did not expire through its TTL: $crashed" + +version=$(run_lab status --json | jq -r '.client.version // "unknown"') +pass "real Herdr $version keeps Pi idle with clean clearing and crash TTL cleanup" diff --git a/tests/fm-supervision-visibility.test.sh b/tests/fm-supervision-visibility.test.sh new file mode 100755 index 000000000..624847af7 --- /dev/null +++ b/tests/fm-supervision-visibility.test.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +# Herdr-visible supervision metadata must describe the watcher without changing +# the primary agent's real idle/working lifecycle state. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-supervision-visibility) +FAKEBIN=$(fm_fakebin "$TMP_ROOT") +HERDR_LOG="$TMP_ROOT/herdr.log" +WATCH="$ROOT/bin/fm-watch.sh" +: > "$HERDR_LOG" + +cat > "$FAKEBIN/herdr" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "${FM_FAKE_HERDR_LOG:?}" +if [ -n "${FM_FAKE_HERDR_FAIL_ONCE_FILE:-}" ] && [ ! -e "$FM_FAKE_HERDR_FAIL_ONCE_FILE" ]; then + touch "$FM_FAKE_HERDR_FAIL_ONCE_FILE" + exit 1 +fi +if [ "${FM_FAKE_HERDR_HANG:-0}" = 1 ]; then + trap '' TERM ALRM + while :; do :; done +fi +printf '%s\n' '{"ok":true}' +SH +chmod +x "$FAKEBIN/herdr" + +HOME_UNIT="$TMP_ROOT/unit-home" +mkdir -p "$HOME_UNIT/state" "$HOME_UNIT/config" +FM_HOME="$HOME_UNIT" +FM_STATE_OVERRIDE="$HOME_UNIT/state" +FM_ROOT_OVERRIDE="$ROOT" +# shellcheck source=bin/fm-watch.sh +. "$WATCH" + +run_visibility() { + PATH="$FAKEBIN:$PATH" FM_FAKE_HERDR_LOG="$HERDR_LOG" "$@" +} + +reset_visibility_globals() { + fm_supervision_visibility_stop + FM_SUPERVISION_VISIBILITY_PUBLISHED=0 + FM_SUPERVISION_VISIBILITY_SOURCE=firstmate-supervision:test + FM_SUPERVISION_VISIBILITY_SESSION= + FM_SUPERVISION_VISIBILITY_PANE= + FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY= +} + +test_refresh_is_independent_of_watcher_cycle() { + local calls=0 attempts=0 + reset_visibility_globals + : > "$HERDR_LOG" + FM_SUPERVISOR_BACKEND=herdr + FM_SUPERVISOR_TARGET=fm-lab-visible:w1:p1 + HERDR_ENV= + TMUX= + FM_HERDR_SUPERVISION_STATUS_TTL_MS=1000 + mkdir -p "$WATCH_LOCK" + printf '%s\n' "$$" > "$WATCH_LOCK/pid" + fm_pid_identity "$$" > "$WATCH_LOCK/pid-identity" + run_visibility fm_supervision_visibility_start "$$" + [ -n "$FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY" ] \ + || fail "refresher process identity was not recorded" + [ "$(fm_pid_identity "$FM_SUPERVISION_VISIBILITY_REFRESH_PID" 2>/dev/null || true)" = "$FM_SUPERVISION_VISIBILITY_REFRESH_IDENTITY" ] \ + || fail "recorded refresher identity did not match its live process" + while [ "$calls" -lt 2 ] && [ "$attempts" -lt 100 ]; do + calls=$(grep -c -- '--custom-status supervised' "$HERDR_LOG" 2>/dev/null || true) + [ "$calls" -ge 2 ] && break + sleep 0.05 + attempts=$((attempts + 1)) + done + fm_supervision_visibility_stop + [ "$calls" -ge 2 ] || fail "visibility TTL was not refreshed independently of the watcher cycle" + FM_HERDR_SUPERVISION_STATUS_TTL_MS=360000 + pass "Herdr visibility refresh cadence is independent of watcher cycle duration" +} + +test_background_publication_is_cleared_on_exit() { + local calls=0 attempts=0 fail_once="$TMP_ROOT/fail-once" + reset_visibility_globals + : > "$HERDR_LOG" + FM_SUPERVISOR_BACKEND=herdr + FM_SUPERVISOR_TARGET=fm-lab-visible:w1:p1 + HERDR_ENV= + TMUX= + FM_HERDR_SUPERVISION_STATUS_TTL_MS=1000 + mkdir -p "$WATCH_LOCK" + printf '%s\n' "$$" > "$WATCH_LOCK/pid" + fm_pid_identity "$$" > "$WATCH_LOCK/pid-identity" + FM_FAKE_HERDR_FAIL_ONCE_FILE="$fail_once" run_visibility fm_supervision_visibility_start "$$" + [ "$FM_SUPERVISION_VISIBILITY_PUBLISHED" -eq 0 ] \ + || fail "failed initial publication was recorded as visible" + while [ "$calls" -lt 2 ] && [ "$attempts" -lt 100 ]; do + calls=$(grep -c -- '--custom-status supervised' "$HERDR_LOG" 2>/dev/null || true) + [ "$calls" -ge 2 ] && break + sleep 0.05 + attempts=$((attempts + 1)) + done + fm_supervision_visibility_stop + run_visibility fm_supervision_visibility_clear + grep -F -- '--source firstmate-supervision:test --clear-custom-status --clear-state-labels' "$HERDR_LOG" >/dev/null \ + || fail "clean exit did not clear metadata published by the background refresher" + FM_HERDR_SUPERVISION_STATUS_TTL_MS=360000 + pass "clean exit clears metadata after a transient initial publication failure" +} + +test_active_and_clean_exit_metadata() { + reset_visibility_globals + : > "$HERDR_LOG" + FM_SUPERVISOR_BACKEND=herdr + FM_SUPERVISOR_TARGET=fm-lab-visible:w1:p1 + HERDR_ENV= + TMUX= + run_visibility fm_supervision_visibility_refresh + grep -Fx "pane report-metadata w1:p1 --source firstmate-supervision:test --custom-status supervised --state-label idle=idle · supervised --ttl-ms 360000 --session fm-lab-visible" "$HERDR_LOG" >/dev/null \ + || fail "active watcher did not publish the truthful Herdr state label" + [ "$FM_SUPERVISION_VISIBILITY_PUBLISHED" -eq 1 ] \ + || fail "successful active metadata publication was not recorded" + + run_visibility fm_supervision_visibility_clear + grep -Fx "pane report-metadata w1:p1 --source firstmate-supervision:test --clear-custom-status --clear-state-labels --session fm-lab-visible" "$HERDR_LOG" >/dev/null \ + || fail "clean watcher exit did not clear its Herdr metadata source" + [ "$FM_SUPERVISION_VISIBILITY_PUBLISHED" -eq 0 ] \ + || fail "clean metadata clear retained the published marker" + pass "Herdr visibility publishes idle-supervised without faking work and clears on clean exit" +} + +test_hung_metadata_call_cannot_wedge_supervision() { + local started elapsed + reset_visibility_globals + : > "$HERDR_LOG" + FM_SUPERVISOR_BACKEND=herdr + FM_SUPERVISOR_TARGET=fm-lab-visible:w1:p1 + HERDR_ENV= + TMUX= + started=$(date +%s) + FM_FAKE_HERDR_HANG=1 run_visibility fm_supervision_visibility_refresh + elapsed=$(( $(date +%s) - started )) + [ "$elapsed" -le 4 ] || fail "hung Herdr metadata call delayed supervision for ${elapsed}s" + [ "$FM_SUPERVISION_VISIBILITY_PUBLISHED" -eq 0 ] \ + || fail "timed-out metadata publication was recorded as visible" + pass "a hung optional Herdr metadata call times out without wedging supervision" +} + +test_inactive_and_non_herdr_runtimes_stay_unpublished() { + reset_visibility_globals + : > "$HERDR_LOG" + FM_SUPERVISOR_BACKEND=tmux + FM_SUPERVISOR_TARGET=firstmate:0 + HERDR_ENV=1 + HERDR_PANE_ID=w1:p1 + HERDR_SESSION=fm-lab-visible + TMUX=/tmp/tmux-1/default,1,0 + run_visibility fm_supervision_visibility_refresh + [ ! -s "$HERDR_LOG" ] || fail "an explicit non-Herdr supervisor published Herdr metadata" + + FM_SUPERVISOR_BACKEND= + FM_SUPERVISOR_TARGET= + run_visibility fm_supervision_visibility_refresh + [ ! -s "$HERDR_LOG" ] || fail "a tmux runtime nested in Herdr published metadata on its outer pane" + + TMUX= + HERDR_ENV= + HERDR_PANE_ID= + run_visibility fm_supervision_visibility_refresh + [ ! -s "$HERDR_LOG" ] || fail "an inactive non-Herdr watcher published Herdr metadata" + pass "inactive, non-Herdr, and nested-tmux supervision do not claim a Herdr indicator" +} + +wait_for_log_match() { # [attempts] + local pattern=$1 attempts=${2:-100} i=0 + while [ "$i" -lt "$attempts" ]; do + grep -F -- "$pattern" "$HERDR_LOG" >/dev/null 2>&1 && return 0 + sleep 0.05 + i=$((i + 1)) + done + return 1 +} + +test_singleton_duplicate_does_not_publish_or_clear() { + local home first_pid duplicate_pid duplicate_out source + home="$TMP_ROOT/singleton-home" + duplicate_out="$TMP_ROOT/duplicate.out" + mkdir -p "$home/state" "$home/config" + : > "$HERDR_LOG" + + PATH="$FAKEBIN:$PATH" FM_FAKE_HERDR_LOG="$HERDR_LOG" \ + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_SUPERVISOR_BACKEND=herdr FM_SUPERVISOR_TARGET=fm-lab-visible:w1:p1 \ + FM_POLL=1 FM_SIGNAL_GRACE=0 FM_HEARTBEAT=600 \ + bash "$WATCH" >"$TMP_ROOT/first.out" 2>&1 & + first_pid=$! + source="firstmate-supervision:$first_pid" + if ! wait_for_log_match "--source $source --custom-status supervised"; then + kill -TERM "$first_pid" 2>/dev/null || true + wait "$first_pid" 2>/dev/null || true + fail "singleton watcher did not publish its active Herdr metadata: $(cat "$TMP_ROOT/first.out" 2>/dev/null)" + fi + + PATH="$FAKEBIN:$PATH" FM_FAKE_HERDR_LOG="$HERDR_LOG" \ + FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_SUPERVISOR_BACKEND=herdr FM_SUPERVISOR_TARGET=fm-lab-visible:w1:p1 \ + FM_POLL=1 FM_SIGNAL_GRACE=0 FM_HEARTBEAT=600 \ + bash "$WATCH" >"$duplicate_out" 2>&1 & + duplicate_pid=$! + wait "$duplicate_pid" || true + grep -F "watcher: already running pid $first_pid" "$duplicate_out" >/dev/null \ + || fail "duplicate watcher did not stand down on the singleton" + if grep -F -- "--source firstmate-supervision:$duplicate_pid" "$HERDR_LOG" >/dev/null; then + kill -TERM "$first_pid" 2>/dev/null || true + wait "$first_pid" 2>/dev/null || true + fail "duplicate watcher published or cleared a second Herdr metadata source" + fi + if grep -F -- "--source $source --clear-custom-status" "$HERDR_LOG" >/dev/null; then + kill -TERM "$first_pid" 2>/dev/null || true + wait "$first_pid" 2>/dev/null || true + fail "duplicate arm cleared the live watcher's Herdr indicator" + fi + + kill -TERM "$first_pid" 2>/dev/null || true + wait "$first_pid" 2>/dev/null || true + wait_for_log_match "--source $source --clear-custom-status --clear-state-labels" \ + || fail "clean singleton exit did not clear its unique Herdr metadata source" + pass "duplicate arm preserves one live supervision cycle and one truthful Herdr indicator" +} + +test_active_and_clean_exit_metadata +test_hung_metadata_call_cannot_wedge_supervision +test_refresh_is_independent_of_watcher_cycle +test_background_publication_is_cleared_on_exit +test_inactive_and_non_herdr_runtimes_stay_unpublished +test_singleton_duplicate_does_not_publish_or_clear diff --git a/tests/fm-watcher-lock.test.sh b/tests/fm-watcher-lock.test.sh index 984824376..7f6fd9242 100755 --- a/tests/fm-watcher-lock.test.sh +++ b/tests/fm-watcher-lock.test.sh @@ -437,14 +437,22 @@ test_watch_restart_rejects_reused_pid() { } test_watch_restart_attaches_to_healthy_peer() { - local dir state fakebin out peer identity armpid status i + local dir state fakebin out ready peer identity armpid status i dir=$(make_case restart-healthy-peer) state="$dir/state" fakebin="$dir/fakebin" out="$dir/restart.out" + ready="$dir/peer.ready" mark_pr_check_migration_complete "$state" - node -e 'process.on("SIGTERM", () => {}); setTimeout(() => {}, 300000)' & + node -e 'process.on("SIGTERM", () => {}); require("fs").writeFileSync(process.argv[1], "ready\n"); setTimeout(() => {}, 300000)' "$ready" & peer=$! + i=0 + while [ "$i" -lt 80 ]; do + [ -f "$ready" ] && break + sleep 0.1 + i=$((i + 1)) + done + [ -f "$ready" ] || fail "TERM-resistant peer did not become ready" identity=$(FM_STATE_OVERRIDE="$state" bash -c '. "$1"; fm_pid_identity "$2"' _ "$LIB" "$peer") || fail "could not identify peer pid" mkdir "$state/.watch.lock" printf '%s\n' "$peer" > "$state/.watch.lock/pid" @@ -722,14 +730,15 @@ test_arm_waits_for_peer_beacon_after_child_stands_down() { printf '%s\n' "$dir" > "$state/.watch.lock/fm-home" printf '%s\n' "$WATCH" > "$state/.watch.lock/watcher-path" printf '%s\n' "$identity" > "$state/.watch.lock/pid-identity" - PATH="$fakebin:$PATH" FM_HOME="$dir" FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_ARM_CONFIRM_TIMEOUT=1 FM_ARM_ATTACH_POLL=0.1 "$WATCH_ARM" > "$armout" & + PATH="$fakebin:$PATH" FM_HOME="$dir" FM_POLL=5 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_ARM_CONFIRM_TIMEOUT=10 FM_ARM_ATTACH_POLL=0.1 "$WATCH_ARM" > "$armout" & armpid=$! # Synchronize on the owned child declining the live peer lock before making # the peer healthy. Sleeping for the same one-second budget as the arm made # this regression fixture race the confirmation deadline under full-suite # load, rather than testing the intended successor-handshake boundary. + # Keep the normal confirmation budget so loaded startup cannot preempt this event gate. i=0 - while [ "$i" -lt 80 ]; do + while [ "$i" -lt 200 ]; do grep -qF "watcher: already running pid $peer" "$state"/.watch-arm-output.* 2>/dev/null && break sleep 0.1 i=$((i + 1)) @@ -749,7 +758,7 @@ test_arm_waits_for_peer_beacon_after_child_stands_down() { # After the peer dies without a successor, the attached arm must fail loudly. kill "$peer" 2>/dev/null || true wait "$peer" 2>/dev/null || true - wait_for_exit "$armpid" 80 + wait_for_exit "$armpid" 200 status=$? [ "$status" -ne 0 ] && [ "$status" -ne 124 ] || fail "attached arm did not fail after peer died (status $status): $(cat "$armout")" grep -qF 'watcher: FAILED - cycle ended without an actionable reason' "$armout" || fail "peer-attached arm did not emit the typed cycle-end failure"