From 6dd01f43ce28a8daa679c93515146b1bfc8c6795 Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Mon, 13 Jul 2026 14:35:20 +0530 Subject: [PATCH 1/2] fix(session): reap unattended runs that end via `report` (v0.141.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An automation/cron/task agent that finishes by calling `report` flips its session row to `done` mid-turn. By the time the turn-end Stop beacon reached `markTurnIdle`, the status was already terminal, so teardown bailed on `status !== 'running'` and the interactive TUI pane kept running forever — dozens of orphaned tmux panes + claude processes piled up on the live fleet (some 20h+ old, all belonging to `done` sessions; `session.reaped` had fired only once). - markTurnIdle: accept `done` as well as `running` for unattended, non-resident runs, and reap a still-alive pane (honoring claimed / attached / blocked). A liveness poll skips an already-dead pane so a stray second beacon can't re-reap. - reapIdleSessions backstop: also sweep `done` unattended rows whose pane is still alive (self-heals orphans predating the fix / lost Stop beacons); only ever acts on a genuinely-alive pane, so a cleanly-reaped row is never re-killed/re-audited, and nothing outside term_sessions (non-agent-os tmux sessions) is ever touched. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 13 +++++++++++- package-lock.json | 4 ++-- package.json | 2 +- src/terminal.ts | 50 +++++++++++++++++++++++++++++++++++------------ 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e700e66..611968e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,18 @@ new version heading in the same commit. ## [Unreleased] -## [0.141.0] — 2026-07-13 +## [0.141.1] — 2026-07-13 +### Fixed +- **Unattended (automation/cron/task) sessions no longer leak a live pane after they finish.** An agent that + ends by calling `report` flips its row to `done` mid-turn, so by the time the turn-end Stop beacon reached + `markTurnIdle` the status was already terminal and the teardown bailed on `status !== 'running'` — the + interactive TUI kept running forever (observed: dozens of orphaned tmux panes + claude processes, some 20h+ + old, all belonging to `done` sessions; `session.reaped` had fired only once). `markTurnIdle` now reaps an + unattended run whose pane is still alive even when `report` already marked it `done` (still honoring + claimed / attached / blocked-on-human), skipping an already-dead pane via a liveness poll so a stray second + beacon can't re-reap. The idle backstop (`reapIdleSessions`) likewise sweeps `done` unattended rows that + still hold a live pane — cleaning up any that predate the fix or whose Stop beacon never landed. Net: a + finished background run stops for real, and the session list stops showing it as "live". ### Added - **Settings → System now shows live host resources.** A new "Host resources" panel reports memory used/free/total with a usage bar, CPU utilization (sampled server-side) + core count + model + load diff --git a/package-lock.json b/package-lock.json index 5edade1..c20c91c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.141.0", + "version": "0.141.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.141.0", + "version": "0.141.1", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index bb65de7..9b123f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.141.0", + "version": "0.141.1", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", diff --git a/src/terminal.ts b/src/terminal.ts index db8fa33..eb1779d 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -1148,17 +1148,31 @@ export class TerminalManager { } catch { /* one bad row must not stop the sweep */ } } - // (2) unattended turn-end backstop. `last_activity IS NOT NULL` = a turn-end beacon has fired, so this - // never reaps a run still working its first turn (that row's last_activity is NULL until markTurnIdle). - const stragglers = this.db.prepare("SELECT id, tmux, run_as, spawned_by, agent FROM term_sessions WHERE headless = 1 AND resident = 0 AND claimed_by IS NULL AND status = 'running' AND last_activity IS NOT NULL AND last_activity < ?") - .all<{ id: string; tmux: string; run_as: string | null; spawned_by: string | null; agent: string }>(cutoff); - for (const r of stragglers) { - try { - const space = this.spaceFor(r.run_as ?? r.spawned_by); - if (this.backend.hasClient(space, r.tmux) === true) continue; // a human is still watching — leave it - if (this.hasPendingHumanBlock(r.id)) continue; // blocked on an answer/approval — keep alive - this.teardownUnattended(r.id, space, r.tmux, 'idle-backstop'); - } catch { /* one bad row must not stop the sweep */ } + // (2) unattended backstop — the safety net for markTurnIdle. Two ways an unattended run leaks a live pane: + // (a) DONE ORPHAN — it ended via `report`, which flips the row to 'done' before the Stop beacon lands, so + // markTurnIdle used to bail. Its pane must die regardless of idle time; a done run should hold no TUI. + // (markTurnIdle now reaps these directly; this sweep also clears any that predate the fix or whose + // Stop beacon never landed.) + // (b) IDLE STRAGGLER — still 'running' with a turn-end beacon seen (`last_activity` set) and idle past the + // timeout: the classic case where the Stop beacon was lost or a human attached then detached. + // Poll TRUE pane liveness once and only ever act on a pane that's ACTUALLY alive, so a cleanly-reaped row + // is never re-killed / re-audited on a later tick (a `done` row keeps its status forever once torn down). + const alive = this.backend.aliveNames(); + if (alive) { + const unattended = this.db.prepare("SELECT id, tmux, run_as, spawned_by, agent, status, last_activity FROM term_sessions WHERE headless = 1 AND resident = 0 AND claimed_by IS NULL AND status IN ('running','done')") + .all<{ id: string; tmux: string; run_as: string | null; spawned_by: string | null; agent: string; status: string; last_activity: number | null }>(); + for (const r of unattended) { + try { + if (!alive.has(r.tmux)) continue; // pane already gone — nothing to reap + // a 'running' straggler is only idle-reaped once it has seen a turn-end beacon AND gone quiet past the + // cutoff; a 'done' orphan is reaped on sight — it should never still be holding an interactive pane. + if (r.status === 'running' && (r.last_activity == null || r.last_activity >= cutoff)) continue; + const space = this.spaceFor(r.run_as ?? r.spawned_by); + if (this.backend.hasClient(space, r.tmux) === true) continue; // a human is still watching — leave it + if (this.hasPendingHumanBlock(r.id)) continue; // blocked on an answer/approval — keep alive + this.teardownUnattended(r.id, space, r.tmux, r.status === 'done' ? 'done-orphan' : 'idle-backstop'); + } catch { /* one bad row must not stop the sweep */ } + } } } @@ -1168,18 +1182,28 @@ export class TerminalManager { * and it isn't blocked on a person, close it NOW — capture the transcript, mark it done, kill the pane so * the automations pile-up guard releases immediately (parity with the old `claude -p` exit). Otherwise * (claimed / attached / blocked) it stays a live TUI; we only stamp the turn-end time so the idle backstop - * has a clock. No-op for interactive/resident runs and for anything not running. + * has a clock. No-op for interactive/resident runs. + * + * We accept a status of BOTH 'running' AND 'done': an agent that ends by calling `report` (the fleet's + * automation prompts all do) flips its row to 'done' MID-turn, so by the time this turn-end beacon lands + * the status is already terminal — but the interactive TUI pane is still live and MUST be reaped, else it + * leaks a claude process forever (the row reads `done` while its pane keeps running). Before the fix this + * bailed on `status !== 'running'` and orphaned every report-ended unattended run. A truly torn-down run + * (pane already gone) is skipped via the liveness poll below, so a stray second beacon can't re-reap. */ markTurnIdle(sessionId: string): void { const r = this.db.prepare('SELECT tmux, status, headless, resident, claimed_by, run_as, spawned_by FROM term_sessions WHERE id = ?') .get<{ tmux: string; status: string; headless: number; resident: number; claimed_by: string | null; run_as: string | null; spawned_by: string | null }>(sessionId); - if (!r || r.status !== 'running' || !r.headless || r.resident) return; // only unattended, still-running runs + if (!r || !r.headless || r.resident) return; // only unattended, non-resident runs + if (r.status !== 'running' && r.status !== 'done') return; // stopped/crashed are already torn down // Record the turn-end time regardless of the decision below — it's the idle backstop's clock and the // signal that this run has completed at least one turn (so the backstop won't reap a mid-turn run). this.db.prepare('UPDATE term_sessions SET last_activity = ? WHERE id = ?').run(Date.now(), sessionId); if (r.claimed_by) return; // taken over → sticky, the human owns its lifecycle if (this.hasPendingHumanBlock(sessionId)) return; // waiting on an answer/approval → keep the pane alive const space = this.spaceFor(r.run_as ?? r.spawned_by); + const alive = this.backend.aliveNames(); + if (alive && !alive.has(r.tmux)) return; // pane already gone (already reaped) — nothing to do if (this.backend.hasClient(space, r.tmux) === true) return; // a human is watching live → don't close on them this.teardownUnattended(sessionId, space, r.tmux, 'turn-end'); } From 5d1768a81f22f3a9b4f67c81e50d3805dfad75b9 Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Mon, 13 Jul 2026 14:37:21 +0530 Subject: [PATCH 2/2] fix(session): keep launcher-backend straggler reaping when liveness unpollable The backstop's 'done'-orphan sweep needs a tmux liveness poll, but aliveNames() returns null on the Linux LauncherSessionBackend (and transient local failures). Gating the whole sweep on a non-null poll would regress running-straggler reaping on launcher backends. Fall back to the classic time-based running-only rule when liveness can't be polled; sweep 'done' orphans only when it can (per docs/concurrency-cap-plan.md, which flagged the aliveNames()===null fail-open). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/terminal.ts | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/terminal.ts b/src/terminal.ts index eb1779d..bb4f684 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -1152,27 +1152,34 @@ export class TerminalManager { // (a) DONE ORPHAN — it ended via `report`, which flips the row to 'done' before the Stop beacon lands, so // markTurnIdle used to bail. Its pane must die regardless of idle time; a done run should hold no TUI. // (markTurnIdle now reaps these directly; this sweep also clears any that predate the fix or whose - // Stop beacon never landed.) + // Stop beacon never landed.) Only detectable when we can poll liveness (see below). // (b) IDLE STRAGGLER — still 'running' with a turn-end beacon seen (`last_activity` set) and idle past the // timeout: the classic case where the Stop beacon was lost or a human attached then detached. - // Poll TRUE pane liveness once and only ever act on a pane that's ACTUALLY alive, so a cleanly-reaped row - // is never re-killed / re-audited on a later tick (a `done` row keeps its status forever once torn down). + // `aliveNames()` returns the live tmux set, or NULL when the backend can't report liveness (the Linux + // LauncherSessionBackend always; a transient local poll failure). When we CAN poll, gate on true pane + // liveness so a cleanly-reaped row is never re-killed / re-audited on a later tick (a `done` row keeps its + // status forever once torn down) — this is what lets us sweep 'done' orphans safely. When we CAN'T, fall + // back to the classic time-based rule for RUNNING rows only (never blind-sweep a 'done' row, or we'd + // re-teardown it every tick with no way to know its pane already died). const alive = this.backend.aliveNames(); - if (alive) { - const unattended = this.db.prepare("SELECT id, tmux, run_as, spawned_by, agent, status, last_activity FROM term_sessions WHERE headless = 1 AND resident = 0 AND claimed_by IS NULL AND status IN ('running','done')") - .all<{ id: string; tmux: string; run_as: string | null; spawned_by: string | null; agent: string; status: string; last_activity: number | null }>(); - for (const r of unattended) { - try { - if (!alive.has(r.tmux)) continue; // pane already gone — nothing to reap + const unattended = this.db.prepare("SELECT id, tmux, run_as, spawned_by, agent, status, last_activity FROM term_sessions WHERE headless = 1 AND resident = 0 AND claimed_by IS NULL AND status IN ('running','done')") + .all<{ id: string; tmux: string; run_as: string | null; spawned_by: string | null; agent: string; status: string; last_activity: number | null }>(); + for (const r of unattended) { + try { + if (alive) { + if (!alive.has(r.tmux)) continue; // pane already gone — nothing to reap // a 'running' straggler is only idle-reaped once it has seen a turn-end beacon AND gone quiet past the // cutoff; a 'done' orphan is reaped on sight — it should never still be holding an interactive pane. if (r.status === 'running' && (r.last_activity == null || r.last_activity >= cutoff)) continue; - const space = this.spaceFor(r.run_as ?? r.spawned_by); - if (this.backend.hasClient(space, r.tmux) === true) continue; // a human is still watching — leave it - if (this.hasPendingHumanBlock(r.id)) continue; // blocked on an answer/approval — keep alive - this.teardownUnattended(r.id, space, r.tmux, r.status === 'done' ? 'done-orphan' : 'idle-backstop'); - } catch { /* one bad row must not stop the sweep */ } - } + } else { + // no liveness signal: classic straggler rule, running-only, so we can't re-sweep a done row blind. + if (r.status !== 'running' || r.last_activity == null || r.last_activity >= cutoff) continue; + } + const space = this.spaceFor(r.run_as ?? r.spawned_by); + if (this.backend.hasClient(space, r.tmux) === true) continue; // a human is still watching — leave it + if (this.hasPendingHumanBlock(r.id)) continue; // blocked on an answer/approval — keep alive + this.teardownUnattended(r.id, space, r.tmux, r.status === 'done' ? 'done-orphan' : 'idle-backstop'); + } catch { /* one bad row must not stop the sweep */ } } }