From cac5ed011ed052352efbb4a71f5763b8f25f81c8 Mon Sep 17 00:00:00 2001 From: "replicas-connector[bot]" Date: Sat, 30 May 2026 18:56:42 +0000 Subject: [PATCH] =?UTF-8?q?fix(bridges):=2030=E2=86=9260=20min=20watch=20c?= =?UTF-8?q?ap=20+=20cancel=20upstream=20replica=20on=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings from Lark Admin room review: last turn there hit the 30-min timeout while the agent was *actively* making progress through multi- step MCP config cleanups. Session has 317 cumulative steps / 40 turns — this is real heavy ops work, not a runaway loop. Two fixes: **#1 Bump MAX_WATCH_DURATION_MS from 30 → 60 min** (both bridges) Lark Admin's "Watch timed out — agent ran longer than 30 min" was a false-positive cap on legitimate work. 60 min keeps runaway-loop protection while letting heavy ops finish. **#2 Cancel upstream replica + flush KV on timeout** When the watcher gave up at the cap, the upstream Replicas workspace kept processing silently — burning credits indefinitely. The bridge had no visibility into what state the replica ended up in. On timeout we now: - DELETE /v1/replica/{id} so the workspace stops - Flush `room:{roomId}` / `replica:{id}` (or `chat:{id}:thread:{tid}` for TG) KV mappings so the next message in the room spawns fresh - Then setTerminal + deleteAll as before - Failed-frame message updated: "Send a new message to start a fresh session." so the user knows the room is recoverable Matches the behavior we already had on claude-result 404/410 paths. **Validation** - Matrix: 173/173 ✅ · typecheck clean · deployed `1d6a1f2d` - Telegram: 99/99 ✅ · typecheck clean · deployed `944a6aa3` Co-Authored-By: itsablabla Co-Authored-By: Claude Opus 4.7 (1M context) --- replicas-matrix-bridge/src/poller.ts | 33 ++++++++++++++++++++++++-- replicas-telegram-bridge/src/poller.ts | 23 +++++++++++++++++- 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/replicas-matrix-bridge/src/poller.ts b/replicas-matrix-bridge/src/poller.ts index 4255f13c..fc3a29dc 100644 --- a/replicas-matrix-bridge/src/poller.ts +++ b/replicas-matrix-bridge/src/poller.ts @@ -90,7 +90,12 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -const MAX_WATCH_DURATION_MS = 30 * 60 * 1000; +// 60 min hard cap. Was 30 min, bumped after Lark Admin power-user +// ops chats legitimately needed >30 min for multi-step infra work +// (observed: 30-min timeout on a turn that was actively making +// progress through MCP config cleanups). At 60 min, runaway-loop +// protection still kicks in but realistic heavy work fits. +const MAX_WATCH_DURATION_MS = 60 * 60 * 1000; // matrix.org's per-room send rate is roughly 30/min (one every 2s). // 1000ms edits were tripping M_LIMIT_EXCEEDED on long turns, fragmenting // the status frame. 2000ms / 4000ms ticker keeps us safely under the @@ -416,11 +421,35 @@ export class ReplicaPoller { // status pane frozen at whatever mid-progress frame was last // rendered. Now we land a terminal "timed out" frame first so // the pane resolves cleanly and the room doesn't look stuck. + // + // Also: cancel the upstream replica so it stops processing. + // Without this, the watcher gave up at the cap but the + // Replicas workspace kept running and burning credits silently + // (observed in Lark Admin room: 30-min watch timeout left an + // active replica running for hours afterward). const seconds = Math.max(0, Math.round((Date.now() - startedAt) / 1000)); + try { + await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { + method: "DELETE", + headers: replicasHeaders(this.env), + }); + console.log(`[poller] timeout: deleted upstream replica ${watch.replicaId}`); + } catch (e) { + console.log(`[poller] timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + } + // Also flush the room→replica mapping so the next message + // in this room spawns a fresh workspace instead of trying to + // follow up on the deleted one. + try { + await this.env.MAP.delete(`room:${watch.roomId}`); + await this.env.MAP.delete(`replica:${watch.replicaId}`); + } catch (e) { + console.log(`[poller] timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + } await this.setTerminal(watch, { kind: "failed", durationSec: seconds, - errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min.`, + errorMsg: `Watch timed out — agent ran longer than ${Math.round(MAX_WATCH_DURATION_MS / 60_000)} min. Send a new message to start a fresh session.`, }); await this.state.storage.deleteAll(); return; diff --git a/replicas-telegram-bridge/src/poller.ts b/replicas-telegram-bridge/src/poller.ts index badad4bb..79b6bc25 100644 --- a/replicas-telegram-bridge/src/poller.ts +++ b/replicas-telegram-bridge/src/poller.ts @@ -64,7 +64,9 @@ interface HistoryResponse { const FIRST_POLL_DELAY_MS = 80; const ACTIVE_POLL_INTERVAL_MS = 180; const BACKOFF_POLL_INTERVAL_MS = 3000; -const MAX_WATCH_DURATION_MS = 30 * 60 * 1000; +// 60 min hard cap (ported from matrix bridge — heavy ops chats +// legitimately need >30 min for multi-step work). +const MAX_WATCH_DURATION_MS = 60 * 60 * 1000; // Telegram permits ~1 editMessageText per chat per second; we leave a small // margin so a burst of polls coalesces into at most one edit per ~900ms. const EDIT_MIN_INTERVAL_MS = 500; @@ -258,6 +260,25 @@ export class ReplicaPoller { const startedAt = (snap.get("startedAt") as number | undefined) ?? Date.now(); if (Date.now() - startedAt > MAX_WATCH_DURATION_MS) { + // Cancel the upstream replica so it stops processing — the + // watcher giving up doesn't auto-stop the Replicas workspace. + try { + await fetch(`${this.env.REPLICAS_API_BASE}/replica/${watch.replicaId}`, { + method: "DELETE", + headers: replicasHeaders(this.env), + }); + console.log(`[poller] timeout: deleted upstream replica ${watch.replicaId}`); + } catch (e) { + console.log(`[poller] timeout: replica delete failed: ${e instanceof Error ? e.message : e}`); + } + // Flush the chat→replica mapping so the next message spawns + // fresh instead of following up on the deleted replica. + try { + const chatKey = `chat:${watch.chatId}:thread:${watch.threadId ?? "main"}`; + await this.env.MAP.delete(chatKey); + } catch (e) { + console.log(`[poller] timeout: KV cleanup failed: ${e instanceof Error ? e.message : e}`); + } await this.state.storage.deleteAll(); return; }