From 6d4a1f8103016e5fce8a0f0ecdb5d895ec7e08c5 Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Tue, 14 Jul 2026 16:03:15 +0530 Subject: [PATCH] feat(terminal): natural drag-select + scheme-less clickable links over claude's mouse-reporting TUI (v0.192.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first-party client strips the app's mouse-*tracking* DECSET modes (1000–1003) out of the PTY output stream before xterm sees them, so a plain drag selects and links are clickable like a web page — no Option-drag. Only the wheel is forwarded back to claude (as SGR events) so its conversation still scrolls. Replaces WebLinksAddon with custom link providers that match bare domains, localhost:port, www, and OSC-8 links (not file.tsx:42). Adds scripts/mouse-tui.mjs + TERMBED_CMD so the test bed can reproduce claude's mouse mode. Verified end-to-end with Playwright against the harness: drag-select, link-click, and wheel all pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 18 +++++ package-lock.json | 4 +- package.json | 2 +- scripts/mouse-tui.mjs | 62 ++++++++++++++++ scripts/termbed.mjs | 8 ++- web/src/Xterm.tsx | 161 +++++++++++++++++++++++++++++++++++++++--- web/src/termbed.tsx | 7 +- 7 files changed, 248 insertions(+), 14 deletions(-) create mode 100644 scripts/mouse-tui.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index e7901ea..f7eb101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ new version heading in the same commit. ## [Unreleased] +## [0.192.0] — 2026-07-14 +### Changed +- **The browser terminal now selects and clicks like a web page — no more Option-drag, and links work.** + claude's fullscreen TUI turns on mouse tracking, which made xterm hand every click/drag to the app: a + plain drag wouldn't select (you had to hold Option), and links couldn't be clicked. But claude only ever + uses the *wheel* (it runs with `DISABLE_MOUSE_CLICKS`) — so our first-party `` client now strips + just the mouse-*tracking* DECSET modes (1000–1003) out of the PTY output stream before xterm sees them + (native drag-select + clickable links come back), and forwards **only the wheel** back to claude as SGR + events so its conversation still scrolls. Uses documented xterm API only (no internals), so it survives + xterm upgrades. (`web/src/Xterm.tsx` `stripMouseTracking` + the custom wheel handler.) +- **Links no longer need a scheme to be clickable.** Replaced the stock `WebLinksAddon` (full `https://…` + only) with custom link providers that also match bare domains (`example.com`), subdomains, `www.…`, + `localhost:PORT`/`127.0.0.1:PORT`, and OSC-8 hyperlinks — opening the right target in a new tab — while + deliberately *not* underlining `file.tsx:42`-style tokens. (`web/src/Xterm.tsx` `makeLinkProvider`.) +- Test bed: `scripts/mouse-tui.mjs` reproduces claude's mouse-reporting condition (plain bash never did), + and `TERMBED_CMD=…` points the test bed at it — so selection/links/wheel can be verified against a real + mouse-mode app. (`scripts/termbed.mjs`, `web/src/termbed.tsx`.) + ## [0.191.0] — 2026-07-14 ### Added - **Chat — a plain-language window onto a claude-code run for non-technical teammates.** A new **Chat** diff --git a/package-lock.json b/package-lock.json index 616620f..4378b39 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.191.0", + "version": "0.192.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.191.0", + "version": "0.192.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index 71b243c..40da21a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.191.0", + "version": "0.192.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", diff --git a/scripts/mouse-tui.mjs b/scripts/mouse-tui.mjs new file mode 100644 index 0000000..21fbdba --- /dev/null +++ b/scripts/mouse-tui.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node +// A tiny fullscreen TUI that reproduces the ONE condition plain bash never did in the test bed: an app +// that turns on mouse tracking (like claude's TUI). It goes to the alternate screen, enables mouse +// reporting (1000 + SGR 1006 — exactly what makes xterm disable selection), prints a few links WITHOUT a +// scheme, and echoes every wheel event it receives back onto a status line. So in the browser terminal +// you can verify, against a real mouse-reporting app: +// • plain drag SELECTS (no Option) — our output-stream mouse-tracking stripper +// • a bare domain / localhost:port is CLICKABLE — our custom link providers +// • the WHEEL still reaches the app — our custom wheel forwarder (the counter ticks) +// +// Run it inside the test bed: TERMBED_CMD='node ../scripts/mouse-tui.mjs' node scripts/termbed.mjs +// (or just `node scripts/mouse-tui.mjs` in any terminal to eyeball the escapes). Ctrl-C / q to quit. +const out = process.stdout +const w = (s) => out.write(s) + +const ALT_ON = '\x1b[?1049h', ALT_OFF = '\x1b[?1049l' +const MOUSE_ON = '\x1b[?1000h\x1b[?1006h', MOUSE_OFF = '\x1b[?1006l\x1b[?1000l' +const HIDE = '\x1b[?25l', SHOW = '\x1b[?25h' +const home = (r, c) => `\x1b[${r};${c}H` +const clear = '\x1b[2J' + +let wheels = 0, lastEvt = '—' + +function draw() { + w(clear + home(1, 1)) + w('\x1b[1;38;5;39m mouse-tui \x1b[0m — reproduces claude\'s mouse-reporting TUI\r\n') + w('\x1b[90m mouse tracking is ON (1000+1006), so without the fix xterm would refuse to select\x1b[0m\r\n\r\n') + w(' Links to click (none carry a scheme except the first):\r\n') + w(' 1. full url https://example.com/path?q=1\r\n') + w(' 2. bare domain example.com\r\n') + w(' 3. subdomain docs.github.io/xterm\r\n') + w(' 4. localhost localhost:5199/foo\r\n') + w(' 5. www www.google.com\r\n') + w(' 6. not a link Component.tsx:42 (should NOT underline)\r\n\r\n') + w(' Drag across any line above → it should highlight and land on your clipboard.\r\n\r\n') + w(`\x1b[7m WHEEL received: ${wheels} last event: ${lastEvt} \x1b[0m\r\n`) + w('\r\n \x1b[90mscroll over the pane to tick the counter · press q or Ctrl-C to quit\x1b[0m') +} + +function cleanup(code = 0) { + w(MOUSE_OFF + SHOW + ALT_OFF) + try { process.stdin.setRawMode(false) } catch { /* not a tty */ } + process.exit(code) +} + +w(ALT_ON + HIDE + MOUSE_ON) +draw() + +try { process.stdin.setRawMode(true) } catch { /* not a tty — still readable */ } +process.stdin.resume() +process.stdin.on('data', (buf) => { + const s = buf.toString('latin1') + if (s === 'q' || s === '\x03') return cleanup(0) // q or Ctrl-C + // SGR mouse: ESC [ < btn ; col ; row (M|m). Wheel up = 64, down = 65 (bit 0x40 set). + const m = /\x1b\[<(\d+);(\d+);(\d+)([Mm])/.exec(s) + if (m) { + const btn = Number(m[1]) + if (btn === 64 || btn === 65) { wheels++; lastEvt = `${btn === 64 ? 'up' : 'down'} @ ${m[2]},${m[3]}`; draw() } + } +}) +process.on('SIGINT', () => cleanup(0)) +process.on('SIGTERM', () => cleanup(0)) diff --git a/scripts/termbed.mjs b/scripts/termbed.mjs index 43ecc4e..a059e0c 100755 --- a/scripts/termbed.mjs +++ b/scripts/termbed.mjs @@ -34,7 +34,11 @@ for (const bin of ['tmux', 'ttyd']) { // host's tmux config. (A `set -g mouse on` there would hand wheel + drag to tmux copy-mode — tmux would // scroll its own history and copy-on-select, clearing the highlight — masking the real client behaviour.) const shell = process.env.SHELL || '/bin/bash' -spawnSync('tmux', ['-f', '/dev/null', '-S', SOCK, 'new-session', '-d', '-s', SESSION, shell], { stdio: 'ignore' }) +// TERMBED_CMD lets you point the session at a mouse-reporting TUI instead of a bare shell — e.g. +// `TERMBED_CMD='node ../scripts/mouse-tui.mjs' node scripts/termbed.mjs` to reproduce claude's mouse +// mode and verify native drag-select + clickable links + wheel forwarding. Default: a login shell. +const prog = process.env.TERMBED_CMD ? ['sh', '-c', process.env.TERMBED_CMD] : [shell] +spawnSync('tmux', ['-f', '/dev/null', '-S', SOCK, 'new-session', '-d', '-s', SESSION, ...prog], { stdio: 'ignore' }) for (const opt of [ ['set', '-g', 'set-clipboard', 'on'], // forward OSC 52 copy-on-select → our client's OSC 52 handler ['set', '-g', 'allow-passthrough', 'on'], @@ -55,7 +59,7 @@ for (const opt of [ const ttyd = spawn('ttyd', [ '-p', String(TTYD_PORT), '-i', '127.0.0.1', '-b', '/pty', '-W', '-t', 'disableLeaveAlert=true', - 'tmux', '-f', '/dev/null', '-u', '-S', SOCK, 'new-session', '-A', '-s', SESSION, shell, + 'tmux', '-f', '/dev/null', '-u', '-S', SOCK, 'new-session', '-A', '-s', SESSION, ...prog, ], { stdio: 'inherit' }) ttyd.on('error', (e) => { console.error('✗ ttyd failed:', e.message); cleanup(1) }) diff --git a/web/src/Xterm.tsx b/web/src/Xterm.tsx index 84abd0d..47b3614 100644 --- a/web/src/Xterm.tsx +++ b/web/src/Xterm.tsx @@ -12,10 +12,21 @@ // '2' PAUSE · '3' RESUME // • server→client: first byte is the command — '0' OUTPUT (raw bytes) · '1' SET_WINDOW_TITLE · // '2' SET_PREFERENCES +// +// Natural selection + links over a mouse-reporting TUI. claude's TUI (fullscreen, alt-screen) turns on +// mouse tracking. The moment xterm sees a tracking mode it DISABLES its own selection service and starts +// forwarding click/drag to the app — so a plain drag no longer selects (you'd need Option held) and a +// link click gets eaten. But claude only actually uses the WHEEL (it runs with DISABLE_MOUSE_CLICKS); +// it ignores the click/drag reports entirely. So we intercept the PTY output stream and strip just the +// mouse-*tracking* DECSET modes (1000/1001/1002/1003) before xterm sees them: xterm stays in its normal, +// no-mouse state (plain drag selects, links are clickable like a web page), and we forward ONLY the +// wheel back to claude ourselves (as SGR mouse events) so its conversation still scrolls. claude's own +// mouse state is unchanged — it enabled tracking on its side and happily parses the wheel events we send; +// it just never gets clicks, which it discards anyway. Uses only documented xterm API (no internals), so +// it survives xterm upgrades. See `stripMouseTracking` + the custom wheel handler below. import { useEffect, useRef } from 'react' -import { Terminal, type ITheme } from '@xterm/xterm' +import { Terminal, type ITheme, type ILink, type ILinkProvider } from '@xterm/xterm' import { FitAddon } from '@xterm/addon-fit' -import { WebLinksAddon } from '@xterm/addon-web-links' import { SearchAddon } from '@xterm/addon-search' import { CanvasAddon } from '@xterm/addon-canvas' import '@xterm/xterm/css/xterm.css' @@ -29,6 +40,109 @@ const S_OUTPUT = 48 // '0' const S_TITLE = 49 // '1' const S_PREFS = 50 // '2' +// ── mouse-tracking stripper ────────────────────────────────────────────────────────────────────────── +// The DECSET private modes that make xterm switch into "forward the mouse to the app" (and thus disable +// selection). We drop these from the output stream; every other private mode — SGR encoding (1006), +// focus (1004), alt-screen (1049), bracketed paste (2004), … — passes through untouched. +const TRACK_MODES = new Set([1000, 1001, 1002, 1003]) +const ESC = 0x1b, LBRACK = 0x5b, QUEST = 0x3f, SET = 0x68 /* h */, RESET = 0x6c /* l */ + +/** A stateful filter over the raw PTY byte stream that removes mouse-tracking DECSET/DECRST sequences + * (`ESC [ ? …h|l`) while preserving all other output verbatim, and reports (via `onWant`) whether the + * app currently wants the mouse — so the wheel handler knows to forward. Handles a sequence split across + * WebSocket frames by carrying an unfinished tail to the next call. */ +function stripMouseTracking(onWant: (want: boolean) => void): (input: Uint8Array) => Uint8Array { + let carry = new Uint8Array(0) + return (input: Uint8Array): Uint8Array => { + let buf = input + if (carry.length) { const j = new Uint8Array(carry.length + input.length); j.set(carry); j.set(input, carry.length); buf = j; carry = new Uint8Array(0) } + const n = buf.length + const out = new Uint8Array(n) + let o = 0, i = 0 + while (i < n) { + const b = buf[i] + if (b !== ESC) { out[o++] = b; i++; continue } + // Need ESC '[' '?' to be a private mode set/reset; if the buffer ends mid-prefix, hold it. + if (i + 2 >= n) { if (buf[i + 1] === LBRACK || i + 1 >= n) { carry = buf.slice(i); break } out[o++] = b; i++; continue } + if (buf[i + 1] !== LBRACK || buf[i + 2] !== QUEST) { out[o++] = b; i++; continue } + // Scan numeric params (digits + ';') then a final byte. + let j = i + 3 + let params = '' + while (j < n) { const c = buf[j]; if ((c >= 0x30 && c <= 0x39) || c === 0x3b) { params += String.fromCharCode(c); j++ } else break } + if (j >= n) { carry = buf.slice(i); break } // incomplete — wait for the rest + const fin = buf[j] + if (fin !== SET && fin !== RESET) { for (let k = i; k <= j; k++) out[o++] = buf[k]; i = j + 1; continue } // e.g. ?…$p — pass through + const modes = params.split(';').filter((s) => s.length) + const kept = modes.filter((m) => !TRACK_MODES.has(Number(m))) + if (modes.some((m) => TRACK_MODES.has(Number(m)))) onWant(fin === SET) + if (kept.length) { out[o++] = ESC; out[o++] = LBRACK; out[o++] = QUEST; const s = kept.join(';'); for (let k = 0; k < s.length; k++) out[o++] = s.charCodeAt(k); out[o++] = fin } + i = j + 1 + } + if (carry.length > 64) { for (let k = 0; k < carry.length; k++) out[o++] = carry[k]; carry = new Uint8Array(0) } // runaway guard + return o === n ? out : out.subarray(0, o) + } +} + +// ── link detection ─────────────────────────────────────────────────────────────────────────────────── +// Matchers run in priority order; earlier matches win over later ones on the same columns, so a full URL +// isn't also picked up as a bare domain. The TLD list is deliberately conservative so `file.ts:42` or +// `Component.tsx` don't get underlined as if they were `example.com`. +const TLD = 'com|net|org|io|ai|dev|app|sh|co|gg|xyz|me|so|to|tv|cloud|tech|edu|gov|info|biz|us|uk|ca|de|fr|jp|in' +const TAIL = `[^\\s"'\`<>)\\]}]` // a link body char — stops at whitespace and common wrappers +const MATCHERS: { re: RegExp; url: (m: string) => string }[] = [ + { re: new RegExp(`(?:https?|file)://${TAIL}+`, 'gi'), url: (m) => m }, + { re: new RegExp(`\\bwww\\.${TAIL}+`, 'gi'), url: (m) => `https://${m}` }, + { re: new RegExp(`\\b(?:localhost|127\\.0\\.0\\.1)(?::\\d+)?(?:/${TAIL}*)?`, 'gi'), url: (m) => `http://${m}` }, + { re: new RegExp(`\\b[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\\.[a-z0-9-]+)*\\.(?:${TLD})(?::\\d+)?(?:/${TAIL}*)?`, 'gi'), url: (m) => `https://${m}` }, +] + +function openUrl(url: string) { try { window.open(url, '_blank', 'noopener,noreferrer') } catch { /* popup blocked */ } } + +/** Trim trailing sentence punctuation that a link body regex greedily swallowed (`…example.com.` → drop + * the period), returning how many chars were shaved so the highlight range can shrink to match. */ +function trimTrail(s: string): { text: string; shaved: number } { + const m = s.match(/[.,;:!?)\]}'"]+$/) + const shaved = m ? m[0].length : 0 + return { text: shaved ? s.slice(0, -shaved) : s, shaved } +} + +/** A link provider spanning the four matchers above over a single buffer row. xterm asks per row; we read + * that row's text, find every non-overlapping match, and hand back clickable ranges (1-based, inclusive) + * that open in a new tab. Wrapped links that span rows are matched on whichever row they start — good + * enough for the URLs claude prints; keeps this simple and allocation-light. */ +function makeLinkProvider(term: Terminal): ILinkProvider { + return { + provideLinks(row, cb) { + const line = term.buffer.active.getLine(row - 1) + if (!line) return cb(undefined) + const text = line.translateToString(true) + if (!text || !/[.:/]/.test(text)) return cb(undefined) + const taken: boolean[] = [] + const links: ILink[] = [] + for (const { re, url } of MATCHERS) { + re.lastIndex = 0 + let m: RegExpExecArray | null + while ((m = re.exec(text))) { + const start = m.index + const { text: raw, shaved } = trimTrail(m[0]) + const end = start + raw.length // exclusive + if (raw.length < 4 || taken[start] || taken[end - 1]) continue + for (let k = start; k < end; k++) taken[k] = true + const target = url(raw) + links.push({ + text: raw, + range: { start: { x: start + 1, y: row }, end: { x: end, y: row } }, + decorations: { pointerCursor: true, underline: true }, + activate: (e) => { e.preventDefault(); openUrl(target) }, + }) + if (shaved && re.lastIndex > end) re.lastIndex = end // don't skip a link glued right after + } + } + cb(links.length ? links : undefined) + }, + } +} + const enc = new TextEncoder() /** Write text to the clipboard in BOTH secure and insecure contexts. `navigator.clipboard` only exists @@ -120,18 +234,22 @@ export function Xterm({ theme, cursorBlink: true, scrollback: 10000, - // Copy/paste ergonomics — the reason we own the frontend: - // hold Option (mac) / any drag with this on lets you select even while claude's mouse-reporting is - // active; right-click pastes the OS clipboard. + // Selection ergonomics — the reason we own the frontend. We strip the app's mouse-tracking modes + // (see `stripMouseTracking`), so a plain drag already selects with no modifier; Option-drag stays a + // harmless fallback in case some future app enables a mode we don't strip. macOptionClickForcesSelection: true, rightClickSelectsWord: false, allowProposedApi: true, + // OSC 8 hyperlinks (ESC ] 8 ; ; ) — claude and modern CLIs emit these; open them in a new tab. + linkHandler: { activate: (_e, uri) => openUrl(uri), allowNonHttpProtocols: false }, }) const fit = new FitAddon() const search = new SearchAddon() term.loadAddon(fit) term.loadAddon(search) - term.loadAddon(new WebLinksAddon()) + // Clickable links for bare domains / localhost:port / www / full URLs (our own provider — the stock + // WebLinksAddon only matches full `https://…`, missing exactly the "rendered link without a scheme"). + term.registerLinkProvider(makeLinkProvider(term)) term.open(host) // Canvas renderer: draw the grid to a instead of the default DOM renderer, whose real // per-cell text is natively selectable by the browser — that native selection competes with xterm's @@ -170,6 +288,30 @@ export function Xterm({ return true }) + // ── mouse forwarding ─────────────────────────────────────────────────────────────────────────── + // We strip the app's mouse-tracking from the output stream (below), so xterm no longer forwards the + // wheel to the app on its own. Re-supply just the wheel: while the app WANTS the mouse (it emitted a + // tracking DECSET — claude's TUI, or tmux at a bare prompt), translate each wheel notch into SGR + // wheel-button events (button 64 up / 65 down) and send them down, so claude scrolls its conversation + // and tmux scrolls its scrollback exactly as if we'd never stripped the mode. When no app wants the + // mouse, return true to let xterm do its own local scrollback scroll. + let appMouseWanted = false + term.attachCustomWheelEventHandler((e) => { + if (!appMouseWanted || e.deltaY === 0) return true + const rect = host.getBoundingClientRect() + const col = Math.min(term.cols, Math.max(1, Math.floor(((e.clientX - rect.left) / rect.width) * term.cols) + 1)) + const rowY = Math.min(term.rows, Math.max(1, Math.floor(((e.clientY - rect.top) / rect.height) * term.rows) + 1)) + const btn = e.deltaY < 0 ? 64 : 65 + const notches = e.deltaMode === 1 ? Math.abs(e.deltaY) // lines + : e.deltaMode === 2 ? Math.abs(e.deltaY) * term.rows // pages + : Math.max(1, Math.round(Math.abs(e.deltaY) / 40)) // pixels + let seq = '' + for (let k = 0; k < Math.min(notches, 8); k++) seq += `\x1b[<${btn};${col};${rowY}M` + send(C_INPUT + seq) + return false // handled — don't let xterm also scroll its (empty, on the alt-screen) local buffer + }) + const mouseFilter = stripMouseTracking((w) => { appMouseWanted = w }) + // ── socket ─────────────────────────────────────────────────────────────────────────────────── const url = wsUrl.startsWith('/') ? `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}${wsUrl}` @@ -203,9 +345,12 @@ export function Xterm({ const cmd = view[0] const body = view.subarray(1) if (cmd === S_OUTPUT) { - pending += body.length + // Strip the app's mouse-tracking DECSET modes before xterm sees them (keeps native drag-select + + // clickable links) — see stripMouseTracking. Flow-control accounts for the cleaned length. + const clean = mouseFilter(body) + pending += clean.length if (pending > HIGH) send(C_PAUSE) - term.write(body, () => { pending -= body.length; if (pending <= HIGH) send(C_RESUME) }) + term.write(clean, () => { pending -= clean.length; if (pending <= HIGH) send(C_RESUME) }) } else if (cmd === S_TITLE) { cbs.current.onTitle?.(new TextDecoder().decode(body)) } else if (cmd === S_PREFS) { diff --git a/web/src/termbed.tsx b/web/src/termbed.tsx index 1ff485b..5492eff 100644 --- a/web/src/termbed.tsx +++ b/web/src/termbed.tsx @@ -15,7 +15,12 @@ function TermBed() { const [menu, setMenu] = useState<{ x: number; y: number } | null>(null) const [copied, setCopied] = useState(false) const h = useRef(null) - const onReady = useCallback((handle: XtermHandle) => { h.current = handle }, []) + const onReady = useCallback((handle: XtermHandle) => { + h.current = handle + // Test hook: expose the live Terminal so the Playwright harness can assert selection/buffer state + // (the canvas renderer draws no readable DOM). Test bed only — never mounted in the real console. + ;(window as unknown as { __aosTerm?: unknown }).__aosTerm = handle.term + }, []) const copyTimer = useRef | undefined>(undefined) const onCopy = useCallback(() => { setCopied(true)