From 995d94ec98bbd0dea1d2c364fe80493f625ca53f Mon Sep 17 00:00:00 2001 From: Cho Young-Hwi Date: Tue, 7 Jul 2026 00:56:51 +0000 Subject: [PATCH] [#957] Lifecycle-aware cleanup of agent backend temp dirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code keeps temp under /tmp/claude-{uid} and never bounds it. On Linux hosts where /tmp has a per-user quota, that dir grows until the quota is exhausted, after which every Claude bash call fails silently (exit 1, no output). QuadWork owns the agent lifecycle, so it cleans at known-safe moments. - server/temp-cleanup.js: pure, never-throwing stale sweep. Deletion is strictly age-based (max mtime/atime/ctime vs cutoff), because /tmp/claude-{uid} is shared across all of a user's claude agents and per-entry ownership isn't resolvable — so a live session's freshly-touched files are always spared. Covers claude-{uid} entries + stray gemini crash dumps; codex state lives under ~/.codex (no /tmp sweep). Config-gated via temp_cleanup.{enabled,max_age_hours} (default enabled, 72h). - server/index.js: sweep on agent teardown (deferred, stale-only) and on an hourly timer + one at boot. Timer handle captured and cleared in shutdown() per #972 clean-shutdown. Agent spawn/PTY path untouched. - Tests: stale removed, live-session file preserved, gemini dumps, missing dir / null uid no-op, never-throws, settings resolution. - Docs: troubleshooting.md (silent-exit-1 post-mortem) + install-vps.md note. Reimplemented fresh against current main (old PR #960 bitrotted after #968/#972/#975 reworked index.js). Co-Authored-By: Claude Opus 4.8 --- docs/install-vps.md | 9 +++ docs/troubleshooting.md | 26 ++++++++ server/index.js | 43 ++++++++++++ server/temp-cleanup.js | 130 ++++++++++++++++++++++++++++++++++++ server/temp-cleanup.test.js | 103 ++++++++++++++++++++++++++++ 5 files changed, 311 insertions(+) create mode 100644 server/temp-cleanup.js create mode 100644 server/temp-cleanup.test.js diff --git a/docs/install-vps.md b/docs/install-vps.md index 41080fd1..9ed61572 100644 --- a/docs/install-vps.md +++ b/docs/install-vps.md @@ -362,3 +362,12 @@ chmod 600 ~/.quadwork/.env 14. Configure nginx reverse proxy + SSL 15. Add HTTP basic auth 16. Verify reboot survival: `sudo reboot`, then check `pm2 list` + +## Note: /tmp quotas and Claude temp + +Some VPS images mount `/tmp` with a per-user quota (`usrquota`). Claude Code +accumulates temp under `/tmp/claude-{uid}`; if the quota fills up, every +Claude bash command starts failing silently with exit 1 (see +[troubleshooting](troubleshooting.md#every-claude-bash-command-fails-silently-exit-1-no-output)). +QuadWork sweeps stale entries automatically (hourly + on agent teardown, +72h age) — configurable via `temp_cleanup` in `~/.quadwork/config.json`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 5ec355b2..ba15d5f5 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -226,3 +226,29 @@ existing network controls (127.0.0.1 bind + SSH tunnel, tailnet ACL, or the nginx Basic Auth from the VPS guide). The token + Origin allowlist are defence-in-depth against browser-based cross-origin attacks, not a substitute for network-level access control. + +## Every Claude bash command fails silently (exit 1, no output) + +**Symptom:** every bash command a `claude` agent runs — even `true` — returns +exit 1 with empty stdout/stderr. The Read tool still works, a plain shell on +the host works, and `codex` agents are unaffected. Easily mistaken for a +bwrap/AppArmor/sandbox failure. + +**Cause:** Claude Code keeps its temp under `/tmp/claude-{uid}` and never +cleans it up. On hosts where `/tmp` is mounted with a per-user quota +(`usrquota`), that dir grows until the quota is exhausted — after which Claude +can't write the temp files it needs before executing any command (#957). + +**Check:** `du -sh /tmp/claude-$(id -u)` and try `dd if=/dev/zero +of=/tmp/probe bs=1M count=10` — a "Disk quota exceeded" error confirms it. + +**Fix:** QuadWork sweeps stale backend temp automatically (hourly, at boot, +and on agent teardown; entries older than 72h). If you hit the quota *before* +a sweep (e.g. the server was down), clear it manually: +`find /tmp/claude-$(id -u)/* -maxdepth 0 -mmin +60 -exec rm -rf {} +` + +Tune or disable via `~/.quadwork/config.json`: + +```json +{ "temp_cleanup": { "enabled": true, "max_age_hours": 72 } } +``` diff --git a/server/index.js b/server/index.js index 58ae4de9..8022b703 100644 --- a/server/index.js +++ b/server/index.js @@ -12,6 +12,7 @@ const fileChat = require("./file-chat"); const { dispatchToAgentPTY, cleanupSession: cleanupPtyDispatcher } = require("./pty-dispatcher"); const { runAcMigration } = require("./migrate-ac"); const selfHeal = require("./self-heal"); +const tempCleanup = require("./temp-cleanup"); // #957: stale backend-temp sweep const { injectModeForCommand } = require("../src/lib/injectMode.js"); const telegramBridge = require("./bridges/telegram"); // #972: stop on shutdown const discordBridge = require("./bridges/discord"); // #972: stop on shutdown @@ -789,6 +790,37 @@ async function stopAgentSession(key, { clearSelfHeal = false } = {}) { session.exitedUnexpectedly = false; const [projectId, agentId] = key.split("/"); if (projectId && agentId) stopMcpProxy(projectId, agentId); + // #957: teardown is a known-safe moment to sweep stale backend temp. It's + // deferred (setImmediate) so it never blocks the stop path, and stale-only, + // so it never touches files a still-live agent on the shared /tmp/claude-{uid} + // is using. This only reads a session's OWN temp when that session is gone. + setImmediate(backendTempSweepTick); +} + +// #957: sweep stale backend temp entries (/tmp/claude-{uid}, stray gemini +// crash dumps). On hosts where /tmp has a per-user quota, unbounded Claude +// temp eventually exhausts it and every Claude bash call fails silently with +// exit 1 — see issue #957 for the full post-mortem. Runs on agent teardown and +// on an hourly timer; opt-out / age via config.json +// `temp_cleanup: { enabled, max_age_hours }`. The reentrancy guard stops a +// teardown-triggered sweep from overlapping the periodic one. +let _tempSweepRunning = false; +function backendTempSweepTick() { + if (_tempSweepRunning) return; + _tempSweepRunning = true; + try { + const settings = tempCleanup.cleanupSettings(readConfig()); + if (!settings.enabled) return; + const r = tempCleanup.sweepBackendTemp({ maxAgeHours: settings.maxAgeHours }); + if (r.removed.length > 0) { + console.log(`[temp-cleanup] removed ${r.removed.length} stale entr${r.removed.length === 1 ? "y" : "ies"} (kept ${r.kept})`); + } + for (const e of r.errors) console.error(`[temp-cleanup] ${e}`); + } catch (err) { + console.error(`[temp-cleanup] sweep failed: ${err.message}`); + } finally { + _tempSweepRunning = false; + } } app.get("/api/agents", (_req, res) => { @@ -2029,6 +2061,16 @@ if (!process.env.QUADWORK_SKIP_LISTEN) { _reseedRetryHandle = setInterval(reseedRetryTick, RESEED_RETRY_INTERVAL_MS); } +// #957: hourly stale-temp sweep, plus one at boot — a server that was down for +// days should reclaim quota immediately, not an hour after start. The handle is +// captured so shutdown() (#972) clears the timer on Ctrl+C / full reset. +const TEMP_SWEEP_INTERVAL_MS = 60 * 60 * 1000; +let _tempSweepHandle = null; +if (!process.env.QUADWORK_SKIP_LISTEN) { + setImmediate(backendTempSweepTick); + _tempSweepHandle = setInterval(backendTempSweepTick, TEMP_SWEEP_INTERVAL_MS); +} + // #422 / quadwork#310: auto-continue after loop guard. // // Per opted-in project, poll AC's /api/status every 10s. When we see @@ -2272,6 +2314,7 @@ function shutdown() { // Polling + watchdog timers. if (_autoStopHandle) { clearInterval(_autoStopHandle); _autoStopHandle = null; } if (_reseedRetryHandle) { clearInterval(_reseedRetryHandle); _reseedRetryHandle = null; } + if (_tempSweepHandle) { clearInterval(_tempSweepHandle); _tempSweepHandle = null; } // #957 if (_watchdogHandle) { clearInterval(_watchdogHandle); _watchdogHandle = null; } // Trigger schedulers (clears each trigger's interval + duration timers). diff --git a/server/temp-cleanup.js b/server/temp-cleanup.js new file mode 100644 index 00000000..c432ff96 --- /dev/null +++ b/server/temp-cleanup.js @@ -0,0 +1,130 @@ +"use strict"; + +// #957: lifecycle-aware cleanup of agent backend temp dirs. +// +// Claude Code overrides TMPDIR to /tmp/claude-{uid} and never cleans it up. +// On Linux hosts where /tmp carries a per-user quota (usrquota), that dir +// accumulates until the quota is exhausted — at which point EVERY Claude bash +// call fails silently (exit 1, no output) because Claude can't write its +// temp/sandbox-setup files before exec. Gemini similarly strands +// /tmp/gemini-client-error-*.json crash dumps. Codex keeps its state under +// ~/.codex (not /tmp) so it needs no sweep here. +// +// Design constraints (the #957 "never delete a live session's temp" invariant): +// - /tmp/claude-{uid} is SHARED by every claude agent of this user, so a +// teardown of one agent must never delete another live agent's files. +// Per-entry ownership isn't resolvable at the dir level, so deletion is +// strictly age-based ("stale entries only") — the same sweep is therefore +// safe from any call site, teardown OR timer. +// - Age uses max(mtime, atime, ctime) like systemd-tmpfiles, so anything a +// live session still touches stays "fresh" and is spared. +// - Never throws: a cleanup bug must not break agent teardown or the server +// boot path. Errors are collected and reported in the result. +// - Cross-platform no-op where it doesn't apply: Windows has no +// process.getuid → the claude dir is skipped; the gemini glob simply +// matches nothing where those files don't exist. + +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +const DEFAULT_MAX_AGE_HOURS = 72; + +// Newest of mtime/atime/ctime — "last time anything observed this entry". +function newestTimeMs(st) { + return Math.max(st.mtimeMs || 0, st.atimeMs || 0, st.ctimeMs || 0); +} + +// Remove `entry` (file or dir) if its newest timestamp is older than the +// cutoff. Returns "removed" | "kept" | "error". +function removeIfStale(entry, cutoffMs, result) { + let st; + try { + st = fs.lstatSync(entry); + } catch { + return "kept"; // raced away — already gone + } + if (newestTimeMs(st) >= cutoffMs) { + result.kept += 1; + return "kept"; + } + try { + fs.rmSync(entry, { recursive: true, force: true }); + result.removed.push(entry); + return "removed"; + } catch (err) { + result.errors.push(`${entry}: ${err.message}`); + return "error"; + } +} + +// Sweep stale backend temp entries. Options (all injectable for tests): +// maxAgeHours - entries with no mtime/atime/ctime newer than this are removed +// tmpRoot - temp root (default os.tmpdir()) +// uid - numeric uid for the claude-{uid} dir (default process.getuid) +// now - epoch ms (default Date.now()) +// Returns { removed: string[], kept: number, errors: string[] }. +function sweepBackendTemp(opts = {}) { + const result = { removed: [], kept: 0, errors: [] }; + try { + const maxAgeHours = Number.isFinite(opts.maxAgeHours) && opts.maxAgeHours > 0 + ? opts.maxAgeHours + : DEFAULT_MAX_AGE_HOURS; + const tmpRoot = opts.tmpRoot || os.tmpdir(); + const now = typeof opts.now === "number" ? opts.now : Date.now(); + const cutoffMs = now - maxAgeHours * 60 * 60 * 1000; + + // Claude: entries directly under /tmp/claude-{uid}. The dir itself is + // kept (claude recreates it anyway; deleting it while a session spawns + // would be a race for no benefit). + const uid = typeof opts.uid === "number" + ? opts.uid + : (typeof process.getuid === "function" ? process.getuid() : null); + if (uid !== null) { + const claudeDir = path.join(tmpRoot, `claude-${uid}`); + let entries = []; + try { + entries = fs.readdirSync(claudeDir); + } catch { + entries = []; // no dir → nothing to do + } + for (const name of entries) { + removeIfStale(path.join(claudeDir, name), cutoffMs, result); + } + } + + // Gemini: stray crash dumps written directly to the temp root. + let rootEntries = []; + try { + rootEntries = fs.readdirSync(tmpRoot); + } catch { + rootEntries = []; + } + for (const name of rootEntries) { + if (name.startsWith("gemini-client-error-") && name.endsWith(".json")) { + removeIfStale(path.join(tmpRoot, name), cutoffMs, result); + } + } + } catch (err) { + result.errors.push(String(err && err.message ? err.message : err)); + } + return result; +} + +// Resolve {enabled, maxAgeHours} from config.json's optional top-level +// `temp_cleanup` block. Defaults: enabled, 72h. `enabled: false` opts out. +function cleanupSettings(cfg) { + const tc = (cfg && cfg.temp_cleanup) || {}; + return { + enabled: tc.enabled !== false, + maxAgeHours: Number.isFinite(tc.max_age_hours) && tc.max_age_hours > 0 + ? tc.max_age_hours + : DEFAULT_MAX_AGE_HOURS, + }; +} + +module.exports = { + sweepBackendTemp, + cleanupSettings, + DEFAULT_MAX_AGE_HOURS, +}; diff --git a/server/temp-cleanup.test.js b/server/temp-cleanup.test.js new file mode 100644 index 00000000..07deed98 --- /dev/null +++ b/server/temp-cleanup.test.js @@ -0,0 +1,103 @@ +"use strict"; + +// #957: unit tests for the stale backend-temp sweep. All filesystem work +// happens inside a throwaway fixture dir; uid/now/tmpRoot are injected so +// nothing ever touches the real /tmp/claude-{uid}. Plain node:assert script — +// auto-discovered by the #836 runner. Run with `node server/temp-cleanup.test.js`. + +const assert = require("node:assert/strict"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); +const { sweepBackendTemp, cleanupSettings, DEFAULT_MAX_AGE_HOURS } = require("./temp-cleanup"); + +const HOUR = 60 * 60 * 1000; +const NOW = 1_800_000_000_000; // fixed epoch for determinism (no Date.now()) + +function makeFixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "qw-957-tempclean-")); + const claudeDir = path.join(root, "claude-1234"); + fs.mkdirSync(claudeDir); + return { root, claudeDir }; +} + +// Create a file or dir and stamp its atime/mtime to `ageHours` before NOW. +function touch(p, ageHours, { dir = false } = {}) { + if (dir) { + fs.mkdirSync(p, { recursive: true }); + fs.writeFileSync(path.join(p, "inner.txt"), "x"); + } else { + fs.writeFileSync(p, "x"); + } + const t = new Date(NOW - ageHours * HOUR); + fs.utimesSync(p, t, t); +} + +// ── stale vs a live session's fresh entries under claude-{uid} ──────────── +{ + const { root, claudeDir } = makeFixture(); + touch(path.join(claudeDir, "old-file"), 100); + touch(path.join(claudeDir, "old-dir"), 100, { dir: true }); + // inner.txt written above bumps the dir mtime to real-now; restamp old. + fs.utimesSync(path.join(claudeDir, "old-dir"), new Date(NOW - 100 * HOUR), new Date(NOW - 100 * HOUR)); + touch(path.join(claudeDir, "live-session-file"), 1); // a live agent just wrote this + + const r = sweepBackendTemp({ tmpRoot: root, uid: 1234, now: NOW, maxAgeHours: 72 }); + + assert.ok(!fs.existsSync(path.join(claudeDir, "old-file")), "stale file under claude-{uid} removed"); + assert.ok(!fs.existsSync(path.join(claudeDir, "old-dir")), "stale directory removed recursively"); + assert.ok(fs.existsSync(path.join(claudeDir, "live-session-file")), "live session's fresh file PRESERVED"); + assert.ok(fs.existsSync(claudeDir), "the claude-{uid} dir itself is kept"); + assert.equal(r.removed.length, 2, "removed count = 2"); + assert.equal(r.kept, 1, "kept count = 1 (the live file)"); + assert.equal(r.errors.length, 0, "no errors"); + fs.rmSync(root, { recursive: true, force: true }); +} + +// ── gemini crash dumps at the temp root ────────────────────────────────── +{ + const { root } = makeFixture(); + const staleDump = path.join(root, "gemini-client-error-Turn.run-2026-06-03T00-28-55-388Z.json"); + touch(staleDump, 100); + touch(path.join(root, "gemini-client-error-fresh.json"), 1); + touch(path.join(root, "unrelated-old-file.json"), 100); + + const r = sweepBackendTemp({ tmpRoot: root, uid: 1234, now: NOW, maxAgeHours: 72 }); + + assert.ok(!fs.existsSync(staleDump), "stale gemini crash dump removed"); + assert.ok(fs.existsSync(path.join(root, "gemini-client-error-fresh.json")), "fresh gemini dump spared"); + assert.ok(fs.existsSync(path.join(root, "unrelated-old-file.json")), "non-gemini root file NEVER touched, even when old"); + assert.equal(r.removed.length, 1, "only the stale gemini dump counts as removed"); + fs.rmSync(root, { recursive: true, force: true }); +} + +// ── missing claude dir / null uid (Windows) → graceful no-op ───────────── +{ + const root = fs.mkdtempSync(path.join(os.tmpdir(), "qw-957-tempclean-")); + const r1 = sweepBackendTemp({ tmpRoot: root, uid: 9999, now: NOW }); + assert.equal(r1.removed.length, 0, "missing claude-{uid} dir → nothing removed"); + assert.equal(r1.errors.length, 0, "missing claude-{uid} dir → no error"); + + const r2 = sweepBackendTemp({ tmpRoot: root, uid: null, now: NOW }); + assert.equal(r2.removed.length, 0, "uid null → claude sweep skipped"); + assert.equal(r2.errors.length, 0, "uid null → no error"); + fs.rmSync(root, { recursive: true, force: true }); +} + +// ── never throws, even on a bogus tmpRoot ──────────────────────────────── +{ + const r = sweepBackendTemp({ tmpRoot: "/nonexistent/qw-nope", uid: 1, now: NOW }); + assert.ok(Array.isArray(r.removed) && r.removed.length === 0, "bogus tmpRoot → no-op, no throw"); +} + +// ── cleanupSettings: defaults + opt-out + custom/invalid age + null cfg ─── +{ + assert.equal(cleanupSettings({}).enabled, true, "default: enabled"); + assert.equal(cleanupSettings({}).maxAgeHours, DEFAULT_MAX_AGE_HOURS, `default: ${DEFAULT_MAX_AGE_HOURS}h`); + assert.equal(cleanupSettings({ temp_cleanup: { enabled: false } }).enabled, false, "enabled:false opts out"); + assert.equal(cleanupSettings({ temp_cleanup: { max_age_hours: 24 } }).maxAgeHours, 24, "custom max_age_hours honored"); + assert.equal(cleanupSettings({ temp_cleanup: { max_age_hours: -5 } }).maxAgeHours, DEFAULT_MAX_AGE_HOURS, "invalid age → default"); + assert.equal(cleanupSettings(null).enabled, true, "null config → defaults, no throw"); +} + +console.log("temp-cleanup.test.js: all assertions passed");