From 67d6b7f3b11ab5b075497a0027daab5faad03d75 Mon Sep 17 00:00:00 2001 From: Austin Kidwell Date: Wed, 15 Jul 2026 05:00:40 -0700 Subject: [PATCH] feat(council): run queue_monitor as a persistent background service (--daemon + installer) queue_monitor.py is a persistent poll loop (in-process tail offsets + per-key alert dedup; --once seeds at EOF and detects nothing), so live monitoring needs a long-lived daemon rather than the one-shot pattern the keepers use. - queue_monitor.py: add --daemon flag + _harden_for_windows_daemon() that redirects stdout/stderr to ~/.claude/logs/queue_monitor.log, so a pythonw Task Scheduler run (whose console fds are discarded) stays diagnosable. Mirrors browser_bridge_keeper._harden_for_windows_daemon. Additive: an interactive `python queue_monitor.py` run is unchanged (still prints to console); hardening only fires under --daemon. - install_queue_monitor_task.ps1: register PerplexityQueueMonitor as a persistent pythonw daemon (--daemon --interval 15). AtLogOn trigger + 5-min repetition with MultipleInstances=IgnoreNew: the repetition is a watchdog (relaunch if dead) that never spawns a duplicate while the daemon is alive. Interactive/Limited principal, restart-on-failure, no time limit; -Uninstall and -NoPushover switches. Sibling of install_bridge_keeper_task.ps1. Verified live: task registers without elevation, daemon runs (pythonw), polls every 15s, and writes the poll table to queue_monitor.log. Existing monitor suites still green (test_queue_monitor + _stall: 30 passed). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XpuwG5kvGqBwZtCKTSdAK6 --- .../install_queue_monitor_task.ps1 | 145 ++++++++++++++++++ council-automation/queue_monitor.py | 41 +++++ 2 files changed, 186 insertions(+) create mode 100644 council-automation/install_queue_monitor_task.ps1 diff --git a/council-automation/install_queue_monitor_task.ps1 b/council-automation/install_queue_monitor_task.ps1 new file mode 100644 index 0000000..efa58ac --- /dev/null +++ b/council-automation/install_queue_monitor_task.ps1 @@ -0,0 +1,145 @@ +# install_queue_monitor_task.ps1 — Register queue_monitor.py as a persistent +# Windows Task Scheduler service (live Pushover alerts + best-effort self-heal). +# +# Why: queue_monitor.py watches the Perplexity research FIFO queue snapshot + +# activity log and fires Pushover alerts (stalled holder, deep queue, long run, +# error/timeout, stalled-near-empty) plus a rate-limited keeper self-heal. To be +# useful it must run continuously in the background. Sibling of the +# PerplexitySessionKeeper / BrowserBridgeKeeper tasks. +# +# ARCHITECTURE DIFFERENCE vs the keepers: those are ONE-SHOT (check → act → exit) +# and rely on task repetition to re-run. queue_monitor.py is a PERSISTENT poll +# loop (its tail offsets + per-key alert dedup live in-process; --once seeds at +# EOF and detects nothing). So this task runs a long-lived pythonw daemon and +# uses MultipleInstances=IgnoreNew: the AtLogOn trigger starts it, and a 5-min +# repetition acts as a watchdog — if the daemon is alive the new instance is +# ignored, if it died the repetition relaunches it. No duplicate daemons. +# +# The daemon is launched with --daemon so pythonw's discarded stdout/stderr are +# redirected to ~/.claude/logs/queue_monitor.log (see _harden_for_windows_daemon). +# +# Run ONCE from PowerShell the first time (elevation only if your policy requires +# it for Register-ScheduledTask; the current-user Interactive principal usually +# does not). Re-running is idempotent (updates the task in place). +# +# Usage: +# PowerShell -ExecutionPolicy Bypass -File install_queue_monitor_task.ps1 +# PowerShell -ExecutionPolicy Bypass -File install_queue_monitor_task.ps1 -Uninstall +# PowerShell -ExecutionPolicy Bypass -File install_queue_monitor_task.ps1 -Interval 15 -NoPushover + +param( + [switch]$Uninstall, + [switch]$NoPushover, + [int]$Interval = 15, + [string]$TaskName = "PerplexityQueueMonitor", + [string]$PythonW = "" # auto-detect if empty +) + +$ErrorActionPreference = "Stop" + +$monitorPath = "$HOME\.claude\council-automation\queue_monitor.py" +$logDir = "$HOME\.claude\logs" + +if ($Uninstall) { + if (Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue) { + Write-Host "Removing scheduled task '$TaskName'..." + # Stop a running instance first so the daemon actually exits. + try { Stop-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue } catch {} + Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false + Write-Host "Done. The queue monitor daemon will no longer auto-start at logon." + Write-Host "Any still-running pythonw daemon: find via" + Write-Host " Get-CimInstance Win32_Process -Filter \"Name='pythonw.exe'\" | Where-Object { `$_.CommandLine -like '*queue_monitor*' }" + } else { + Write-Host "Task '$TaskName' not registered. Nothing to remove." + } + return +} + +# Validate monitor script exists +if (-not (Test-Path $monitorPath)) { + Write-Error "queue_monitor.py not found at: $monitorPath" + exit 1 +} + +# Find pythonw.exe (no-console Python interpreter) +if (-not $PythonW) { + $pythonExe = (Get-Command python -ErrorAction SilentlyContinue).Source + if (-not $pythonExe) { + Write-Error "python not found on PATH. Pass -PythonW explicitly." + exit 1 + } + $PythonW = Join-Path (Split-Path $pythonExe -Parent) "pythonw.exe" + if (-not (Test-Path $PythonW)) { + Write-Error "pythonw.exe not found next to python ($pythonExe). Pass -PythonW ." + exit 1 + } +} +Write-Host "pythonw.exe: $PythonW" +Write-Host "monitor: $monitorPath" + +# Create log directory +New-Item -ItemType Directory -Path $logDir -Force | Out-Null + +# Build the action: pythonw.exe -u queue_monitor.py --daemon --interval N [--no-pushover] +$argList = "-u `"$monitorPath`" --daemon --interval $Interval" +if ($NoPushover) { $argList += " --no-pushover" } +$action = New-ScheduledTaskAction ` + -Execute $PythonW ` + -Argument $argList ` + -WorkingDirectory (Split-Path $monitorPath -Parent) + +# Triggers: at logon (start when the user signs in) + every 5 minutes (watchdog +# relaunch if the daemon ever died). With MultipleInstances=IgnoreNew below, the +# repetition is a no-op while the daemon is alive. +$logonTrigger = New-ScheduledTaskTrigger -AtLogOn -User "$env:USERNAME" +$periodicTrigger = New-ScheduledTaskTrigger -Once -At (Get-Date).AddMinutes(2) ` + -RepetitionInterval (New-TimeSpan -Minutes 5) ` + -RepetitionDuration (New-TimeSpan -Days 365) + +# Run as the current user, "only when logged on" (Interactive) — same principal as +# the keepers. The monitor reads local queue files + imports research_monitor for +# Pushover/keeper-trigger, so it needs the interactive user's environment. +$principal = New-ScheduledTaskPrincipal ` + -UserId "$env:USERDOMAIN\$env:USERNAME" ` + -LogonType Interactive ` + -RunLevel Limited + +# Settings: never spawn a second daemon (IgnoreNew), restart on failure, no time +# limit (persistent loop), allow on battery. +$settings = New-ScheduledTaskSettingsSet ` + -MultipleInstances IgnoreNew ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -RestartCount 5 ` + -RestartInterval (New-TimeSpan -Minutes 1) ` + -ExecutionTimeLimit (New-TimeSpan -Days 0) # 0 = no time limit + +# Remove existing task if present (idempotent install) +if (Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue) { + Write-Host "Removing existing task '$TaskName' before re-registering..." + try { Stop-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue } catch {} + Unregister-ScheduledTask -TaskName $TaskName -Confirm:$false +} + +Register-ScheduledTask ` + -TaskName $TaskName ` + -Description "Persistent background monitor for the Perplexity research FIFO queue: live Pushover alerts (stalls, deep queue, long runs, errors, stalled-near-empty) + rate-limited keeper self-heal. Long-lived pythonw daemon; 5-min repetition is a watchdog (IgnoreNew)." ` + -Action $action ` + -Trigger @($logonTrigger, $periodicTrigger) ` + -Principal $principal ` + -Settings $settings | Out-Null + +Write-Host "" +Write-Host "[OK] Task '$TaskName' registered." +Write-Host "" +Write-Host "Next steps:" +Write-Host " 1. Start it now (one-time):" +Write-Host " Start-ScheduledTask -TaskName $TaskName" +Write-Host " 2. Verify it's running:" +Write-Host " Get-ScheduledTaskInfo -TaskName $TaskName" +Write-Host " 3. Watch the daemon log:" +Write-Host " Get-Content `"$logDir\queue_monitor.log`" -Wait -Tail 20" +Write-Host "" +Write-Host "Uninstall later with:" +Write-Host " PowerShell -ExecutionPolicy Bypass -File `"$PSCommandPath`" -Uninstall" diff --git a/council-automation/queue_monitor.py b/council-automation/queue_monitor.py index e593968..97fd935 100644 --- a/council-automation/queue_monitor.py +++ b/council-automation/queue_monitor.py @@ -27,7 +27,9 @@ import argparse import json import logging +import os import re +import signal import sys import time from dataclasses import dataclass @@ -47,6 +49,10 @@ SNAPSHOT_PATH: Path = research_queue.SNAPSHOT ACTIVITY_LOG_PATH: Path = research_queue.ACTIVITY_LOG LOG_WORKDIR: Path = Path.home() / ".claude" / "council-logs" / "queue-monitor" +# Daemon stdio sink: when run under pythonw.exe by Task Scheduler (--daemon), +# fds 1/2 are redirected here so the otherwise-discarded poll table + logging +# output stay diagnosable. Mirrors browser_bridge_keeper._harden_for_windows_daemon. +LOG_PATH: Path = Path.home() / ".claude" / "logs" / "queue_monitor.log" # -------------------------------------------------------------------------- # Thresholds (monkeypatchable module globals -- see evaluate()). @@ -448,6 +454,34 @@ def _maybe_self_heal(alerts: Sequence[Alert], last_fired_ts: float) -> float: # -------------------------------------------------------------------------- # Poll loop # -------------------------------------------------------------------------- +def _harden_for_windows_daemon() -> None: + """Redirect fds 1/2 to LOG_PATH so pythonw Task Scheduler runs are diagnosable. + + Called from main() ONLY for --daemon (persistent background service), so an + interactive `python queue_monitor.py` run still prints its table to the + console. Mirrors browser_bridge_keeper._harden_for_windows_daemon. + """ + if sys.platform != "win32": + return + try: + LOG_PATH.parent.mkdir(parents=True, exist_ok=True) + log_fd = os.open(str(LOG_PATH), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) + os.dup2(log_fd, 1) + os.dup2(log_fd, 2) + os.close(log_fd) + sys.stdout = os.fdopen(1, "w", buffering=1) + sys.stderr = os.fdopen(2, "w", buffering=1) + except (OSError, ValueError): + pass + for sig_name in ("SIGBREAK", "SIGINT"): + sig = getattr(signal, sig_name, None) + if sig is not None: + try: + signal.signal(sig, signal.SIG_IGN) + except (ValueError, OSError): + pass + + def run_loop(interval_s: float = 10.0, once: bool = False, use_pushover: bool = True) -> None: """Poll SNAPSHOT_PATH/ACTIVITY_LOG_PATH, evaluate, print, and notify. @@ -511,8 +545,15 @@ def main(argv: Optional[List[str]] = None) -> int: help="Perform a single evaluation pass and exit.") parser.add_argument("--no-pushover", action="store_true", help="Disable Pushover notifications (still prints + logs).") + parser.add_argument("--daemon", action="store_true", + help="Persistent background service mode: redirect stdout/stderr " + "to LOG_PATH (~/.claude/logs/queue_monitor.log) so a pythonw " + "Task Scheduler run stays diagnosable. Implies looping.") args = parser.parse_args(argv) + if args.daemon: + _harden_for_windows_daemon() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") run_loop(interval_s=args.interval, once=args.once, use_pushover=not args.no_pushover) return 0