From 7f067c144ec97c3d2203725e18d6a66ef245806c Mon Sep 17 00:00:00 2001 From: Alexander Nicolay Date: Thu, 16 Jul 2026 22:14:35 +0200 Subject: [PATCH] Optional camera module: phone-camera page records shots automatically --- AGENTS.md | 4 +- README.md | 22 +++++ src/matebot/camera.py | 141 +++++++++++++++++++++++++++++++++ src/matebot/cli.py | 64 ++++++++++++++- src/matebot/config.py | 13 ++- src/matebot/web_cam/index.html | 107 +++++++++++++++++++++++++ tests/test_camera.py | 73 +++++++++++++++++ 7 files changed, 419 insertions(+), 5 deletions(-) create mode 100644 src/matebot/camera.py create mode 100644 src/matebot/web_cam/index.html create mode 100644 tests/test_camera.py diff --git a/AGENTS.md b/AGENTS.md index c9c16c7..bff193c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,7 +43,9 @@ The decision tree: 6. **Optional extras**, in ascending effort: journal repo + GitHub Pages (README "Publishing your journal"), dial-in hints (on by default), bean bag tracking (`/newbag`, multiple bags, `/tossbag`), smart plug hooks for cold-start `/wake` - (README "Smart plug cold start"). + (README "Smart plug cold start"), shot-video camera module (README + "Camera module" — MATEBOT_CAMERA=1, needs ffmpeg + HTTPS in front of the + camera page; getUserMedia refuses plain HTTP). Common failure modes: token/chat id swapped or quoted wrong; bot and machine on different networks/VLANs; machine powered off at a dumb power strip; diff --git a/README.md b/README.md index 6aba7ae..a623daf 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,28 @@ Any shell command works (Home Assistant webhook, `tinytuya`, zigbee2mqtt…). NixOS module users: write `%` as `%%` in the hook options — the values pass through a systemd unit, which treats single `%` as a specifier. +## Camera module (optional): shot videos synced with the charts + +Opt in with `MATEBOT_CAMERA=1` and MATEbot serves a phone-camera page on +`MATEBOT_CAMERA_PORT` (default 8877). Open it on a phone near the machine, +allow camera access, and leave the page open — recording starts and stops +automatically with each shot. Clips are transcoded (720p, with audio), +committed to the journal repo, and the shot page plays them in sync with the +curves (scrub the chart, the video follows). + +**HTTPS is required** — browsers only grant camera access on secure pages. +MATEbot itself serves plain HTTP; put it behind whatever TLS you have: + +- a reverse proxy with certificates (nginx/caddy/traefik), e.g. + `matebot.example.com` → `127.0.0.1:8877` (WebSocket support needed) +- or zero-config on a tailnet: `tailscale serve https / http://127.0.0.1:8877` + +Sync calibration: detection and stream latency mean the video usually starts +~1 s after the true shot start. `MATEBOT_CAMERA_OFFSET` (default `-1.0`) +corrects this globally; watch one replay and nudge per shot with +`/vsync +0.5` if needed. Videos rotate out after `MATEBOT_VIDEO_KEEP` shots +(git history keeps every clip recoverable). + ## Configuration Environment variables, or the same keys in `~/.config/matebot/config.toml`: diff --git a/src/matebot/camera.py b/src/matebot/camera.py new file mode 100644 index 0000000..69a39e4 --- /dev/null +++ b/src/matebot/camera.py @@ -0,0 +1,141 @@ +"""Optional camera module: MATEbot serves a phone-camera page and records shots. + +Strictly opt-in (``MATEBOT_CAMERA=1``). The phone opens the page (HTTPS via +the user's reverse proxy — getUserMedia requires a secure context), grants +camera access and keeps the page open near the machine. When a shot starts, +the server tells the page to record; MediaRecorder chunks stream back over +the WebSocket. When the shot ends (+ a short tail), the clip is transcoded +and attached to the shot via the video pipeline. + +Best-effort by design: no page connected → no recording, no errors. +""" + +from __future__ import annotations + +import asyncio +import importlib.resources +import json +import logging +import tempfile +from pathlib import Path + +from aiohttp import WSMsgType, web + +log = logging.getLogger(__name__) + +TAIL_SECONDS = 5.0 +MAX_RECORD_SECONDS = 240 + + +class CameraServer: + def __init__(self, port: int, on_clip) -> None: + """*on_clip* is ``async (webm_path: Path) -> None`` — called per finished clip.""" + self.port = port + self.on_clip = on_clip + self._ws: web.WebSocketResponse | None = None + self._chunks: list[bytes] = [] + self._recording = False + self._runner: web.AppRunner | None = None + self._stop_task: asyncio.Task | None = None + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + app = web.Application() + app.router.add_get("/", self._page) + app.router.add_get("/ws", self._websocket) + self._runner = web.AppRunner(app) + await self._runner.setup() + site = web.TCPSite(self._runner, "0.0.0.0", self.port) + await site.start() + log.info("camera server listening on :%d", self.port) + + async def stop(self) -> None: + if self._runner: + await self._runner.cleanup() + + @property + def page_connected(self) -> bool: + return self._ws is not None and not self._ws.closed + + # ------------------------------------------------------------ recording + + async def shot_started(self) -> None: + if not self.page_connected or self._recording: + return + self._chunks = [] + self._recording = True + await self._send({"cmd": "start"}) + log.info("camera: recording started") + self._stop_task = asyncio.create_task(self._safety_stop()) + + async def shot_ended(self) -> None: + if not self._recording: + return + await asyncio.sleep(TAIL_SECONDS) + await self._finish() + + async def _safety_stop(self) -> None: + await asyncio.sleep(MAX_RECORD_SECONDS) + if self._recording: + log.warning("camera: safety stop after %ds", MAX_RECORD_SECONDS) + await self._finish() + + async def _finish(self) -> None: + if not self._recording: + return + self._recording = False + if self._stop_task and not self._stop_task.done(): + self._stop_task.cancel() + await self._send({"cmd": "stop"}) + # give the page a moment to flush its final chunks + for _ in range(20): + await asyncio.sleep(0.25) + if not self.page_connected: + break + chunks, self._chunks = self._chunks, [] + if not chunks: + log.info("camera: no video data received") + return + with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp: + for chunk in chunks: + tmp.write(chunk) + path = Path(tmp.name) + log.info("camera: clip complete (%.1f MB)", path.stat().st_size / 1e6) + try: + await self.on_clip(path) + finally: + path.unlink(missing_ok=True) + + # ------------------------------------------------------------ http + + async def _page(self, request: web.Request) -> web.Response: + html = (importlib.resources.files("matebot") / "web_cam" / "index.html").read_text() + return web.Response(text=html, content_type="text/html") + + async def _websocket(self, request: web.Request) -> web.WebSocketResponse: + ws = web.WebSocketResponse(max_msg_size=8 * 2**20) + await ws.prepare(request) + if self._ws and not self._ws.closed: + await self._ws.close() # newest page wins + self._ws = ws + log.info("camera page connected from %s", request.remote) + try: + async for msg in ws: + if msg.type == WSMsgType.BINARY: + if self._recording or self._chunks is not None: + self._chunks.append(msg.data) + elif msg.type == WSMsgType.TEXT: + log.debug("camera page: %s", msg.data[:100]) + finally: + if self._ws is ws: + self._ws = None + log.info("camera page disconnected") + return ws + + async def _send(self, payload: dict) -> None: + if self.page_connected: + try: + await self._ws.send_str(json.dumps(payload)) + except Exception as exc: # noqa: BLE001 + log.warning("camera send failed: %s", exc) diff --git a/src/matebot/cli.py b/src/matebot/cli.py index 9e42cd7..4e3b360 100644 --- a/src/matebot/cli.py +++ b/src/matebot/cli.py @@ -150,6 +150,52 @@ async def save_notes(shot_id: int, notes: dict) -> bool: convo = Conversation(messenger, state, save_notes) cache_frame, latest_frame = make_frame_cache() router = CommandRouter(client, state, convo, messenger, config, latest_frame) + + camera = None + pending_clip: dict = {"path": None, "ts": 0.0} + attach_lock = asyncio.Lock() + + async def try_attach_clip(shot_id: int | None = None) -> None: + """Rendezvous: a finished clip and a resolved shot id arrive in + either order; whichever side is second completes the attach.""" + import time as _time + + from .video import VideoError, attach_video, get_offset + + async with attach_lock: + clip = pending_clip["path"] + if clip is None or _time.monotonic() - pending_clip["ts"] > 300: + return + sid = shot_id if shot_id is not None else state.get("last_shot_id") + if not sid or get_offset(config.data_repo, sid) is not None: + return # no shot yet, or it already has a video + pending_clip["path"] = None + try: + await attach_video(config.data_repo, sid, clip, + offset=config.camera_offset) + log.info("camera clip attached to shot %06d", sid) + schedule_sync(quiet=True) + except VideoError as exc: + log.warning("clip attach failed: %s", exc) + await messenger.send(f"🎬 Couldn't process the shot video: {exc}") + finally: + pathlib.Path(clip).unlink(missing_ok=True) + + if config.camera_enabled and config.data_repo: + import shutil as _shutil + import time as _time + + from .camera import CameraServer + + async def on_clip(webm_path): + keep = pathlib.Path(config.state_dir) / "pending_clip.webm" + _shutil.copy(webm_path, keep) + pending_clip["path"] = str(keep) + pending_clip["ts"] = _time.monotonic() + await try_attach_clip() + + camera = CameraServer(config.camera_port, on_clip) + await camera.start() # messenger APIs can be flaky at boot; retry instead of crash-looping for attempt in range(8): try: @@ -179,12 +225,25 @@ async def pump_shots(): last_frame_at = 0.0 + shot_active = False + async def tee(source): - nonlocal last_frame_at + nonlocal last_frame_at, shot_active async for frame in source: now = _time.monotonic() gap = now - last_frame_at if last_frame_at else None last_frame_at = now + if camera is not None and frame.get("tp") == "evt:status": + active = ( + frame.get("m") == 1 + and (frame.get("process") or {}).get("a") == 1 + and not _re.search(config.ignore_profiles, frame.get("p") or "") + ) + if active and not shot_active: + await camera.shot_started() + elif shot_active and not active: + asyncio.create_task(camera.shot_ended()) + shot_active = active if gap is None or gap > 60: await router.on_machine_online(frame) if state.get("sync_pending"): @@ -253,6 +312,7 @@ async def on_utility(profile): ) except Exception: # noqa: BLE001 - keep watching even if messaging fails log.exception("questionnaire start failed (state kept for resume)") + await try_attach_clip(shot.entry.id) schedule_sync() # archive the .slog right away async def weekly_digest(): @@ -285,6 +345,8 @@ async def weekly_digest(): if exc: raise exc finally: + if camera is not None: + await camera.stop() await messenger.stop() return 0 diff --git a/src/matebot/config.py b/src/matebot/config.py index 1708945..09bae90 100644 --- a/src/matebot/config.py +++ b/src/matebot/config.py @@ -32,6 +32,9 @@ "water_warn_pct": "MATEBOT_WATER_WARN", "autoheat_window": "MATEBOT_AUTOHEAT", "video_keep": "MATEBOT_VIDEO_KEEP", + "camera_enabled": "MATEBOT_CAMERA", + "camera_port": "MATEBOT_CAMERA_PORT", + "camera_offset": "MATEBOT_CAMERA_OFFSET", "plots_enabled": "MATEBOT_PLOTS", "wake_hook": "MATEBOT_WAKE_HOOK", "sleep_hook": "MATEBOT_SLEEP_HOOK", @@ -65,6 +68,9 @@ class Config: # e.g. "06:30-07:00": machine powered on in this window switches to brew, once a day autoheat_window: str = "" video_keep: int = 15 # newest N shot videos kept in the journal; 0 = keep all + camera_enabled: bool = False # opt-in: serve the phone-camera page + record shots + camera_port: int = 8877 + camera_offset: float = -1.0 # shot t=0 relative to video t=0 (detection+stream latency) plots_enabled: bool = True # send the shot chart as a photo (needs matplotlib) wake_hook: str = "" # shell command to power the machine's smart plug ON sleep_hook: str = "" # shell command to power the smart plug OFF @@ -83,11 +89,12 @@ def load(cls, path: str | Path | None = None) -> Config: value = env if value is None: continue - if f.name == "min_shot_s": + if f.name in ("min_shot_s", "camera_offset"): value = float(value) - elif f.name in ("clean_every", "water_warn_pct", "video_keep"): + elif f.name in ("clean_every", "water_warn_pct", "video_keep", "camera_port"): value = int(value) - elif f.name in ("sync_enabled", "hints_enabled", "digest_enabled", "plots_enabled"): + elif f.name in ("sync_enabled", "hints_enabled", "digest_enabled", "plots_enabled", + "camera_enabled"): value = str(value).lower() in ("1", "true", "yes", "on") kwargs[f.name] = value config = cls(**kwargs) diff --git a/src/matebot/web_cam/index.html b/src/matebot/web_cam/index.html new file mode 100644 index 0000000..f35163f --- /dev/null +++ b/src/matebot/web_cam/index.html @@ -0,0 +1,107 @@ + + + + + +MATEbot Cam + + + + +
+

🎥 MATEbot Cam

+
+ starting… +
+ + + + + + diff --git a/tests/test_camera.py b/tests/test_camera.py new file mode 100644 index 0000000..4e7f9e7 --- /dev/null +++ b/tests/test_camera.py @@ -0,0 +1,73 @@ +import asyncio +import socket + +import aiohttp +import pytest + +from matebot import camera as camera_mod +from matebot.camera import CameraServer + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.mark.asyncio +async def test_full_recording_cycle(monkeypatch): + monkeypatch.setattr(camera_mod, "TAIL_SECONDS", 0.05) + clips = [] + + async def on_clip(path): + clips.append(path.read_bytes()) + + port = free_port() + server = CameraServer(port, on_clip) + await server.start() + try: + async with aiohttp.ClientSession() as session: + # the page itself is served + async with session.get(f"http://127.0.0.1:{port}/") as resp: + assert resp.status == 200 + assert "MATEbot Cam" in await resp.text() + + async with session.ws_connect(f"http://127.0.0.1:{port}/ws") as ws: + assert server.page_connected + + await server.shot_started() + msg = await asyncio.wait_for(ws.receive_json(), 5) + assert msg == {"cmd": "start"} + + await ws.send_bytes(b"CHUNK1") + await ws.send_bytes(b"CHUNK2") + await asyncio.sleep(0.1) # let the server ingest + + ended = asyncio.create_task(server.shot_ended()) + msg = await asyncio.wait_for(ws.receive_json(), 5) + assert msg == {"cmd": "stop"} + await ws.close() # page done flushing + await asyncio.wait_for(ended, 10) + + assert clips == [b"CHUNK1CHUNK2"] + finally: + await server.stop() + + +@pytest.mark.asyncio +async def test_no_page_no_recording(monkeypatch): + monkeypatch.setattr(camera_mod, "TAIL_SECONDS", 0.05) + clips = [] + + async def on_clip(path): + clips.append(path) + + server = CameraServer(free_port(), on_clip) + await server.start() + try: + await server.shot_started() # no page connected: silently ignored + await server.shot_ended() + assert clips == [] + assert not server.page_connected + finally: + await server.stop()