diff --git a/AGENTS.md b/AGENTS.md index 810cf0d..51130f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,8 @@ src/ │ └── recovery.rs # Error recovery and user guidance ├── streaming/ # HLS pipeline │ ├── hls_generator.rs # FFmpeg subprocess per camera (HLS muxer) +│ ├── supervisor.rs # Polls FFmpeg every 2s, respawns with exponential backoff +│ │ # (1s→30s), reports Streaming/Restarting/Failed into Dashboard │ ├── hls_uploader.rs # Watches HLS dir, drives playlist updates + motion event channel │ ├── segment_uploader.rs # Posts each .ts to POST /push-segment with retry/backoff │ ├── motion_detector.rs # Parallel FFmpeg scene-change scorer @@ -100,7 +102,7 @@ src/ 2. Detect cameras (`camera::detect_cameras()`) 3. Register with Command Center (`api_client.register()`) 4. Detect hardware encoder once (NVENC/QSV/AMF), persist to DB -5. Create HLS generator per camera (FFmpeg subprocess) +5. Spawn an FFmpeg **supervisor** per camera (`streaming/supervisor.rs`) that owns the `HlsGenerator`, polls the child every 2s, respawns it with exponential backoff (1s → 2s → 4s → … capped at 30s) when it dies, and trips the camera into `Failed` state if the backoff window sees 5+ crashes in 60s. Each transition (`Starting` / `Streaming` / `Restarting { reason }` / `Failed { reason }`) is pushed into the `Dashboard` so the heartbeat / WS messages carry the real pipeline state (with `last_error`) rather than the old hardcoded `"streaming"`. 6. Spawn HLS uploader tasks (segment push + playlist update + codec detection) 7. Spawn motion detector per camera (second FFmpeg probe for scene-change scoring) 8. Launch local HTTP server (port 8080) + WebSocket client @@ -182,8 +184,8 @@ All outbound calls use `ApiClient` in `src/api/client.rs`: | Method | Path | Header | Body | When | |--------|------|--------|------|------| -| POST | `/api/nodes/register` | `X-API-Key` | `RegisterRequest` JSON | Startup | -| POST | `/api/nodes/heartbeat` | `X-API-Key` | `HeartbeatRequest` JSON | Every `heartbeat_interval` s (fallback path; WS heartbeat is primary) | +| POST | `/api/nodes/register` | `X-Node-API-Key` | `RegisterRequest` JSON | Startup | +| POST | `/api/nodes/heartbeat` | `X-Node-API-Key` | `HeartbeatRequest` JSON (includes per-camera `CameraStatus { camera_id, status, last_error }` with real pipeline state) | Every `heartbeat_interval` s (fallback path; WS heartbeat is primary) | | POST | `/api/cameras/{id}/codec` | `X-Node-API-Key` | `{video_codec, audio_codec}` JSON | After first segment or codec change | | POST | `/api/cameras/{id}/push-segment?filename=…` | `X-Node-API-Key` | raw `.ts` bytes (`video/mp2t`) | Every segment | | POST | `/api/cameras/{id}/playlist` | `X-Node-API-Key` | playlist text (`text/plain`) | Every playlist rewrite | diff --git a/README.md b/README.md index f1d791d..a34ec35 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,8 @@ docker run -d \ **Hardware encoding:** At startup, CloudNode probes for a hardware encoder (NVENC, QSV, AMF) and caches the result in the database. Falls back to `libx264` if none is found. +**FFmpeg supervisor:** Each camera's FFmpeg pipeline is wrapped in a supervisor (`streaming/supervisor.rs`) that polls the child every 2 seconds. If FFmpeg exits (disk full, V4L2 disconnect, segment-writer failure, etc.) the supervisor respawns it with exponential backoff — 1s → 2s → 4s → … capped at 30s. A pipeline that crashes 5+ times inside a 60-second window is flagged `Failed` and stops retrying, so a permanently broken camera can't spin forever. Real pipeline state (`starting` / `streaming` / `restarting` / `failed`) is reported on every heartbeat (HTTP and WebSocket) together with the human-readable failure reason, which is what the Command Center dashboard surfaces — it replaces the legacy hardcoded `"streaming"` status that used to make every node look healthy regardless of what FFmpeg was doing. + --- ## API Endpoints @@ -317,6 +319,7 @@ src/ ├── setup/ # Interactive TUI setup wizard (crossterm + inquire) ├── streaming/ # HLS pipeline │ ├── hls_generator.rs # FFmpeg subprocess per camera (HLS muxer) +│ ├── supervisor.rs # Polls FFmpeg, respawns with exponential backoff, reports real pipeline state │ ├── hls_uploader.rs # Watches HLS dir, hands segments to SegmentUploader, updates playlist, drives motion events │ ├── segment_uploader.rs# Posts each .ts to POST /push-segment with retry │ ├── motion_detector.rs # Parallel FFmpeg scene-change scorer diff --git a/config.example.yaml b/config.example.yaml index 2693da8..56842ad 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -27,18 +27,50 @@ cameras: streaming: # Target frames per second fps: 30 - + # JPEG quality (1-100) jpeg_quality: 85 + # Video encoder override. Leave empty for auto-detect (NVENC/QSV/AMF/libx264). + # Examples: "h264_nvenc", "h264_qsv", "h264_amf", "libx264" + encoder: "" + + # HLS muxer settings + hls: + # Enable HLS streaming (must be true for cloud streaming) + enabled: true + + # Segment duration in seconds + segment_duration: 1 + + # Number of segments kept in the rolling playlist + playlist_size: 15 + + # Video bitrate (e.g. "2500k") + bitrate: "2500k" + +# Motion detection (second FFmpeg process per camera runs a scene-change filter) +motion: + # Enable motion detection + enabled: true + + # Scene-change threshold (0.0 = identical frames, 1.0 = totally different). + # Lower values are more sensitive — 0.02 is a good default for typical indoor + # lighting. Raise if you get noise from compression/autoexposure flicker. + threshold: 0.02 + + # Minimum seconds between reported motion events per camera (cooldown). + cooldown_secs: 30 + # Recording settings recording: # Enable local recording enabled: true - + # Recording format: "mp4" or "mkv" format: "mp4" - + + storage: # Base path for recordings and snapshots path: "./data"