feat(status): ingest + surface real pipeline state from CloudNode

CloudNode's FFmpeg supervisor now reports "starting" / "streaming" /
"restarting" / "failed" / "error" per camera, with an optional
last_error reason when things are broken. Backend ingests these in
both the HTTP and WebSocket heartbeat paths, persists them on the
Camera row (new last_error column, auto-migrated on startup), and
exposes them through to_dict so the dashboard can tell the user *why*
a camera they expect to be live isn't showing video.

Frontend: CameraCard renders new badges for the supervised states,
puts the supervisor's last_error inline under "Pipeline Failed", and
shows a yellow "Reconnecting — …" overlay while restarting so the
user doesn't wonder if the camera just hung.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: Sbussiso

backend/app/api/nodes.py

@@ -225,6 +225,13 @@ async def node_heartbeat(
             if cam:
                 cam.status = cam_status.status
                 cam.last_seen = now
+                # Record (or clear) the pipeline failure reason. Healthy
+                # states wipe the field so stale errors don't linger in
+                # the API response after the supervisor recovers.
+                if cam_status.status in ("restarting", "failed", "error"):
+                    cam.last_error = cam_status.last_error
+                else:
+                    cam.last_error = None
 
     db.commit()
 

backend/app/api/ws.py

@@ -246,6 +246,13 @@ async def _handle_heartbeat(node_id: str, node_db_id: int, org_id: str, payload:
                         new_cam_status = cam_data.get("status", "online")
                         cam.status = new_cam_status
                         cam.last_seen = now
+                        # Record (or clear) the pipeline failure reason.
+                        # Healthy states wipe the field so stale errors
+                        # don't linger once the supervisor recovers.
+                        if new_cam_status in ("restarting", "failed", "error"):
+                            cam.last_error = cam_data.get("last_error")
+                        else:
+                            cam.last_error = None
                         if (
                             prev_cam_status != new_cam_status
                             and new_cam_status in ("online", "offline")

backend/app/models/models.py

@@ -26,7 +26,15 @@ class Camera(Base):
     capabilities = Column(String(500), default="streaming")
     group_id = Column(Integer, ForeignKey("camera_groups.id"), nullable=True)
     last_seen = Column(DateTime)
+    # Pipeline state. In addition to the legacy "online" / "offline", the
+    # CloudNode's FFmpeg supervisor now reports "starting", "streaming",
+    # "restarting", "failed", and "error" so the UI can tell the user
+    # why a camera they expect to be live isn't showing video.
     status = Column(String(20), default="offline")
+    # Human-readable failure reason that goes alongside `status` when the
+    # pipeline is `restarting` / `failed` / `error`. Cleared whenever the
+    # node reports a healthy status.
+    last_error = Column(String(500), nullable=True)
     created_at = Column(DateTime, default=lambda: datetime.now(tz=timezone.utc).replace(tzinfo=None))
 
     # Codec detection fields
@@ -49,13 +57,19 @@ def effective_status(self) -> str:
         return self.status
 
     def to_dict(self):
+        eff = self.effective_status
+        # Only surface last_error when the camera is actually in a
+        # broken state — once it flips back to streaming, the stale
+        # reason would just confuse anyone reading the API response.
+        err = self.last_error if eff in ("restarting", "failed", "error") else None
         return {
             "camera_id": self.camera_id,
             "name": self.name,
             "node_type": self.node_type,
             "capabilities": self.capabilities.split(",") if self.capabilities else [],
             "group": self.group.name if self.group else None,
-            "status": self.effective_status,
+            "status": eff,
+            "last_error": err,
             "last_seen": self.last_seen.isoformat() if self.last_seen else None,
         }
 

backend/app/schemas/schemas.py

@@ -39,6 +39,10 @@ class NodeRegister(BaseModel):
 class CameraStatus(BaseModel):
     camera_id: str = Field(..., max_length=150)
     status: str = Field(..., max_length=20)
+    # Optional failure reason — sent by the CloudNode supervisor when
+    # the pipeline is restarting, failed, or errored. Old nodes that
+    # predate the supervisor simply omit it (the field is Optional).
+    last_error: Optional[str] = Field(None, max_length=500)
 
 
 class NodeHeartbeat(BaseModel):

frontend/src/components/CameraCard.jsx

@@ -45,16 +45,44 @@ function CameraCard({
     }
   }, [cameraId, getToken, recording, showToast])
 
-  const isOffline = camera.status === "offline"
+  // "failed" (supervisor gave up) and "error" mean the pipeline is
+  // producing nothing — treat them as offline for playback purposes.
+  // "restarting" and "starting" are transient; the HLS player will just
+  // show its buffering state until segments arrive.
+  const status = camera.status
+  const isDown = status === "offline" || status === "failed" || status === "error"
+  const isTransient = status === "starting" || status === "restarting"
 
   const nodeTypeLabel = camera.node_type || "Camera"
   const nodeTypeIcon = "📹"
 
-  const statusClass = camera.status === "online" ? "online" :
-                      camera.status === "streaming" ? "streaming" :
-                      camera.status === "recording" ? "recording" : "offline"
+  const statusClass =
+    status === "online"     ? "online" :
+    status === "streaming"  ? "streaming" :
+    status === "recording"  ? "recording" :
+    status === "starting"   ? "starting" :
+    status === "restarting" ? "restarting" :
+    status === "failed"     ? "failed" :
+    status === "error"      ? "error" : "offline"
 
-  const cardClasses = `camera-card ${isOffline ? "offline" : ""}`
+  // Down-state messages used inside the feed placeholder. "failed" and
+  // "error" render the supervisor's last_error if we have it so the user
+  // isn't left guessing why the camera went dark.
+  const downLabel =
+    status === "failed" ? "Pipeline Failed" :
+    status === "error"  ? "Pipeline Error"  : "Camera Offline"
+  const downDetail =
+    (status === "failed" || status === "error") && camera.last_error
+      ? camera.last_error
+      : null
+
+  // Tooltip content for the status badge. Shows the reason inline so the
+  // user doesn't have to hover into the feed to see what's wrong.
+  const badgeTitle = camera.last_error
+    ? `${status}: ${camera.last_error}`
+    : status || "unknown"
+
+  const cardClasses = `camera-card ${isDown ? "offline" : ""}`
 
   return (
     <div className={cardClasses}>
@@ -67,25 +95,33 @@ function CameraCard({
             <span className="node-type">{nodeTypeLabel}</span>
           </div>
         </div>
-        <div className={`status-badge ${statusClass}`}>
+        <div className={`status-badge ${statusClass}`} title={badgeTitle}>
           <span className="dot"></span>
-          <span className="status-text">{camera.status || "unknown"}</span>
+          <span className="status-text">{status || "unknown"}</span>
         </div>
       </div>
 
       <div className="camera-feed-container">
-        {isOffline ? (
+        {isDown ? (
           <div className="feed-loading error">
             <span className="status-icon">⚠️</span>
-            <span>Camera Offline</span>
+            <span>{downLabel}</span>
+            {downDetail && <span className="feed-detail">{downDetail}</span>}
           </div>
         ) : (
           <HlsPlayer
             cameraId={cameraId}
             cameraName={camera.name || `Camera ${cameraId.slice(-4)}`}
-            status={camera.status}
+            status={status}
           />
         )}
+        {isTransient && (
+          <div className="camera-feed-overlay-banner">
+            {status === "restarting"
+              ? `Reconnecting${camera.last_error ? ` — ${camera.last_error}` : "…"}`
+              : "Starting up…"}
+          </div>
+        )}
       </div>
 
       <div className="camera-controls">
@@ -120,6 +156,7 @@ export default memo(CameraCard, (prevProps, nextProps) => {
   return (
     prevProps.cameraId === nextProps.cameraId &&
     prevProps.camera.status === nextProps.camera.status &&
-    prevProps.camera.name === nextProps.camera.name
+    prevProps.camera.name === nextProps.camera.name &&
+    prevProps.camera.last_error === nextProps.camera.last_error
   )
 })

frontend/src/index.css

@@ -395,6 +395,39 @@ body {
     animation: pulse-fast 1s ease-in-out infinite;
 }
 
+/* Supervisor pipeline states — added alongside the FFmpeg auto-restart
+   work so operators can tell apart "camera unplugged" (offline) from
+   "pipeline respawning" (restarting) from "pipeline gave up" (failed). */
+.status-badge.starting {
+    background: rgba(245, 158, 11, 0.15);
+    color: var(--accent-amber);
+}
+.status-badge.starting .dot {
+    background: var(--accent-amber);
+    box-shadow: 0 0 8px var(--accent-amber-glow);
+    animation: pulse 2s ease-in-out infinite;
+}
+
+.status-badge.restarting {
+    background: rgba(245, 158, 11, 0.15);
+    color: var(--accent-amber);
+}
+.status-badge.restarting .dot {
+    background: var(--accent-amber);
+    box-shadow: 0 0 8px var(--accent-amber-glow);
+    animation: pulse-fast 1s ease-in-out infinite;
+}
+
+.status-badge.failed, .status-badge.error {
+    background: rgba(239, 68, 68, 0.15);
+    color: var(--accent-red);
+}
+.status-badge.failed .dot, .status-badge.error .dot {
+    background: var(--accent-red);
+    box-shadow: 0 0 8px var(--accent-red-glow);
+    animation: pulse-fast 1s ease-in-out infinite;
+}
+
 /* Video Feed */
 .camera-feed-container {
     position: relative;
@@ -446,6 +479,39 @@ body {
     margin-top: 4px;
 }
 
+/* Error detail shown under "Pipeline Failed" / "Pipeline Error" — gives
+   the operator the supervisor's last_error so they don't have to go
+   hunting through the CloudNode TUI to figure out what's wrong. */
+.feed-loading .feed-detail {
+    font-size: 0.7rem;
+    color: var(--text-muted);
+    margin-top: 4px;
+    max-width: 80%;
+    text-align: center;
+    word-break: break-word;
+    opacity: 0.85;
+}
+
+/* Small banner overlaid on the HLS player when the pipeline is
+   transient (starting / restarting). Keeps video visible if segments
+   are still trickling in, but flags that the state isn't steady. */
+.camera-feed-overlay-banner {
+    position: absolute;
+    top: 8px;
+    left: 8px;
+    right: 8px;
+    padding: 6px 10px;
+    background: rgba(245, 158, 11, 0.85);
+    color: #000;
+    font-size: 0.7rem;
+    font-weight: 600;
+    border-radius: 4px;
+    text-align: center;
+    pointer-events: none;
+    z-index: 2;
+    text-shadow: none;
+}
+
 .loading-spinner {
     width: 32px;
     height: 32px;