From 2ff12f308a4a688526ba1793780b037d9e35533a Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Thu, 9 Jul 2026 14:24:44 -0700 Subject: [PATCH 01/14] feat(camera_viz): add video-file replay source (type: video) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Play a recording through the FrameSource contract so Televiz can be tested and previewed without camera hardware. Decode via OpenCV's FFmpeg backend on the v4l2 hot path (pinned staging, async H2D, GPU BGR→RGBA); monotonic-deadline pacing at the file's native FPS (or a YAML override); loops by default, holds the last frame otherwise. stereo: true splits side-by-side recordings into per-eye halves on the GPU (viewer-only, like SyntheticStereoSource). width/height are optional — the file is probed at build time — so plane aspect and encoder/sender geometry now come from the built source's spec instead of the YAML. Relative path: values resolve against the config's directory in both entry points. Also works as camera_streamer's capture side (mono), exercising the full NVENC → RTP chain camera-free; verified end-to-end offscreen render + a 30 fps RTP send on hardware. Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 16 +- examples/camera_viz/README.md | 13 +- examples/camera_viz/camera_streamer.py | 19 +- examples/camera_viz/camera_viz.py | 36 ++- examples/camera_viz/configs/video.yaml | 51 +++++ examples/camera_viz/pyproject.toml | 3 + examples/camera_viz/sources/__init__.py | 33 ++- examples/camera_viz/sources/video_file.py | 212 ++++++++++++++++++ examples/camera_viz/tests/pyproject.toml | 2 + .../tests/test_video_file_source.py | 202 +++++++++++++++++ 10 files changed, 557 insertions(+), 30 deletions(-) create mode 100644 examples/camera_viz/configs/video.yaml create mode 100644 examples/camera_viz/sources/video_file.py create mode 100644 examples/camera_viz/tests/test_video_file_source.py diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 6eeec8982..3665fc502 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -58,6 +58,10 @@ The camera kind is selected per entry by the YAML ``type:`` field: - OAK-D mono RGB / LEFT / RIGHT (stereo not yet wired). * - ``zed`` - ZED 2 / Mini / X One; mono or ``stereo: true`` (per-eye SDK retrieve, zero-copy on the GPU). + * - ``video`` + - Video-file replay (anything OpenCV's FFmpeg backend reads) — preview / Televiz testing + without a camera. Loops by default; ``stereo: true`` splits side-by-side recordings into + eyes (viewer only). Output goes to a window or an XR headset. Stereo cameras render true side-by-side stereo in XR; window mode shows the left eye. XR placement lock modes are ``world`` / ``head`` / ``lazy``. @@ -77,7 +81,7 @@ Televiz as the compositor at the end of the chain: ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) ├── pipeline/ — source ABC + threaded runner ├── placements/ — XR lock-mode strategies (world / head / lazy) - ├── sources/ — V4L2 / OAK-D / ZED / synthetic / rtp_h264 + ├── sources/ — V4L2 / OAK-D / ZED / synthetic / video replay / rtp_h264 ├── transports/ — RTP sender + receiver (native + GStreamer) ├── codec/ — native NVENC / NVDEC pybind module ├── configs/ — one YAML per camera kind @@ -130,8 +134,8 @@ Set ``source: local`` in the config and run the viewer: ./camera_viz.sh run configs/v4l2.yaml -Swap the config for ``oakd.yaml``, ``zed.yaml``, ``synthetic.yaml``, ``synthetic_stereo.yaml``, or -``multi_camera.yaml``. +Swap the config for ``oakd.yaml``, ``zed.yaml``, ``synthetic.yaml``, ``synthetic_stereo.yaml``, +``multi_camera.yaml``, or ``video.yaml`` (file replay — point ``path:`` at any recording). Split (robot → workstation over RTP) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -180,11 +184,11 @@ A single YAML drives both capture and visualization. Each ``cameras:`` entry bec cameras: - name: cam enabled: true - type: v4l2 # v4l2 | oakd | zed | synthetic - width: 2560 + type: v4l2 # v4l2 | oakd | zed | synthetic | video + width: 2560 # video: optional — defaults to the file's size height: 720 fps: 30 - stereo: false # zed / synthetic only — per-eye capture + SBS in XR + stereo: false # zed / synthetic / video only — per-eye capture + SBS in XR rtp: port: 5000 # left eye when stereo port_right: 5001 # required when stereo + source: rtp diff --git a/examples/camera_viz/README.md b/examples/camera_viz/README.md index 7cf156253..34ef59250 100644 --- a/examples/camera_viz/README.md +++ b/examples/camera_viz/README.md @@ -20,6 +20,7 @@ SPDX-License-Identifier: Apache-2.0 | `v4l2` | USB / UVC — anything `v4l2-ctl --list-formats-ext` shows | | `oakd` | OAK-D mono RGB / LEFT / RIGHT (stereo not yet wired) | | `zed` | ZED 2 / Mini / X One; mono or `stereo: true` (per-eye SDK retrieve, zero-copy GPU) | +| `video` | Video-file replay (anything OpenCV/FFmpeg reads) — preview / testing without a camera. Loops by default; `stereo: true` splits side-by-side files into eyes (viewer only) | Output: window or XR headset; one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`. @@ -46,7 +47,7 @@ Flags: `--no-{v4l2,oakd,rtp}`, `--with-zed`, `--sender-only`, `--jetson`. Pass ` ./camera_viz.sh run configs/v4l2.yaml ``` -Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`. +Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`, `video.yaml` (file replay — point `path:` at any recording). ## Mode 2 — Split (robot → workstation, RTP) @@ -92,12 +93,12 @@ encoder: auto | native | gstreamer cameras: - name: cam enabled: true - type: v4l2 # v4l2 | oakd | zed | synthetic - width: 2560 + type: v4l2 # v4l2 | oakd | zed | synthetic | video + width: 2560 # video: optional — defaults to the file's size height: 720 fps: 30 - stereo: false # zed / synthetic only — enables per-eye capture + SBS XR - # … type-specific fields (e.g. synthetic: disparity_px) + stereo: false # zed / synthetic / video only — per-eye capture + SBS XR + # … type-specific fields (e.g. synthetic: disparity_px; video: path, loop) rtp: port: 5000 # left eye when stereo port_right: 5001 # required when stereo + source: rtp @@ -144,7 +145,7 @@ camera_viz/ ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) ├── pipeline/ — source ABC + threaded runner ├── placements/ — XR lock-mode strategies -├── sources/ — V4L2 / OAK-D / ZED / synthetic / rtp_h264 +├── sources/ — V4L2 / OAK-D / ZED / synthetic / video replay / rtp_h264 ├── transports/ — RTP sender + receiver, native + GStreamer ├── codec/ — native NVENC/NVDEC pybind module ├── configs/ — one YAML per camera kind diff --git a/examples/camera_viz/camera_streamer.py b/examples/camera_viz/camera_streamer.py index 9978dd1fe..76cc5e4e2 100755 --- a/examples/camera_viz/camera_streamer.py +++ b/examples/camera_viz/camera_streamer.py @@ -30,7 +30,12 @@ import yaml from pipeline import FrameSource -from sources import PairedFrameSource, build_local_camera, set_verbose +from sources import ( + PairedFrameSource, + build_local_camera, + resolve_video_paths, + set_verbose, +) from transports import RtpH264Sender, make_encoder logger = logging.getLogger("camera_streamer") @@ -136,10 +141,13 @@ def _build_senders(self) -> List[RtpH264Sender]: ) def build_one(source: FrameSource, port: int) -> RtpH264Sender: + # Encoder / sender geometry comes from the built source's spec, + # not the YAML — sources that size themselves (video replay + # probing its file) may omit width/height from the config. encoder = make_encoder( rtp.get("encoder", self._default_encoder), - width=int(self._cfg["width"]), - height=int(self._cfg["height"]), + width=source.spec.width, + height=source.spec.height, bitrate=int(rtp.get("bitrate_mbps", 15)) * 1_000_000, fps=int(self._cfg.get("fps", 30)), gop=int(rtp["gop"]) if "gop" in rtp else None, @@ -150,8 +158,8 @@ def build_one(source: FrameSource, port: int) -> RtpH264Sender: encoder=encoder, host=self._host, port=port, - width=int(self._cfg["width"]), - height=int(self._cfg["height"]), + width=source.spec.width, + height=source.spec.height, fps=int(self._cfg.get("fps", 30)), mtu=int(rtp.get("mtu", 1400)), ) @@ -278,6 +286,7 @@ def main(argv: Optional[List[str]] = None) -> int: # Top-level ``verbose:`` enables per-source periodic breadcrumbs. set_verbose(bool(cfg.get("verbose", False))) + resolve_video_paths(cfg, args.config.parent) streaming = cfg.get("streaming", {}) host = args.host or streaming.get("host") diff --git a/examples/camera_viz/camera_viz.py b/examples/camera_viz/camera_viz.py index e0fc143f1..291e17940 100755 --- a/examples/camera_viz/camera_viz.py +++ b/examples/camera_viz/camera_viz.py @@ -30,7 +30,13 @@ from pipeline import FrameSource, VizRunner from placements import PlacementConfig, PlacementStrategy, build as build_placement -from sources import PairedFrameSource, RtpH264Source, build_local_camera, set_verbose +from sources import ( + PairedFrameSource, + RtpH264Source, + build_local_camera, + resolve_video_paths, + set_verbose, +) @dataclass @@ -75,18 +81,15 @@ def _enabled_cameras(cfg: dict) -> List[dict]: def _placement_with_aspect( - spec: Optional[dict], cam: dict, is_xr: bool + spec: Optional[dict], width: int, height: int, is_xr: bool ) -> Optional[PlacementStrategy]: - """Build the placement for ``cam``, filling in ``size`` from the - camera's aspect ratio when the YAML doesn't pin it. Width defaults - to 1.0 m so a 16:9 camera lands at 1.0 x 0.5625, a 3.55:1 SBS at - 1.0 x 0.281.""" + """Build the placement, filling in ``size`` from the source's aspect + ratio when the YAML doesn't pin it. Width defaults to 1.0 m so a + 16:9 source lands at 1.0 x 0.5625, a 3.55:1 SBS at 1.0 x 0.281.""" if spec is not None and "size" not in spec: - w = int(cam["width"]) - h = int(cam["height"]) spec = { **spec, - "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * h / w], + "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * height / width], } return _build_placement(spec, is_xr) @@ -104,9 +107,15 @@ def _build_local_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: placements_cfg = cfg.get("display", {}).get("placements", {}) entries: List[SourceEntry] = [] for cam in _enabled_cameras(cfg): - placement = _placement_with_aspect(placements_cfg.get(cam["name"]), cam, is_xr) + cam_sources = build_local_camera(cam) + # Aspect comes from the built source's spec, not the YAML — video + # sources may omit width/height and size themselves from the file. + first = cam_sources[0].spec + placement = _placement_with_aspect( + placements_cfg.get(cam["name"]), first.width, first.height, is_xr + ) stereo, baseline_mm = _stereo_for(cam, placements_cfg) - for source in build_local_camera(cam): + for source in cam_sources: entries.append( SourceEntry( source=source, @@ -130,7 +139,9 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: f"camera_viz: camera {cam.get('name')!r} missing rtp.port; " "required when source: rtp" ) - placement = _placement_with_aspect(placements_cfg.get(cam["name"]), cam, is_xr) + placement = _placement_with_aspect( + placements_cfg.get(cam["name"]), int(cam["width"]), int(cam["height"]), is_xr + ) stereo, baseline_mm = _stereo_for(cam, placements_cfg) if stereo: @@ -225,6 +236,7 @@ def main(argv: Optional[list[str]] = None) -> int: # Top-level ``verbose:`` enables per-source periodic breadcrumbs. set_verbose(bool(cfg.get("verbose", False))) + resolve_video_paths(cfg, args.config.parent) source_mode = cfg.get("source", "local").lower() if source_mode not in ("local", "rtp"): diff --git a/examples/camera_viz/configs/video.yaml b/examples/camera_viz/configs/video.yaml new file mode 100644 index 000000000..ccd887a9c --- /dev/null +++ b/examples/camera_viz/configs/video.yaml @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Video-file replay — plays a recording through the same source → +# QuadLayer → Televiz path as a live camera. Preview / Televiz testing +# when no camera is attached; also works as camera_streamer's capture +# side (mono only) to exercise the full RTP path camera-free. +# +# Point ``path`` at any file OpenCV's FFmpeg backend reads (mp4 / mkv / +# webm, H.264 / HEVC / AV1 / ...). No sample clip ships with the repo; +# generate a 10 s test pattern with: +# ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 recording.mp4 + +source: local # local | rtp — see real-camera configs for rtp +streaming: + host: 127.0.0.1 # only used by camera_streamer (rtp mode) +encoder: auto # auto | native | gstreamer + +cameras: + - name: replay + enabled: true + type: video + path: recording.mp4 # relative paths resolve against this YAML's dir + loop: true # rewind at end of file (false = hold last frame) + fps: 0 # 0 = the file's native rate. With source: rtp, + # set this explicitly — the sender's encode loop + # paces at fps (default 30) regardless of the file. + # width/height default to the file's native size (per eye when stereo); + # set both to resize each frame on the CPU before upload. + # width: 1280 + # height: 720 + stereo: false # true = side-by-side file, split into eyes. + # Viewer-only: the RTP sender needs per-eye + # streams and rejects single-source stereo. + rtp: # only used when source: rtp + camera_streamer + port: 5000 + bitrate_mbps: 15 + +display: + mode: window # window | xr + window: + width: 1280 + height: 720 + xr: + near_z: 0.05 + far_z: 100.0 + clear_color: [0.05, 0.05, 0.08, 1.0] + placements: + replay: # size defaults to 1.0 m wide × aspect-derived height + lock_mode: lazy + distance: 1.5 diff --git a/examples/camera_viz/pyproject.toml b/examples/camera_viz/pyproject.toml index fd1d74bb4..f6433da43 100644 --- a/examples/camera_viz/pyproject.toml +++ b/examples/camera_viz/pyproject.toml @@ -26,6 +26,9 @@ dependencies = [ # extras by default; pass ``--no-`` to skip. [project.optional-dependencies] v4l2 = ["opencv-python>=4.5"] +# type: video (file replay) decodes via OpenCV's FFmpeg backend — same +# wheel as v4l2, listed separately so --no-v4l2 setups can still ask for it. +video = ["opencv-python>=4.5"] oakd = ["depthai>=3.0"] # pyzed isn't on PyPI — the ZED SDK's bundled ``get_python_api.py`` # produces a Python-version-specific wheel. ``camera_viz.sh setup --with-zed`` diff --git a/examples/camera_viz/sources/__init__.py b/examples/camera_viz/sources/__init__.py index f1a7a53d0..defdadbb3 100644 --- a/examples/camera_viz/sources/__init__.py +++ b/examples/camera_viz/sources/__init__.py @@ -8,6 +8,7 @@ from __future__ import annotations +from pathlib import Path from typing import List from pipeline import FrameSource @@ -17,6 +18,7 @@ from .rtp_h264 import RtpH264Source from .synthetic import SyntheticSource, SyntheticStereoSource from .v4l2 import V4l2Source +from .video_file import VideoFileSource from .zed import ZedSource __all__ = [ @@ -26,12 +28,26 @@ "SyntheticSource", "SyntheticStereoSource", "V4l2Source", + "VideoFileSource", "ZedSource", "build_local_camera", + "resolve_video_paths", "set_verbose", ] +def resolve_video_paths(cfg: dict, base_dir) -> None: + """Anchor relative ``path:`` values of ``type: video`` cameras to the + YAML file's directory (in place), so playback doesn't depend on the + process CWD. Call right after loading the config.""" + for cam in cfg.get("cameras", []): + if cam.get("type") == "video" and "path" in cam: + p = Path(str(cam["path"])).expanduser() + if not p.is_absolute(): + p = Path(base_dir) / p + cam["path"] = str(p) + + def build_local_camera(spec: dict) -> List[FrameSource]: """Build local FrameSource(s) for one ``cameras:`` entry. @@ -104,6 +120,21 @@ def build_local_camera(spec: dict) -> List[FrameSource]: ) return [PairedFrameSource(name=name, left=eyes[0], right=eyes[1])] return eyes + if kind == "video": + # Stereo (side-by-side file) emits both eyes from one source, like + # SyntheticStereoSource — viewer-only; camera_streamer's + # _eye_sources rejects it because there are no per-eye streams. + return [ + VideoFileSource( + name=name, + path=spec["path"], + width=int(spec.get("width", 0)), + height=int(spec.get("height", 0)), + fps=float(spec.get("fps", 0.0)), + loop=bool(spec.get("loop", True)), + stereo=stereo, + ) + ] if kind == "zed": eyes = list( ZedSource.build( @@ -126,5 +157,5 @@ def build_local_camera(spec: dict) -> List[FrameSource]: return eyes raise ValueError( f"build_local_camera: unknown camera type {kind!r} " - "(known: synthetic, v4l2, oakd, zed)" + "(known: synthetic, v4l2, oakd, zed, video)" ) diff --git a/examples/camera_viz/sources/video_file.py b/examples/camera_viz/sources/video_file.py new file mode 100644 index 000000000..f0e13b224 --- /dev/null +++ b/examples/camera_viz/sources/video_file.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Video-file replay source for camera_viz. + +Plays a recording through the same FrameSource contract as a live +camera — preview / Televiz testing when no hardware is attached, and a +deterministic stand-in for a camera on the RTP sender side. + +Decode is OpenCV's FFmpeg backend (any container / codec it reads), +same hot path as :mod:`sources.v4l2`: ``cap.read()`` → pinned host +staging → async H2D → GPU BGR→RGBA. Frames are paced against an +absolute monotonic schedule so long clips don't drift. + +``stereo: true`` treats the file as side-by-side (a ZED recording's +natural layout) and splits each frame into per-eye halves on the GPU — +one source emits both eyes, so eye sync is frame-perfect. Like +SyntheticStereoSource, this doesn't expose per-eye streams, so stereo +replay is viewer-only (camera_streamer rejects it loudly). +""" + +from __future__ import annotations + +import math +import time +from pathlib import Path +from typing import Optional + +import numpy as np + +from ._helpers import PolledSource, alloc_pinned_host, notify + +# Playback rate used when the container reports a missing / absurd FPS. +_FALLBACK_FPS = 30.0 + + +def _probe(path: Path) -> tuple: + """Return (width, height, fps) of ``path``. Raises ValueError when the + file is missing or unreadable — a configuration error, not a device + blip, so it fails at build time rather than retrying forever.""" + import cv2 + + if not path.is_file(): + raise ValueError(f"video source: no such file: {path}") + cap = cv2.VideoCapture(str(path)) + try: + if not cap.isOpened(): + raise ValueError( + f"video source: cannot open {path} — unsupported container/" + "codec for OpenCV's FFmpeg backend?" + ) + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = float(cap.get(cv2.CAP_PROP_FPS)) + if w <= 0 or h <= 0: + # Some containers only report dimensions after a decode. + ret, frame = cap.read() + if not ret or frame is None: + raise ValueError(f"video source: {path} contains no decodable frames") + h, w = frame.shape[:2] + finally: + cap.release() + return w, h, fps + + +class VideoFileSource(PolledSource): + """File playback as a FrameSource; mono or side-by-side stereo.""" + + _kind = "video" + + def __init__( + self, + name: str, + path: str, + width: int = 0, + height: int = 0, + fps: float = 0.0, + loop: bool = True, + stereo: bool = False, + ) -> None: + try: + import cv2 + except ImportError as e: + raise RuntimeError( + "VideoFileSource requires opencv-python. Install via " + "`uv pip install opencv-python`." + ) from e + self._cv2 = cv2 + self._path = Path(path) + + native_w, native_h, native_fps = _probe(self._path) + eyes = 2 if stereo else 1 + if stereo and native_w % 2: + raise ValueError( + f"video source '{name}': stereo needs an even frame width to " + f"split side-by-side, got {native_w}" + ) + + # Output size: YAML width/height win; otherwise the file's native + # size (per eye when stereo). + out_w = int(width) if width else native_w // eyes + out_h = int(height) if height else native_h + + play_fps = float(fps) if fps else native_fps + if not math.isfinite(play_fps) or not 0.0 < play_fps <= 1000.0: + notify( + self._kind, + f"'{name}': no usable FPS in {self._path.name} " + f"({native_fps!r}); playing at {_FALLBACK_FPS:g}", + ) + play_fps = _FALLBACK_FPS + self._frame_interval_s = 1.0 / play_fps + self._loop = bool(loop) + self._stereo = bool(stereo) + + super().__init__(name=name, width=out_w, height=out_h, staging_channels=3) + cp = self._cp + + # The decoded frame is the full (SBS when stereo) picture; staging + # and the GPU BGR landing zone are sized to it. The base class's + # per-eye staging is replaced — its (h, w, 3) allocation only fits + # the mono layout. + self._frame_w = out_w * eyes + if stereo: + self._host_staging = alloc_pinned_host( + (out_h, self._frame_w, 3), np.uint8 + ) + # Right-eye output buffers, rotated in lock-step with the base + # class's (left-eye) triple-buffer. + self._gpu_right = [ + cp.empty((out_h, out_w, 4), dtype=cp.uint8) for _ in range(3) + ] + for buf in self._gpu_right: + buf[..., 3] = 255 + self._gpu_bgr = cp.empty((out_h, self._frame_w, 3), dtype=cp.uint8) + + self._cap = None + self._finished = False + self._deadline = 0.0 + + def _open_device(self) -> bool: + cap = self._cv2.VideoCapture(str(self._path)) + if not cap.isOpened(): + cap.release() + return False + self._cap = cap + self._finished = False + self._deadline = time.monotonic() + return True + + def _close_device(self) -> None: + if self._cap is not None: + try: + self._cap.release() + except Exception: + pass + self._cap = None + + def _grab(self) -> Optional[np.ndarray]: + if self._finished: + # loop: false and the clip has ended — hold the last published + # frame. Idle here (not None-spin) until stop(). + self._stop.wait(timeout=0.25) + return None + + wait = self._deadline - time.monotonic() + if wait > 0 and self._stop.wait(timeout=wait): + return None + + cv2 = self._cv2 + ret, frame = self._cap.read() + if not ret or frame is None: + if not self._loop: + self._finished = True + notify(self._kind, "playback finished (holding last frame)") + return None + self._cap.set(cv2.CAP_PROP_POS_FRAMES, 0) + ret, frame = self._cap.read() + if not ret or frame is None: + # Rewind failed — reopen the file via the reconnect path. + raise RuntimeError("rewind failed") + + self._deadline += self._frame_interval_s + now = time.monotonic() + if self._deadline < now: + # Fell behind (slow decode, stall) — resync rather than burst. + self._deadline = now + self._frame_interval_s + + h, fw = self._host_staging.shape[:2] + if frame.shape[0] != h or frame.shape[1] != fw: + frame = cv2.resize(frame, (fw, h), interpolation=cv2.INTER_LINEAR) + self._host_staging[...] = frame + return self._host_staging + + def _upload_and_convert(self, gpu_buf) -> None: + # Async H2D + GPU channel reverse (BGR → RGB); alpha stays 255 from + # construction. Stereo slices the SBS landing zone into eyes. + with self._stream: + self._gpu_bgr.set(self._host_staging) + if not self._stereo: + gpu_buf[..., :3] = self._gpu_bgr[..., ::-1] + return + w = self._spec.width + gpu_buf[..., :3] = self._gpu_bgr[:, :w, ::-1] + self._gpu_right[self._write_idx][..., :3] = self._gpu_bgr[:, w:, ::-1] + + def latest(self): + frame = super().latest() + if frame is not None and self._stereo: + # _consumed_idx is the slot latest() just returned; only the + # consumer thread reads it, so no extra locking needed. + frame.image_right = self._gpu_right[self._consumed_idx] + return frame diff --git a/examples/camera_viz/tests/pyproject.toml b/examples/camera_viz/tests/pyproject.toml index 4ee7cdc74..2b9b77779 100644 --- a/examples/camera_viz/tests/pyproject.toml +++ b/examples/camera_viz/tests/pyproject.toml @@ -17,6 +17,8 @@ dev = [ "pytest", "numpy", "pyyaml>=6.0", + # Encodes + decodes the synthetic clips in test_video_file_source.py. + "opencv-python>=4.5", ] gpu = [ "cupy-cuda12x", diff --git a/examples/camera_viz/tests/test_video_file_source.py b/examples/camera_viz/tests/test_video_file_source.py new file mode 100644 index 000000000..a0b075094 --- /dev/null +++ b/examples/camera_viz/tests/test_video_file_source.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the video-file replay source (``type: video``). + +Clips are synthesized in tmp_path with cv2.VideoWriter — no binary +fixtures in the repo. Probe / path-resolution tests run everywhere; +playback tests need cupy + a CUDA device and skip cleanly without one. +""" + +from __future__ import annotations + +import time + +import numpy as np +import pytest + +cv2 = pytest.importorskip("cv2") + +from sources import build_local_camera, resolve_video_paths # noqa: E402 + + +def _cuda_available() -> bool: + try: + import cupy as cp + except ImportError: + return False + try: + return cp.cuda.runtime.getDeviceCount() > 0 + except Exception: + return False + + +requires_gpu = pytest.mark.skipif( + not _cuda_available(), reason="needs cupy + a CUDA device" +) + + +def _write_clip(path, frames, fps: float = 30.0) -> None: + h, w = frames[0].shape[:2] + vw = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + if not vw.isOpened(): + pytest.skip("cv2.VideoWriter cannot encode mp4v here") + for f in frames: + vw.write(f) + vw.release() + + +def _solid(h: int, w: int, bgr) -> np.ndarray: + frame = np.empty((h, w, 3), dtype=np.uint8) + frame[...] = bgr + return frame + + +def _collect(source, n: int, timeout_s: float = 10.0) -> list: + """Poll latest() until ``n`` frames arrive or the timeout hits.""" + frames = [] + deadline = time.monotonic() + timeout_s + while len(frames) < n and time.monotonic() < deadline: + f = source.latest() + if f is not None: + frames.append(f) + time.sleep(0.002) + return frames + + +# ── build-time behavior (no GPU needed) ───────────────────────────── + + +def test_missing_file_raises(tmp_path): + with pytest.raises(ValueError, match="no such file"): + build_local_camera( + {"name": "x", "type": "video", "path": str(tmp_path / "nope.mp4")} + ) + + +def test_resolve_video_paths_anchors_to_yaml_dir(tmp_path): + cfg = { + "cameras": [ + {"type": "video", "path": "clip.mp4"}, + {"type": "video", "path": "/abs/clip.mp4"}, + {"type": "v4l2", "device": "/dev/video0"}, + ] + } + resolve_video_paths(cfg, tmp_path) + assert cfg["cameras"][0]["path"] == str(tmp_path / "clip.mp4") + assert cfg["cameras"][1]["path"] == "/abs/clip.mp4" + assert "path" not in cfg["cameras"][2] + + +# ── playback (GPU) ─────────────────────────────────────────────────── + + +@requires_gpu +def test_probe_size_playback_and_color(tmp_path): + """Omitting width/height sizes the source from the file; frames come + out RGBA with BGR→RGB applied.""" + import cupy as cp + + clip = tmp_path / "blue.mp4" + _write_clip(clip, [_solid(48, 64, (255, 0, 0))] * 10, fps=60.0) # BGR blue + + (source,) = build_local_camera({"name": "replay", "type": "video", "path": str(clip)}) + assert source.spec.width == 64 + assert source.spec.height == 48 + + with source: + frames = _collect(source, 3) + assert len(frames) == 3 + img = cp.asnumpy(frames[-1].image) + assert img.shape == (48, 64, 4) + assert img[..., 2].mean() > 200 # blue landed in the R..B order's B slot + assert img[..., 0].mean() < 60 # red channel stays dark + assert (img[..., 3] == 255).all() + + +@requires_gpu +def test_loop_wraps_past_end_of_file(tmp_path): + clip = tmp_path / "short.mp4" + _write_clip(clip, [_solid(32, 32, (0, 255, 0))] * 6, fps=60.0) + + (source,) = build_local_camera( + {"name": "looper", "type": "video", "path": str(clip), "fps": 60} + ) + with source: + # 6-frame clip: anything past 6 proves the rewind worked. + frames = _collect(source, 20) + assert len(frames) == 20 + + +@requires_gpu +def test_no_loop_holds_after_last_frame(tmp_path): + clip = tmp_path / "once.mp4" + _write_clip(clip, [_solid(32, 32, (0, 0, 255))] * 5, fps=100.0) + + (source,) = build_local_camera( + { + "name": "oneshot", + "type": "video", + "path": str(clip), + "loop": False, + "fps": 100, + } + ) + with source: + frames = _collect(source, 5, timeout_s=5.0) + assert 1 <= len(frames) <= 5 + # Clip is done — no new frames may appear, and the source must + # stay alive (holding) rather than reconnect-spinning. + time.sleep(0.3) + assert source.latest() is None + + +@requires_gpu +def test_stereo_sbs_splits_eyes(tmp_path): + """SBS file → per-eye halves: left green, right red, both eyes in + one Frame.""" + import cupy as cp + + h, eye_w = 32, 48 + sbs = np.empty((h, eye_w * 2, 3), dtype=np.uint8) + sbs[:, :eye_w] = (0, 255, 0) # BGR green left + sbs[:, eye_w:] = (0, 0, 255) # BGR red right + clip = tmp_path / "sbs.mp4" + _write_clip(clip, [sbs] * 10, fps=60.0) + + (source,) = build_local_camera( + {"name": "sbs", "type": "video", "path": str(clip), "stereo": True} + ) + assert source.spec.width == eye_w + assert source.spec.height == h + + with source: + frames = _collect(source, 3) + assert len(frames) == 3 + left = cp.asnumpy(frames[-1].image) + right = cp.asnumpy(frames[-1].image_right) + assert left.shape == right.shape == (h, eye_w, 4) + assert left[..., 1].mean() > 200 # green + assert right[..., 0].mean() > 200 # red + assert (right[..., 3] == 255).all() + + +@requires_gpu +def test_resize_override(tmp_path): + clip = tmp_path / "big.mp4" + _write_clip(clip, [_solid(48, 64, (128, 128, 128))] * 5, fps=60.0) + + (source,) = build_local_camera( + { + "name": "small", + "type": "video", + "path": str(clip), + "width": 32, + "height": 24, + } + ) + assert source.spec.width == 32 + assert source.spec.height == 24 + with source: + frames = _collect(source, 2) + assert len(frames) == 2 + assert frames[-1].image.shape == (24, 32, 4) From 3abd8d386e7ad6249933dc47bc27b6422bf34044 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Thu, 9 Jul 2026 15:19:54 -0700 Subject: [PATCH 02/14] docs(camera_streaming): restructure as a QA walkthrough; feature video replay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the page in test order: requirements → setup → a no-camera first run (video replay, with the exact expected terminal output) → real cameras → XR → split mode → configuration → troubleshooting. Video replay is now the featured hardware-free entry point; synthetic is demoted to a note and table row calling it a render-path debugging tool. Split mode gains a warning that its extra NVENC/NVDEC hop adds latency and that direct mode is preferred whenever the camera can attach to the viewer machine. Supporting changes: camera_viz now raises a clear error when source: rtp entries omit width/height (previously a bare KeyError — video configs make those keys optional); video.yaml documents where the ffmpeg test clip lands; a camera_viz .gitignore keeps generated / user-supplied clips out of git. Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 293 ++++++++++++-------- examples/camera_viz/.gitignore | 7 + examples/camera_viz/camera_viz.py | 6 + examples/camera_viz/configs/video.yaml | 7 +- 4 files changed, 194 insertions(+), 119 deletions(-) create mode 100644 examples/camera_viz/.gitignore diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 3665fc502..fc1e72e16 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -5,13 +5,15 @@ Camera Streaming ================ ``camera_viz`` is the reference camera-streaming sample built on :doc:`Televiz -` (``isaacteleop.viz``). It captures frames from one or more cameras and -visualizes them on a desktop window or an XR headset — one plane per camera, aspect-fit — and can -stream a robot's cameras to a remote workstation over the network. +` (``isaacteleop.viz``). It captures frames from one or more cameras — +or replays a video file — and visualizes them on a desktop window or an XR headset, one plane per +camera, aspect-fit. It can also stream a robot's cameras to a workstation over the network +(split mode). -The sample lives at :code-dir:`examples/camera_viz/ `; this page summarizes how -it works. For the exact command surface and flags, see the -:code-file:`README `. +The sample lives at :code-dir:`examples/camera_viz/ `. This page walks +through it in the order you would test it: setup, a first run that needs no camera, real cameras, +XR output, then the robot → workstation split mode. For the exact command surface and flags, see +the :code-file:`README `. .. figure:: ../_static/televiz_2d.gif :alt: camera_viz in window mode with synthetic multi-camera feeds @@ -24,25 +26,78 @@ it works. For the exact command surface and flags, see the :local: :depth: 2 -Modes +Requirements +------------ + +- Linux x86_64 or Jetson with an NVIDIA GPU and a CUDA 12 driver (``nvidia-smi`` works) — every + source hands frames to the renderer GPU-resident via CuPy. +- ``display.mode: window`` needs a local display attached; ``display.mode: xr`` needs an OpenXR + runtime (see :doc:`/getting_started/televiz`). +- No IsaacTeleop source build and no camera required — setup installs everything from PyPI, and + the video-replay source runs hardware-free. + +Setup ----- -.. list-table:: - :header-rows: 1 - :widths: 16 84 +One-time setup installs the sample's Python environment: - * - Mode - - What it does - * - **Direct** - - The workstation runs the viewer with cameras attached locally (``source: local``). - * - **Split** - - The robot runs a sender that ships RTP H.264 to a workstation receiver (``source: rtp``). - Wired Ethernet only. - -Supported cameras +.. code-block:: bash + + examples/camera_viz/camera_viz.sh setup + source examples/camera_viz/.venv/bin/activate + +``setup`` installs ``isaacteleop`` (which bundles Televiz) and every other Python dependency from +PyPI into ``.venv/`` via ``uv``, builds the native NVENC/NVDEC codec, and probes system packages +(GStreamer plugins, cairo / girepository headers, JetPack ``cuda-nvrtc`` + ``ld.so`` wiring). When +something is missing it prints the exact ``apt-get`` line and prompts ``[y/N]`` — answering ``n`` +or running non-interactively aborts. + +Useful flags: ``--no-{v4l2,oakd,rtp}``, ``--with-zed``, ``--sender-only``, ``--jetson``. Pass +``--venv PATH`` to install into an existing virtual environment. To develop against a locally +built wheel instead of the PyPI release, pass ``--wheel `` (see +:doc:`/getting_started/build_from_source/index`). + +First run — no camera required +------------------------------ + +The video-replay source (``type: video``) plays a recording through exactly the same source → +QuadLayer → Televiz path a live camera uses, so it doubles as the quickest end-to-end check and +as a stand-in feed while the real camera isn't available: + +.. code-block:: bash + + cd examples/camera_viz + # Any file OpenCV's FFmpeg backend reads works (mp4 / mkv / webm, H.264 / + # HEVC / AV1, ...); or generate a 10 s test pattern: + ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 configs/recording.mp4 + + ./camera_viz.sh run configs/video.yaml + +**Expected result** — the terminal reports the session and the source coming up:: + + camera_viz: source=local, mode=window, xr=False, 1 layer(s) + [video] opening... + [video] connected + [video] streaming + +and a window opens looping the clip. + +To replay your own recording, edit ``path:`` in :code-file:`configs/video.yaml +` — relative paths resolve against the YAML's directory, +not your working directory. ``loop: false`` holds the last frame instead of rewinding; +``stereo: true`` splits a side-by-side recording (e.g. from a ZED) into per-eye views; ``width`` +/ ``height`` default to the file's native size. + +.. note:: + + There is also ``type: synthetic`` (``configs/synthetic.yaml``) — a GPU-generated moving test + pattern. It's a debugging tool for isolating the render path: no file, no decoder, no + hardware involved. + +Supported sources ----------------- -The camera kind is selected per entry by the YAML ``type:`` field: +The source kind is selected per ``cameras:`` entry by the YAML ``type:`` field: .. list-table:: :header-rows: 1 @@ -50,8 +105,6 @@ The camera kind is selected per entry by the YAML ``type:`` field: * - ``type:`` - Notes - * - ``synthetic`` - - GPU test pattern — no hardware. ``stereo: true`` adds a ``disparity_px`` offset between eyes. * - ``v4l2`` - USB / UVC cameras — anything ``v4l2-ctl --list-formats-ext`` reports. * - ``oakd`` @@ -59,92 +112,68 @@ The camera kind is selected per entry by the YAML ``type:`` field: * - ``zed`` - ZED 2 / Mini / X One; mono or ``stereo: true`` (per-eye SDK retrieve, zero-copy on the GPU). * - ``video`` - - Video-file replay (anything OpenCV's FFmpeg backend reads) — preview / Televiz testing - without a camera. Loops by default; ``stereo: true`` splits side-by-side recordings into - eyes (viewer only). - -Output goes to a window or an XR headset. Stereo cameras render true side-by-side stereo in XR; -window mode shows the left eye. XR placement lock modes are ``world`` / ``head`` / ``lazy``. - -How it works ------------- - -The sample is organized so that capture, transport, and visualization are cleanly separated, with -Televiz as the compositor at the end of the chain: - -.. code-block:: text - :class: code-100col - - camera_viz/ - ├── camera_viz.sh — CLI: setup / loopback / run / deploy / service-* - ├── camera_viz.py — receiver / viewer (drives a Televiz VizSession) - ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) - ├── pipeline/ — source ABC + threaded runner - ├── placements/ — XR lock-mode strategies (world / head / lazy) - ├── sources/ — V4L2 / OAK-D / ZED / synthetic / video replay / rtp_h264 - ├── transports/ — RTP sender + receiver (native + GStreamer) - ├── codec/ — native NVENC / NVDEC pybind module - ├── configs/ — one YAML per camera kind - └── scripts/ — installer + systemd unit template - -- **Sources** (:code-dir:`sources/ `) implement a common source ABC and - hand frames to a threaded runner in :code-dir:`pipeline/ `. Each - source produces GPU frames where possible — e.g. the ZED source uses ``retrieve_image(MEM.GPU)`` so - BGRA8 stays in VRAM and a CUDA kernel channel-swaps into contiguous RGBA with no host round-trip. -- **The viewer** (:code-file:`camera_viz.py `) creates a - ``VizSession`` and adds one ``QuadLayer`` per enabled camera, then submits each frame to its layer - and calls ``render()`` once per frame. Stereo cameras submit both eyes. -- **Transport** (:code-dir:`transports/ `) carries the split mode: - an RTP H.264 sender on the robot and a receiver on the workstation, with a native NVENC/NVDEC codec - module (:code-dir:`codec/ `) or a GStreamer fallback. -- **Placement** (:code-dir:`placements/ `) holds the XR lock-mode - strategies. Placement is application policy — Televiz only renders a layer at whatever pose the app - sets each frame. + - Video-file replay (anything OpenCV's FFmpeg backend reads). Loops by default; + ``stereo: true`` splits side-by-side recordings into eyes (viewer only). + * - ``synthetic`` + - Debugging tool — GPU-generated test pattern, no hardware or file. -Setup ------ +Running with a real camera +-------------------------- -One-time setup installs the sample's Python environment — no source build required: +Attach the camera to the machine that runs the viewer, keep ``source: local`` in the config, and +run with the matching config: .. code-block:: bash - examples/camera_viz/camera_viz.sh setup - source examples/camera_viz/.venv/bin/activate + ./camera_viz.sh run configs/v4l2.yaml # or oakd.yaml / zed.yaml -``setup`` installs ``isaacteleop`` (which bundles Televiz) and every other Python dependency from -PyPI into ``.venv/`` via ``uv``, builds the native NVENC/NVDEC codec, and probes system packages -(GStreamer plugins, cairo / girepository headers, JetPack ``cuda-nvrtc`` + ``ld.so`` wiring). When -something is missing it prints the exact ``apt-get`` line and prompts ``[y/N]`` — answering ``n`` or -running non-interactively aborts. +**Expected result** — the same startup lines as above with the camera's tag (``[v4l2]``, +``[oakd]``, ``[zed]``) and a live feed in the window. Multiple ``cameras:`` entries render as one +plane each in the same window. -Useful flags: ``--no-{v4l2,oakd,rtp}``, ``--with-zed``, ``--sender-only``, ``--jetson``. Pass -``--venv PATH`` to install into an existing virtual environment. To develop against a locally built -wheel instead of the PyPI release, pass ``--wheel `` (see -:doc:`/getting_started/build_from_source/index`). +XR output +--------- -Running -------- +Set ``display.mode: xr`` in the YAML — or pass ``--mode xr`` to override the config — and run +with an OpenXR runtime active. Each camera renders as its own plane in the headset; stereo +sources (``stereo: true``) render true side-by-side stereo. Window mode shows the left eye. -Direct (cameras on the workstation) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How a plane follows the operator's head is the per-camera ``lock_mode`` under +``display.placements.``: -Set ``source: local`` in the config and run the viewer: +.. list-table:: + :header-rows: 1 + :widths: 16 84 -.. code-block:: bash + * - Mode + - Behavior + * - ``world`` + - Placed once in front of you and stays put. + * - ``head`` + - Follows your head every frame. + * - ``lazy`` + - World-locked, but re-snaps in front of you when you look away (default). - ./camera_viz.sh run configs/v4l2.yaml +Lazy-mode knobs live under ``placements.``: ``look_away_angle_deg``, +``reposition_distance``, ``reposition_delay_s``, ``transition_duration_s``. -Swap the config for ``oakd.yaml``, ``zed.yaml``, ``synthetic.yaml``, ``synthetic_stereo.yaml``, -``multi_camera.yaml``, or ``video.yaml`` (file replay — point ``path:`` at any recording). +Split mode — robot → workstation over RTP +----------------------------------------- -Split (robot → workstation over RTP) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Split mode runs the capture side on the robot (``camera_streamer.py``) and ships RTP H.264 to +the viewer on the workstation (``source: rtp``). .. warning:: - **Wired networks only.** There is no retransmit or FEC — one lost packet corrupts one frame until + Split mode adds a full extra encode/decode hop — NVENC on the robot, UDP, NVDEC on the + workstation — so it is **not recommended** when a camera can attach directly to the viewer + machine; run direct instead. Reach for it only when the cameras are physically on the robot. + Wired networks only: there is no retransmit or FEC — one lost packet corrupts one frame until the next IDR (default every 5 s). +In split mode every camera entry must pin ``width``, ``height``, and ``fps`` in the YAML — the +receiver sizes its decoder from the config, not from the wire. + Set ``source: rtp`` in the config, export the robot/streaming credentials once per shell, then deploy the sender and run the viewer: @@ -157,22 +186,23 @@ deploy the sender and run the viewer: ./camera_viz.sh run configs/v4l2.yaml # viewer on the workstation ``deploy`` rsyncs the source to the robot, installs sender dependencies, renders a -``camera-streamer.service`` systemd user unit (injecting ``--host`` from ``$STREAMING_HOST`` without -editing the YAML on disk), and enables it. Operate the running unit with +``camera-streamer.service`` systemd user unit (injecting ``--host`` from ``$STREAMING_HOST`` +without editing the YAML on disk), and enables it. Operate the running unit with ``./camera_viz.sh service-{status,logs,restart}``. The sender retries forever across unplug, SDK errors, and network blips. Loopback ^^^^^^^^ -``./camera_viz.sh loopback configs/v4l2.yaml`` runs the sender and viewer together on ``127.0.0.1`` -— the quickest way to smoke-test the RTP path on one machine. +``./camera_viz.sh loopback configs/v4l2.yaml`` runs the sender and viewer together on +``127.0.0.1`` — the quickest way to smoke-test the RTP path on one machine. It also works +camera-free with a mono ``type: video`` entry (set ``width`` / ``height`` / ``fps``). Configuration ------------- -A single YAML drives both capture and visualization. Each ``cameras:`` entry becomes its own plane -(and, in split mode, its own RTP port). Abbreviated: +A single YAML drives both capture and visualization. Each ``cameras:`` entry becomes its own +plane (and, in split mode, its own RTP port). Abbreviated: .. code-block:: yaml @@ -184,11 +214,13 @@ A single YAML drives both capture and visualization. Each ``cameras:`` entry bec cameras: - name: cam enabled: true - type: v4l2 # v4l2 | oakd | zed | synthetic | video + type: v4l2 # v4l2 | oakd | zed | video | synthetic width: 2560 # video: optional — defaults to the file's size - height: 720 + height: 720 # (required when source: rtp) fps: 30 - stereo: false # zed / synthetic / video only — per-eye capture + SBS in XR + stereo: false # zed / video / synthetic — per-eye capture + SBS in XR + path: clip.mp4 # video only — file to replay, relative to this YAML + loop: true # video only — rewind at end of file rtp: port: 5000 # left eye when stereo port_right: 5001 # required when stereo + source: rtp @@ -206,29 +238,58 @@ A single YAML drives both capture and visualization. Each ``cameras:`` entry bec # size: [w_m, h_m] # stereo_baseline_mm: 0 -See the :code-dir:`configs/ ` directory for a complete, commented YAML -per camera kind. +See the :code-dir:`configs/ ` directory for a complete, commented +YAML per source kind. -Lock modes (XR) -^^^^^^^^^^^^^^^ +Troubleshooting +--------------- -How a camera plane is positioned relative to the operator's head each frame: +- **No window appears over SSH** — ``display.mode: window`` needs a local display; run on the + machine you're sitting at, or use a video-capable remote desktop. +- **"video source: no such file"** — relative ``path:`` values resolve against the YAML's + directory (``configs/``), not the directory you launched from. +- **A source fails asking for CuPy / CUDA** — check ``nvidia-smi`` works and setup completed; + all sources allocate their frame buffers on the GPU. +- **Split mode renders nothing** — check the sender is up (``./camera_viz.sh service-status``), + ``$STREAMING_HOST`` was the workstation's IP at deploy time, and UDP ports (default 5000+) + aren't firewalled. +- **Not sure which side is stuck?** — set ``verbose: true`` at the top of the YAML for periodic + per-source breadcrumbs on both ends. -.. list-table:: - :header-rows: 1 - :widths: 16 84 +How it works +------------ - * - Mode - - Behavior - * - ``world`` - - Placed once in front of you and stays put. - * - ``head`` - - Follows your head every frame. - * - ``lazy`` - - World-locked, but re-snaps in front of you when you look away (default). +The sample is organized so that capture, transport, and visualization are cleanly separated, with +Televiz as the compositor at the end of the chain: -Lazy-mode knobs live under ``placements.``: ``look_away_angle_deg``, ``reposition_distance``, -``reposition_delay_s``, ``transition_duration_s``. +.. code-block:: text + :class: code-100col + + camera_viz/ + ├── camera_viz.sh — CLI: setup / loopback / run / deploy / service-* + ├── camera_viz.py — receiver / viewer (drives a Televiz VizSession) + ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) + ├── pipeline/ — source ABC + threaded runner + ├── placements/ — XR lock-mode strategies (world / head / lazy) + ├── sources/ — V4L2 / OAK-D / ZED / video replay / synthetic / rtp_h264 + ├── transports/ — RTP sender + receiver (native + GStreamer) + ├── codec/ — native NVENC / NVDEC pybind module + ├── configs/ — one YAML per source kind + └── scripts/ — installer + systemd unit template + +- **Sources** (:code-dir:`sources/ `) implement a common source ABC and + hand frames to a threaded runner in :code-dir:`pipeline/ `. Each + source produces GPU frames where possible — e.g. the ZED source uses ``retrieve_image(MEM.GPU)`` so + BGRA8 stays in VRAM and a CUDA kernel channel-swaps into contiguous RGBA with no host round-trip. +- **The viewer** (:code-file:`camera_viz.py `) creates a + ``VizSession`` and adds one ``QuadLayer`` per enabled camera, then submits each frame to its layer + and calls ``render()`` once per frame. Stereo cameras submit both eyes. +- **Transport** (:code-dir:`transports/ `) carries the split mode: + an RTP H.264 sender on the robot and a receiver on the workstation, with a native NVENC/NVDEC codec + module (:code-dir:`codec/ `) or a GStreamer fallback. +- **Placement** (:code-dir:`placements/ `) holds the XR lock-mode + strategies. Placement is application policy — Televiz only renders a layer at whatever pose the app + sets each frame. Sharing the XR session with TeleopSession ----------------------------------------- diff --git a/examples/camera_viz/.gitignore b/examples/camera_viz/.gitignore new file mode 100644 index 000000000..aa3714e34 --- /dev/null +++ b/examples/camera_viz/.gitignore @@ -0,0 +1,7 @@ +# Replay clips for ``type: video`` — user-supplied or generated +# (e.g. the ffmpeg test pattern from the docs), never committed. +*.mp4 +*.mkv +*.webm +*.avi +*.mov diff --git a/examples/camera_viz/camera_viz.py b/examples/camera_viz/camera_viz.py index 291e17940..8363af360 100755 --- a/examples/camera_viz/camera_viz.py +++ b/examples/camera_viz/camera_viz.py @@ -139,6 +139,12 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: f"camera_viz: camera {cam.get('name')!r} missing rtp.port; " "required when source: rtp" ) + if "width" not in cam or "height" not in cam: + raise ValueError( + f"camera_viz: camera {cam.get('name')!r} needs explicit " + "width/height when source: rtp — the receiver sizes its " + "decoder from the YAML, not from the wire" + ) placement = _placement_with_aspect( placements_cfg.get(cam["name"]), int(cam["width"]), int(cam["height"]), is_xr ) diff --git a/examples/camera_viz/configs/video.yaml b/examples/camera_viz/configs/video.yaml index ccd887a9c..59ec53405 100644 --- a/examples/camera_viz/configs/video.yaml +++ b/examples/camera_viz/configs/video.yaml @@ -7,9 +7,10 @@ # side (mono only) to exercise the full RTP path camera-free. # # Point ``path`` at any file OpenCV's FFmpeg backend reads (mp4 / mkv / -# webm, H.264 / HEVC / AV1 / ...). No sample clip ships with the repo; -# generate a 10 s test pattern with: -# ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 recording.mp4 +# webm, H.264 / HEVC / AV1 / ...). Relative paths resolve against this +# YAML's directory. No sample clip ships with the repo; generate a 10 s +# test pattern with (from examples/camera_viz/): +# ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 configs/recording.mp4 source: local # local | rtp — see real-camera configs for rtp streaming: From a5dddd61dd952d115d6663b47a5d3b3db7e367c0 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Thu, 9 Jul 2026 15:31:32 -0700 Subject: [PATCH 03/14] feat(camera_viz): default display mode to XR; document run --mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit XR is the product experience, so it becomes the default everywhere: the code default when a config omits display.mode, and the shipped configs' explicit value. camera_viz.sh run already forwarded extra args to camera_viz.py — document --mode xr|window in the usage text and examples so the desktop-window fallback is discoverable. Reframe the camera-streaming doc for public developers (it read like an internal QA script): soften the walkthrough framing, present the first run with both the XR default and the --mode window fallback, fold the XR section into a Display modes section, and add an XR- runtime-missing troubleshooting entry. Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 51 +++++++++++-------- examples/camera_viz/README.md | 7 +-- examples/camera_viz/camera_viz.py | 7 +-- examples/camera_viz/camera_viz.sh | 10 ++-- examples/camera_viz/configs/dellwebcam.yaml | 2 +- examples/camera_viz/configs/multi_camera.yaml | 2 +- examples/camera_viz/configs/oakd.yaml | 2 +- examples/camera_viz/configs/realsense.yaml | 2 +- examples/camera_viz/configs/synthetic.yaml | 2 +- examples/camera_viz/configs/v4l2.yaml | 2 +- examples/camera_viz/configs/video.yaml | 2 +- examples/camera_viz/configs/zed.yaml | 2 +- 12 files changed, 52 insertions(+), 39 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index fc1e72e16..b1e2a9fd8 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -10,10 +10,10 @@ or replays a video file — and visualizes them on a desktop window or an XR hea camera, aspect-fit. It can also stream a robot's cameras to a workstation over the network (split mode). -The sample lives at :code-dir:`examples/camera_viz/ `. This page walks -through it in the order you would test it: setup, a first run that needs no camera, real cameras, -XR output, then the robot → workstation split mode. For the exact command surface and flags, see -the :code-file:`README `. +The sample lives at :code-dir:`examples/camera_viz/ `. This page walks you +from setup and a hardware-free first run to real cameras and the robot → workstation split mode. +For the exact command surface and flags, see the +:code-file:`README `. .. figure:: ../_static/televiz_2d.gif :alt: camera_viz in window mode with synthetic multi-camera feeds @@ -31,8 +31,9 @@ Requirements - Linux x86_64 or Jetson with an NVIDIA GPU and a CUDA 12 driver (``nvidia-smi`` works) — every source hands frames to the renderer GPU-resident via CuPy. -- ``display.mode: window`` needs a local display attached; ``display.mode: xr`` needs an OpenXR - runtime (see :doc:`/getting_started/televiz`). +- The default display mode is XR, which needs an OpenXR runtime (see + :doc:`/getting_started/televiz`). No headset handy? ``--mode window`` renders to a desktop + window instead and only needs a local display. - No IsaacTeleop source build and no camera required — setup installs everything from PyPI, and the video-replay source runs hardware-free. @@ -71,16 +72,18 @@ as a stand-in feed while the real camera isn't available: # HEVC / AV1, ...); or generate a 10 s test pattern: ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 configs/recording.mp4 - ./camera_viz.sh run configs/video.yaml + ./camera_viz.sh run configs/video.yaml # XR headset (default) + ./camera_viz.sh run configs/video.yaml --mode window # desktop window instead -**Expected result** — the terminal reports the session and the source coming up:: +**You should see** the terminal report the session and the source coming up:: - camera_viz: source=local, mode=window, xr=False, 1 layer(s) + camera_viz: source=local, mode=xr, xr=True, 1 layer(s) [video] opening... [video] connected [video] streaming -and a window opens looping the clip. +and the clip looping on a plane in the headset — or in a desktop window (``mode=window, +xr=False``) with the ``--mode window`` override. To replay your own recording, edit ``path:`` in :code-file:`configs/video.yaml ` — relative paths resolve against the YAML's directory, @@ -127,18 +130,19 @@ run with the matching config: ./camera_viz.sh run configs/v4l2.yaml # or oakd.yaml / zed.yaml -**Expected result** — the same startup lines as above with the camera's tag (``[v4l2]``, -``[oakd]``, ``[zed]``) and a live feed in the window. Multiple ``cameras:`` entries render as one -plane each in the same window. +**You should see** the same startup lines as above with the camera's tag (``[v4l2]``, +``[oakd]``, ``[zed]``) and the live feed. Multiple ``cameras:`` entries render as one plane +each. -XR output ---------- +Display modes +------------- -Set ``display.mode: xr`` in the YAML — or pass ``--mode xr`` to override the config — and run -with an OpenXR runtime active. Each camera renders as its own plane in the headset; stereo -sources (``stereo: true``) render true side-by-side stereo. Window mode shows the left eye. +XR is the default: each camera renders as its own plane in the headset via the active OpenXR +runtime, and stereo sources (``stereo: true``) render true side-by-side stereo. Pass +``--mode window`` (or set ``display.mode: window`` in the YAML) to render to a desktop window +instead — no headset or runtime needed; stereo shows the left eye. -How a plane follows the operator's head is the per-camera ``lock_mode`` under +In XR, how a plane follows the operator's head is the per-camera ``lock_mode`` under ``display.placements.``: .. list-table:: @@ -227,7 +231,7 @@ plane (and, in split mode, its own RTP port). Abbreviated: bitrate_mbps: 15 display: - mode: window | xr + mode: xr | window # default: xr window: { width, height } xr: { near_z, far_z } clear_color: [r, g, b, a] @@ -244,8 +248,11 @@ YAML per source kind. Troubleshooting --------------- -- **No window appears over SSH** — ``display.mode: window`` needs a local display; run on the - machine you're sitting at, or use a video-capable remote desktop. +- **The XR session fails to create** — the default mode needs an active OpenXR runtime (see + :doc:`/getting_started/televiz`); pass ``--mode window`` to render to a desktop window + instead. +- **No window appears over SSH** — ``--mode window`` needs a local display; run on the machine + you're sitting at, or use a video-capable remote desktop. - **"video source: no such file"** — relative ``path:`` values resolve against the YAML's directory (``configs/``), not the directory you launched from. - **A source fails asking for CuPy / CUDA** — check ``nvidia-smi`` works and setup completed; diff --git a/examples/camera_viz/README.md b/examples/camera_viz/README.md index 34ef59250..0d6329b34 100644 --- a/examples/camera_viz/README.md +++ b/examples/camera_viz/README.md @@ -22,7 +22,7 @@ SPDX-License-Identifier: Apache-2.0 | `zed` | ZED 2 / Mini / X One; mono or `stereo: true` (per-eye SDK retrieve, zero-copy GPU) | | `video` | Video-file replay (anything OpenCV/FFmpeg reads) — preview / testing without a camera. Loops by default; `stereo: true` splits side-by-side files into eyes (viewer only) | -Output: window or XR headset; one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`. +Output: XR headset (default) or desktop window (`run CONFIG --mode window`); one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`. --- @@ -44,7 +44,8 @@ Flags: `--no-{v4l2,oakd,rtp}`, `--with-zed`, `--sender-only`, `--jetson`. Pass ` ## Mode 1 — Direct ```bash -./camera_viz.sh run configs/v4l2.yaml +./camera_viz.sh run configs/v4l2.yaml # XR headset (default) +./camera_viz.sh run configs/v4l2.yaml --mode window # desktop window instead ``` Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`, `video.yaml` (file replay — point `path:` at any recording). @@ -107,7 +108,7 @@ cameras: # gpu_id: 0 # multi-GPU pin display: # camera_viz only - mode: window | xr + mode: xr | window # default: xr window: { width, height } xr: { near_z, far_z } clear_color: [r, g, b, a] diff --git a/examples/camera_viz/camera_viz.py b/examples/camera_viz/camera_viz.py index 8363af360..77a22061a 100755 --- a/examples/camera_viz/camera_viz.py +++ b/examples/camera_viz/camera_viz.py @@ -199,7 +199,7 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: def _make_session(cfg: dict, mode_override: Optional[str] = None) -> viz.VizSession: display = cfg.get("display", {}) # --mode overrides display.mode when given. - mode_str = (mode_override or display.get("mode", "window")).lower() + mode_str = (mode_override or display.get("mode", "xr")).lower() session_cfg = viz.VizSessionConfig() if mode_str == "window": session_cfg.mode = viz.DisplayMode.kWindow @@ -228,7 +228,8 @@ def main(argv: Optional[list[str]] = None) -> int: "--mode", choices=("window", "xr"), default=None, - help="Override display.mode from the config (default: use the config's value).", + help="Override display.mode from the config " + "(default: the config's value, or xr when the config omits it).", ) args = parser.parse_args(argv) @@ -248,7 +249,7 @@ def main(argv: Optional[list[str]] = None) -> int: if source_mode not in ("local", "rtp"): raise ValueError(f"camera_viz: source must be local|rtp, got {source_mode!r}") - effective_mode = (args.mode or cfg.get("display", {}).get("mode", "window")).lower() + effective_mode = (args.mode or cfg.get("display", {}).get("mode", "xr")).lower() session = _make_session(cfg, mode_override=args.mode) is_xr = session.is_xr_mode() diff --git a/examples/camera_viz/camera_viz.sh b/examples/camera_viz/camera_viz.sh index 64756fd2b..1052d9ed3 100755 --- a/examples/camera_viz/camera_viz.sh +++ b/examples/camera_viz/camera_viz.sh @@ -8,7 +8,7 @@ # Local: # ./camera_viz.sh setup [--sender-only] install deps + build codec # ./camera_viz.sh loopback CONFIG run streamer + viz on 127.0.0.1 -# ./camera_viz.sh run CONFIG run the viewer (honors source:) +# ./camera_viz.sh run CONFIG [--mode M] run the viewer (honors source:) # # Remote (Jetson robot): # ./camera_viz.sh deploy --host H --user U [--password P] CONFIG @@ -401,10 +401,13 @@ LOCAL loopback CONFIG Run camera_streamer + camera_viz on 127.0.0.1. - run CONFIG Run the viewer with the YAML as-is. ``source: + run CONFIG [--mode xr|window] + Run the viewer with the YAML as-is. ``source: local`` opens cameras directly; ``source: rtp`` listens on rtp.port (sender IP irrelevant — the - receiver binds 0.0.0.0). + receiver binds 0.0.0.0). --mode overrides + display.mode: xr renders to the headset (the + default), window to a desktop window. REMOTE (Jetson robot) deploy [--host H --user U [--password P]] @@ -437,6 +440,7 @@ EXAMPLES ./camera_viz.sh loopback configs/v4l2.yaml ./camera_viz.sh deploy --host 10.29.90.127 --user nvidia configs/v4l2.yaml ./camera_viz.sh run configs/v4l2.yaml + ./camera_viz.sh run configs/video.yaml --mode window # no headset needed # Env-var style (avoids passwords in shell history / argv): export REMOTE_HOST=10.29.90.127 REMOTE_USER=nvidia diff --git a/examples/camera_viz/configs/dellwebcam.yaml b/examples/camera_viz/configs/dellwebcam.yaml index 6c285f532..de3d42d6a 100644 --- a/examples/camera_viz/configs/dellwebcam.yaml +++ b/examples/camera_viz/configs/dellwebcam.yaml @@ -40,7 +40,7 @@ cameras: # gop: # default: fps*5 (IDR every 5 s, ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1920 height: 1080 diff --git a/examples/camera_viz/configs/multi_camera.yaml b/examples/camera_viz/configs/multi_camera.yaml index 5c1bd679d..bc468fbec 100644 --- a/examples/camera_viz/configs/multi_camera.yaml +++ b/examples/camera_viz/configs/multi_camera.yaml @@ -45,7 +45,7 @@ cameras: bitrate_mbps: 15 display: - mode: xr # window | xr + mode: xr # xr | window (xr is the default) window: # used only if mode: window width: 2560 height: 720 diff --git a/examples/camera_viz/configs/oakd.yaml b/examples/camera_viz/configs/oakd.yaml index 68c3557a6..97854b610 100644 --- a/examples/camera_viz/configs/oakd.yaml +++ b/examples/camera_viz/configs/oakd.yaml @@ -35,7 +35,7 @@ cameras: # gop: # default: fps*5 (ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 diff --git a/examples/camera_viz/configs/realsense.yaml b/examples/camera_viz/configs/realsense.yaml index 9ea683805..c7e4ffa0f 100644 --- a/examples/camera_viz/configs/realsense.yaml +++ b/examples/camera_viz/configs/realsense.yaml @@ -37,7 +37,7 @@ cameras: # gop: # default: fps*5 (IDR every 5 s, ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1920 height: 1080 diff --git a/examples/camera_viz/configs/synthetic.yaml b/examples/camera_viz/configs/synthetic.yaml index 1459c8364..61147e3bd 100644 --- a/examples/camera_viz/configs/synthetic.yaml +++ b/examples/camera_viz/configs/synthetic.yaml @@ -25,7 +25,7 @@ cameras: # gop: # default: fps*5 (ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 diff --git a/examples/camera_viz/configs/v4l2.yaml b/examples/camera_viz/configs/v4l2.yaml index 90a0ac151..b0730056c 100644 --- a/examples/camera_viz/configs/v4l2.yaml +++ b/examples/camera_viz/configs/v4l2.yaml @@ -32,7 +32,7 @@ cameras: # gop: # default: fps*5 (IDR every 5 s, ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 2560 height: 720 diff --git a/examples/camera_viz/configs/video.yaml b/examples/camera_viz/configs/video.yaml index 59ec53405..c4443ab16 100644 --- a/examples/camera_viz/configs/video.yaml +++ b/examples/camera_viz/configs/video.yaml @@ -38,7 +38,7 @@ cameras: bitrate_mbps: 15 display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 diff --git a/examples/camera_viz/configs/zed.yaml b/examples/camera_viz/configs/zed.yaml index 3db9d3ccd..701b26c9f 100644 --- a/examples/camera_viz/configs/zed.yaml +++ b/examples/camera_viz/configs/zed.yaml @@ -34,7 +34,7 @@ cameras: # gop: # default: fps*5 (ULL tuning) display: - mode: xr # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 From 3279d8415a20a99a8e2e9149b43e697f340d02db Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Thu, 9 Jul 2026 16:11:31 -0700 Subject: [PATCH 04/14] chore(camera_viz): remove dellwebcam config Device-specific duplicate of v4l2.yaml; nothing referenced it. Signed-off-by: Farbod Motlagh --- examples/camera_viz/configs/dellwebcam.yaml | 54 --------------------- 1 file changed, 54 deletions(-) delete mode 100644 examples/camera_viz/configs/dellwebcam.yaml diff --git a/examples/camera_viz/configs/dellwebcam.yaml b/examples/camera_viz/configs/dellwebcam.yaml deleted file mode 100644 index de3d42d6a..000000000 --- a/examples/camera_viz/configs/dellwebcam.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# V4L2 source configured for the DELL Display 4MP Webcam (USB ID 413c:d005). -# Same YAML drives both camera_viz (receiver) and camera_streamer (RTP sender). -# Flip top-level ``source: local|rtp`` to control what camera_viz opens; -# camera_streamer always reads cameras + rtp ports + streaming.host. -# -# DELL Display 4MP Webcam V4L2 node layout (on THIS machine): -# /dev/video0 — main capture node ← used here -# /dev/video1 — metadata (no capture) -# /dev/video2 — secondary capture (NV12 only) -# /dev/video3 — metadata (no capture) -# -# Format note: this webcam offers 1920x1080 ONLY via MJPG. Its YUYV mode -# tops out at 640x480, so MJPG is selected below for 1080p30. -# -# To inspect available resolutions and formats on the capture node: -# sudo apt install v4l-utils -# v4l2-ctl --device=/dev/video0 --list-formats-ext - -source: local # local | rtp -streaming: - host: 127.0.0.1 # camera_streamer target (override with --host) -encoder: auto # auto | native | gstreamer - # auto picks native NVENC on desktop, GStreamer on Jetson - -cameras: - - name: cam - enabled: true - type: v4l2 - device: /dev/video0 # DELL Display 4MP Webcam main capture node - width: 1920 - height: 1080 - fps: 30 - fourcc: MJPG # webcam only offers 1920x1080 via MJPG (YUYV maxes at 640x480) - rtp: # used by camera_streamer + source: rtp receivers - port: 5000 - bitrate_mbps: 15 # camera_streamer default; lower for tight uplinks - # gop: # default: fps*5 (IDR every 5 s, ULL tuning) - -display: - mode: xr # xr | window (xr is the default) - window: - width: 1920 - height: 1080 - xr: - near_z: 0.05 - far_z: 100.0 - clear_color: [0.0, 0.0, 0.0, 0.0] - placements: - cam: # size defaults to 1.0 m wide × aspect-derived height - lock_mode: lazy - distance: 1.5 From 370f196c3db519c4f03d2498db09ba41379dc9b8 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Thu, 9 Jul 2026 16:19:19 -0700 Subject: [PATCH 05/14] feat(camera_viz): ship a sample replay clip in test_data/ via Git LFS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit configs/video.yaml now points at test_data/recording.mp4 (10 s testsrc2 pattern, ~1 MB) so the hardware-free first run works straight after clone — no ffmpeg step. Track *.mp4 via LFS in .gitattributes, carve the test_data exception out of the camera_viz .gitignore, and document the git lfs pull recovery in troubleshooting. Signed-off-by: Farbod Motlagh --- .gitattributes | 3 +++ docs/source/references/camera_streaming.rst | 15 ++++++++------- examples/camera_viz/.gitignore | 6 ++++-- examples/camera_viz/README.md | 1 + examples/camera_viz/configs/video.yaml | 8 ++++---- examples/camera_viz/test_data/recording.mp4 | 3 +++ 6 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 examples/camera_viz/test_data/recording.mp4 diff --git a/.gitattributes b/.gitattributes index 1cff09735..ce629912c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,6 +18,9 @@ *.usdc filter=lfs diff=lfs merge=lfs -text *.usdz filter=lfs diff=lfs merge=lfs -text +# Video assets are tracked by default. +*.mp4 filter=lfs diff=lfs merge=lfs -text + # # Exceptions for small source SVGs (webpack-inlined; not binary assets). deps/cloudxr/webxr_client/src/icons/*.svg -filter -diff -merge text diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index b1e2a9fd8..9287e7e17 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -63,15 +63,12 @@ First run — no camera required The video-replay source (``type: video``) plays a recording through exactly the same source → QuadLayer → Televiz path a live camera uses, so it doubles as the quickest end-to-end check and -as a stand-in feed while the real camera isn't available: +as a stand-in feed while the real camera isn't available. A 10 s test clip ships with the repo +(``test_data/recording.mp4``, via Git LFS) and ``configs/video.yaml`` already points at it: .. code-block:: bash cd examples/camera_viz - # Any file OpenCV's FFmpeg backend reads works (mp4 / mkv / webm, H.264 / - # HEVC / AV1, ...); or generate a 10 s test pattern: - ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 configs/recording.mp4 - ./camera_viz.sh run configs/video.yaml # XR headset (default) ./camera_viz.sh run configs/video.yaml --mode window # desktop window instead @@ -85,8 +82,9 @@ as a stand-in feed while the real camera isn't available: and the clip looping on a plane in the headset — or in a desktop window (``mode=window, xr=False``) with the ``--mode window`` override. -To replay your own recording, edit ``path:`` in :code-file:`configs/video.yaml -` — relative paths resolve against the YAML's directory, +To replay your own recording — any file OpenCV's FFmpeg backend reads (mp4 / mkv / webm, H.264 / +HEVC / AV1, ...) — edit ``path:`` in :code-file:`configs/video.yaml +`; relative paths resolve against the YAML's directory, not your working directory. ``loop: false`` holds the last frame instead of rewinding; ``stereo: true`` splits a side-by-side recording (e.g. from a ZED) into per-eye views; ``width`` / ``height`` default to the file's native size. @@ -255,6 +253,8 @@ Troubleshooting you're sitting at, or use a video-capable remote desktop. - **"video source: no such file"** — relative ``path:`` values resolve against the YAML's directory (``configs/``), not the directory you launched from. +- **The shipped test clip won't open** — it lives in Git LFS; if ``test_data/recording.mp4`` is + a tiny text file, run ``git lfs install && git lfs pull``. - **A source fails asking for CuPy / CUDA** — check ``nvidia-smi`` works and setup completed; all sources allocate their frame buffers on the GPU. - **Split mode renders nothing** — check the sender is up (``./camera_viz.sh service-status``), @@ -282,6 +282,7 @@ Televiz as the compositor at the end of the chain: ├── transports/ — RTP sender + receiver (native + GStreamer) ├── codec/ — native NVENC / NVDEC pybind module ├── configs/ — one YAML per source kind + ├── test_data/ — sample replay clip (Git LFS) └── scripts/ — installer + systemd unit template - **Sources** (:code-dir:`sources/ `) implement a common source ABC and diff --git a/examples/camera_viz/.gitignore b/examples/camera_viz/.gitignore index aa3714e34..cbf20a0c3 100644 --- a/examples/camera_viz/.gitignore +++ b/examples/camera_viz/.gitignore @@ -1,7 +1,9 @@ -# Replay clips for ``type: video`` — user-supplied or generated -# (e.g. the ffmpeg test pattern from the docs), never committed. +# Replay clips for ``type: video`` — user-supplied or generated, +# never committed. The shipped sample clip in test_data/ (Git LFS) +# is the exception. *.mp4 *.mkv *.webm *.avi *.mov +!test_data/*.mp4 diff --git a/examples/camera_viz/README.md b/examples/camera_viz/README.md index 0d6329b34..39e803b3c 100644 --- a/examples/camera_viz/README.md +++ b/examples/camera_viz/README.md @@ -150,6 +150,7 @@ camera_viz/ ├── transports/ — RTP sender + receiver, native + GStreamer ├── codec/ — native NVENC/NVDEC pybind module ├── configs/ — one YAML per camera kind +├── test_data/ — sample replay clip (Git LFS) └── scripts/ ├── _install_deps.sh — installer (setup + deploy) └── camera-streamer.service.in — systemd unit template diff --git a/examples/camera_viz/configs/video.yaml b/examples/camera_viz/configs/video.yaml index c4443ab16..8e6b59613 100644 --- a/examples/camera_viz/configs/video.yaml +++ b/examples/camera_viz/configs/video.yaml @@ -8,9 +8,9 @@ # # Point ``path`` at any file OpenCV's FFmpeg backend reads (mp4 / mkv / # webm, H.264 / HEVC / AV1 / ...). Relative paths resolve against this -# YAML's directory. No sample clip ships with the repo; generate a 10 s -# test pattern with (from examples/camera_viz/): -# ffmpeg -f lavfi -i testsrc2=size=1280x720:rate=30 -t 10 configs/recording.mp4 +# YAML's directory. The default is the 10 s test clip shipped at +# ../test_data/recording.mp4 (Git LFS — run ``git lfs pull`` if it opens +# as a tiny text pointer). source: local # local | rtp — see real-camera configs for rtp streaming: @@ -21,7 +21,7 @@ cameras: - name: replay enabled: true type: video - path: recording.mp4 # relative paths resolve against this YAML's dir + path: ../test_data/recording.mp4 # relative to this YAML's dir loop: true # rewind at end of file (false = hold last frame) fps: 0 # 0 = the file's native rate. With source: rtp, # set this explicitly — the sender's encode loop diff --git a/examples/camera_viz/test_data/recording.mp4 b/examples/camera_viz/test_data/recording.mp4 new file mode 100644 index 000000000..1fcfa39f1 --- /dev/null +++ b/examples/camera_viz/test_data/recording.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c96cd61dcf8dcb293cc0db47b6bc1c3905b8d95b3844fb49ae34beb8f24ad96 +size 1026163 From f90ed4c1998c2298142351b130200b6bfb897349 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Thu, 9 Jul 2026 16:27:44 -0700 Subject: [PATCH 06/14] docs(camera_streaming): lead with cameras-to-headset; frame replay/window as debug aids Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 9287e7e17..84e0b7faa 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -5,10 +5,10 @@ Camera Streaming ================ ``camera_viz`` is the reference camera-streaming sample built on :doc:`Televiz -` (``isaacteleop.viz``). It captures frames from one or more cameras — -or replays a video file — and visualizes them on a desktop window or an XR headset, one plane per -camera, aspect-fit. It can also stream a robot's cameras to a workstation over the network -(split mode). +` (``isaacteleop.viz``). It captures frames from one or more cameras +and streams them to an XR headset — one plane per camera, aspect-fit — either directly or from a +robot to a workstation over the network (split mode). For development and debugging it can also +replay a video file in place of a camera, and render to a desktop window instead of the headset. The sample lives at :code-dir:`examples/camera_viz/ `. This page walks you from setup and a hardware-free first run to real cameras and the robot → workstation split mode. From 94f5e3422c69d866c382733bf2e4ff714b3cce53 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 11:52:04 -0700 Subject: [PATCH 07/14] docs(camera_streaming): defer requirements/setup to the standard pages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Point system requirements at /references/requirements and the XR prerequisites (CloudXR server, headset connection) at the quick-start step anchors instead of restating them. Setup now says explicitly that the pip-install step from the quick start isn't needed — the sample's setup script creates its own environment. Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 31 +++++++++++---------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 84e0b7faa..ae25231d4 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -29,29 +29,32 @@ For the exact command surface and flags, see the Requirements ------------ -- Linux x86_64 or Jetson with an NVIDIA GPU and a CUDA 12 driver (``nvidia-smi`` works) — every - source hands frames to the renderer GPU-resident via CuPy. -- The default display mode is XR, which needs an OpenXR runtime (see - :doc:`/getting_started/televiz`). No headset handy? ``--mode window`` renders to a desktop - window instead and only needs a local display. +- A workstation meeting the :doc:`system requirements ` (Ubuntu, NVIDIA + GPU, CUDA driver) — every source hands frames to the renderer GPU-resident via CuPy. +- For the default XR mode, a running CloudXR server with a connected headset — follow the + :doc:`quick start ` steps :ref:`run-cloudxr-server` and + :ref:`connect-xr-headset`. No headset handy? ``--mode window`` renders to a desktop window + instead and only needs a local display. - No IsaacTeleop source build and no camera required — setup installs everything from PyPI, and the video-replay source runs hardware-free. Setup ----- -One-time setup installs the sample's Python environment: +Clone the repository if you haven't already (quick start step :ref:`check-out-code-base`), then +run the sample's one-time setup: .. code-block:: bash examples/camera_viz/camera_viz.sh setup source examples/camera_viz/.venv/bin/activate -``setup`` installs ``isaacteleop`` (which bundles Televiz) and every other Python dependency from -PyPI into ``.venv/`` via ``uv``, builds the native NVENC/NVDEC codec, and probes system packages -(GStreamer plugins, cairo / girepository headers, JetPack ``cuda-nvrtc`` + ``ld.so`` wiring). When -something is missing it prints the exact ``apt-get`` line and prompts ``[y/N]`` — answering ``n`` -or running non-interactively aborts. +There is no need to install the ``isaacteleop`` pip package yourself — ``setup`` creates the +sample's own environment: it installs ``isaacteleop`` (which bundles Televiz) and every other +Python dependency from PyPI into ``.venv/`` via ``uv``, builds the native NVENC/NVDEC codec, and +probes system packages (GStreamer plugins, cairo / girepository headers, JetPack ``cuda-nvrtc`` + +``ld.so`` wiring). When something is missing it prints the exact ``apt-get`` line and prompts +``[y/N]`` — answering ``n`` or running non-interactively aborts. Useful flags: ``--no-{v4l2,oakd,rtp}``, ``--with-zed``, ``--sender-only``, ``--jetson``. Pass ``--venv PATH`` to install into an existing virtual environment. To develop against a locally @@ -246,9 +249,9 @@ YAML per source kind. Troubleshooting --------------- -- **The XR session fails to create** — the default mode needs an active OpenXR runtime (see - :doc:`/getting_started/televiz`); pass ``--mode window`` to render to a desktop window - instead. +- **The XR session fails to create** — the default mode needs the CloudXR server running and a + headset connected (quick start steps :ref:`run-cloudxr-server` and :ref:`connect-xr-headset`); + pass ``--mode window`` to render to a desktop window instead. - **No window appears over SSH** — ``--mode window`` needs a local display; run on the machine you're sitting at, or use a video-capable remote desktop. - **"video source: no such file"** — relative ``path:`` values resolve against the YAML's From ff918a466d8d1c844de3fcff5400629690094562 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 12:17:02 -0700 Subject: [PATCH 08/14] docs(camera_streaming): flag table, tighter first run, loopback framing Expand the setup flags into a table explaining what each one does (semantics verified against _install_deps.sh). Trim the first-run section: drop the LFS/OpenCV asides and the synthetic note, keeping just 'set a custom path for your own video'. Drop the redundant third requirements bullet, reword '``cameras:``' prose to 'the cameras list', and state that loopback is a testing/debugging aid. Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 76 +++++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index ae25231d4..e99759258 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -35,8 +35,6 @@ Requirements :doc:`quick start ` steps :ref:`run-cloudxr-server` and :ref:`connect-xr-headset`. No headset handy? ``--mode window`` renders to a desktop window instead and only needs a local display. -- No IsaacTeleop source build and no camera required — setup installs everything from PyPI, and - the video-replay source runs hardware-free. Setup ----- @@ -56,18 +54,43 @@ probes system packages (GStreamer plugins, cairo / girepository headers, JetPack ``ld.so`` wiring). When something is missing it prints the exact ``apt-get`` line and prompts ``[y/N]`` — answering ``n`` or running non-interactively aborts. -Useful flags: ``--no-{v4l2,oakd,rtp}``, ``--with-zed``, ``--sender-only``, ``--jetson``. Pass -``--venv PATH`` to install into an existing virtual environment. To develop against a locally -built wheel instead of the PyPI release, pass ``--wheel `` (see -:doc:`/getting_started/build_from_source/index`). +By default ``setup`` provisions everything except ZED support; flags trim or extend that: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Flag + - Effect + * - ``--no-v4l2`` + - Skip USB / UVC webcam support (``opencv-python``). + * - ``--no-oakd`` + - Skip OAK-D support (``depthai``). + * - ``--no-rtp`` + - Skip split-mode dependencies: the GStreamer system packages and the native NVENC/NVDEC + codec build. Direct mode still works. + * - ``--with-zed`` + - Also build + install the ZED SDK's Python API (``pyzed``). Requires the ZED SDK on the + machine (default ``/usr/local/zed``; override with ``--zed-sdk PATH``). + * - ``--sender-only`` + - Robot-side install: only what ``camera_streamer.py`` needs — skips ``isaacteleop`` and the + Vulkan viewer dependencies. ``deploy`` uses this on the robot automatically. + * - ``--jetson`` + - JetPack-specific provisioning: installs ``cuda-nvrtc`` and creates the unversioned CUDA + library symlinks + ``ld.so`` wiring that JetPack images skip. Leave off on desktops. + * - ``--venv PATH`` + - Install into an existing virtual environment instead of creating ``.venv/``. + * - ``--wheel PATH`` + - Install a locally built ``isaacteleop`` wheel instead of the PyPI release — for developing + Isaac Teleop itself (see :doc:`/getting_started/build_from_source/index`). First run — no camera required ------------------------------ -The video-replay source (``type: video``) plays a recording through exactly the same source → -QuadLayer → Televiz path a live camera uses, so it doubles as the quickest end-to-end check and -as a stand-in feed while the real camera isn't available. A 10 s test clip ships with the repo -(``test_data/recording.mp4``, via Git LFS) and ``configs/video.yaml`` already points at it: +The video-replay source (``type: video``) plays a recording through exactly the same path a live +camera uses, so it doubles as the quickest end-to-end check and as a stand-in feed while the real +camera isn't available. A test clip ships with the repo and ``configs/video.yaml`` already points +at it: .. code-block:: bash @@ -85,23 +108,15 @@ as a stand-in feed while the real camera isn't available. A 10 s test clip ships and the clip looping on a plane in the headset — or in a desktop window (``mode=window, xr=False``) with the ``--mode window`` override. -To replay your own recording — any file OpenCV's FFmpeg backend reads (mp4 / mkv / webm, H.264 / -HEVC / AV1, ...) — edit ``path:`` in :code-file:`configs/video.yaml -`; relative paths resolve against the YAML's directory, -not your working directory. ``loop: false`` holds the last frame instead of rewinding; -``stereo: true`` splits a side-by-side recording (e.g. from a ZED) into per-eye views; ``width`` -/ ``height`` default to the file's native size. - -.. note:: - - There is also ``type: synthetic`` (``configs/synthetic.yaml``) — a GPU-generated moving test - pattern. It's a debugging tool for isolating the render path: no file, no decoder, no - hardware involved. +To replay your own video, set a custom ``path:`` in :code-file:`configs/video.yaml +` — relative paths resolve against the YAML's directory. +``loop: false`` holds the last frame instead of rewinding; ``stereo: true`` splits a side-by-side +recording (e.g. from a ZED) into per-eye views. Supported sources ----------------- -The source kind is selected per ``cameras:`` entry by the YAML ``type:`` field: +The source kind is selected by the ``type`` field of each entry in the YAML ``cameras`` list: .. list-table:: :header-rows: 1 @@ -132,8 +147,8 @@ run with the matching config: ./camera_viz.sh run configs/v4l2.yaml # or oakd.yaml / zed.yaml **You should see** the same startup lines as above with the camera's tag (``[v4l2]``, -``[oakd]``, ``[zed]``) and the live feed. Multiple ``cameras:`` entries render as one plane -each. +``[oakd]``, ``[zed]``) and the live feed. Multiple entries in the ``cameras`` list render as one +plane each. Display modes ------------- @@ -199,15 +214,16 @@ errors, and network blips. Loopback ^^^^^^^^ -``./camera_viz.sh loopback configs/v4l2.yaml`` runs the sender and viewer together on -``127.0.0.1`` — the quickest way to smoke-test the RTP path on one machine. It also works -camera-free with a mono ``type: video`` entry (set ``width`` / ``height`` / ``fps``). +Loopback is a testing / debugging aid, not a deployment mode: ``./camera_viz.sh loopback +configs/v4l2.yaml`` runs the sender and viewer together on ``127.0.0.1`` — the quickest way to +smoke-test the RTP path on one machine. It also works camera-free with a mono ``type: video`` +entry (set ``width`` / ``height`` / ``fps``). Configuration ------------- -A single YAML drives both capture and visualization. Each ``cameras:`` entry becomes its own -plane (and, in split mode, its own RTP port). Abbreviated: +A single YAML drives both capture and visualization. Each entry in the ``cameras`` list becomes +its own plane (and, in split mode, its own RTP port). Abbreviated: .. code-block:: yaml From a0015c5b7a70681f117cbfed9b643d1d14e39b11 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 12:22:30 -0700 Subject: [PATCH 09/14] docs(camera_streaming): shorten --sender-only/--jetson flag rows to split-mode one-liners Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index e99759258..23311b1a6 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -73,11 +73,9 @@ By default ``setup`` provisions everything except ZED support; flags trim or ext - Also build + install the ZED SDK's Python API (``pyzed``). Requires the ZED SDK on the machine (default ``/usr/local/zed``; override with ``--zed-sdk PATH``). * - ``--sender-only`` - - Robot-side install: only what ``camera_streamer.py`` needs — skips ``isaacteleop`` and the - Vulkan viewer dependencies. ``deploy`` uses this on the robot automatically. + - Split mode only — robot-side install of just the sender's dependencies. * - ``--jetson`` - - JetPack-specific provisioning: installs ``cuda-nvrtc`` and creates the unversioned CUDA - library symlinks + ``ld.so`` wiring that JetPack images skip. Leave off on desktops. + - Split mode only — extra CUDA wiring JetPack images need on the robot. * - ``--venv PATH`` - Install into an existing virtual environment instead of creating ``.venv/``. * - ``--wheel PATH`` From 1a5cd1391211313dcc8b865b9dea7e01336032ae Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 12:24:54 -0700 Subject: [PATCH 10/14] =?UTF-8?q?docs(camera=5Fstreaming):=20explain=20whe?= =?UTF-8?q?n=20split=20mode=20exists=20=E2=80=94=20cameras=20must=20stream?= =?UTF-8?q?=20to=20where=20Isaac=20Teleop=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 23311b1a6..08a53d51b 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -183,11 +183,12 @@ the viewer on the workstation (``source: rtp``). .. warning:: - Split mode adds a full extra encode/decode hop — NVENC on the robot, UDP, NVDEC on the - workstation — so it is **not recommended** when a camera can attach directly to the viewer - machine; run direct instead. Reach for it only when the cameras are physically on the robot. - Wired networks only: there is no retransmit or FEC — one lost packet corrupts one frame until - the next IDR (default every 5 s). + Split mode exists for one situation: the cameras are on the robot, but Isaac Teleop runs on a + workstation, so the frames must be streamed to where Isaac Teleop is running. That costs a + full extra encode/decode hop — NVENC on the robot, UDP, NVDEC on the workstation — so + whenever a camera can attach directly to the machine running Isaac Teleop, run direct mode + instead. Wired networks only: there is no retransmit or FEC — one lost packet corrupts one + frame until the next IDR (default every 5 s). In split mode every camera entry must pin ``width``, ``height``, and ``fps`` in the YAML — the receiver sizes its decoder from the config, not from the wire. From 1ac29b0951622706ded8369ce3a318873c716a7d Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 12:31:48 -0700 Subject: [PATCH 11/14] docs(camera_streaming): state split mode is not recommended in most cases Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 08a53d51b..01d8a2415 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -183,11 +183,11 @@ the viewer on the workstation (``source: rtp``). .. warning:: - Split mode exists for one situation: the cameras are on the robot, but Isaac Teleop runs on a - workstation, so the frames must be streamed to where Isaac Teleop is running. That costs a - full extra encode/decode hop — NVENC on the robot, UDP, NVDEC on the workstation — so - whenever a camera can attach directly to the machine running Isaac Teleop, run direct mode - instead. Wired networks only: there is no retransmit or FEC — one lost packet corrupts one + Split mode is **not recommended in most cases**. It exists for one situation: the cameras are + on the robot, but Isaac Teleop runs on a workstation, so the frames must be streamed to where + Isaac Teleop is running. That costs a full extra encode/decode hop — NVENC on the robot, UDP, + NVDEC on the workstation — so whenever a camera can attach directly to the machine running + Isaac Teleop, run direct mode instead. Wired networks only: there is no retransmit or FEC — one lost packet corrupts one frame until the next IDR (default every 5 s). In split mode every camera entry must pin ``width``, ``height``, and ``fps`` in the YAML — the From 5abbb1b050225400f47688473c4e63331d55ad3a Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 13:56:42 -0700 Subject: [PATCH 12/14] docs(camera_streaming): drop LFS troubleshooting bullet Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 01d8a2415..02cc689ca 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -271,8 +271,6 @@ Troubleshooting you're sitting at, or use a video-capable remote desktop. - **"video source: no such file"** — relative ``path:`` values resolve against the YAML's directory (``configs/``), not the directory you launched from. -- **The shipped test clip won't open** — it lives in Git LFS; if ``test_data/recording.mp4`` is - a tiny text file, run ``git lfs install && git lfs pull``. - **A source fails asking for CuPy / CUDA** — check ``nvidia-smi`` works and setup completed; all sources allocate their frame buffers on the GPU. - **Split mode renders nothing** — check the sender is up (``./camera_viz.sh service-status``), From 26df443303f6771d31d672217be3cbae686ee9e9 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 10 Jul 2026 14:14:52 -0700 Subject: [PATCH 13/14] style(camera_viz): apply ruff-format Signed-off-by: Farbod Motlagh --- examples/camera_viz/camera_viz.py | 5 ++++- examples/camera_viz/sources/video_file.py | 4 +--- examples/camera_viz/tests/test_video_file_source.py | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/camera_viz/camera_viz.py b/examples/camera_viz/camera_viz.py index 77a22061a..9f2af2df3 100755 --- a/examples/camera_viz/camera_viz.py +++ b/examples/camera_viz/camera_viz.py @@ -146,7 +146,10 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: "decoder from the YAML, not from the wire" ) placement = _placement_with_aspect( - placements_cfg.get(cam["name"]), int(cam["width"]), int(cam["height"]), is_xr + placements_cfg.get(cam["name"]), + int(cam["width"]), + int(cam["height"]), + is_xr, ) stereo, baseline_mm = _stereo_for(cam, placements_cfg) diff --git a/examples/camera_viz/sources/video_file.py b/examples/camera_viz/sources/video_file.py index f0e13b224..b2ef9eeb9 100644 --- a/examples/camera_viz/sources/video_file.py +++ b/examples/camera_viz/sources/video_file.py @@ -121,9 +121,7 @@ def __init__( # the mono layout. self._frame_w = out_w * eyes if stereo: - self._host_staging = alloc_pinned_host( - (out_h, self._frame_w, 3), np.uint8 - ) + self._host_staging = alloc_pinned_host((out_h, self._frame_w, 3), np.uint8) # Right-eye output buffers, rotated in lock-step with the base # class's (left-eye) triple-buffer. self._gpu_right = [ diff --git a/examples/camera_viz/tests/test_video_file_source.py b/examples/camera_viz/tests/test_video_file_source.py index a0b075094..f2af155cf 100644 --- a/examples/camera_viz/tests/test_video_file_source.py +++ b/examples/camera_viz/tests/test_video_file_source.py @@ -99,7 +99,9 @@ def test_probe_size_playback_and_color(tmp_path): clip = tmp_path / "blue.mp4" _write_clip(clip, [_solid(48, 64, (255, 0, 0))] * 10, fps=60.0) # BGR blue - (source,) = build_local_camera({"name": "replay", "type": "video", "path": str(clip)}) + (source,) = build_local_camera( + {"name": "replay", "type": "video", "path": str(clip)} + ) assert source.spec.width == 64 assert source.spec.height == 48 From 165e77c94388c576000e6e5c339ff3a0996b1156 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Mon, 13 Jul 2026 15:31:17 -0700 Subject: [PATCH 14/14] fix(camera_viz): rename video.yaml to replay.yaml; portable abs-path test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address MR review: 'video' was too generic a config name — rename to replay.yaml and update all references (docs, README, camera_viz.sh). Clarify the stereo comment/doc: side-by-side replay renders as stereo in direct mode only (the RTP sender rejects single-source stereo). Also fix a Windows CI failure in test_resolve_video_paths: a POSIX '/abs' string isn't absolute on Windows and got re-anchored to the drive. Build the absolute test input from tmp_path so it's absolute on any OS. Signed-off-by: Farbod Motlagh --- docs/source/references/camera_streaming.rst | 14 +++++++------- examples/camera_viz/.gitignore | 3 +++ examples/camera_viz/README.md | 2 +- examples/camera_viz/camera_viz.sh | 2 +- .../camera_viz/configs/{video.yaml => replay.yaml} | 7 ++++--- .../camera_viz/tests/test_video_file_source.py | 8 ++++++-- 6 files changed, 22 insertions(+), 14 deletions(-) rename examples/camera_viz/configs/{video.yaml => replay.yaml} (86%) diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 02cc689ca..9e245189e 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -87,14 +87,14 @@ First run — no camera required The video-replay source (``type: video``) plays a recording through exactly the same path a live camera uses, so it doubles as the quickest end-to-end check and as a stand-in feed while the real -camera isn't available. A test clip ships with the repo and ``configs/video.yaml`` already points +camera isn't available. A test clip ships with the repo and ``configs/replay.yaml`` already points at it: .. code-block:: bash cd examples/camera_viz - ./camera_viz.sh run configs/video.yaml # XR headset (default) - ./camera_viz.sh run configs/video.yaml --mode window # desktop window instead + ./camera_viz.sh run configs/replay.yaml # XR headset (default) + ./camera_viz.sh run configs/replay.yaml --mode window # desktop window instead **You should see** the terminal report the session and the source coming up:: @@ -106,10 +106,10 @@ at it: and the clip looping on a plane in the headset — or in a desktop window (``mode=window, xr=False``) with the ``--mode window`` override. -To replay your own video, set a custom ``path:`` in :code-file:`configs/video.yaml -` — relative paths resolve against the YAML's directory. -``loop: false`` holds the last frame instead of rewinding; ``stereo: true`` splits a side-by-side -recording (e.g. from a ZED) into per-eye views. +To replay your own video, set a custom ``path:`` in :code-file:`configs/replay.yaml +` — relative paths resolve against the YAML's directory. +``loop: false`` holds the last frame instead of rewinding; ``stereo: true`` replays a side-by-side +recording (e.g. from a ZED) as stereo, splitting each frame into per-eye views (direct mode only). Supported sources ----------------- diff --git a/examples/camera_viz/.gitignore b/examples/camera_viz/.gitignore index cbf20a0c3..0e1069186 100644 --- a/examples/camera_viz/.gitignore +++ b/examples/camera_viz/.gitignore @@ -7,3 +7,6 @@ *.avi *.mov !test_data/*.mp4 + +# Vendor SDK scratch (depthai telemetry, etc.) written at runtime. +.cache/ diff --git a/examples/camera_viz/README.md b/examples/camera_viz/README.md index 39e803b3c..c2fe4ba54 100644 --- a/examples/camera_viz/README.md +++ b/examples/camera_viz/README.md @@ -48,7 +48,7 @@ Flags: `--no-{v4l2,oakd,rtp}`, `--with-zed`, `--sender-only`, `--jetson`. Pass ` ./camera_viz.sh run configs/v4l2.yaml --mode window # desktop window instead ``` -Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`, `video.yaml` (file replay — point `path:` at any recording). +Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`, `replay.yaml` (file replay — point `path:` at any recording). ## Mode 2 — Split (robot → workstation, RTP) diff --git a/examples/camera_viz/camera_viz.sh b/examples/camera_viz/camera_viz.sh index 1052d9ed3..a3dc5a424 100755 --- a/examples/camera_viz/camera_viz.sh +++ b/examples/camera_viz/camera_viz.sh @@ -440,7 +440,7 @@ EXAMPLES ./camera_viz.sh loopback configs/v4l2.yaml ./camera_viz.sh deploy --host 10.29.90.127 --user nvidia configs/v4l2.yaml ./camera_viz.sh run configs/v4l2.yaml - ./camera_viz.sh run configs/video.yaml --mode window # no headset needed + ./camera_viz.sh run configs/replay.yaml --mode window # no headset needed # Env-var style (avoids passwords in shell history / argv): export REMOTE_HOST=10.29.90.127 REMOTE_USER=nvidia diff --git a/examples/camera_viz/configs/video.yaml b/examples/camera_viz/configs/replay.yaml similarity index 86% rename from examples/camera_viz/configs/video.yaml rename to examples/camera_viz/configs/replay.yaml index 8e6b59613..633146b9d 100644 --- a/examples/camera_viz/configs/video.yaml +++ b/examples/camera_viz/configs/replay.yaml @@ -30,9 +30,10 @@ cameras: # set both to resize each frame on the CPU before upload. # width: 1280 # height: 720 - stereo: false # true = side-by-side file, split into eyes. - # Viewer-only: the RTP sender needs per-eye - # streams and rejects single-source stereo. + stereo: false # true = replay a side-by-side recording as + # stereo, splitting each frame into left/right + # eyes. Supported in direct mode only; the RTP + # sender needs per-eye streams and rejects it. rtp: # only used when source: rtp + camera_streamer port: 5000 bitrate_mbps: 15 diff --git a/examples/camera_viz/tests/test_video_file_source.py b/examples/camera_viz/tests/test_video_file_source.py index f2af155cf..0b1186218 100644 --- a/examples/camera_viz/tests/test_video_file_source.py +++ b/examples/camera_viz/tests/test_video_file_source.py @@ -74,16 +74,20 @@ def test_missing_file_raises(tmp_path): def test_resolve_video_paths_anchors_to_yaml_dir(tmp_path): + # An already-absolute path must be left untouched. Build it from + # tmp_path so it's absolute on any OS (a POSIX "/abs" string is not + # absolute on Windows and would get re-anchored to the drive). + abs_path = str(tmp_path / "elsewhere" / "clip.mp4") cfg = { "cameras": [ {"type": "video", "path": "clip.mp4"}, - {"type": "video", "path": "/abs/clip.mp4"}, + {"type": "video", "path": abs_path}, {"type": "v4l2", "device": "/dev/video0"}, ] } resolve_video_paths(cfg, tmp_path) assert cfg["cameras"][0]["path"] == str(tmp_path / "clip.mp4") - assert cfg["cameras"][1]["path"] == "/abs/clip.mp4" + assert cfg["cameras"][1]["path"] == abs_path assert "path" not in cfg["cameras"][2]