diff --git a/.gitattributes b/.gitattributes index 1cff09735..ce629912c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,6 +18,9 @@ *.usdc filter=lfs diff=lfs merge=lfs -text *.usdz filter=lfs diff=lfs merge=lfs -text +# Video assets are tracked by default. +*.mp4 filter=lfs diff=lfs merge=lfs -text + # # Exceptions for small source SVGs (webpack-inlined; not binary assets). deps/cloudxr/webxr_client/src/icons/*.svg -filter -diff -merge text diff --git a/docs/source/references/camera_streaming.rst b/docs/source/references/camera_streaming.rst index 6eeec8982..9e245189e 100644 --- a/docs/source/references/camera_streaming.rst +++ b/docs/source/references/camera_streaming.rst @@ -5,12 +5,14 @@ Camera Streaming ================ ``camera_viz`` is the reference camera-streaming sample built on :doc:`Televiz -` (``isaacteleop.viz``). It captures frames from one or more cameras and -visualizes them on a desktop window or an XR headset — one plane per camera, aspect-fit — and can -stream a robot's cameras to a remote workstation over the network. - -The sample lives at :code-dir:`examples/camera_viz/ `; this page summarizes how -it works. For the exact command surface and flags, see the +` (``isaacteleop.viz``). It captures frames from one or more cameras +and streams them to an XR headset — one plane per camera, aspect-fit — either directly or from a +robot to a workstation over the network (split mode). For development and debugging it can also +replay a video file in place of a camera, and render to a desktop window instead of the headset. + +The sample lives at :code-dir:`examples/camera_viz/ `. This page walks you +from setup and a hardware-free first run to real cameras and the robot → workstation split mode. +For the exact command surface and flags, see the :code-file:`README `. .. figure:: ../_static/televiz_2d.gif @@ -24,25 +26,95 @@ it works. For the exact command surface and flags, see the :local: :depth: 2 -Modes +Requirements +------------ + +- A workstation meeting the :doc:`system requirements ` (Ubuntu, NVIDIA + GPU, CUDA driver) — every source hands frames to the renderer GPU-resident via CuPy. +- For the default XR mode, a running CloudXR server with a connected headset — follow the + :doc:`quick start ` steps :ref:`run-cloudxr-server` and + :ref:`connect-xr-headset`. No headset handy? ``--mode window`` renders to a desktop window + instead and only needs a local display. + +Setup ----- +Clone the repository if you haven't already (quick start step :ref:`check-out-code-base`), then +run the sample's one-time setup: + +.. code-block:: bash + + examples/camera_viz/camera_viz.sh setup + source examples/camera_viz/.venv/bin/activate + +There is no need to install the ``isaacteleop`` pip package yourself — ``setup`` creates the +sample's own environment: it installs ``isaacteleop`` (which bundles Televiz) and every other +Python dependency from PyPI into ``.venv/`` via ``uv``, builds the native NVENC/NVDEC codec, and +probes system packages (GStreamer plugins, cairo / girepository headers, JetPack ``cuda-nvrtc`` + +``ld.so`` wiring). When something is missing it prints the exact ``apt-get`` line and prompts +``[y/N]`` — answering ``n`` or running non-interactively aborts. + +By default ``setup`` provisions everything except ZED support; flags trim or extend that: + .. list-table:: :header-rows: 1 - :widths: 16 84 + :widths: 22 78 + + * - Flag + - Effect + * - ``--no-v4l2`` + - Skip USB / UVC webcam support (``opencv-python``). + * - ``--no-oakd`` + - Skip OAK-D support (``depthai``). + * - ``--no-rtp`` + - Skip split-mode dependencies: the GStreamer system packages and the native NVENC/NVDEC + codec build. Direct mode still works. + * - ``--with-zed`` + - Also build + install the ZED SDK's Python API (``pyzed``). Requires the ZED SDK on the + machine (default ``/usr/local/zed``; override with ``--zed-sdk PATH``). + * - ``--sender-only`` + - Split mode only — robot-side install of just the sender's dependencies. + * - ``--jetson`` + - Split mode only — extra CUDA wiring JetPack images need on the robot. + * - ``--venv PATH`` + - Install into an existing virtual environment instead of creating ``.venv/``. + * - ``--wheel PATH`` + - Install a locally built ``isaacteleop`` wheel instead of the PyPI release — for developing + Isaac Teleop itself (see :doc:`/getting_started/build_from_source/index`). + +First run — no camera required +------------------------------ + +The video-replay source (``type: video``) plays a recording through exactly the same path a live +camera uses, so it doubles as the quickest end-to-end check and as a stand-in feed while the real +camera isn't available. A test clip ships with the repo and ``configs/replay.yaml`` already points +at it: - * - Mode - - What it does - * - **Direct** - - The workstation runs the viewer with cameras attached locally (``source: local``). - * - **Split** - - The robot runs a sender that ships RTP H.264 to a workstation receiver (``source: rtp``). - Wired Ethernet only. - -Supported cameras +.. code-block:: bash + + cd examples/camera_viz + ./camera_viz.sh run configs/replay.yaml # XR headset (default) + ./camera_viz.sh run configs/replay.yaml --mode window # desktop window instead + +**You should see** the terminal report the session and the source coming up:: + + camera_viz: source=local, mode=xr, xr=True, 1 layer(s) + [video] opening... + [video] connected + [video] streaming + +and the clip looping on a plane in the headset — or in a desktop window (``mode=window, +xr=False``) with the ``--mode window`` override. + +To replay your own video, set a custom ``path:`` in :code-file:`configs/replay.yaml +` — relative paths resolve against the YAML's directory. +``loop: false`` holds the last frame instead of rewinding; ``stereo: true`` replays a side-by-side +recording (e.g. from a ZED) as stereo, splitting each frame into per-eye views (direct mode only). + +Supported sources ----------------- -The camera kind is selected per entry by the YAML ``type:`` field: +The source kind is selected by the ``type`` field of each entry in the YAML ``cameras`` list: .. list-table:: :header-rows: 1 @@ -50,96 +122,76 @@ The camera kind is selected per entry by the YAML ``type:`` field: * - ``type:`` - Notes - * - ``synthetic`` - - GPU test pattern — no hardware. ``stereo: true`` adds a ``disparity_px`` offset between eyes. * - ``v4l2`` - USB / UVC cameras — anything ``v4l2-ctl --list-formats-ext`` reports. * - ``oakd`` - OAK-D mono RGB / LEFT / RIGHT (stereo not yet wired). * - ``zed`` - ZED 2 / Mini / X One; mono or ``stereo: true`` (per-eye SDK retrieve, zero-copy on the GPU). + * - ``video`` + - Video-file replay (anything OpenCV's FFmpeg backend reads). Loops by default; + ``stereo: true`` splits side-by-side recordings into eyes (viewer only). + * - ``synthetic`` + - Debugging tool — GPU-generated test pattern, no hardware or file. -Output goes to a window or an XR headset. Stereo cameras render true side-by-side stereo in XR; -window mode shows the left eye. XR placement lock modes are ``world`` / ``head`` / ``lazy``. - -How it works ------------- - -The sample is organized so that capture, transport, and visualization are cleanly separated, with -Televiz as the compositor at the end of the chain: - -.. code-block:: text - :class: code-100col - - camera_viz/ - ├── camera_viz.sh — CLI: setup / loopback / run / deploy / service-* - ├── camera_viz.py — receiver / viewer (drives a Televiz VizSession) - ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) - ├── pipeline/ — source ABC + threaded runner - ├── placements/ — XR lock-mode strategies (world / head / lazy) - ├── sources/ — V4L2 / OAK-D / ZED / synthetic / rtp_h264 - ├── transports/ — RTP sender + receiver (native + GStreamer) - ├── codec/ — native NVENC / NVDEC pybind module - ├── configs/ — one YAML per camera kind - └── scripts/ — installer + systemd unit template - -- **Sources** (:code-dir:`sources/ `) implement a common source ABC and - hand frames to a threaded runner in :code-dir:`pipeline/ `. Each - source produces GPU frames where possible — e.g. the ZED source uses ``retrieve_image(MEM.GPU)`` so - BGRA8 stays in VRAM and a CUDA kernel channel-swaps into contiguous RGBA with no host round-trip. -- **The viewer** (:code-file:`camera_viz.py `) creates a - ``VizSession`` and adds one ``QuadLayer`` per enabled camera, then submits each frame to its layer - and calls ``render()`` once per frame. Stereo cameras submit both eyes. -- **Transport** (:code-dir:`transports/ `) carries the split mode: - an RTP H.264 sender on the robot and a receiver on the workstation, with a native NVENC/NVDEC codec - module (:code-dir:`codec/ `) or a GStreamer fallback. -- **Placement** (:code-dir:`placements/ `) holds the XR lock-mode - strategies. Placement is application policy — Televiz only renders a layer at whatever pose the app - sets each frame. - -Setup ------ +Running with a real camera +-------------------------- -One-time setup installs the sample's Python environment — no source build required: +Attach the camera to the machine that runs the viewer, keep ``source: local`` in the config, and +run with the matching config: .. code-block:: bash - examples/camera_viz/camera_viz.sh setup - source examples/camera_viz/.venv/bin/activate + ./camera_viz.sh run configs/v4l2.yaml # or oakd.yaml / zed.yaml -``setup`` installs ``isaacteleop`` (which bundles Televiz) and every other Python dependency from -PyPI into ``.venv/`` via ``uv``, builds the native NVENC/NVDEC codec, and probes system packages -(GStreamer plugins, cairo / girepository headers, JetPack ``cuda-nvrtc`` + ``ld.so`` wiring). When -something is missing it prints the exact ``apt-get`` line and prompts ``[y/N]`` — answering ``n`` or -running non-interactively aborts. +**You should see** the same startup lines as above with the camera's tag (``[v4l2]``, +``[oakd]``, ``[zed]``) and the live feed. Multiple entries in the ``cameras`` list render as one +plane each. -Useful flags: ``--no-{v4l2,oakd,rtp}``, ``--with-zed``, ``--sender-only``, ``--jetson``. Pass -``--venv PATH`` to install into an existing virtual environment. To develop against a locally built -wheel instead of the PyPI release, pass ``--wheel `` (see -:doc:`/getting_started/build_from_source/index`). +Display modes +------------- -Running -------- +XR is the default: each camera renders as its own plane in the headset via the active OpenXR +runtime, and stereo sources (``stereo: true``) render true side-by-side stereo. Pass +``--mode window`` (or set ``display.mode: window`` in the YAML) to render to a desktop window +instead — no headset or runtime needed; stereo shows the left eye. -Direct (cameras on the workstation) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +In XR, how a plane follows the operator's head is the per-camera ``lock_mode`` under +``display.placements.``: -Set ``source: local`` in the config and run the viewer: +.. list-table:: + :header-rows: 1 + :widths: 16 84 -.. code-block:: bash + * - Mode + - Behavior + * - ``world`` + - Placed once in front of you and stays put. + * - ``head`` + - Follows your head every frame. + * - ``lazy`` + - World-locked, but re-snaps in front of you when you look away (default). - ./camera_viz.sh run configs/v4l2.yaml +Lazy-mode knobs live under ``placements.``: ``look_away_angle_deg``, +``reposition_distance``, ``reposition_delay_s``, ``transition_duration_s``. -Swap the config for ``oakd.yaml``, ``zed.yaml``, ``synthetic.yaml``, ``synthetic_stereo.yaml``, or -``multi_camera.yaml``. +Split mode — robot → workstation over RTP +----------------------------------------- -Split (robot → workstation over RTP) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Split mode runs the capture side on the robot (``camera_streamer.py``) and ships RTP H.264 to +the viewer on the workstation (``source: rtp``). .. warning:: - **Wired networks only.** There is no retransmit or FEC — one lost packet corrupts one frame until - the next IDR (default every 5 s). + Split mode is **not recommended in most cases**. It exists for one situation: the cameras are + on the robot, but Isaac Teleop runs on a workstation, so the frames must be streamed to where + Isaac Teleop is running. That costs a full extra encode/decode hop — NVENC on the robot, UDP, + NVDEC on the workstation — so whenever a camera can attach directly to the machine running + Isaac Teleop, run direct mode instead. Wired networks only: there is no retransmit or FEC — one lost packet corrupts one + frame until the next IDR (default every 5 s). + +In split mode every camera entry must pin ``width``, ``height``, and ``fps`` in the YAML — the +receiver sizes its decoder from the config, not from the wire. Set ``source: rtp`` in the config, export the robot/streaming credentials once per shell, then deploy the sender and run the viewer: @@ -153,22 +205,24 @@ deploy the sender and run the viewer: ./camera_viz.sh run configs/v4l2.yaml # viewer on the workstation ``deploy`` rsyncs the source to the robot, installs sender dependencies, renders a -``camera-streamer.service`` systemd user unit (injecting ``--host`` from ``$STREAMING_HOST`` without -editing the YAML on disk), and enables it. Operate the running unit with +``camera-streamer.service`` systemd user unit (injecting ``--host`` from ``$STREAMING_HOST`` +without editing the YAML on disk), and enables it. Operate the running unit with ``./camera_viz.sh service-{status,logs,restart}``. The sender retries forever across unplug, SDK errors, and network blips. Loopback ^^^^^^^^ -``./camera_viz.sh loopback configs/v4l2.yaml`` runs the sender and viewer together on ``127.0.0.1`` -— the quickest way to smoke-test the RTP path on one machine. +Loopback is a testing / debugging aid, not a deployment mode: ``./camera_viz.sh loopback +configs/v4l2.yaml`` runs the sender and viewer together on ``127.0.0.1`` — the quickest way to +smoke-test the RTP path on one machine. It also works camera-free with a mono ``type: video`` +entry (set ``width`` / ``height`` / ``fps``). Configuration ------------- -A single YAML drives both capture and visualization. Each ``cameras:`` entry becomes its own plane -(and, in split mode, its own RTP port). Abbreviated: +A single YAML drives both capture and visualization. Each entry in the ``cameras`` list becomes +its own plane (and, in split mode, its own RTP port). Abbreviated: .. code-block:: yaml @@ -180,18 +234,20 @@ A single YAML drives both capture and visualization. Each ``cameras:`` entry bec cameras: - name: cam enabled: true - type: v4l2 # v4l2 | oakd | zed | synthetic - width: 2560 - height: 720 + type: v4l2 # v4l2 | oakd | zed | video | synthetic + width: 2560 # video: optional — defaults to the file's size + height: 720 # (required when source: rtp) fps: 30 - stereo: false # zed / synthetic only — per-eye capture + SBS in XR + stereo: false # zed / video / synthetic — per-eye capture + SBS in XR + path: clip.mp4 # video only — file to replay, relative to this YAML + loop: true # video only — rewind at end of file rtp: port: 5000 # left eye when stereo port_right: 5001 # required when stereo + source: rtp bitrate_mbps: 15 display: - mode: window | xr + mode: xr | window # default: xr window: { width, height } xr: { near_z, far_z } clear_color: [r, g, b, a] @@ -202,29 +258,62 @@ A single YAML drives both capture and visualization. Each ``cameras:`` entry bec # size: [w_m, h_m] # stereo_baseline_mm: 0 -See the :code-dir:`configs/ ` directory for a complete, commented YAML -per camera kind. +See the :code-dir:`configs/ ` directory for a complete, commented +YAML per source kind. + +Troubleshooting +--------------- + +- **The XR session fails to create** — the default mode needs the CloudXR server running and a + headset connected (quick start steps :ref:`run-cloudxr-server` and :ref:`connect-xr-headset`); + pass ``--mode window`` to render to a desktop window instead. +- **No window appears over SSH** — ``--mode window`` needs a local display; run on the machine + you're sitting at, or use a video-capable remote desktop. +- **"video source: no such file"** — relative ``path:`` values resolve against the YAML's + directory (``configs/``), not the directory you launched from. +- **A source fails asking for CuPy / CUDA** — check ``nvidia-smi`` works and setup completed; + all sources allocate their frame buffers on the GPU. +- **Split mode renders nothing** — check the sender is up (``./camera_viz.sh service-status``), + ``$STREAMING_HOST`` was the workstation's IP at deploy time, and UDP ports (default 5000+) + aren't firewalled. +- **Not sure which side is stuck?** — set ``verbose: true`` at the top of the YAML for periodic + per-source breadcrumbs on both ends. -Lock modes (XR) -^^^^^^^^^^^^^^^ +How it works +------------ -How a camera plane is positioned relative to the operator's head each frame: +The sample is organized so that capture, transport, and visualization are cleanly separated, with +Televiz as the compositor at the end of the chain: -.. list-table:: - :header-rows: 1 - :widths: 16 84 +.. code-block:: text + :class: code-100col - * - Mode - - Behavior - * - ``world`` - - Placed once in front of you and stays put. - * - ``head`` - - Follows your head every frame. - * - ``lazy`` - - World-locked, but re-snaps in front of you when you look away (default). + camera_viz/ + ├── camera_viz.sh — CLI: setup / loopback / run / deploy / service-* + ├── camera_viz.py — receiver / viewer (drives a Televiz VizSession) + ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) + ├── pipeline/ — source ABC + threaded runner + ├── placements/ — XR lock-mode strategies (world / head / lazy) + ├── sources/ — V4L2 / OAK-D / ZED / video replay / synthetic / rtp_h264 + ├── transports/ — RTP sender + receiver (native + GStreamer) + ├── codec/ — native NVENC / NVDEC pybind module + ├── configs/ — one YAML per source kind + ├── test_data/ — sample replay clip (Git LFS) + └── scripts/ — installer + systemd unit template -Lazy-mode knobs live under ``placements.``: ``look_away_angle_deg``, ``reposition_distance``, -``reposition_delay_s``, ``transition_duration_s``. +- **Sources** (:code-dir:`sources/ `) implement a common source ABC and + hand frames to a threaded runner in :code-dir:`pipeline/ `. Each + source produces GPU frames where possible — e.g. the ZED source uses ``retrieve_image(MEM.GPU)`` so + BGRA8 stays in VRAM and a CUDA kernel channel-swaps into contiguous RGBA with no host round-trip. +- **The viewer** (:code-file:`camera_viz.py `) creates a + ``VizSession`` and adds one ``QuadLayer`` per enabled camera, then submits each frame to its layer + and calls ``render()`` once per frame. Stereo cameras submit both eyes. +- **Transport** (:code-dir:`transports/ `) carries the split mode: + an RTP H.264 sender on the robot and a receiver on the workstation, with a native NVENC/NVDEC codec + module (:code-dir:`codec/ `) or a GStreamer fallback. +- **Placement** (:code-dir:`placements/ `) holds the XR lock-mode + strategies. Placement is application policy — Televiz only renders a layer at whatever pose the app + sets each frame. Sharing the XR session with TeleopSession ----------------------------------------- diff --git a/examples/camera_viz/.gitignore b/examples/camera_viz/.gitignore new file mode 100644 index 000000000..0e1069186 --- /dev/null +++ b/examples/camera_viz/.gitignore @@ -0,0 +1,12 @@ +# Replay clips for ``type: video`` — user-supplied or generated, +# never committed. The shipped sample clip in test_data/ (Git LFS) +# is the exception. +*.mp4 +*.mkv +*.webm +*.avi +*.mov +!test_data/*.mp4 + +# Vendor SDK scratch (depthai telemetry, etc.) written at runtime. +.cache/ diff --git a/examples/camera_viz/README.md b/examples/camera_viz/README.md index 7cf156253..c2fe4ba54 100644 --- a/examples/camera_viz/README.md +++ b/examples/camera_viz/README.md @@ -20,8 +20,9 @@ SPDX-License-Identifier: Apache-2.0 | `v4l2` | USB / UVC — anything `v4l2-ctl --list-formats-ext` shows | | `oakd` | OAK-D mono RGB / LEFT / RIGHT (stereo not yet wired) | | `zed` | ZED 2 / Mini / X One; mono or `stereo: true` (per-eye SDK retrieve, zero-copy GPU) | +| `video` | Video-file replay (anything OpenCV/FFmpeg reads) — preview / testing without a camera. Loops by default; `stereo: true` splits side-by-side files into eyes (viewer only) | -Output: window or XR headset; one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`. +Output: XR headset (default) or desktop window (`run CONFIG --mode window`); one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`. --- @@ -43,10 +44,11 @@ Flags: `--no-{v4l2,oakd,rtp}`, `--with-zed`, `--sender-only`, `--jetson`. Pass ` ## Mode 1 — Direct ```bash -./camera_viz.sh run configs/v4l2.yaml +./camera_viz.sh run configs/v4l2.yaml # XR headset (default) +./camera_viz.sh run configs/v4l2.yaml --mode window # desktop window instead ``` -Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`. +Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`, `replay.yaml` (file replay — point `path:` at any recording). ## Mode 2 — Split (robot → workstation, RTP) @@ -92,12 +94,12 @@ encoder: auto | native | gstreamer cameras: - name: cam enabled: true - type: v4l2 # v4l2 | oakd | zed | synthetic - width: 2560 + type: v4l2 # v4l2 | oakd | zed | synthetic | video + width: 2560 # video: optional — defaults to the file's size height: 720 fps: 30 - stereo: false # zed / synthetic only — enables per-eye capture + SBS XR - # … type-specific fields (e.g. synthetic: disparity_px) + stereo: false # zed / synthetic / video only — per-eye capture + SBS XR + # … type-specific fields (e.g. synthetic: disparity_px; video: path, loop) rtp: port: 5000 # left eye when stereo port_right: 5001 # required when stereo + source: rtp @@ -106,7 +108,7 @@ cameras: # gpu_id: 0 # multi-GPU pin display: # camera_viz only - mode: window | xr + mode: xr | window # default: xr window: { width, height } xr: { near_z, far_z } clear_color: [r, g, b, a] @@ -144,10 +146,11 @@ camera_viz/ ├── camera_streamer.py — robot-side RTP sender (per-camera supervisor) ├── pipeline/ — source ABC + threaded runner ├── placements/ — XR lock-mode strategies -├── sources/ — V4L2 / OAK-D / ZED / synthetic / rtp_h264 +├── sources/ — V4L2 / OAK-D / ZED / synthetic / video replay / rtp_h264 ├── transports/ — RTP sender + receiver, native + GStreamer ├── codec/ — native NVENC/NVDEC pybind module ├── configs/ — one YAML per camera kind +├── test_data/ — sample replay clip (Git LFS) └── scripts/ ├── _install_deps.sh — installer (setup + deploy) └── camera-streamer.service.in — systemd unit template diff --git a/examples/camera_viz/camera_streamer.py b/examples/camera_viz/camera_streamer.py index 9978dd1fe..76cc5e4e2 100755 --- a/examples/camera_viz/camera_streamer.py +++ b/examples/camera_viz/camera_streamer.py @@ -30,7 +30,12 @@ import yaml from pipeline import FrameSource -from sources import PairedFrameSource, build_local_camera, set_verbose +from sources import ( + PairedFrameSource, + build_local_camera, + resolve_video_paths, + set_verbose, +) from transports import RtpH264Sender, make_encoder logger = logging.getLogger("camera_streamer") @@ -136,10 +141,13 @@ def _build_senders(self) -> List[RtpH264Sender]: ) def build_one(source: FrameSource, port: int) -> RtpH264Sender: + # Encoder / sender geometry comes from the built source's spec, + # not the YAML — sources that size themselves (video replay + # probing its file) may omit width/height from the config. encoder = make_encoder( rtp.get("encoder", self._default_encoder), - width=int(self._cfg["width"]), - height=int(self._cfg["height"]), + width=source.spec.width, + height=source.spec.height, bitrate=int(rtp.get("bitrate_mbps", 15)) * 1_000_000, fps=int(self._cfg.get("fps", 30)), gop=int(rtp["gop"]) if "gop" in rtp else None, @@ -150,8 +158,8 @@ def build_one(source: FrameSource, port: int) -> RtpH264Sender: encoder=encoder, host=self._host, port=port, - width=int(self._cfg["width"]), - height=int(self._cfg["height"]), + width=source.spec.width, + height=source.spec.height, fps=int(self._cfg.get("fps", 30)), mtu=int(rtp.get("mtu", 1400)), ) @@ -278,6 +286,7 @@ def main(argv: Optional[List[str]] = None) -> int: # Top-level ``verbose:`` enables per-source periodic breadcrumbs. set_verbose(bool(cfg.get("verbose", False))) + resolve_video_paths(cfg, args.config.parent) streaming = cfg.get("streaming", {}) host = args.host or streaming.get("host") diff --git a/examples/camera_viz/camera_viz.py b/examples/camera_viz/camera_viz.py index e0fc143f1..9f2af2df3 100755 --- a/examples/camera_viz/camera_viz.py +++ b/examples/camera_viz/camera_viz.py @@ -30,7 +30,13 @@ from pipeline import FrameSource, VizRunner from placements import PlacementConfig, PlacementStrategy, build as build_placement -from sources import PairedFrameSource, RtpH264Source, build_local_camera, set_verbose +from sources import ( + PairedFrameSource, + RtpH264Source, + build_local_camera, + resolve_video_paths, + set_verbose, +) @dataclass @@ -75,18 +81,15 @@ def _enabled_cameras(cfg: dict) -> List[dict]: def _placement_with_aspect( - spec: Optional[dict], cam: dict, is_xr: bool + spec: Optional[dict], width: int, height: int, is_xr: bool ) -> Optional[PlacementStrategy]: - """Build the placement for ``cam``, filling in ``size`` from the - camera's aspect ratio when the YAML doesn't pin it. Width defaults - to 1.0 m so a 16:9 camera lands at 1.0 x 0.5625, a 3.55:1 SBS at - 1.0 x 0.281.""" + """Build the placement, filling in ``size`` from the source's aspect + ratio when the YAML doesn't pin it. Width defaults to 1.0 m so a + 16:9 source lands at 1.0 x 0.5625, a 3.55:1 SBS at 1.0 x 0.281.""" if spec is not None and "size" not in spec: - w = int(cam["width"]) - h = int(cam["height"]) spec = { **spec, - "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * h / w], + "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * height / width], } return _build_placement(spec, is_xr) @@ -104,9 +107,15 @@ def _build_local_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: placements_cfg = cfg.get("display", {}).get("placements", {}) entries: List[SourceEntry] = [] for cam in _enabled_cameras(cfg): - placement = _placement_with_aspect(placements_cfg.get(cam["name"]), cam, is_xr) + cam_sources = build_local_camera(cam) + # Aspect comes from the built source's spec, not the YAML — video + # sources may omit width/height and size themselves from the file. + first = cam_sources[0].spec + placement = _placement_with_aspect( + placements_cfg.get(cam["name"]), first.width, first.height, is_xr + ) stereo, baseline_mm = _stereo_for(cam, placements_cfg) - for source in build_local_camera(cam): + for source in cam_sources: entries.append( SourceEntry( source=source, @@ -130,7 +139,18 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: f"camera_viz: camera {cam.get('name')!r} missing rtp.port; " "required when source: rtp" ) - placement = _placement_with_aspect(placements_cfg.get(cam["name"]), cam, is_xr) + if "width" not in cam or "height" not in cam: + raise ValueError( + f"camera_viz: camera {cam.get('name')!r} needs explicit " + "width/height when source: rtp — the receiver sizes its " + "decoder from the YAML, not from the wire" + ) + placement = _placement_with_aspect( + placements_cfg.get(cam["name"]), + int(cam["width"]), + int(cam["height"]), + is_xr, + ) stereo, baseline_mm = _stereo_for(cam, placements_cfg) if stereo: @@ -182,7 +202,7 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]: def _make_session(cfg: dict, mode_override: Optional[str] = None) -> viz.VizSession: display = cfg.get("display", {}) # --mode overrides display.mode when given. - mode_str = (mode_override or display.get("mode", "window")).lower() + mode_str = (mode_override or display.get("mode", "xr")).lower() session_cfg = viz.VizSessionConfig() if mode_str == "window": session_cfg.mode = viz.DisplayMode.kWindow @@ -211,7 +231,8 @@ def main(argv: Optional[list[str]] = None) -> int: "--mode", choices=("window", "xr"), default=None, - help="Override display.mode from the config (default: use the config's value).", + help="Override display.mode from the config " + "(default: the config's value, or xr when the config omits it).", ) args = parser.parse_args(argv) @@ -225,12 +246,13 @@ def main(argv: Optional[list[str]] = None) -> int: # Top-level ``verbose:`` enables per-source periodic breadcrumbs. set_verbose(bool(cfg.get("verbose", False))) + resolve_video_paths(cfg, args.config.parent) source_mode = cfg.get("source", "local").lower() if source_mode not in ("local", "rtp"): raise ValueError(f"camera_viz: source must be local|rtp, got {source_mode!r}") - effective_mode = (args.mode or cfg.get("display", {}).get("mode", "window")).lower() + effective_mode = (args.mode or cfg.get("display", {}).get("mode", "xr")).lower() session = _make_session(cfg, mode_override=args.mode) is_xr = session.is_xr_mode() diff --git a/examples/camera_viz/camera_viz.sh b/examples/camera_viz/camera_viz.sh index 64756fd2b..a3dc5a424 100755 --- a/examples/camera_viz/camera_viz.sh +++ b/examples/camera_viz/camera_viz.sh @@ -8,7 +8,7 @@ # Local: # ./camera_viz.sh setup [--sender-only] install deps + build codec # ./camera_viz.sh loopback CONFIG run streamer + viz on 127.0.0.1 -# ./camera_viz.sh run CONFIG run the viewer (honors source:) +# ./camera_viz.sh run CONFIG [--mode M] run the viewer (honors source:) # # Remote (Jetson robot): # ./camera_viz.sh deploy --host H --user U [--password P] CONFIG @@ -401,10 +401,13 @@ LOCAL loopback CONFIG Run camera_streamer + camera_viz on 127.0.0.1. - run CONFIG Run the viewer with the YAML as-is. ``source: + run CONFIG [--mode xr|window] + Run the viewer with the YAML as-is. ``source: local`` opens cameras directly; ``source: rtp`` listens on rtp.port (sender IP irrelevant — the - receiver binds 0.0.0.0). + receiver binds 0.0.0.0). --mode overrides + display.mode: xr renders to the headset (the + default), window to a desktop window. REMOTE (Jetson robot) deploy [--host H --user U [--password P]] @@ -437,6 +440,7 @@ EXAMPLES ./camera_viz.sh loopback configs/v4l2.yaml ./camera_viz.sh deploy --host 10.29.90.127 --user nvidia configs/v4l2.yaml ./camera_viz.sh run configs/v4l2.yaml + ./camera_viz.sh run configs/replay.yaml --mode window # no headset needed # Env-var style (avoids passwords in shell history / argv): export REMOTE_HOST=10.29.90.127 REMOTE_USER=nvidia diff --git a/examples/camera_viz/configs/dellwebcam.yaml b/examples/camera_viz/configs/dellwebcam.yaml deleted file mode 100644 index 6c285f532..000000000 --- a/examples/camera_viz/configs/dellwebcam.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# V4L2 source configured for the DELL Display 4MP Webcam (USB ID 413c:d005). -# Same YAML drives both camera_viz (receiver) and camera_streamer (RTP sender). -# Flip top-level ``source: local|rtp`` to control what camera_viz opens; -# camera_streamer always reads cameras + rtp ports + streaming.host. -# -# DELL Display 4MP Webcam V4L2 node layout (on THIS machine): -# /dev/video0 — main capture node ← used here -# /dev/video1 — metadata (no capture) -# /dev/video2 — secondary capture (NV12 only) -# /dev/video3 — metadata (no capture) -# -# Format note: this webcam offers 1920x1080 ONLY via MJPG. Its YUYV mode -# tops out at 640x480, so MJPG is selected below for 1080p30. -# -# To inspect available resolutions and formats on the capture node: -# sudo apt install v4l-utils -# v4l2-ctl --device=/dev/video0 --list-formats-ext - -source: local # local | rtp -streaming: - host: 127.0.0.1 # camera_streamer target (override with --host) -encoder: auto # auto | native | gstreamer - # auto picks native NVENC on desktop, GStreamer on Jetson - -cameras: - - name: cam - enabled: true - type: v4l2 - device: /dev/video0 # DELL Display 4MP Webcam main capture node - width: 1920 - height: 1080 - fps: 30 - fourcc: MJPG # webcam only offers 1920x1080 via MJPG (YUYV maxes at 640x480) - rtp: # used by camera_streamer + source: rtp receivers - port: 5000 - bitrate_mbps: 15 # camera_streamer default; lower for tight uplinks - # gop: # default: fps*5 (IDR every 5 s, ULL tuning) - -display: - mode: window # window | xr - window: - width: 1920 - height: 1080 - xr: - near_z: 0.05 - far_z: 100.0 - clear_color: [0.0, 0.0, 0.0, 0.0] - placements: - cam: # size defaults to 1.0 m wide × aspect-derived height - lock_mode: lazy - distance: 1.5 diff --git a/examples/camera_viz/configs/multi_camera.yaml b/examples/camera_viz/configs/multi_camera.yaml index 5c1bd679d..bc468fbec 100644 --- a/examples/camera_viz/configs/multi_camera.yaml +++ b/examples/camera_viz/configs/multi_camera.yaml @@ -45,7 +45,7 @@ cameras: bitrate_mbps: 15 display: - mode: xr # window | xr + mode: xr # xr | window (xr is the default) window: # used only if mode: window width: 2560 height: 720 diff --git a/examples/camera_viz/configs/oakd.yaml b/examples/camera_viz/configs/oakd.yaml index 68c3557a6..97854b610 100644 --- a/examples/camera_viz/configs/oakd.yaml +++ b/examples/camera_viz/configs/oakd.yaml @@ -35,7 +35,7 @@ cameras: # gop: # default: fps*5 (ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 diff --git a/examples/camera_viz/configs/realsense.yaml b/examples/camera_viz/configs/realsense.yaml index 9ea683805..c7e4ffa0f 100644 --- a/examples/camera_viz/configs/realsense.yaml +++ b/examples/camera_viz/configs/realsense.yaml @@ -37,7 +37,7 @@ cameras: # gop: # default: fps*5 (IDR every 5 s, ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1920 height: 1080 diff --git a/examples/camera_viz/configs/replay.yaml b/examples/camera_viz/configs/replay.yaml new file mode 100644 index 000000000..633146b9d --- /dev/null +++ b/examples/camera_viz/configs/replay.yaml @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Video-file replay — plays a recording through the same source → +# QuadLayer → Televiz path as a live camera. Preview / Televiz testing +# when no camera is attached; also works as camera_streamer's capture +# side (mono only) to exercise the full RTP path camera-free. +# +# Point ``path`` at any file OpenCV's FFmpeg backend reads (mp4 / mkv / +# webm, H.264 / HEVC / AV1 / ...). Relative paths resolve against this +# YAML's directory. The default is the 10 s test clip shipped at +# ../test_data/recording.mp4 (Git LFS — run ``git lfs pull`` if it opens +# as a tiny text pointer). + +source: local # local | rtp — see real-camera configs for rtp +streaming: + host: 127.0.0.1 # only used by camera_streamer (rtp mode) +encoder: auto # auto | native | gstreamer + +cameras: + - name: replay + enabled: true + type: video + path: ../test_data/recording.mp4 # relative to this YAML's dir + loop: true # rewind at end of file (false = hold last frame) + fps: 0 # 0 = the file's native rate. With source: rtp, + # set this explicitly — the sender's encode loop + # paces at fps (default 30) regardless of the file. + # width/height default to the file's native size (per eye when stereo); + # set both to resize each frame on the CPU before upload. + # width: 1280 + # height: 720 + stereo: false # true = replay a side-by-side recording as + # stereo, splitting each frame into left/right + # eyes. Supported in direct mode only; the RTP + # sender needs per-eye streams and rejects it. + rtp: # only used when source: rtp + camera_streamer + port: 5000 + bitrate_mbps: 15 + +display: + mode: xr # xr | window (xr is the default) + window: + width: 1280 + height: 720 + xr: + near_z: 0.05 + far_z: 100.0 + clear_color: [0.05, 0.05, 0.08, 1.0] + placements: + replay: # size defaults to 1.0 m wide × aspect-derived height + lock_mode: lazy + distance: 1.5 diff --git a/examples/camera_viz/configs/synthetic.yaml b/examples/camera_viz/configs/synthetic.yaml index 1459c8364..61147e3bd 100644 --- a/examples/camera_viz/configs/synthetic.yaml +++ b/examples/camera_viz/configs/synthetic.yaml @@ -25,7 +25,7 @@ cameras: # gop: # default: fps*5 (ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 diff --git a/examples/camera_viz/configs/v4l2.yaml b/examples/camera_viz/configs/v4l2.yaml index 90a0ac151..b0730056c 100644 --- a/examples/camera_viz/configs/v4l2.yaml +++ b/examples/camera_viz/configs/v4l2.yaml @@ -32,7 +32,7 @@ cameras: # gop: # default: fps*5 (IDR every 5 s, ULL tuning) display: - mode: window # window | xr + mode: xr # xr | window (xr is the default) window: width: 2560 height: 720 diff --git a/examples/camera_viz/configs/zed.yaml b/examples/camera_viz/configs/zed.yaml index 3db9d3ccd..701b26c9f 100644 --- a/examples/camera_viz/configs/zed.yaml +++ b/examples/camera_viz/configs/zed.yaml @@ -34,7 +34,7 @@ cameras: # gop: # default: fps*5 (ULL tuning) display: - mode: xr # window | xr + mode: xr # xr | window (xr is the default) window: width: 1280 height: 720 diff --git a/examples/camera_viz/pyproject.toml b/examples/camera_viz/pyproject.toml index fd1d74bb4..f6433da43 100644 --- a/examples/camera_viz/pyproject.toml +++ b/examples/camera_viz/pyproject.toml @@ -26,6 +26,9 @@ dependencies = [ # extras by default; pass ``--no-`` to skip. [project.optional-dependencies] v4l2 = ["opencv-python>=4.5"] +# type: video (file replay) decodes via OpenCV's FFmpeg backend — same +# wheel as v4l2, listed separately so --no-v4l2 setups can still ask for it. +video = ["opencv-python>=4.5"] oakd = ["depthai>=3.0"] # pyzed isn't on PyPI — the ZED SDK's bundled ``get_python_api.py`` # produces a Python-version-specific wheel. ``camera_viz.sh setup --with-zed`` diff --git a/examples/camera_viz/sources/__init__.py b/examples/camera_viz/sources/__init__.py index f1a7a53d0..defdadbb3 100644 --- a/examples/camera_viz/sources/__init__.py +++ b/examples/camera_viz/sources/__init__.py @@ -8,6 +8,7 @@ from __future__ import annotations +from pathlib import Path from typing import List from pipeline import FrameSource @@ -17,6 +18,7 @@ from .rtp_h264 import RtpH264Source from .synthetic import SyntheticSource, SyntheticStereoSource from .v4l2 import V4l2Source +from .video_file import VideoFileSource from .zed import ZedSource __all__ = [ @@ -26,12 +28,26 @@ "SyntheticSource", "SyntheticStereoSource", "V4l2Source", + "VideoFileSource", "ZedSource", "build_local_camera", + "resolve_video_paths", "set_verbose", ] +def resolve_video_paths(cfg: dict, base_dir) -> None: + """Anchor relative ``path:`` values of ``type: video`` cameras to the + YAML file's directory (in place), so playback doesn't depend on the + process CWD. Call right after loading the config.""" + for cam in cfg.get("cameras", []): + if cam.get("type") == "video" and "path" in cam: + p = Path(str(cam["path"])).expanduser() + if not p.is_absolute(): + p = Path(base_dir) / p + cam["path"] = str(p) + + def build_local_camera(spec: dict) -> List[FrameSource]: """Build local FrameSource(s) for one ``cameras:`` entry. @@ -104,6 +120,21 @@ def build_local_camera(spec: dict) -> List[FrameSource]: ) return [PairedFrameSource(name=name, left=eyes[0], right=eyes[1])] return eyes + if kind == "video": + # Stereo (side-by-side file) emits both eyes from one source, like + # SyntheticStereoSource — viewer-only; camera_streamer's + # _eye_sources rejects it because there are no per-eye streams. + return [ + VideoFileSource( + name=name, + path=spec["path"], + width=int(spec.get("width", 0)), + height=int(spec.get("height", 0)), + fps=float(spec.get("fps", 0.0)), + loop=bool(spec.get("loop", True)), + stereo=stereo, + ) + ] if kind == "zed": eyes = list( ZedSource.build( @@ -126,5 +157,5 @@ def build_local_camera(spec: dict) -> List[FrameSource]: return eyes raise ValueError( f"build_local_camera: unknown camera type {kind!r} " - "(known: synthetic, v4l2, oakd, zed)" + "(known: synthetic, v4l2, oakd, zed, video)" ) diff --git a/examples/camera_viz/sources/video_file.py b/examples/camera_viz/sources/video_file.py new file mode 100644 index 000000000..b2ef9eeb9 --- /dev/null +++ b/examples/camera_viz/sources/video_file.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Video-file replay source for camera_viz. + +Plays a recording through the same FrameSource contract as a live +camera — preview / Televiz testing when no hardware is attached, and a +deterministic stand-in for a camera on the RTP sender side. + +Decode is OpenCV's FFmpeg backend (any container / codec it reads), +same hot path as :mod:`sources.v4l2`: ``cap.read()`` → pinned host +staging → async H2D → GPU BGR→RGBA. Frames are paced against an +absolute monotonic schedule so long clips don't drift. + +``stereo: true`` treats the file as side-by-side (a ZED recording's +natural layout) and splits each frame into per-eye halves on the GPU — +one source emits both eyes, so eye sync is frame-perfect. Like +SyntheticStereoSource, this doesn't expose per-eye streams, so stereo +replay is viewer-only (camera_streamer rejects it loudly). +""" + +from __future__ import annotations + +import math +import time +from pathlib import Path +from typing import Optional + +import numpy as np + +from ._helpers import PolledSource, alloc_pinned_host, notify + +# Playback rate used when the container reports a missing / absurd FPS. +_FALLBACK_FPS = 30.0 + + +def _probe(path: Path) -> tuple: + """Return (width, height, fps) of ``path``. Raises ValueError when the + file is missing or unreadable — a configuration error, not a device + blip, so it fails at build time rather than retrying forever.""" + import cv2 + + if not path.is_file(): + raise ValueError(f"video source: no such file: {path}") + cap = cv2.VideoCapture(str(path)) + try: + if not cap.isOpened(): + raise ValueError( + f"video source: cannot open {path} — unsupported container/" + "codec for OpenCV's FFmpeg backend?" + ) + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = float(cap.get(cv2.CAP_PROP_FPS)) + if w <= 0 or h <= 0: + # Some containers only report dimensions after a decode. + ret, frame = cap.read() + if not ret or frame is None: + raise ValueError(f"video source: {path} contains no decodable frames") + h, w = frame.shape[:2] + finally: + cap.release() + return w, h, fps + + +class VideoFileSource(PolledSource): + """File playback as a FrameSource; mono or side-by-side stereo.""" + + _kind = "video" + + def __init__( + self, + name: str, + path: str, + width: int = 0, + height: int = 0, + fps: float = 0.0, + loop: bool = True, + stereo: bool = False, + ) -> None: + try: + import cv2 + except ImportError as e: + raise RuntimeError( + "VideoFileSource requires opencv-python. Install via " + "`uv pip install opencv-python`." + ) from e + self._cv2 = cv2 + self._path = Path(path) + + native_w, native_h, native_fps = _probe(self._path) + eyes = 2 if stereo else 1 + if stereo and native_w % 2: + raise ValueError( + f"video source '{name}': stereo needs an even frame width to " + f"split side-by-side, got {native_w}" + ) + + # Output size: YAML width/height win; otherwise the file's native + # size (per eye when stereo). + out_w = int(width) if width else native_w // eyes + out_h = int(height) if height else native_h + + play_fps = float(fps) if fps else native_fps + if not math.isfinite(play_fps) or not 0.0 < play_fps <= 1000.0: + notify( + self._kind, + f"'{name}': no usable FPS in {self._path.name} " + f"({native_fps!r}); playing at {_FALLBACK_FPS:g}", + ) + play_fps = _FALLBACK_FPS + self._frame_interval_s = 1.0 / play_fps + self._loop = bool(loop) + self._stereo = bool(stereo) + + super().__init__(name=name, width=out_w, height=out_h, staging_channels=3) + cp = self._cp + + # The decoded frame is the full (SBS when stereo) picture; staging + # and the GPU BGR landing zone are sized to it. The base class's + # per-eye staging is replaced — its (h, w, 3) allocation only fits + # the mono layout. + self._frame_w = out_w * eyes + if stereo: + self._host_staging = alloc_pinned_host((out_h, self._frame_w, 3), np.uint8) + # Right-eye output buffers, rotated in lock-step with the base + # class's (left-eye) triple-buffer. + self._gpu_right = [ + cp.empty((out_h, out_w, 4), dtype=cp.uint8) for _ in range(3) + ] + for buf in self._gpu_right: + buf[..., 3] = 255 + self._gpu_bgr = cp.empty((out_h, self._frame_w, 3), dtype=cp.uint8) + + self._cap = None + self._finished = False + self._deadline = 0.0 + + def _open_device(self) -> bool: + cap = self._cv2.VideoCapture(str(self._path)) + if not cap.isOpened(): + cap.release() + return False + self._cap = cap + self._finished = False + self._deadline = time.monotonic() + return True + + def _close_device(self) -> None: + if self._cap is not None: + try: + self._cap.release() + except Exception: + pass + self._cap = None + + def _grab(self) -> Optional[np.ndarray]: + if self._finished: + # loop: false and the clip has ended — hold the last published + # frame. Idle here (not None-spin) until stop(). + self._stop.wait(timeout=0.25) + return None + + wait = self._deadline - time.monotonic() + if wait > 0 and self._stop.wait(timeout=wait): + return None + + cv2 = self._cv2 + ret, frame = self._cap.read() + if not ret or frame is None: + if not self._loop: + self._finished = True + notify(self._kind, "playback finished (holding last frame)") + return None + self._cap.set(cv2.CAP_PROP_POS_FRAMES, 0) + ret, frame = self._cap.read() + if not ret or frame is None: + # Rewind failed — reopen the file via the reconnect path. + raise RuntimeError("rewind failed") + + self._deadline += self._frame_interval_s + now = time.monotonic() + if self._deadline < now: + # Fell behind (slow decode, stall) — resync rather than burst. + self._deadline = now + self._frame_interval_s + + h, fw = self._host_staging.shape[:2] + if frame.shape[0] != h or frame.shape[1] != fw: + frame = cv2.resize(frame, (fw, h), interpolation=cv2.INTER_LINEAR) + self._host_staging[...] = frame + return self._host_staging + + def _upload_and_convert(self, gpu_buf) -> None: + # Async H2D + GPU channel reverse (BGR → RGB); alpha stays 255 from + # construction. Stereo slices the SBS landing zone into eyes. + with self._stream: + self._gpu_bgr.set(self._host_staging) + if not self._stereo: + gpu_buf[..., :3] = self._gpu_bgr[..., ::-1] + return + w = self._spec.width + gpu_buf[..., :3] = self._gpu_bgr[:, :w, ::-1] + self._gpu_right[self._write_idx][..., :3] = self._gpu_bgr[:, w:, ::-1] + + def latest(self): + frame = super().latest() + if frame is not None and self._stereo: + # _consumed_idx is the slot latest() just returned; only the + # consumer thread reads it, so no extra locking needed. + frame.image_right = self._gpu_right[self._consumed_idx] + return frame diff --git a/examples/camera_viz/test_data/recording.mp4 b/examples/camera_viz/test_data/recording.mp4 new file mode 100644 index 000000000..1fcfa39f1 --- /dev/null +++ b/examples/camera_viz/test_data/recording.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c96cd61dcf8dcb293cc0db47b6bc1c3905b8d95b3844fb49ae34beb8f24ad96 +size 1026163 diff --git a/examples/camera_viz/tests/pyproject.toml b/examples/camera_viz/tests/pyproject.toml index 4ee7cdc74..2b9b77779 100644 --- a/examples/camera_viz/tests/pyproject.toml +++ b/examples/camera_viz/tests/pyproject.toml @@ -17,6 +17,8 @@ dev = [ "pytest", "numpy", "pyyaml>=6.0", + # Encodes + decodes the synthetic clips in test_video_file_source.py. + "opencv-python>=4.5", ] gpu = [ "cupy-cuda12x", diff --git a/examples/camera_viz/tests/test_video_file_source.py b/examples/camera_viz/tests/test_video_file_source.py new file mode 100644 index 000000000..0b1186218 --- /dev/null +++ b/examples/camera_viz/tests/test_video_file_source.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the video-file replay source (``type: video``). + +Clips are synthesized in tmp_path with cv2.VideoWriter — no binary +fixtures in the repo. Probe / path-resolution tests run everywhere; +playback tests need cupy + a CUDA device and skip cleanly without one. +""" + +from __future__ import annotations + +import time + +import numpy as np +import pytest + +cv2 = pytest.importorskip("cv2") + +from sources import build_local_camera, resolve_video_paths # noqa: E402 + + +def _cuda_available() -> bool: + try: + import cupy as cp + except ImportError: + return False + try: + return cp.cuda.runtime.getDeviceCount() > 0 + except Exception: + return False + + +requires_gpu = pytest.mark.skipif( + not _cuda_available(), reason="needs cupy + a CUDA device" +) + + +def _write_clip(path, frames, fps: float = 30.0) -> None: + h, w = frames[0].shape[:2] + vw = cv2.VideoWriter(str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + if not vw.isOpened(): + pytest.skip("cv2.VideoWriter cannot encode mp4v here") + for f in frames: + vw.write(f) + vw.release() + + +def _solid(h: int, w: int, bgr) -> np.ndarray: + frame = np.empty((h, w, 3), dtype=np.uint8) + frame[...] = bgr + return frame + + +def _collect(source, n: int, timeout_s: float = 10.0) -> list: + """Poll latest() until ``n`` frames arrive or the timeout hits.""" + frames = [] + deadline = time.monotonic() + timeout_s + while len(frames) < n and time.monotonic() < deadline: + f = source.latest() + if f is not None: + frames.append(f) + time.sleep(0.002) + return frames + + +# ── build-time behavior (no GPU needed) ───────────────────────────── + + +def test_missing_file_raises(tmp_path): + with pytest.raises(ValueError, match="no such file"): + build_local_camera( + {"name": "x", "type": "video", "path": str(tmp_path / "nope.mp4")} + ) + + +def test_resolve_video_paths_anchors_to_yaml_dir(tmp_path): + # An already-absolute path must be left untouched. Build it from + # tmp_path so it's absolute on any OS (a POSIX "/abs" string is not + # absolute on Windows and would get re-anchored to the drive). + abs_path = str(tmp_path / "elsewhere" / "clip.mp4") + cfg = { + "cameras": [ + {"type": "video", "path": "clip.mp4"}, + {"type": "video", "path": abs_path}, + {"type": "v4l2", "device": "/dev/video0"}, + ] + } + resolve_video_paths(cfg, tmp_path) + assert cfg["cameras"][0]["path"] == str(tmp_path / "clip.mp4") + assert cfg["cameras"][1]["path"] == abs_path + assert "path" not in cfg["cameras"][2] + + +# ── playback (GPU) ─────────────────────────────────────────────────── + + +@requires_gpu +def test_probe_size_playback_and_color(tmp_path): + """Omitting width/height sizes the source from the file; frames come + out RGBA with BGR→RGB applied.""" + import cupy as cp + + clip = tmp_path / "blue.mp4" + _write_clip(clip, [_solid(48, 64, (255, 0, 0))] * 10, fps=60.0) # BGR blue + + (source,) = build_local_camera( + {"name": "replay", "type": "video", "path": str(clip)} + ) + assert source.spec.width == 64 + assert source.spec.height == 48 + + with source: + frames = _collect(source, 3) + assert len(frames) == 3 + img = cp.asnumpy(frames[-1].image) + assert img.shape == (48, 64, 4) + assert img[..., 2].mean() > 200 # blue landed in the R..B order's B slot + assert img[..., 0].mean() < 60 # red channel stays dark + assert (img[..., 3] == 255).all() + + +@requires_gpu +def test_loop_wraps_past_end_of_file(tmp_path): + clip = tmp_path / "short.mp4" + _write_clip(clip, [_solid(32, 32, (0, 255, 0))] * 6, fps=60.0) + + (source,) = build_local_camera( + {"name": "looper", "type": "video", "path": str(clip), "fps": 60} + ) + with source: + # 6-frame clip: anything past 6 proves the rewind worked. + frames = _collect(source, 20) + assert len(frames) == 20 + + +@requires_gpu +def test_no_loop_holds_after_last_frame(tmp_path): + clip = tmp_path / "once.mp4" + _write_clip(clip, [_solid(32, 32, (0, 0, 255))] * 5, fps=100.0) + + (source,) = build_local_camera( + { + "name": "oneshot", + "type": "video", + "path": str(clip), + "loop": False, + "fps": 100, + } + ) + with source: + frames = _collect(source, 5, timeout_s=5.0) + assert 1 <= len(frames) <= 5 + # Clip is done — no new frames may appear, and the source must + # stay alive (holding) rather than reconnect-spinning. + time.sleep(0.3) + assert source.latest() is None + + +@requires_gpu +def test_stereo_sbs_splits_eyes(tmp_path): + """SBS file → per-eye halves: left green, right red, both eyes in + one Frame.""" + import cupy as cp + + h, eye_w = 32, 48 + sbs = np.empty((h, eye_w * 2, 3), dtype=np.uint8) + sbs[:, :eye_w] = (0, 255, 0) # BGR green left + sbs[:, eye_w:] = (0, 0, 255) # BGR red right + clip = tmp_path / "sbs.mp4" + _write_clip(clip, [sbs] * 10, fps=60.0) + + (source,) = build_local_camera( + {"name": "sbs", "type": "video", "path": str(clip), "stereo": True} + ) + assert source.spec.width == eye_w + assert source.spec.height == h + + with source: + frames = _collect(source, 3) + assert len(frames) == 3 + left = cp.asnumpy(frames[-1].image) + right = cp.asnumpy(frames[-1].image_right) + assert left.shape == right.shape == (h, eye_w, 4) + assert left[..., 1].mean() > 200 # green + assert right[..., 0].mean() > 200 # red + assert (right[..., 3] == 255).all() + + +@requires_gpu +def test_resize_override(tmp_path): + clip = tmp_path / "big.mp4" + _write_clip(clip, [_solid(48, 64, (128, 128, 128))] * 5, fps=60.0) + + (source,) = build_local_camera( + { + "name": "small", + "type": "video", + "path": str(clip), + "width": 32, + "height": 24, + } + ) + assert source.spec.width == 32 + assert source.spec.height == 24 + with source: + frames = _collect(source, 2) + assert len(frames) == 2 + assert frames[-1].image.shape == (24, 32, 4)