[TeleViz] video-file replay source, XR-default display, and camera-streaming doc overhaul

.gitattributes

@@ -18,6 +18,9 @@
 *.usdc filter=lfs diff=lfs merge=lfs -text
 *.usdz filter=lfs diff=lfs merge=lfs -text
 
+# Video assets are tracked by default.
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+
 #
 # Exceptions for small source SVGs (webpack-inlined; not binary assets).
 deps/cloudxr/webxr_client/src/icons/*.svg -filter -diff -merge text

examples/camera_viz/.gitignore

@@ -0,0 +1,12 @@
+# Replay clips for ``type: video`` — user-supplied or generated,
+# never committed. The shipped sample clip in test_data/ (Git LFS)
+# is the exception.
+*.mp4
+*.mkv
+*.webm
+*.avi
+*.mov
+!test_data/*.mp4
+
+# Vendor SDK scratch (depthai telemetry, etc.) written at runtime.
+.cache/

examples/camera_viz/README.md

@@ -20,8 +20,9 @@ SPDX-License-Identifier: Apache-2.0
 | `v4l2`      | USB / UVC — anything `v4l2-ctl --list-formats-ext` shows |
 | `oakd`      | OAK-D mono RGB / LEFT / RIGHT (stereo not yet wired) |
 | `zed`       | ZED 2 / Mini / X One; mono or `stereo: true` (per-eye SDK retrieve, zero-copy GPU) |
+| `video`     | Video-file replay (anything OpenCV/FFmpeg reads) — preview / testing without a camera. Loops by default; `stereo: true` splits side-by-side files into eyes (viewer only) |
 
-Output: window or XR headset; one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`.
+Output: XR headset (default) or desktop window (`run CONFIG --mode window`); one plane per camera, aspect-fit. Stereo cameras render true SBS in XR; window mode shows the left eye. XR placements: `world` / `head` / `lazy`.
 
 ---
 
@@ -43,10 +44,11 @@ Flags: `--no-{v4l2,oakd,rtp}`, `--with-zed`, `--sender-only`, `--jetson`. Pass `
 ## Mode 1 — Direct
 
 ```bash
-./camera_viz.sh run configs/v4l2.yaml
+./camera_viz.sh run configs/v4l2.yaml                  # XR headset (default)
+./camera_viz.sh run configs/v4l2.yaml --mode window    # desktop window instead
 ```
 
-Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`.
+Set `source: local`. Swap config for `oakd.yaml`, `zed.yaml`, `synthetic.yaml`, `synthetic_stereo.yaml`, `multi_camera.yaml`, `replay.yaml` (file replay — point `path:` at any recording).
 
 ## Mode 2 — Split (robot → workstation, RTP)
 
@@ -92,12 +94,12 @@ encoder: auto | native | gstreamer
 cameras:
   - name: cam
     enabled: true
-    type: v4l2                # v4l2 | oakd | zed | synthetic
-    width: 2560
+    type: v4l2                # v4l2 | oakd | zed | synthetic | video
+    width: 2560               # video: optional — defaults to the file's size
     height: 720
     fps: 30
-    stereo: false             # zed / synthetic only — enables per-eye capture + SBS XR
-    # … type-specific fields (e.g. synthetic: disparity_px)
+    stereo: false             # zed / synthetic / video only — per-eye capture + SBS XR
+    # … type-specific fields (e.g. synthetic: disparity_px; video: path, loop)
     rtp:
       port: 5000              # left eye when stereo
       port_right: 5001        # required when stereo + source: rtp
@@ -106,7 +108,7 @@ cameras:
       # gpu_id: 0             # multi-GPU pin
 
 display:                      # camera_viz only
-  mode: window | xr
+  mode: xr | window           # default: xr
   window: { width, height }
   xr:     { near_z, far_z }
   clear_color: [r, g, b, a]
@@ -144,10 +146,11 @@ camera_viz/
 ├── camera_streamer.py   — robot-side RTP sender (per-camera supervisor)
 ├── pipeline/            — source ABC + threaded runner
 ├── placements/          — XR lock-mode strategies
-├── sources/             — V4L2 / OAK-D / ZED / synthetic / rtp_h264
+├── sources/             — V4L2 / OAK-D / ZED / synthetic / video replay / rtp_h264
 ├── transports/          — RTP sender + receiver, native + GStreamer
 ├── codec/               — native NVENC/NVDEC pybind module
 ├── configs/             — one YAML per camera kind
+├── test_data/           — sample replay clip (Git LFS)
 └── scripts/
     ├── _install_deps.sh             — installer (setup + deploy)
     └── camera-streamer.service.in   — systemd unit template

examples/camera_viz/camera_streamer.py

@@ -30,7 +30,12 @@
 import yaml
 
 from pipeline import FrameSource
-from sources import PairedFrameSource, build_local_camera, set_verbose
+from sources import (
+    PairedFrameSource,
+    build_local_camera,
+    resolve_video_paths,
+    set_verbose,
+)
 from transports import RtpH264Sender, make_encoder
 
 logger = logging.getLogger("camera_streamer")
@@ -136,10 +141,13 @@ def _build_senders(self) -> List[RtpH264Sender]:
             )
 
         def build_one(source: FrameSource, port: int) -> RtpH264Sender:
+            # Encoder / sender geometry comes from the built source's spec,
+            # not the YAML — sources that size themselves (video replay
+            # probing its file) may omit width/height from the config.
             encoder = make_encoder(
                 rtp.get("encoder", self._default_encoder),
-                width=int(self._cfg["width"]),
-                height=int(self._cfg["height"]),
+                width=source.spec.width,
+                height=source.spec.height,
                 bitrate=int(rtp.get("bitrate_mbps", 15)) * 1_000_000,
                 fps=int(self._cfg.get("fps", 30)),
                 gop=int(rtp["gop"]) if "gop" in rtp else None,
@@ -150,8 +158,8 @@ def build_one(source: FrameSource, port: int) -> RtpH264Sender:
                 encoder=encoder,
                 host=self._host,
                 port=port,
-                width=int(self._cfg["width"]),
-                height=int(self._cfg["height"]),
+                width=source.spec.width,
+                height=source.spec.height,
                 fps=int(self._cfg.get("fps", 30)),
                 mtu=int(rtp.get("mtu", 1400)),
             )
@@ -278,6 +286,7 @@ def main(argv: Optional[List[str]] = None) -> int:
 
     # Top-level ``verbose:`` enables per-source periodic breadcrumbs.
     set_verbose(bool(cfg.get("verbose", False)))
+    resolve_video_paths(cfg, args.config.parent)
 
     streaming = cfg.get("streaming", {})
     host = args.host or streaming.get("host")

examples/camera_viz/camera_viz.py

@@ -30,7 +30,13 @@
 
 from pipeline import FrameSource, VizRunner
 from placements import PlacementConfig, PlacementStrategy, build as build_placement
-from sources import PairedFrameSource, RtpH264Source, build_local_camera, set_verbose
+from sources import (
+    PairedFrameSource,
+    RtpH264Source,
+    build_local_camera,
+    resolve_video_paths,
+    set_verbose,
+)
 
 
 @dataclass
@@ -75,18 +81,15 @@ def _enabled_cameras(cfg: dict) -> List[dict]:
 
 
 def _placement_with_aspect(
-    spec: Optional[dict], cam: dict, is_xr: bool
+    spec: Optional[dict], width: int, height: int, is_xr: bool
 ) -> Optional[PlacementStrategy]:
-    """Build the placement for ``cam``, filling in ``size`` from the
-    camera's aspect ratio when the YAML doesn't pin it. Width defaults
-    to 1.0 m so a 16:9 camera lands at 1.0 x 0.5625, a 3.55:1 SBS at
-    1.0 x 0.281."""
+    """Build the placement, filling in ``size`` from the source's aspect
+    ratio when the YAML doesn't pin it. Width defaults to 1.0 m so a
+    16:9 source lands at 1.0 x 0.5625, a 3.55:1 SBS at 1.0 x 0.281."""
     if spec is not None and "size" not in spec:
-        w = int(cam["width"])
-        h = int(cam["height"])
         spec = {
             **spec,
-            "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * h / w],
+            "size": [_DEFAULT_PLANE_WIDTH_M, _DEFAULT_PLANE_WIDTH_M * height / width],
         }
     return _build_placement(spec, is_xr)
 
@@ -104,9 +107,15 @@ def _build_local_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]:
     placements_cfg = cfg.get("display", {}).get("placements", {})
     entries: List[SourceEntry] = []
     for cam in _enabled_cameras(cfg):
-        placement = _placement_with_aspect(placements_cfg.get(cam["name"]), cam, is_xr)
+        cam_sources = build_local_camera(cam)
+        # Aspect comes from the built source's spec, not the YAML — video
+        # sources may omit width/height and size themselves from the file.
+        first = cam_sources[0].spec
+        placement = _placement_with_aspect(
+            placements_cfg.get(cam["name"]), first.width, first.height, is_xr
+        )
         stereo, baseline_mm = _stereo_for(cam, placements_cfg)
-        for source in build_local_camera(cam):
+        for source in cam_sources:
             entries.append(
                 SourceEntry(
                     source=source,
@@ -130,7 +139,18 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]:
                 f"camera_viz: camera {cam.get('name')!r} missing rtp.port; "
                 "required when source: rtp"
             )
-        placement = _placement_with_aspect(placements_cfg.get(cam["name"]), cam, is_xr)
+        if "width" not in cam or "height" not in cam:
+            raise ValueError(
+                f"camera_viz: camera {cam.get('name')!r} needs explicit "
+                "width/height when source: rtp — the receiver sizes its "
+                "decoder from the YAML, not from the wire"
+            )
+        placement = _placement_with_aspect(
+            placements_cfg.get(cam["name"]),
+            int(cam["width"]),
+            int(cam["height"]),
+            is_xr,
+        )
         stereo, baseline_mm = _stereo_for(cam, placements_cfg)
 
         if stereo:
@@ -182,7 +202,7 @@ def _build_rtp_entries(cfg: dict, is_xr: bool) -> List[SourceEntry]:
 def _make_session(cfg: dict, mode_override: Optional[str] = None) -> viz.VizSession:
     display = cfg.get("display", {})
     # --mode overrides display.mode when given.
-    mode_str = (mode_override or display.get("mode", "window")).lower()
+    mode_str = (mode_override or display.get("mode", "xr")).lower()
     session_cfg = viz.VizSessionConfig()
     if mode_str == "window":
         session_cfg.mode = viz.DisplayMode.kWindow
@@ -211,7 +231,8 @@ def main(argv: Optional[list[str]] = None) -> int:
         "--mode",
         choices=("window", "xr"),
         default=None,
-        help="Override display.mode from the config (default: use the config's value).",
+        help="Override display.mode from the config "
+        "(default: the config's value, or xr when the config omits it).",
     )
     args = parser.parse_args(argv)
 
@@ -225,12 +246,13 @@ def main(argv: Optional[list[str]] = None) -> int:
 
     # Top-level ``verbose:`` enables per-source periodic breadcrumbs.
     set_verbose(bool(cfg.get("verbose", False)))
+    resolve_video_paths(cfg, args.config.parent)
 
     source_mode = cfg.get("source", "local").lower()
     if source_mode not in ("local", "rtp"):
         raise ValueError(f"camera_viz: source must be local|rtp, got {source_mode!r}")
 
-    effective_mode = (args.mode or cfg.get("display", {}).get("mode", "window")).lower()
+    effective_mode = (args.mode or cfg.get("display", {}).get("mode", "xr")).lower()
     session = _make_session(cfg, mode_override=args.mode)
     is_xr = session.is_xr_mode()
 

examples/camera_viz/camera_viz.sh

@@ -8,7 +8,7 @@
 # Local:
 #   ./camera_viz.sh setup [--sender-only]      install deps + build codec
 #   ./camera_viz.sh loopback CONFIG            run streamer + viz on 127.0.0.1
-#   ./camera_viz.sh run CONFIG                 run the viewer (honors source:)
+#   ./camera_viz.sh run CONFIG [--mode M]      run the viewer (honors source:)
 #
 # Remote (Jetson robot):
 #   ./camera_viz.sh deploy --host H --user U [--password P] CONFIG
@@ -401,10 +401,13 @@ LOCAL
 
     loopback CONFIG       Run camera_streamer + camera_viz on 127.0.0.1.
 
-    run CONFIG            Run the viewer with the YAML as-is. ``source:
+    run CONFIG [--mode xr|window]
+                          Run the viewer with the YAML as-is. ``source:
                           local`` opens cameras directly; ``source: rtp``
                           listens on rtp.port (sender IP irrelevant — the
-                          receiver binds 0.0.0.0).
+                          receiver binds 0.0.0.0). --mode overrides
+                          display.mode: xr renders to the headset (the
+                          default), window to a desktop window.
 
 REMOTE (Jetson robot)
     deploy [--host H --user U [--password P]]
@@ -437,6 +440,7 @@ EXAMPLES
     ./camera_viz.sh loopback configs/v4l2.yaml
     ./camera_viz.sh deploy --host 10.29.90.127 --user nvidia configs/v4l2.yaml
     ./camera_viz.sh run configs/v4l2.yaml
+    ./camera_viz.sh run configs/replay.yaml --mode window   # no headset needed
 
     # Env-var style (avoids passwords in shell history / argv):
     export REMOTE_HOST=10.29.90.127 REMOTE_USER=nvidia

examples/camera_viz/configs/multi_camera.yaml

@@ -45,7 +45,7 @@ cameras:
       bitrate_mbps: 15
 
 display:
-  mode: xr                      # window | xr
+  mode: xr                      # xr | window (xr is the default)
   window:                       # used only if mode: window
     width: 2560
     height: 720

examples/camera_viz/configs/oakd.yaml

@@ -35,7 +35,7 @@ cameras:
       # gop:                    # default: fps*5 (ULL tuning)
 
 display:
-  mode: window                  # window | xr
+  mode: xr                      # xr | window (xr is the default)
   window:
     width: 1280
     height: 720

examples/camera_viz/configs/realsense.yaml

@@ -37,7 +37,7 @@ cameras:
       # gop:                    # default: fps*5 (IDR every 5 s, ULL tuning)
 
 display:
-  mode: window                  # window | xr
+  mode: xr                      # xr | window (xr is the default)
   window:
     width: 1920
     height: 1080