Vtk streaming encoding

.github/workflows/test_and_release.yml

@@ -64,8 +64,7 @@ jobs:
 
       - name: Install and Run Tests
         run: |
-          pip install ".[turbo]"
-          pip install -r tests/requirements.txt
+          pip install ".[turbo, avif, vtkstreaming, dev]"
           # Install requirements for playwright
           playwright install
           pytest -s ./tests

README.rst

@@ -45,6 +45,9 @@ Install the component
 Optional dependencies
 -----------------------------------------------------------
 
+TurboJPEG
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Faster Jpeg encoding using TurboJPEG.
 
 **macOS system install**
@@ -77,3 +80,18 @@ Once your system is ready, you can try our code example:
 
     # other encoders: jpeg, avif, turbo-jpeg, png, webp
     python ./examples/01_vtk/vtk_cone_simple.py --encoder turbo-jpeg
+
+Video encoding with VTKStreaming
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+VTKStreaming provides tools for encoding and streaming frames from a VTK OpenGL render window using video codecs.
+When NVENC is available, it enables H.264/H.265 hardware encoding; otherwise, software encoding falls back to VP9 via libvpx.
+
+You can try our code example:
+
+.. code-block:: console
+
+    pip install trame trame-vuetify vtk
+    pip install "trame-rca[vtkstreaming]"
+
+    python ./examples/05_video/vtk_cone_simple_video.py

examples/00_vanilla/vanilla_example.py

@@ -94,9 +94,7 @@ def _build_ui(self):
                         display="image",
                         image_style=({},),  # restore default style with width: 100%
                     )
-                    self.view_handler = view.create_view_handler(
-                        self.window,
-                    )
+                    self.view_handler = view.create_view_handler(self.window)
 
 
 # -----------------------------------------------------------------------------

examples/01_vtk/vtk_cone_simple_video.py

@@ -0,0 +1,113 @@
+import vtkmodules.vtkRenderingOpenGL2  # noqa
+from trame.app import TrameApp
+from trame.decorators import change
+from trame.widgets import rca
+from trame.widgets import vuetify3 as v3
+from trame.ui.vuetify3 import SinglePageLayout
+
+from vtkmodules.vtkCommonCore import vtkLogger
+from vtkmodules.vtkFiltersSources import vtkConeSource
+
+from vtkmodules.vtkInteractionStyle import vtkInteractorStyleSwitch  # noqa
+from vtkmodules.vtkRenderingCore import (
+    vtkActor,
+    vtkPolyDataMapper,
+    vtkRenderer,
+    vtkRenderWindow,
+    vtkRenderWindowInteractor,
+)
+
+DEFAULT_RESOLUTION = 6
+
+
+class ConeApp(TrameApp):
+    def __init__(self, server=None):
+        super().__init__(server)
+        self.server.cli.add_argument(
+            "-v",
+            "--verbosity",
+            help="Set the VTK log verbosity (default: INFO). Change to TRACE to see more verbose logs",
+            required=False,
+            default="INFO",
+        )
+        self.server.cli.add_argument(
+            "-f",
+            "--fps",
+            # fps can mean a number of things, so be honest to the user about what it really means in this context.
+            help="Target frame rate. The RcaVideoRenderScheduler encodes one frame every (1000/fps) ms."
+            "You would ideally want to match the client's display refresh rate.",
+            type=float,
+            required=False,
+            default=60,
+        )
+        args, _ = self.server.cli.parse_known_args()
+
+        # Map logging options to vtkLogger
+        vtkLogger.SetStderrVerbosity(vtkLogger.ConvertToVerbosity(args.verbosity))
+        self.render_window = self.setup_vtk()
+        self.target_fps = args.fps
+
+        self._build_ui()
+
+    def setup_vtk(self):
+        renderer = vtkRenderer()
+        render_window = vtkRenderWindow()
+
+        render_window.AddRenderer(renderer)
+
+        render_window_interactor = vtkRenderWindowInteractor()
+        render_window_interactor.SetRenderWindow(render_window)
+        render_window_interactor.GetInteractorStyle().SetCurrentStyleToTrackballCamera()
+
+        self.cone_source = vtkConeSource()
+        mapper = vtkPolyDataMapper()
+        mapper.SetInputConnection(self.cone_source.GetOutputPort())
+        actor = vtkActor()
+        actor.SetMapper(mapper)
+
+        renderer.AddActor(actor)
+        renderer.ResetCamera()
+        render_window.Render()
+
+        return render_window
+
+    @change("resolution")
+    def update_cone(self, resolution, **kwargs):
+        self.cone_source.SetResolution(resolution)
+        self.view_handler.update()
+
+    def update_reset_resolution(self):
+        self.state.resolution = DEFAULT_RESOLUTION
+
+    def _build_ui(self):
+        with SinglePageLayout(self.server, full_height=True) as layout:
+            layout.title.set_text("Video Encoding")
+            with layout.toolbar:
+                v3.VSpacer()
+                v3.VSlider(
+                    v_model=("resolution", DEFAULT_RESOLUTION),
+                    min=3,
+                    max=60,
+                    step=1,
+                    hide_details=True,
+                    density="compact",
+                    style="max-width: 300px",
+                )
+
+                v3.VBtn(icon="mdi-undo-variant", click=self.update_reset_resolution)
+
+            with layout.content:
+                view = rca.RemoteControlledArea(
+                    display="video-decoder",
+                    event_throttle_ms=(500 / self.target_fps,),
+                )
+                self.view_handler = view.create_view_handler(self.render_window)
+                self.view_handler.target_fps = self.target_fps
+
+
+# -----------------------------------------------------------------------------
+# Main
+# -----------------------------------------------------------------------------
+if __name__ == "__main__":
+    app = ConeApp()
+    app.server.start()

pyproject.toml

@@ -30,16 +30,26 @@ classifiers = [
 
 [project.optional-dependencies]
 dev = [
+    "pixelmatch",
     "pre-commit",
-    "ruff",
     "pytest",
+    "pytest-asyncio",
+    "pytest-playwright",
+    "pytest-xprocess",
+    "ruff",
+    "trame-vuetify",
+    "trame>=3.6",
 ]
 avif = [
     "pillow-avif-plugin",
 ]
 turbo = [
     "PyTurboJPEG", # Requires libjpeg-turbo to be available on the system
 ]
+vtkstreaming = [
+    "vtk",
+    "vtk-streaming>=0.3.2",
+]
 
 
 [build-system]

src/trame_rca/encoders/__init__.py

@@ -0,0 +1,32 @@
+import logging
+import os
+import sys
+
+from .image_encoder import RcaImageEncoder
+
+logger = logging.getLogger(__name__)
+
+if sys.platform == "linux":
+    # Disabling vsync avoids capping the render loop at the display refresh
+    # rate (e.g. ~16ms/frame on a 60 Hz display). trame-rca does not override
+    # these itself: it is not its responsibility to mutate global OpenGL (or
+    # other) configuration, and trame-rca can be used without VTK. Set these in
+    # the environment that launches the server for optimal performance.
+    _vsync_vars = ("__GL_SYNC_TO_VBLANK", "vblank_mode")
+    if not all(var in os.environ for var in _vsync_vars):
+        logger.warning(
+            "For optimal rendering performance on Linux, disable vsync by "
+            "setting these environment variables before launching the server: "
+            "%s. Otherwise rendering may be capped at the display refresh rate.",
+            ", ".join(f"{var}=0" for var in _vsync_vars),
+        )
+
+try:
+    from .video_encoder import RcaVideoEncoder
+except ModuleNotFoundError:
+    logger.warning(
+        "VTKStreaming Video encoding is NOT AVAILABLE (missing Python package)"
+    )
+
+
+__all__ = ["RcaVideoEncoder", "RcaImageEncoder"]

src/trame_rca/encoders/image_encoder.py

@@ -0,0 +1,49 @@
+import logging
+from enum import Enum
+from time import time_ns
+
+from numpy.typing import NDArray
+from trame_common.utils import profiler
+from .pil import encode as encode_pil
+
+
+logger = logging.getLogger(__name__)
+try:
+    from .turbo_jpeg import encode as encode_turbo
+except RuntimeError:
+    logger.warning("Turbo JPEG - NOT AVAILABLE (System Library)")
+    encode_turbo = encode_pil
+except ModuleNotFoundError:
+    logger.warning("Turbo JPEG - NOT AVAILABLE (Python package)")
+    encode_turbo = encode_pil
+
+
+class RcaImageEncoder(Enum):
+    JPEG = "jpeg"
+    TURBO_JPEG = "turbo-jpeg"
+    PNG = "png"
+    WEBP = "webp"
+    AVIF = "avif"
+
+    def __init__(self, value: str):
+        self.__value__ = value
+        self._timer_msg = f"rca.encode.{self.value}"
+
+    @property
+    def _impl(self):
+        """Return encoding method"""
+        if self is RcaImageEncoder.TURBO_JPEG:
+            return encode_turbo
+
+        return encode_pil
+
+    def encode(
+        self,
+        np_image: NDArray,
+        cols: int,
+        rows: int,
+        quality: int,
+    ) -> tuple[bytes, dict, int]:
+        now_ms = int(time_ns() / 1000000)
+        with profiler.timer(self._timer_msg):
+            return self._impl(np_image, self.value, cols, rows, quality, now_ms)