From 0fe089c5666763080a9b73ff1676d38cc009771b Mon Sep 17 00:00:00 2001 From: Justine Antoine Date: Thu, 28 May 2026 14:57:25 +0200 Subject: [PATCH 1/6] chore(dependencies): move test requirements in pyproject.toml --- .github/workflows/test_and_release.yml | 3 +-- pyproject.toml | 11 ++++++++++- tests/requirements.txt | 12 ------------ 3 files changed, 11 insertions(+), 15 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml index 462e748..8f3b4cc 100644 --- a/.github/workflows/test_and_release.yml +++ b/.github/workflows/test_and_release.yml @@ -64,8 +64,7 @@ jobs: - name: Install and Run Tests run: | - pip install ".[turbo]" - pip install -r tests/requirements.txt + pip install ".[turbo, avif, test]" # Install requirements for playwright playwright install pytest -s ./tests diff --git a/pyproject.toml b/pyproject.toml index 4419880..d22439c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,10 +29,19 @@ classifiers = [ ] [project.optional-dependencies] +test = [ + "pixelmatch", + "pytest", + "pytest-asyncio", + "pytest-playwright", + "pytest-xprocess", + "trame-vuetify", + "trame>=3.6", + "vtk", +] dev = [ "pre-commit", "ruff", - "pytest", ] avif = [ "pillow-avif-plugin", diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 08a28e9..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -numpy -pillow -pillow-avif-plugin -pixelmatch -pytest -pytest-asyncio -pytest-playwright -pytest-xprocess -trame-server>=3 -trame-vuetify -trame>=3.6 -vtk \ No newline at end of file From fee73ff3e7345e9961393b98947472efc3fbd746 Mon Sep 17 00:00:00 2001 From: Lucie Macron Date: Wed, 27 May 2026 10:24:03 +0200 Subject: [PATCH 2/6] feat(video encoder): support video encoding using vtk streaming chore: rename window protocols and restructure video encoder feat(css): fix css for video decoder and add class names to components feat(logger): removed prints for logger info or warning feat(css): create css file that leverages new class names fix(rca): fixes video encoder with new wheel and update examples --- .github/workflows/test_and_release.yml | 2 +- README.rst | 18 + examples/00_vanilla/vanilla_example.py | 4 +- examples/01_vtk/vtk_cone_simple_video.py | 86 ++++ pyproject.toml | 13 +- src/trame_rca/encoders/__init__.py | 14 + src/trame_rca/encoders/image_encoder.py | 49 +++ src/trame_rca/encoders/video_encoder.py | 129 ++++++ src/trame_rca/module/__init__.py | 1 + src/trame_rca/protocol.py | 43 -- src/trame_rca/rca/__init__.py | 37 ++ src/trame_rca/rca/protocol.py | 43 ++ src/trame_rca/rca/vtk_rca.py | 65 +++ src/trame_rca/schedulers/__init__.py | 17 + src/trame_rca/schedulers/image_scheduler.py | 148 +++++++ src/trame_rca/schedulers/protocol.py | 63 +++ src/trame_rca/schedulers/video_scheduler.py | 86 ++++ src/trame_rca/utils.py | 398 +----------------- src/trame_rca/view_adapter.py | 189 +++++++++ src/trame_rca/vtk_utils.py | 55 +-- src/trame_rca/widgets/rca.py | 38 +- tests/conftest.py | 31 ++ .../{test_rca_utils.py => test_rca_image.py} | 119 ++---- tests/test_rca_interaction.py | 36 ++ tests/test_rca_video.py | 67 +++ vue-components/src/components/DisplayArea.js | 2 +- .../src/components/ImageDisplayArea.js | 2 +- vue-components/src/components/ImageRegion.js | 6 +- vue-components/src/components/ImageStream.js | 2 +- .../src/components/MediaSourceDisplayArea.js | 2 +- .../src/components/RawImageDisplayArea.js | 5 +- .../src/components/RemoteControlledArea.js | 6 +- .../src/components/VideoDecoderDisplayArea.js | 2 +- vue-components/src/style.css | 44 ++ vue-components/src/use.js | 1 + 35 files changed, 1220 insertions(+), 603 deletions(-) create mode 100644 examples/01_vtk/vtk_cone_simple_video.py create mode 100644 src/trame_rca/encoders/image_encoder.py create mode 100644 src/trame_rca/encoders/video_encoder.py create mode 100644 src/trame_rca/rca/__init__.py create mode 100644 src/trame_rca/rca/protocol.py create mode 100644 src/trame_rca/rca/vtk_rca.py create mode 100644 src/trame_rca/schedulers/__init__.py create mode 100644 src/trame_rca/schedulers/image_scheduler.py create mode 100644 src/trame_rca/schedulers/protocol.py create mode 100644 src/trame_rca/schedulers/video_scheduler.py create mode 100644 src/trame_rca/view_adapter.py rename tests/{test_rca_utils.py => test_rca_image.py} (53%) create mode 100644 tests/test_rca_interaction.py create mode 100644 tests/test_rca_video.py create mode 100644 vue-components/src/style.css diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml index 8f3b4cc..95c76c6 100644 --- a/.github/workflows/test_and_release.yml +++ b/.github/workflows/test_and_release.yml @@ -64,7 +64,7 @@ jobs: - name: Install and Run Tests run: | - pip install ".[turbo, avif, test]" + pip install ".[turbo, avif, vtkstreaming, dev]" # Install requirements for playwright playwright install pytest -s ./tests diff --git a/README.rst b/README.rst index 7e50d54..8f340ae 100644 --- a/README.rst +++ b/README.rst @@ -45,6 +45,9 @@ Install the component Optional dependencies ----------------------------------------------------------- +TurboJPEG +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Faster Jpeg encoding using TurboJPEG. **macOS system install** @@ -77,3 +80,18 @@ Once your system is ready, you can try our code example: # other encoders: jpeg, avif, turbo-jpeg, png, webp python ./examples/01_vtk/vtk_cone_simple.py --encoder turbo-jpeg + +Video encoding with VTKStreaming +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +VTKStreaming provides tools for encoding and streaming frames from a VTK OpenGL render window using video codecs. +When NVENC is available, it enables H.264/H.265 hardware encoding; otherwise, software encoding falls back to VP9 via libvpx. + +You can try our code example: + +.. code-block:: console + + pip install trame trame-vuetify vtk + pip install "trame-rca[vtkstreaming]" + + python ./examples/05_video/vtk_cone_simple_video.py diff --git a/examples/00_vanilla/vanilla_example.py b/examples/00_vanilla/vanilla_example.py index 1d8cf50..c2737b5 100644 --- a/examples/00_vanilla/vanilla_example.py +++ b/examples/00_vanilla/vanilla_example.py @@ -94,9 +94,7 @@ def _build_ui(self): display="image", image_style=({},), # restore default style with width: 100% ) - self.view_handler = view.create_view_handler( - self.window, - ) + self.view_handler = view.create_view_handler(self.window) # ----------------------------------------------------------------------------- diff --git a/examples/01_vtk/vtk_cone_simple_video.py b/examples/01_vtk/vtk_cone_simple_video.py new file mode 100644 index 0000000..6b7f5d0 --- /dev/null +++ b/examples/01_vtk/vtk_cone_simple_video.py @@ -0,0 +1,86 @@ +import vtkmodules.vtkRenderingOpenGL2 # noqa +from trame.app import TrameApp +from trame.decorators import change +from trame.widgets import rca +from trame.widgets import vuetify3 as v3 +from trame.ui.vuetify3 import SinglePageLayout + +from vtkmodules.vtkFiltersSources import vtkConeSource + +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleSwitch # noqa +from vtkmodules.vtkRenderingCore import ( + vtkActor, + vtkPolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, +) + +DEFAULT_RESOLUTION = 6 + + +class ConeApp(TrameApp): + def __init__(self, server=None): + super().__init__(server) + self.render_window = self.setup_vtk() + + self._build_ui() + + def setup_vtk(self): + renderer = vtkRenderer() + render_window = vtkRenderWindow() + + render_window.AddRenderer(renderer) + + render_window_interactor = vtkRenderWindowInteractor() + render_window_interactor.SetRenderWindow(render_window) + render_window_interactor.GetInteractorStyle().SetCurrentStyleToTrackballCamera() + + self.cone_source = vtkConeSource() + mapper = vtkPolyDataMapper() + mapper.SetInputConnection(self.cone_source.GetOutputPort()) + actor = vtkActor() + actor.SetMapper(mapper) + + renderer.AddActor(actor) + renderer.ResetCamera() + render_window.Render() + + return render_window + + @change("resolution") + def update_cone(self, resolution, **kwargs): + self.cone_source.SetResolution(resolution) + self.view_handler.update() + + def update_reset_resolution(self): + self.state.resolution = DEFAULT_RESOLUTION + + def _build_ui(self): + with SinglePageLayout(self.server, full_height=True) as layout: + layout.title.set_text("Video Encoding") + with layout.toolbar: + v3.VSpacer() + v3.VSlider( + v_model=("resolution", DEFAULT_RESOLUTION), + min=3, + max=60, + step=1, + hide_details=True, + density="compact", + style="max-width: 300px", + ) + + v3.VBtn(icon="mdi-undo-variant", click=self.update_reset_resolution) + + with layout.content: + view = rca.RemoteControlledArea(display="video-decoder") + self.view_handler = view.create_view_handler(self.render_window) + + +# ----------------------------------------------------------------------------- +# Main +# ----------------------------------------------------------------------------- +if __name__ == "__main__": + app = ConeApp() + app.server.start() diff --git a/pyproject.toml b/pyproject.toml index d22439c..430a4d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,19 +29,16 @@ classifiers = [ ] [project.optional-dependencies] -test = [ +dev = [ "pixelmatch", + "pre-commit", "pytest", "pytest-asyncio", "pytest-playwright", "pytest-xprocess", + "ruff", "trame-vuetify", "trame>=3.6", - "vtk", -] -dev = [ - "pre-commit", - "ruff", ] avif = [ "pillow-avif-plugin", @@ -49,6 +46,10 @@ avif = [ turbo = [ "PyTurboJPEG", # Requires libjpeg-turbo to be available on the system ] +vtkstreaming = [ + "vtk", + "vtk-streaming>=0.3.2", +] [build-system] diff --git a/src/trame_rca/encoders/__init__.py b/src/trame_rca/encoders/__init__.py index e69de29..4735fa4 100644 --- a/src/trame_rca/encoders/__init__.py +++ b/src/trame_rca/encoders/__init__.py @@ -0,0 +1,14 @@ +import logging + +from .image_encoder import RcaImageEncoder + +try: + from .video_encoder import RcaVideoEncoder +except ModuleNotFoundError: + logger = logging.getLogger(__name__) + logger.warning( + "VTKStreaming Video encoding is NOT AVAILABLE (missing Python package)" + ) + + +__all__ = ["RcaVideoEncoder", "RcaImageEncoder"] diff --git a/src/trame_rca/encoders/image_encoder.py b/src/trame_rca/encoders/image_encoder.py new file mode 100644 index 0000000..1951606 --- /dev/null +++ b/src/trame_rca/encoders/image_encoder.py @@ -0,0 +1,49 @@ +import logging +from enum import Enum +from time import time_ns + +from numpy.typing import NDArray +from trame_common.utils import profiler +from .pil import encode as encode_pil + + +logger = logging.getLogger(__name__) +try: + from .turbo_jpeg import encode as encode_turbo +except RuntimeError: + logger.warning("Turbo JPEG - NOT AVAILABLE (System Library)") + encode_turbo = encode_pil +except ModuleNotFoundError: + logger.warning("Turbo JPEG - NOT AVAILABLE (Python package)") + encode_turbo = encode_pil + + +class RcaImageEncoder(Enum): + JPEG = "jpeg" + TURBO_JPEG = "turbo-jpeg" + PNG = "png" + WEBP = "webp" + AVIF = "avif" + + def __init__(self, value: str): + self.__value__ = value + self._timer_msg = f"rca.encode.{self.value}" + + @property + def _impl(self): + """Return encoding method""" + if self is RcaImageEncoder.TURBO_JPEG: + return encode_turbo + + return encode_pil + + def encode( + self, + np_image: NDArray, + cols: int, + rows: int, + quality: int, + ) -> tuple[bytes, dict, int]: + now_ms = int(time_ns() / 1000000) + with profiler.timer(self._timer_msg): + return self._impl(np_image, self.value, cols, rows, quality, now_ms) diff --git a/src/trame_rca/encoders/video_encoder.py b/src/trame_rca/encoders/video_encoder.py new file mode 100644 index 0000000..a70be85 --- /dev/null +++ b/src/trame_rca/encoders/video_encoder.py @@ -0,0 +1,129 @@ +from time import time_ns +import logging + +from vtkmodules.vtkCommonCore import vtkUnsignedCharArray +from vtkmodules.vtkRenderingCore import vtkRenderWindow +from vtkmodules.util.misc import calldata_type +from vtkmodules.util.vtkConstants import VTK_OBJECT + +from vtk_streaming.vtkStreamingCore import ( + VTKPF_IYUV, + VTKVC_H264, + VTKVC_VP9, + vtkCompressedVideoPacket, +) +from vtk_streaming.vtkStreamingEncode import vtkVideoEncoder +from vtk_streaming.vtkStreamingNvEncode import vtkNvEncoderGL +from vtk_streaming.vtkStreamingOpenGL2 import vtkOpenGLVideoFrame +from vtk_streaming.vtkStreamingVpxEncode import vtkVpxEncoder + +logger = logging.getLogger(__name__) + + +def create_vpx_encoder() -> vtkVpxEncoder: + encoder = vtkVpxEncoder() + encoder.SetCodec(VTKVC_VP9) + encoder.SetTargetCPUUsage(9) + encoder.SetRowBasedMultiThreading(False) # crashes if True + return encoder + + +def create_nvenc_encoder() -> vtkNvEncoderGL: + encoder = vtkNvEncoderGL() + encoder.SetCodec(VTKVC_H264) + return encoder + + +def encode( + video_packet: vtkCompressedVideoPacket, now_ms: int +) -> tuple[bytes, dict, int]: + frame_data: vtkUnsignedCharArray = video_packet.GetData() + meta = { + "type": "application/octet-stream", + "codec": video_packet.GetCodecLongName(), + "w": video_packet.GetDisplayWidth(), + "h": video_packet.GetDisplayHeight(), + "st": now_ms, + "key": ("key" if video_packet.GetIsKeyFrame() else "delta"), + } + + return (bytes(frame_data), meta, now_ms) + + +class RcaVideoEncoder: + def __init__(self, render_window: vtkRenderWindow) -> None: + self._render_window = render_window + self.encoder = None + self.frame = None + self._push_callback = None + self._window_size = None + + if vtkNvEncoderGL.CheckAvailability(): + logger.info("Using H264 through NVENC") + self._create_encoder = create_nvenc_encoder + else: + logger.info("Using VP9 through libvpx") + self._create_encoder = create_vpx_encoder + + self._initialize(render_window) + + def _set_size(self, render_window_size: tuple[int]): + self._window_size = render_window_size + width = self._window_size[0] + self._window_size[0] % 4 + height = self._window_size[1] + self._window_size[1] % 4 + + self.encoder.SetWidth(width) + self.encoder.SetHeight(height) + self.frame.SetWidth(width) + self.frame.SetHeight(height) + self.frame.AllocateDataStore() + + def _initialize(self, render_window: vtkRenderWindow): + self.encoder = self._create_encoder() + self.encoder.SetGraphicsContext(render_window) + self.encoder.SetInputPixelFormat(VTKPF_IYUV) + self.encoder.SetBitRateControlMode(3) # Constant quantization + self.encoder.SetQuantizationParameter(5) # 0 (high quality) - 63 (low quality) + + self.frame = vtkOpenGLVideoFrame() + self.frame.SetContext(render_window) + self.frame.SetPixelFormat(VTKPF_IYUV) + + self._set_size(render_window.GetSize()) + self.encoder.Initialize() + self.encoder.AddObserver( + vtkVideoEncoder.EncodedVideoChunkEvent, self._on_encoded_chunk + ) + self.encoder.ForceIFrameOn() + + def _reset(self, render_window: vtkRenderWindow) -> None: + self.release() + self._initialize(render_window) + + def set_push_callback(self, callback): + self._push_callback = callback + + @calldata_type(VTK_OBJECT) + def _on_encoded_chunk( + self, + _encoder: vtkNvEncoderGL | vtkVpxEncoder, + _event: str, + video_packet: vtkCompressedVideoPacket, + ) -> None: + now_ms = int(time_ns() / 1000000) + content, meta, _ = encode(video_packet, now_ms) + if self._push_callback is not None: + self._push_callback(content, meta) + + def encode(self, render_window: vtkRenderWindow): + if self.encoder is None or self.frame is None: + return + if self._window_size != render_window.GetSize(): + # Ideally we would just call _set_size here + self._reset(render_window) + self.frame.Capture(render_window) + self.encoder.Encode(self.frame) + + def release(self): + self.encoder = None + self.frame = None diff --git a/src/trame_rca/module/__init__.py b/src/trame_rca/module/__init__.py index bdba094..a472822 100644 --- a/src/trame_rca/module/__init__.py +++ b/src/trame_rca/module/__init__.py @@ -7,6 +7,7 @@ serve_path = str(Path(__file__).with_name("serve").resolve()) serve = {f"__trame_rca_{__version__}": serve_path} scripts = [f"__trame_rca_{__version__}/trame-rca.umd.js"] +styles = [f"__trame_rca_{__version__}/style.css"] vue_use = ["trame_rca"] diff --git a/src/trame_rca/protocol.py b/src/trame_rca/protocol.py index a649ce7..d042e47 100644 --- a/src/trame_rca/protocol.py +++ b/src/trame_rca/protocol.py @@ -1,51 +1,8 @@ -from typing import Protocol, runtime_checkable - -from numpy.typing import NDArray from trame_common.utils import profiler from wslink import register as exportRpc from wslink.websocket import LinkProtocol -@runtime_checkable -class AbstractWindow(Protocol): - """ - Protocol defining the interface for interacting with a remote window through RCA. - - Any class matching this interface can be used as a remote window, regardless of inheritance. - Implementing classes must define the required methods and properties to enable window interaction. - """ - - @property - def img_cols_rows(self) -> tuple[NDArray, int, int]: - """ - Returns a tuple containing: - - the window content as a NumPy array, - - the number of columns, - - and the number of rows. - - Called by the scheduler to render the current window view. - """ - pass - - def process_resize_event(self, width: int, height: int) -> None: - """ - Handle a resize event for the RCA (RenderWindowInteractor). - - This method is triggered by the adapter whenever the window is resized. - """ - pass - - def process_interaction_event(self, event: dict) -> None: - """ - Handle an interaction event from the RCA (RenderWindowInteractor). - - This method is invoked by the adapter whenever an interaction event occurs. - Refer to the event types defined in: - https://github.com/Kitware/vtk-js/blob/master/Sources/Rendering/Core/RenderWindowInteractor/index.js - """ - pass - - class AreaAdapter: def __init__(self, name): self.area_name = name diff --git a/src/trame_rca/rca/__init__.py b/src/trame_rca/rca/__init__.py new file mode 100644 index 0000000..2886693 --- /dev/null +++ b/src/trame_rca/rca/__init__.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + +from .protocol import RemoteControlledAreaProtocol + +logger = logging.getLogger(__name__) +try: + from .vtk_rca import VtkRemoteControlledArea +except ModuleNotFoundError as e: + logger.info(e.msg) + +__all__ = [ + "RemoteControlledAreaProtocol", + "VtkRemoteControlledArea", + "window_wrapper", +] + + +def window_wrapper( + window: RemoteControlledAreaProtocol | vtkRenderWindow, +) -> RemoteControlledAreaProtocol: + if isinstance(window, RemoteControlledAreaProtocol): + return window + + from vtkmodules.vtkRenderingCore import vtkRenderWindow + + if isinstance(window, vtkRenderWindow): + return VtkRemoteControlledArea(window) + + raise RuntimeError( + "Invalid window object provided: expected an instance of RemoteControlledAreaProtocol" + ) diff --git a/src/trame_rca/rca/protocol.py b/src/trame_rca/rca/protocol.py new file mode 100644 index 0000000..8f703ef --- /dev/null +++ b/src/trame_rca/rca/protocol.py @@ -0,0 +1,43 @@ +from typing import Protocol, runtime_checkable + +from numpy.typing import NDArray + + +@runtime_checkable +class RemoteControlledAreaProtocol(Protocol): + """ + Protocol defining the interface for interacting with a remote controlled area (RCA). + + Any class matching this interface can be used as a RCA, regardless of inheritance. + Implementing classes must define the required methods and properties to enable RCA interaction. + """ + + @property + def img_cols_rows(self) -> tuple[NDArray, int, int]: + """ + Returns a tuple containing: + - the RCA content as a NumPy array, + - the number of columns, + - and the number of rows. + + Called by the scheduler to render the current window view. + """ + ... + + def process_resize_event(self, width: int, height: int) -> None: + """ + Handle a resize event (RenderWindowInteractor). + + This method is triggered by the adapter whenever the window is resized. + """ + ... + + def process_interaction_event(self, event: dict) -> None: + """ + Handle an interaction event (RenderWindowInteractor). + + This method is invoked by the adapter whenever an interaction event occurs. + Refer to the event types defined in: + https://github.com/Kitware/vtk-js/blob/master/Sources/Rendering/Core/RenderWindowInteractor/index.js + """ + ... diff --git a/src/trame_rca/rca/vtk_rca.py b/src/trame_rca/rca/vtk_rca.py new file mode 100644 index 0000000..1e8a936 --- /dev/null +++ b/src/trame_rca/rca/vtk_rca.py @@ -0,0 +1,65 @@ +import json + +import vtkmodules.vtkRenderingOpenGL2 # noqa +from packaging.version import Version +from trame_common.utils import profiler +from vtkmodules.util.numpy_support import vtk_to_numpy +from vtkmodules.vtkCommonCore import vtkCommand, vtkVersion +from vtkmodules.vtkRenderingCore import vtkRenderWindow, vtkWindowToImageFilter +from vtkmodules.vtkWebCore import vtkRemoteInteractionAdapter + + +VTK_NEED_RESIZE_EVENT = Version(vtkVersion().vtk_version) < Version("9.5") + + +class VtkRemoteControlledArea: + def __init__(self, vtk_render_window: vtkRenderWindow): + self._timer_render = profiler.Timer("rca.vtk.render") + self._timer_capture = profiler.Timer("rca.vtk.capture") + self._vtk_render_window = vtk_render_window + self._iren = self._vtk_render_window.GetInteractor() + self._iren.EnableRenderOff() + self._vtk_render_window.ShowWindowOff() + + self._window_to_image = vtkWindowToImageFilter() + self._window_to_image.SetInput(vtk_render_window) + self._window_to_image.SetScale(1) + self._window_to_image.ReadFrontBufferOff() + self._window_to_image.ShouldRerenderOff() + self._window_to_image.FixBoundaryOn() + + @property + def img_cols_rows(self): + self._render() + with self._timer_capture: + self._window_to_image.Modified() + self._window_to_image.Update() + + image_data = self._window_to_image.GetOutput() + rows, cols, _ = image_data.GetDimensions() + scalars = image_data.GetPointData().GetScalars() + np_image = vtk_to_numpy(scalars) + np_image = np_image.reshape((cols, rows, -1)) + np_image[:] = np_image[::-1, :, :] + return np_image, cols, rows + + @property + def render_window(self): + self._render() + return self._vtk_render_window + + def _render(self): + with self._timer_render: + self._vtk_render_window.Render() + + def process_resize_event(self, width, height): + self._iren.UpdateSize(width, height) + if VTK_NEED_RESIZE_EVENT: + self._iren.InvokeEvent(vtkCommand.WindowResizeEvent) + + def process_interaction_event(self, event): + event_type = event["type"] + if event_type in ["StartInteractionEvent", "EndInteractionEvent"]: + return + + vtkRemoteInteractionAdapter.ProcessEvent(self._iren, json.dumps(event)) diff --git a/src/trame_rca/schedulers/__init__.py b/src/trame_rca/schedulers/__init__.py new file mode 100644 index 0000000..3accf67 --- /dev/null +++ b/src/trame_rca/schedulers/__init__.py @@ -0,0 +1,17 @@ +import logging + +from .image_scheduler import RcaImageRenderScheduler +from .protocol import RcaRenderSchedulerProtocol + +logger = logging.getLogger(__name__) +try: + from .video_scheduler import RcaVideoRenderScheduler +except (ModuleNotFoundError, ImportError) as e: + logger.info(e.msg) + + +__all__ = [ + "RcaImageRenderScheduler", + "RcaRenderSchedulerProtocol", + "RcaVideoRenderScheduler", +] diff --git a/src/trame_rca/schedulers/image_scheduler.py b/src/trame_rca/schedulers/image_scheduler.py new file mode 100644 index 0000000..519ba12 --- /dev/null +++ b/src/trame_rca/schedulers/image_scheduler.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import os +from asyncio import Queue, sleep, wrap_future +from typing import TYPE_CHECKING, Callable +from time import time_ns + +from concurrent.futures import Executor +from concurrent.futures.thread import ThreadPoolExecutor +from trame.app import asynchronous + +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + +from trame_rca.encoders import RcaImageEncoder +from trame_rca.rca import RemoteControlledAreaProtocol, window_wrapper + + +ENCODING_POOL = ThreadPoolExecutor(max(4, os.cpu_count())) + + +class RcaImageRenderScheduler: + """ + Image-based implementation of :class:`RcaRenderSchedulerProtocol`. + + Captures rendered frames, encodes them to an image format (:class:`RcaImageEncoder`) asynchronously, and forwards + the encoded images and metadata to a callback. Frames are initially encoded using interactive quality settings + and may be re-encoded at higher quality shortly afterwards. + + Supports multiple rendering backends, including VTK. + + Call :meth:`close` before discarding the scheduler to release resources and stop background tasks. + """ + + def __init__( + self, + window: RemoteControlledAreaProtocol | vtkRenderWindow, + *, + push_callback: Callable[[bytes, dict], None] | None = None, + encode_pool: Executor = None, + target_fps: float = 30.0, + interactive_quality: int = 50, + still_quality: int = 90, + rca_encoder: RcaImageEncoder | str = "jpeg", + **_, + ): + self._rca = window_wrapper(window) + self._rca_encoder = RcaImageEncoder(rca_encoder) + + self._push_callback = push_callback + + self._target_fps = target_fps + self._interactive_quality = interactive_quality + self._still_quality = still_quality + + self._n_period_until_still_render = 5 + self._last_push_time_ms = int(time_ns() / 1000000) + self._request_render_queue = Queue() + self._render_quality_queue = Queue() + self._push_queue = Queue() + + self._is_closing = False + self._encode_pool: Executor = encode_pool or ENCODING_POOL + self._render_quality_task = asynchronous.create_task(self._render_quality()) + self._render_task = asynchronous.create_task(self._render()) + self._push_task = asynchronous.create_task(self._push()) + + def update_quality(self, interactive, still): + self._interactive_quality = interactive + self._still_quality = still + + def set_push_callback(self, callback: Callable[[bytes, dict], None]): + self._push_callback = callback + + @property + def rca(self): + return self._rca + + @property + def target_fps(self): + return self._target_fps + + @target_fps.setter + def target_fps(self, v): + self._target_fps = v + + @property + def _target_period_s(self): + return 1.0 / self._target_fps + + async def close(self): + # Set closing flag to true and push one final render to make sure every task will have a chance to be canceled. + if self._is_closing: + return + + self._is_closing = True + await self.async_schedule_render() + await sleep(1) + for task in [self._render_task, self._render_quality_task, self._push_task]: + await task + + def schedule_render(self): + asynchronous.create_task(self.async_schedule_render()) + + async def async_schedule_render(self): + await self._request_render_queue.put(True) + + async def _render_quality(self): + while not self._is_closing: + await self._request_render_queue.get() + await self._render_quality_queue.put(self._interactive_quality) + await self._schedule_still_render() + + async def _schedule_still_render(self): + await self._empty_request_render_queue() + for _ in range(self._n_period_until_still_render): + await sleep(self._target_period_s) + if not self._request_render_queue.empty(): + return + await self._render_quality_queue.put(self._still_quality) + + async def _empty_request_render_queue(self): + while not self._request_render_queue.empty(): + await self._request_render_queue.get() + + async def _render(self): + while not self._is_closing: + quality = await self._render_quality_queue.get() + np_img, cols, rows = self._rca.img_cols_rows + await self._push_queue.put( + wrap_future( + self._encode_pool.submit( + self._rca_encoder.encode, + np_img, + cols, + rows, + quality, + ) + ) + ) + + async def _push(self): + while not self._is_closing: + result = await self._push_queue.get() + img, meta, m_time = await result + if m_time >= self._last_push_time_ms and self._push_callback is not None: + self._last_push_time_ms = m_time + self._push_callback(img, meta) diff --git a/src/trame_rca/schedulers/protocol.py b/src/trame_rca/schedulers/protocol.py new file mode 100644 index 0000000..e81288b --- /dev/null +++ b/src/trame_rca/schedulers/protocol.py @@ -0,0 +1,63 @@ +from typing import Protocol, Callable, runtime_checkable + +from trame_rca.rca import RemoteControlledAreaProtocol + + +@runtime_checkable +class RcaRenderSchedulerProtocol(Protocol): + """ + Protocol defining the interface for scheduling renders. + + Implementations are responsible for coordinating render requests, controlling the render rate, + and forwarding rendered frames to the associated RCA. The scheduler typically serializes render operations + and ensures that rendering does not exceed the configured target frame rate. + + Any class matching this interface can be used as a RCA render scheduler, regardless of inheritance. + Implementing classes must define the required methods and properties to enable window interaction. + """ + + @property + def rca(self) -> RemoteControlledAreaProtocol: + """The remote controlled area (RCA) associated with this scheduler""" + ... + + @property + def target_fps(self) -> float: + """Target rendering frame rate""" + ... + + @target_fps.setter + def target_fps(self, value: float) -> None: + """Update the target rendering frame rate.""" + ... + + def set_push_callback( + self, + callback: Callable[[bytes, dict], None], + ) -> None: + """ + Set the callback used to deliver rendered frames. + + The callback is invoked whenever a new frame is available and receives + the encoded frame payload together with metadata describing the frame. + """ + ... + + def schedule_render(self) -> None: + """ + Request a render operation. + + Implementations may execute the render immediately or defer it according + to their scheduling strategy. Multiple calls may be coalesced to avoid + redundant renders. + """ + ... + + async def close(self) -> None: + """ + Release resources held by the scheduler. + + This method should stop any background tasks, cancel pending renders, + and perform any cleanup required before the scheduler is discarded. + """ + ... diff --git a/src/trame_rca/schedulers/video_scheduler.py b/src/trame_rca/schedulers/video_scheduler.py new file mode 100644 index 0000000..760f49b --- /dev/null +++ b/src/trame_rca/schedulers/video_scheduler.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from asyncio import sleep +from typing import Callable, TYPE_CHECKING + +from trame.app import asynchronous + +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + from trame_rca.rca import VtkRemoteControlledArea + +from trame_rca.rca import window_wrapper +from trame_rca.encoders import RcaVideoEncoder + + +class RcaVideoRenderScheduler: + """ + Video-based implementation of :class:`RcaRenderSchedulerProtocol`. + + This scheduler encodes rendered frames using an :class:`RcaVideoEncoder` and forwards the encoded video + data to a callback. Render requests are coalesced and processed by a background task running at the + configured target frame rate, preventing the encoder from producing frames faster than the desired FPS. + + Supports only VTK rendering backend (:class:`vtkRenderWindow` or :class:`VtkRemoteControlledArea`). + + Call :meth:`close` before discarding the scheduler to stop the background task and release encoder resources. + """ + + def __init__( + self, + window: VtkRemoteControlledArea | vtkRenderWindow, + *, + push_callback: Callable[[bytes, dict]] | None = None, + target_fps: float = 30.0, + ): + self._rca: VtkRemoteControlledArea = window_wrapper(window) + self._rca_encoder = RcaVideoEncoder(self._rca.render_window) + self._is_closing = False + self._render_requested = False + + self._target_fps = target_fps + self._render_task = asynchronous.create_task(self._render()) + + if push_callback is not None: + self.set_push_callback(push_callback) + + @property + def rca(self) -> VtkRemoteControlledArea: + return self._rca + + def set_push_callback(self, callback: Callable[[bytes, dict], None]): + self._rca_encoder.set_push_callback(callback) + + @property + def target_fps(self): + return self._target_fps + + @target_fps.setter + def target_fps(self, v): + self._target_fps = v + + @property + def _target_period_s(self): + return 1.0 / self._target_fps + + async def close(self): + # Set closing flag to true and push one final render to make sure every task will have a chance to be canceled. + if self._is_closing: + return + + self._is_closing = True + await sleep(1) + await self._render_task + self._rca_encoder.release() + + def schedule_render(self): + self._render_requested = True + + async def _render(self): + while not self._is_closing: + if self._render_requested: + self._render_requested = False + render_window = self._rca.render_window + self._rca_encoder.encode(render_window) + + await sleep(self._target_period_s) diff --git a/src/trame_rca/utils.py b/src/trame_rca/utils.py index c2ee38d..fe33cfb 100644 --- a/src/trame_rca/utils.py +++ b/src/trame_rca/utils.py @@ -1,399 +1,19 @@ -from __future__ import annotations +import logging -import asyncio -import math -import os -import time -from asyncio import Queue -from concurrent.futures import Executor -from concurrent.futures.thread import ThreadPoolExecutor -from enum import Enum -from typing import TYPE_CHECKING, Callable, Optional - -from numpy.typing import NDArray -from trame.app import asynchronous -from trame_common.utils import profiler - -from trame_rca.encoders.pil import encode as encode_pil - -if TYPE_CHECKING: - from vtkmodules.vtkRenderingCore import vtkRenderWindow - -from trame_rca.protocol import AbstractWindow - -try: - from trame_rca.vtk_utils import VtkWindow -except ModuleNotFoundError as e: - print(e.msg) +from trame_rca.encoders import RcaImageEncoder as RcaEncoder +from trame_rca.schedulers import RcaImageRenderScheduler as RcaRenderScheduler +from trame_rca.view_adapter import RcaViewAdapter +logger = logging.getLogger(__name__) try: - from trame_rca.encoders.turbo_jpeg import encode as encode_turbo -except RuntimeError: - print("Turbo JPEG - NOT AVAILABLE (System Library)") - encode_turbo = encode_pil -except ModuleNotFoundError: - print("Turbo JPEG - NOT AVAILABLE (Python package)") - encode_turbo = encode_pil + from trame_rca.rca import VtkRemoteControlledArea as VtkWindow +except (ModuleNotFoundError, ImportError) as e: + logger.info(e.msg) -ENCODING_POOL = ThreadPoolExecutor(max(4, os.cpu_count())) __all__ = [ - "AbstractWindow", "RcaEncoder", - "RcaViewAdapter", "RcaRenderScheduler", + "RcaViewAdapter", "VtkWindow", ] - - -def time_now_ms() -> int: - return int(time.time_ns() / 1000000) - - -def window_wrapper(window: AbstractWindow | vtkRenderWindow) -> AbstractWindow: - if isinstance(window, AbstractWindow): - return window - - from vtkmodules.vtkRenderingCore import vtkRenderWindow - - if isinstance(window, vtkRenderWindow): - return VtkWindow(window) - - raise RuntimeError( - "Invalid window object provided: expected an instance of AbstractWindow" - ) - - -class RcaEncoder(Enum): - AVIF = "avif" - JPEG = "jpeg" - TURBO_JPEG = "turbo-jpeg" - PNG = "png" - WEBP = "webp" - - def __init__(self, value): - self._value_ = value - self._timer_msg = f"rca.encode.{self.value}" - - @property - def _impl(self): - """Return encoding method""" - if self is RcaEncoder.TURBO_JPEG: - return encode_turbo - - return encode_pil - - def encode( - self, - np_image: NDArray, - cols: int, - rows: int, - quality: int, - ) -> tuple[bytes, dict, int]: - now_ms = time_now_ms() - with profiler.timer(self._timer_msg): - return self._impl(np_image, self.value, cols, rows, quality, now_ms) - - -class RcaRenderScheduler: - """ - Render scheduler which renders images and pushing the encoded output to a given callback function. - Image metadata is also provided alongside the encoded image. - - The scheduler supports multiple rendering backends, including VTK. - It encodes images to JPEG asynchronously, initially using interactive quality, followed - by a higher-quality encoding after a few ticks. - - Rendering speed is regulated according to a target FPS to balance performance and quality. - - Call the `close` method to properly stop the scheduler before deleting the object. - """ - - def __init__( - self, - window: AbstractWindow | vtkRenderWindow, - *, - push_callback: Optional[Callable[[bytes, dict], None]] = None, - encode_pool: Executor = None, - target_fps: Optional[float] = None, - interactive_quality: Optional[int] = None, - still_quality: Optional[int] = None, - rca_encoder: Optional[RcaEncoder | str] = None, - **_, - ): - self._window = window_wrapper(window) - self._rca_encoder = RcaEncoder(rca_encoder or RcaEncoder.JPEG) - self._push_callback = push_callback - self._target_fps = target_fps or 30.0 - self._interactive_quality = interactive_quality - if self._interactive_quality is None: - self._interactive_quality = 50 - - self._still_quality = still_quality - if self._still_quality is None: - self._still_quality = 90 - - self._n_period_until_still_render = 5 - - self._last_push_time_ms = time_now_ms() - self._request_render_queue = Queue() - self._render_quality_queue = Queue() - self._push_queue = Queue() - - self._is_closing = False - self._encode_pool: Executor = encode_pool or ENCODING_POOL - self._render_quality_task = asynchronous.create_task(self._render_quality()) - self._render_task = asynchronous.create_task(self._render()) - self._push_task = asynchronous.create_task(self._push()) - - def set_push_callback(self, callback: Callable[[bytes, dict], None]): - self._push_callback = callback - - @property - def target_fps(self): - return self._target_fps - - @target_fps.setter - def target_fps(self, v): - self._target_fps = v - - @property - def _target_period_s(self): - return 1.0 / self._target_fps - - async def close(self): - # Set closing flag to true and push one final render to make sure every task will have a chance to be canceled. - if self._is_closing: - return - - self._is_closing = True - await self.async_schedule_render() - await asyncio.sleep(1) - for task in [self._render_task, self._render_quality_task, self._push_task]: - await task - - def schedule_render(self): - asynchronous.create_task(self.async_schedule_render()) - - async def async_schedule_render(self): - await self._request_render_queue.put(True) - - async def _render_quality(self): - while not self._is_closing: - await self._request_render_queue.get() - await self._render_quality_queue.put(self._interactive_quality) - await self._schedule_still_render() - - async def _schedule_still_render(self): - await self._empty_request_render_queue() - for _ in range(self._n_period_until_still_render): - await asyncio.sleep(self._target_period_s) - if not self._request_render_queue.empty(): - return - await self._render_quality_queue.put(self._still_quality) - - async def _empty_request_render_queue(self): - while not self._request_render_queue.empty(): - await self._request_render_queue.get() - - async def _render(self): - while not self._is_closing: - quality = await self._render_quality_queue.get() - np_img, cols, rows = self._window.img_cols_rows - await self._push_queue.put( - asyncio.wrap_future( - self._encode_pool.submit( - self._rca_encoder.encode, - np_img, - cols, - rows, - quality, - ) - ) - ) - - async def _push(self): - while not self._is_closing: - result = await self._push_queue.get() - img, meta, m_time = await result - if m_time >= self._last_push_time_ms and self._push_callback is not None: - self._last_push_time_ms = m_time - self._push_callback(img, meta) - - -class RcaViewAdapter: - """ - Adapter for Remote Control Area. - Uses an RCA render scheduler to serialize window to the RCA. - """ - - def __init__( - self, - window: AbstractWindow | vtkRenderWindow, - name: str, - *, - scheduler: RcaRenderScheduler = None, - do_schedule_render_on_interaction=True, - **_, - ): - self._window = window_wrapper(window) - if scheduler is None: - scheduler = RcaRenderScheduler( - self._window, - target_fps=30, - rca_encoder="turbo-jpeg", # will fallback to jpeg if turbo not available - encode_pool=ENCODING_POOL, - ) - - self._scheduler = scheduler - self._scheduler.set_push_callback(self.push) - self.area_name = name - self.streamer = None - self._prev_data_m_time = None - - self._press_set = set() - self._do_render_on_interaction = do_schedule_render_on_interaction - self._scale = 1 - self._max_pixel_count = 0 - self._current_size = None - - def update_quality(self, interactive=50, still=90): - self._scheduler._interactive_quality = interactive - self._scheduler._still_quality = still - - @property - def target_fps(self): - return self._scheduler.target_fps - - @target_fps.setter - def target_fps(self, value): - self._scheduler.target_fps = value - - @property - def max_pixel_count(self): - """Use 0 to disable capping of pixel count""" - return self._max_pixel_count - - @max_pixel_count.setter - def max_pixel_count(self, value): - if self._max_pixel_count != value: - self._max_pixel_count = value - - if self._current_size is not None: - self.update_size( - "self", - { - "w": self._current_size[0], - "h": self._current_size[1], - "p": self._current_size[2], - }, - ) - - @property - def image_size(self): - if self._current_size is None: - return (300, 300) - size_with_scale = ( - int(self._current_size[0] * self._current_size[2] * self._scale), - int(self._current_size[1] * self._current_size[2] * self._scale), - ) - if self._max_pixel_count: - total = size_with_scale[0] * size_with_scale[1] - if total > self._max_pixel_count: - # not perfect but close enough - rescale = math.sqrt(self._max_pixel_count / total) - return ( - int(size_with_scale[0] * rescale), - int(size_with_scale[1] * rescale), - ) - return size_with_scale - - @property - def scale(self): - return self._scale - - @scale.setter - def scale(self, value): - if self._scale != value: - self._scale = value - - if self._current_size is not None: - self.update_size( - "self", - { - "w": self._current_size[0], - "h": self._current_size[1], - "p": self._current_size[2], - }, - ) - - def set_streamer(self, stream_manager): - self.streamer = stream_manager - - def update_size(self, origin, size): - # Resize to ten pixel min to avoid rendering problems - width = max(10, int(size.get("w", 300))) - height = max(10, int(size.get("h", 300))) - pixel_ratio = size.get("p", 1) - self._current_size = (width, height, pixel_ratio) - width = int(width * pixel_ratio * self._scale) - height = int(height * pixel_ratio * self._scale) - - # Handle count cap - if self._max_pixel_count: - total = width * height - if total > self._max_pixel_count: - # not perfect but close enough - rescale = math.sqrt(self._max_pixel_count / total) - width = int(width * rescale) - height = int(height * rescale) - - self._window.process_resize_event(width, height) - self._scheduler.schedule_render() - - def push(self, content, meta: dict): - if not self.streamer: - return - - if content is None: - return - - self.streamer.push_content(self.area_name, meta, content) - - def do_discard_extra_release_event(self, event): - """ - Ignores mouse release events which have not been preceded by a previous mouse press. - """ - event_type = event["type"] - if "Press" in event_type: - self._press_set.add(event_type) - return False - - if not event_type.endswith("Release"): - return False - - press_event = event_type.replace("Release", "Press") - if press_event in self._press_set: - self._press_set.remove(press_event) - return False - - return True - - def on_interaction(self, _, event): - if self.do_discard_extra_release_event(event): - return - self._window.process_interaction_event(event) - if self._do_render_on_interaction: - self._scheduler.schedule_render() - - def schedule_render(self): - """ - Schedule a render and push to the RCA view when rendering is ready. - """ - self._scheduler.schedule_render() - - def update(self): - self.schedule_render() - - async def close(self): - await self._scheduler.close() diff --git a/src/trame_rca/view_adapter.py b/src/trame_rca/view_adapter.py new file mode 100644 index 0000000..153044a --- /dev/null +++ b/src/trame_rca/view_adapter.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import math + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + +from trame_rca.rca import RemoteControlledAreaProtocol +from trame_rca.schedulers import RcaImageRenderScheduler, RcaRenderSchedulerProtocol + + +class RcaViewAdapter: + """ + Adapter for Remote Control Area. + Uses an RCA render scheduler to serialize window to the RCA. + """ + + def __init__( + self, + window: RemoteControlledAreaProtocol | vtkRenderWindow, + name: str, + *, + scheduler: RcaRenderSchedulerProtocol | None = None, + do_schedule_render_on_interaction: bool = True, + **_, + ): + if scheduler is None: + scheduler = RcaImageRenderScheduler(window, push_callback=self.push) + + else: + if scheduler.rca is not window: + raise ValueError("window does not match scheduler.rca") + scheduler.set_push_callback(self.push) + + self._scheduler = scheduler + self._rca = scheduler.rca + + self.area_name = name + self.streamer = None + self._prev_data_m_time = None + + self._press_set = set() + self._do_render_on_interaction = do_schedule_render_on_interaction + self._scale = 1 + self._max_pixel_count = 0 + self._current_size = None + + def update_quality(self, interactive=50, still=90) -> None: + if isinstance(self._scheduler, RcaImageRenderScheduler): + self._scheduler.update_quality(interactive, still) + + @property + def target_fps(self) -> float: + return self._scheduler.target_fps + + @target_fps.setter + def target_fps(self, value) -> None: + self._scheduler.target_fps = value + + @property + def max_pixel_count(self): + """Use 0 to disable capping of pixel count""" + return self._max_pixel_count + + @max_pixel_count.setter + def max_pixel_count(self, value): + if self._max_pixel_count != value: + self._max_pixel_count = value + + if self._current_size is not None: + self.update_size( + "self", + { + "w": self._current_size[0], + "h": self._current_size[1], + "p": self._current_size[2], + }, + ) + + @property + def image_size(self): + if self._current_size is None: + return (300, 300) + size_with_scale = ( + int(self._current_size[0] * self._current_size[2] * self._scale), + int(self._current_size[1] * self._current_size[2] * self._scale), + ) + if self._max_pixel_count: + total = size_with_scale[0] * size_with_scale[1] + if total > self._max_pixel_count: + # not perfect but close enough + rescale = math.sqrt(self._max_pixel_count / total) + return ( + int(size_with_scale[0] * rescale), + int(size_with_scale[1] * rescale), + ) + return size_with_scale + + @property + def scale(self): + return self._scale + + @scale.setter + def scale(self, value): + if self._scale != value: + self._scale = value + + if self._current_size is not None: + self.update_size( + "self", + { + "w": self._current_size[0], + "h": self._current_size[1], + "p": self._current_size[2], + }, + ) + + def set_streamer(self, stream_manager): + self.streamer = stream_manager + + def update_size(self, origin, size): + # Resize to ten pixel min to avoid rendering problems + width = max(10, int(size.get("w", 300))) + height = max(10, int(size.get("h", 300))) + pixel_ratio = size.get("p", 1) + self._current_size = (width, height, pixel_ratio) + width = int(width * pixel_ratio * self._scale) + height = int(height * pixel_ratio * self._scale) + + # Handle count cap + if self._max_pixel_count: + total = width * height + if total > self._max_pixel_count: + # not perfect but close enough + rescale = math.sqrt(self._max_pixel_count / total) + width = int(width * rescale) + height = int(height * rescale) + + self._rca.process_resize_event(width, height) + self._scheduler.schedule_render() + + def push(self, content: bytes, meta: dict): + if not self.streamer: + return + + if content is None: + return + + self.streamer.push_content(self.area_name, meta, content) + + def do_discard_extra_release_event(self, event): + """ + Ignores mouse release events which have not been preceded by a previous mouse press. + """ + event_type = event["type"] + if "Press" in event_type: + self._press_set.add(event_type) + return False + + if not event_type.endswith("Release"): + return False + + press_event = event_type.replace("Release", "Press") + if press_event in self._press_set: + self._press_set.remove(press_event) + return False + + return True + + def on_interaction(self, _, event): + if self.do_discard_extra_release_event(event): + return + self._rca.process_interaction_event(event) + if self._do_render_on_interaction: + self._scheduler.schedule_render() + + def schedule_render(self): + """ + Schedule a render and push to the RCA view when rendering is ready. + """ + self._scheduler.schedule_render() + + def update(self): + self.schedule_render() + + async def close(self): + await self._scheduler.close() diff --git a/src/trame_rca/vtk_utils.py b/src/trame_rca/vtk_utils.py index 30313c3..b707c7c 100644 --- a/src/trame_rca/vtk_utils.py +++ b/src/trame_rca/vtk_utils.py @@ -1,54 +1,3 @@ -import json +from trame_rca.rca import VtkRemoteControlledArea as VtkWindow -import vtkmodules.vtkRenderingOpenGL2 # noqa -from packaging.version import Version -from trame_common.utils import profiler -from vtkmodules.util.numpy_support import vtk_to_numpy -from vtkmodules.vtkCommonCore import vtkCommand, vtkVersion -from vtkmodules.vtkRenderingCore import vtkRenderWindow, vtkWindowToImageFilter -from vtkmodules.vtkWebCore import vtkRemoteInteractionAdapter - - -class VtkWindow: - def __init__(self, vtk_render_window: vtkRenderWindow): - self._timer_render = profiler.Timer("rca.vtk.render") - self._timer_capture = profiler.Timer("rca.vtk.capture") - self._vtk_render_window = vtk_render_window - self._window_to_image = vtkWindowToImageFilter() - self._window_to_image.SetInput(vtk_render_window) - self._window_to_image.SetScale(1) - self._window_to_image.ReadFrontBufferOff() - self._window_to_image.ShouldRerenderOff() - self._window_to_image.FixBoundaryOn() - self._iren = self._vtk_render_window.GetInteractor() - self._iren.EnableRenderOff() - self._vtk_render_window.ShowWindowOff() - - @property - def img_cols_rows(self): - with self._timer_render: - self._vtk_render_window.Render() - - with self._timer_capture: - self._window_to_image.Modified() - self._window_to_image.Update() - - image_data = self._window_to_image.GetOutput() - rows, cols, _ = image_data.GetDimensions() - scalars = image_data.GetPointData().GetScalars() - np_image = vtk_to_numpy(scalars) - np_image = np_image.reshape((cols, rows, -1)) - np_image[:] = np_image[::-1, :, :] - return np_image, cols, rows - - def process_resize_event(self, width, height): - self._iren.UpdateSize(width, height) - if Version(vtkVersion().vtk_version) < Version("9.5"): - self._iren.InvokeEvent(vtkCommand.WindowResizeEvent) - - def process_interaction_event(self, event): - event_type = event["type"] - if event_type in ["StartInteractionEvent", "EndInteractionEvent"]: - return - - vtkRemoteInteractionAdapter.ProcessEvent(self._iren, json.dumps(event)) +__all__ = ["VtkWindow"] diff --git a/src/trame_rca/widgets/rca.py b/src/trame_rca/widgets/rca.py index b8c3a43..be93f72 100644 --- a/src/trame_rca/widgets/rca.py +++ b/src/trame_rca/widgets/rca.py @@ -1,11 +1,20 @@ """RCA Widgets support both vue2 and vue3.""" +from __future__ import annotations + import warnings +from typing import TYPE_CHECKING from weakref import WeakKeyDictionary, WeakValueDictionary from trame_client.widgets.core import AbstractElement -from trame_rca.utils import RcaRenderScheduler, RcaViewAdapter +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + +from trame_rca.encoders import RcaImageEncoder +from trame_rca.utils import RcaViewAdapter +from trame_rca.schedulers import RcaImageRenderScheduler +from trame_rca.rca import RemoteControlledAreaProtocol, window_wrapper from .. import module @@ -54,6 +63,7 @@ def __init__(self, **kwargs): ] self.name = kwargs.get("name") or f"trame_rca_{RemoteControlledArea._next_id}" + self.display = kwargs.get("display") self._handlers = [] self.ctrl.on_server_ready.add(self._on_ready) @@ -68,22 +78,32 @@ def add_view_handler(self, view_handler): def create_view_handler( self, - render_window, - encoder=None, + window: RemoteControlledAreaProtocol | "vtkRenderWindow", + encoder: RcaImageEncoder | str | None = None, target_fps=30, interactive_quality=60, still_quality=90, ): scheduler = None - if encoder: - scheduler = RcaRenderScheduler( - render_window, + window = window_wrapper(window) + if self.display == "video-decoder": + from trame_rca.rca import VtkRemoteControlledArea + from trame_rca.schedulers import RcaVideoRenderScheduler + + if not isinstance(window, VtkRemoteControlledArea): + raise TypeError("Only VTK backends are supported by video decoder") + scheduler = RcaVideoRenderScheduler(window, target_fps=target_fps) + + elif encoder: + scheduler = RcaImageRenderScheduler( + window, target_fps=target_fps, interactive_quality=interactive_quality, still_quality=still_quality, rca_encoder=encoder, ) - view_handler = RcaViewAdapter(render_window, self.name, scheduler=scheduler) + + view_handler = RcaViewAdapter(window, self.name, scheduler=scheduler) self.add_view_handler(view_handler) return view_handler @@ -233,7 +253,7 @@ def _get_rw_handler( if name in cls.HANDLERS: return cls.HANDLERS[name] - scheduler = RcaRenderScheduler( + scheduler = RcaImageRenderScheduler( render_window, target_fps=target_fps, interactive_quality=interactive_quality, @@ -242,7 +262,7 @@ def _get_rw_handler( **kwargs, ) handler = RcaViewAdapter( - render_window, + scheduler.rca, name, scheduler=scheduler, **kwargs, diff --git a/tests/conftest.py b/tests/conftest.py index f653e3c..808ae6d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,15 @@ from pathlib import Path from trame_client.utils.testing import FixtureHelper +from vtkmodules.vtkFiltersSources import vtkConeSource +from vtkmodules.vtkRenderingCore import ( + vtkRenderer, + vtkRenderWindow, + vtkPolyDataMapper, + vtkActor, + vtkRenderWindowInteractor, +) + ROOT_PATH = Path(__file__).parent.parent.absolute() HELPER = FixtureHelper(ROOT_PATH) @@ -17,3 +26,25 @@ def server(xprocess, server_path): finally: # clean up whole process tree afterwards xprocess.getinfo(name).terminate() + + +@pytest.fixture() +def a_render_window(): + renderer = vtkRenderer() + render_window = vtkRenderWindow() + render_window.AddRenderer(renderer) + render_window.SetSize(300, 300) + render_window.ShowWindowOff() + + render_window_interactor = vtkRenderWindowInteractor() + render_window_interactor.SetRenderWindow(render_window) + + cone_source = vtkConeSource() + mapper = vtkPolyDataMapper() + mapper.SetInputConnection(cone_source.GetOutputPort()) + actor = vtkActor() + actor.SetMapper(mapper) + + renderer.AddActor(actor) + renderer.ResetCamera() + yield render_window diff --git a/tests/test_rca_utils.py b/tests/test_rca_image.py similarity index 53% rename from tests/test_rca_utils.py rename to tests/test_rca_image.py index 9a3e462..97ab273 100644 --- a/tests/test_rca_utils.py +++ b/tests/test_rca_image.py @@ -1,29 +1,17 @@ import asyncio import os import sys +import time from multiprocessing import Pool from pathlib import Path from unittest.mock import MagicMock -from playwright.sync_api import expect, sync_playwright import pytest from PIL import Image -from trame_rca.utils import ( - RcaEncoder, - RcaRenderScheduler, - VtkWindow, - time_now_ms, -) - -from vtkmodules.vtkFiltersSources import vtkConeSource -from vtkmodules.vtkRenderingCore import ( - vtkRenderer, - vtkRenderWindow, - vtkPolyDataMapper, - vtkActor, - vtkRenderWindowInteractor, -) +from trame_rca.encoders import RcaImageEncoder +from trame_rca.schedulers import RcaImageRenderScheduler +from trame_rca.rca import VtkRemoteControlledArea if os.environ.get("CI") is not None and sys.platform != "linux": @@ -33,30 +21,10 @@ ) -@pytest.fixture() -def a_threed_view(): - renderer = vtkRenderer() - render_window = vtkRenderWindow() - render_window.AddRenderer(renderer) - - renderWindowInteractor = vtkRenderWindowInteractor() - renderWindowInteractor.SetRenderWindow(render_window) - - cone_source = vtkConeSource() - mapper = vtkPolyDataMapper() - mapper.SetInputConnection(cone_source.GetOutputPort()) - actor = vtkActor() - actor.SetMapper(mapper) - - renderer.AddActor(actor) - renderer.ResetCamera() - yield render_window - - @pytest.mark.parametrize("img_format", ["jpeg", "png", "avif", "webp"]) -def test_a_view_can_be_encoded_to_format(a_threed_view, tmpdir, img_format): - img, *_ = RcaEncoder(img_format).encode( - *VtkWindow(a_threed_view).img_cols_rows, 100 +def test_a_view_can_be_encoded_to_format(a_render_window, tmpdir, img_format): + img, *_ = RcaImageEncoder(img_format).encode( + *VtkRemoteControlledArea(a_render_window).img_cols_rows, 100 ) dest_file = Path(tmpdir) / f"test_img.{img_format}" dest_file.write_bytes(img) @@ -67,10 +35,10 @@ def test_a_view_can_be_encoded_to_format(a_threed_view, tmpdir, img_format): @pytest.mark.parametrize("img_format", ["jpeg", "png", "avif", "webp"]) -def test_np_encode_can_be_done_using_multiprocess(a_threed_view, img_format): - encoder = RcaEncoder(img_format) - array, cols, rows = VtkWindow(a_threed_view).img_cols_rows - now_ms = time_now_ms() +def test_np_encode_can_be_done_using_multiprocess(a_render_window, img_format): + encoder = RcaImageEncoder(img_format) + array, cols, rows = VtkRemoteControlledArea(a_render_window).img_cols_rows + now_ms = int(time.time_ns() / 1000000) with Pool(1) as p: encoded, meta, ret_now_ms = p.apply( @@ -84,14 +52,14 @@ def test_np_encode_can_be_done_using_multiprocess(a_threed_view, img_format): @pytest.mark.asyncio -@pytest.mark.parametrize("encoder", list(RcaEncoder)) +@pytest.mark.parametrize("encoder", list(RcaImageEncoder)) async def test_after_request_render_pushes_render_followed_by_still_render( encoder, - a_threed_view, + a_render_window, ): a_mock_push = MagicMock() - scheduler = RcaRenderScheduler( - a_threed_view, + scheduler = RcaImageRenderScheduler( + a_render_window, push_callback=a_mock_push, target_fps=20, interactive_quality=0, @@ -110,14 +78,14 @@ async def test_after_request_render_pushes_render_followed_by_still_render( @pytest.mark.asyncio -@pytest.mark.parametrize("encoder", list(RcaEncoder)) +@pytest.mark.parametrize("encoder", list(RcaImageEncoder)) async def test_when_schedule_render_called_before_still_render_keeps_animating( encoder, - a_threed_view, + a_render_window, ): a_mock_push = MagicMock() - scheduler = RcaRenderScheduler( - a_threed_view, + scheduler = RcaImageRenderScheduler( + a_render_window, push_callback=a_mock_push, target_fps=20, interactive_quality=0, @@ -137,14 +105,14 @@ async def test_when_schedule_render_called_before_still_render_keeps_animating( @pytest.mark.asyncio -@pytest.mark.parametrize("encoder", list(RcaEncoder)) +@pytest.mark.parametrize("encoder", list(RcaImageEncoder)) async def test_if_no_render_is_scheduled_doesnt_push( encoder, - a_threed_view, + a_render_window, ): a_mock_push = MagicMock() - scheduler = RcaRenderScheduler( - a_threed_view, + scheduler = RcaImageRenderScheduler( + a_render_window, push_callback=a_mock_push, target_fps=20, interactive_quality=0, @@ -159,14 +127,14 @@ async def test_if_no_render_is_scheduled_doesnt_push( @pytest.mark.asyncio -@pytest.mark.parametrize("encoder", list(RcaEncoder)) +@pytest.mark.parametrize("encoder", list(RcaImageEncoder)) async def test_groups_close_request_render_together( encoder, - a_threed_view, + a_render_window, ): a_mock_push = MagicMock() - scheduler = RcaRenderScheduler( - a_threed_view, + scheduler = RcaImageRenderScheduler( + a_render_window, push_callback=a_mock_push, target_fps=20, interactive_quality=0, @@ -182,35 +150,10 @@ async def test_groups_close_request_render_together( await scheduler.close() -@pytest.mark.parametrize("server_path", ["examples/01_vtk/vtk_cone_simple.py"]) -def test_rca_view_is_interactive(server): - with sync_playwright() as p: - url = f"http://127.0.0.1:{server.port}/" - browser = p.chromium.launch() - page = browser.new_page() - page.goto(url) - - element = page.locator("img") - expect(element).to_be_visible() - initial_img_url = element.get_attribute("src") - - box = element.bounding_box() - assert box is not None - - page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2) - page.mouse.down() - page.mouse.move(box["x"] + box["width"] / 2 + 100, box["y"] + box["height"] / 2) - page.mouse.up() - page.wait_for_timeout(100) - new_img_url = element.get_attribute("src") - - assert initial_img_url != new_img_url - - -@pytest.mark.parametrize("encoder", [e.value for e in RcaEncoder]) -def test_scheduler_is_compatible_with_string_encoder_format(encoder, a_threed_view): - RcaRenderScheduler( - a_threed_view, +@pytest.mark.parametrize("encoder", list(RcaImageEncoder)) +def test_scheduler_is_compatible_with_string_encoder_format(encoder, a_render_window): + RcaImageRenderScheduler( + a_render_window, push_callback=MagicMock(), target_fps=20, interactive_quality=0, diff --git a/tests/test_rca_interaction.py b/tests/test_rca_interaction.py new file mode 100644 index 0000000..e781bed --- /dev/null +++ b/tests/test_rca_interaction.py @@ -0,0 +1,36 @@ +import os +import pytest +import sys +from playwright.sync_api import expect, sync_playwright + + +if os.environ.get("CI") is not None and sys.platform != "linux": + pytest.skip( + "Rendering tests are disabled on CI for non Linux platforms.", + allow_module_level=True, + ) + + +@pytest.mark.parametrize("server_path", ["examples/01_vtk/vtk_cone_simple.py"]) +def test_rca_view_is_interactive(server): + with sync_playwright() as p: + url = f"http://127.0.0.1:{server.port}/" + browser = p.chromium.launch() + page = browser.new_page() + page.goto(url) + + element = page.locator("img") + expect(element).to_be_visible() + initial_img_url = element.get_attribute("src") + + box = element.bounding_box() + assert box is not None + + page.mouse.move(box["x"] + box["width"] / 2, box["y"] + box["height"] / 2) + page.mouse.down() + page.mouse.move(box["x"] + box["width"] / 2 + 100, box["y"] + box["height"] / 2) + page.mouse.up() + page.wait_for_timeout(100) + new_img_url = element.get_attribute("src") + + assert initial_img_url != new_img_url diff --git a/tests/test_rca_video.py b/tests/test_rca_video.py new file mode 100644 index 0000000..c7094bc --- /dev/null +++ b/tests/test_rca_video.py @@ -0,0 +1,67 @@ +import asyncio +import os +import sys +import time +from unittest.mock import MagicMock + + +import pytest +from trame_rca.encoders import RcaVideoEncoder +from trame_rca.schedulers import RcaVideoRenderScheduler + + +if os.environ.get("CI") is not None and sys.platform != "linux": + pytest.skip( + "Rendering tests are disabled on CI for non Linux platforms.", + allow_module_level=True, + ) + + +def test_a_view_can_be_encoded_to_format(a_render_window, tmpdir): + rca_encoder = RcaVideoEncoder(a_render_window) + a_mock_push = MagicMock() + + rca_encoder.set_push_callback(a_mock_push) + rca_encoder.encode(a_render_window) + + time.sleep(1) + a_mock_push.assert_called_once() + + args, _ = a_mock_push.call_args + img_bytes = args[0] + assert isinstance(img_bytes, (bytes, bytearray)) + assert len(img_bytes) > 0 + + +@pytest.mark.asyncio +async def test_if_no_render_is_scheduled_doesnt_push(a_render_window): + a_mock_push = MagicMock() + scheduler = RcaVideoRenderScheduler( + a_render_window, + push_callback=a_mock_push, + target_fps=20, + ) + + try: + await asyncio.sleep(1) + assert a_mock_push.call_count == 0 + finally: + await scheduler.close() + + +@pytest.mark.asyncio +async def test_groups_close_request_render_together(a_render_window): + a_mock_push = MagicMock() + scheduler = RcaVideoRenderScheduler( + a_render_window, + push_callback=a_mock_push, + target_fps=30, + ) + + try: + for _ in range(30): + scheduler.schedule_render() + await asyncio.sleep(1) + assert a_mock_push.call_count == 1 + finally: + await scheduler.close() diff --git a/vue-components/src/components/DisplayArea.js b/vue-components/src/components/DisplayArea.js index a7caf64..ad8783d 100644 --- a/vue-components/src/components/DisplayArea.js +++ b/vue-components/src/components/DisplayArea.js @@ -33,7 +33,7 @@ export default { }, }, template: ` -
+
diff --git a/vue-components/src/components/ImageDisplayArea.js b/vue-components/src/components/ImageDisplayArea.js index 8e93279..39f8f27 100644 --- a/vue-components/src/components/ImageDisplayArea.js +++ b/vue-components/src/components/ImageDisplayArea.js @@ -157,5 +157,5 @@ export default { this.cleanup(); }, inject: ['trame'], - template: ``, + template: ``, }; diff --git a/vue-components/src/components/ImageRegion.js b/vue-components/src/components/ImageRegion.js index b773ce6..6b48a7b 100644 --- a/vue-components/src/components/ImageRegion.js +++ b/vue-components/src/components/ImageRegion.js @@ -206,5 +206,9 @@ export default { canvas, }; }, - template: `
`, + template: ` +
+ +
+ `, }; diff --git a/vue-components/src/components/ImageStream.js b/vue-components/src/components/ImageStream.js index d04ab4a..1db49f0 100644 --- a/vue-components/src/components/ImageStream.js +++ b/vue-components/src/components/ImageStream.js @@ -83,5 +83,5 @@ export default { image, }; }, - template: ``, + template: ``, }; diff --git a/vue-components/src/components/MediaSourceDisplayArea.js b/vue-components/src/components/MediaSourceDisplayArea.js index bbbd34a..a02249c 100644 --- a/vue-components/src/components/MediaSourceDisplayArea.js +++ b/vue-components/src/components/MediaSourceDisplayArea.js @@ -156,7 +156,7 @@ export default { }, inject: ['trame', 'rcaPushSize'], template: ` -