From 3499ac6fb65a9e5bd112b4b2d11a8bded826a536 Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 11 May 2026 14:09:25 -0700 Subject: [PATCH 1/5] add frame timing headers --- system/webrtc/device/video.py | 27 ++++++++++++++++++++++++--- system/webrtc/webrtcd.py | 31 +++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/system/webrtc/device/video.py b/system/webrtc/device/video.py index 50feab4f4a910d..35036d7b505973 100644 --- a/system/webrtc/device/video.py +++ b/system/webrtc/device/video.py @@ -1,4 +1,5 @@ import asyncio +import struct import time import av @@ -7,6 +8,13 @@ from cereal import messaging from openpilot.common.realtime import DT_MDL, DT_DMON +# arbitrary 16-byte UUID identifying openpilot frame-timing SEI messages +TIMING_SEI_UUID = bytes([ + 0xa5, 0xe0, 0xc4, 0xa4, 0x5b, 0x6e, 0x4e, 0x1e, + 0x9c, 0x7e, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, +]) +_SEI_PREFIX = b'\x00\x00\x00\x01\x06\x05\x30' + TIMING_SEI_UUID + class LiveStreamVideoStreamTrack(TiciVideoStreamTrack): camera_to_sock_mapping = { @@ -22,6 +30,21 @@ def __init__(self, camera_type: str): self._sock = messaging.sub_sock(self.camera_to_sock_mapping[camera_type], conflate=True) self._pts = 0 self._t0_ns = time.monotonic_ns() + self.timing_sei_enabled = False + + def _build_frame_data(self, msg) -> bytes: + encode_data = getattr(msg, msg.which()) + if not self.timing_sei_enabled: + return encode_data.header + encode_data.data + + idx = encode_data.idx + sei_nal = _SEI_PREFIX + struct.pack('>4d', + (idx.timestampEof - idx.timestampSof) / 1e6, + (msg.logMonoTime - idx.timestampEof) / 1e6, + (time.monotonic_ns() - msg.logMonoTime) / 1e6, + time.time() * 1000, # noqa: TID251 + ) + b'\x80' + return encode_data.header + sei_nal + encode_data.data async def recv(self): while True: @@ -30,9 +53,7 @@ async def recv(self): break await asyncio.sleep(0.005) - evta = getattr(msg, msg.which()) - - packet = av.Packet(evta.header + evta.data) + packet = av.Packet(self._build_frame_data(msg)) packet.time_base = self._time_base self._pts = ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000 diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index 5f66e62bb85fde..176fb6e3dc2948 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import time import argparse import asyncio import contextlib @@ -132,10 +133,13 @@ def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], o config = parse_info_from_offer(sdp) builder = WebRTCAnswerBuilder(sdp) + self.video_tracks = [] assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks" for cam in cameras: - builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()) + track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack() + self.video_tracks.append(track) + builder.add_video_stream(cam, track) self.stream = builder.stream() self.identifier = str(uuid.uuid4()) @@ -174,9 +178,28 @@ async def get_answer(self): return await self.stream.start() def message_handler(self, message: bytes): - assert self.incoming_bridge is not None try: - self.incoming_bridge.send(message) + payload = json.loads(message) if isinstance(message, (bytes, str)) else None + if isinstance(payload, dict): + msg_type = payload.get("type") + + if msg_type == "clockSync": + data = payload.get("data", {}) + pong = json.dumps({"type": "clockSync", "data": { + "action": "pong", "browserSendTime": data.get("browserSendTime"), "deviceTime": time.time() * 1000, # noqa: TID251 + }}) + self.stream.get_messaging_channel().send(pong) + return + + if msg_type == "enableTimingSei": + enabled = bool(payload.get("data", {}).get("enabled")) + for track in self.video_tracks: + if hasattr(track, 'timing_sei_enabled'): + track.timing_sei_enabled = enabled + return + + if self.incoming_bridge is not None: + self.incoming_bridge.send(message) except Exception: self.logger.exception("Cereal incoming proxy failure") @@ -186,7 +209,7 @@ async def run(self): if self.stream.has_messaging_channel(): if self.incoming_bridge is not None: await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services) - self.stream.set_message_handler(self.message_handler) + self.stream.set_message_handler(self.message_handler) if self.outgoing_bridge_runner is not None: channel = self.stream.get_messaging_channel() self.outgoing_bridge_runner.proxy.add_channel(channel) From 9ae526b5e4e9f45409a1fd8fe3a0356b63b4de7d Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 11 May 2026 16:48:46 -0700 Subject: [PATCH 2/5] rfctr --- system/webrtc/webrtcd.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index 176fb6e3dc2948..7e44ba6851e4d2 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -198,8 +198,7 @@ def message_handler(self, message: bytes): track.timing_sei_enabled = enabled return - if self.incoming_bridge is not None: - self.incoming_bridge.send(message) + self.incoming_bridge.send(message) except Exception: self.logger.exception("Cereal incoming proxy failure") @@ -209,7 +208,7 @@ async def run(self): if self.stream.has_messaging_channel(): if self.incoming_bridge is not None: await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services) - self.stream.set_message_handler(self.message_handler) + self.stream.set_message_handler(self.message_handler) if self.outgoing_bridge_runner is not None: channel = self.stream.get_messaging_channel() self.outgoing_bridge_runner.proxy.add_channel(channel) From 753a6a17f591d48553f9e9d276e36893d354f65f Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 11 May 2026 16:49:33 -0700 Subject: [PATCH 3/5] diff --- system/webrtc/webrtcd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index 7e44ba6851e4d2..e684771c571a7f 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -137,9 +137,7 @@ def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], o assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks" for cam in cameras: - track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack() - self.video_tracks.append(track) - builder.add_video_stream(cam, track) + builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack())) self.stream = builder.stream() self.identifier = str(uuid.uuid4()) From 48c56fa7f7577c5aace4187da0436c808ddbf5e4 Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 11 May 2026 16:49:51 -0700 Subject: [PATCH 4/5] clean --- system/webrtc/webrtcd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index e684771c571a7f..25dcadfc0c91df 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -137,7 +137,7 @@ def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], o assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks" for cam in cameras: - builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack())) + builder.add_video_stream(cam, LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()) self.stream = builder.stream() self.identifier = str(uuid.uuid4()) From dfe118238ab30c629e68d6ac4e07ab7d89f11f08 Mon Sep 17 00:00:00 2001 From: stefpi <19478336+stefpi@users.noreply.github.com> Date: Mon, 11 May 2026 19:31:06 -0700 Subject: [PATCH 5/5] remove video_tracks unused --- system/webrtc/webrtcd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/system/webrtc/webrtcd.py b/system/webrtc/webrtcd.py index 25dcadfc0c91df..08b26c47b585f0 100755 --- a/system/webrtc/webrtcd.py +++ b/system/webrtc/webrtcd.py @@ -133,7 +133,6 @@ def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], o config = parse_info_from_offer(sdp) builder = WebRTCAnswerBuilder(sdp) - self.video_tracks = [] assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks" for cam in cameras: