Add pre-encoded video passthrough publish path

.changeset/preencoded-passthrough-core.md

@@ -0,0 +1,7 @@
+---
+"webrtc-sys": patch
+"libwebrtc": patch
+"livekit": patch
+---
+
+Add a pre-encoded video publish path: a passthrough video encoder and encoded video frame buffer in webrtc-sys, and `EncodedVideoFrame`/`EncodedVideoCodec`/`EncodedFrameType` publish APIs with a `VideoEncoderBackend::PreEncoded` backend in libwebrtc. WebRTC rate-control targets and keyframe requests are forwarded to encoded sources, and pre-encoded AV1 and H265 access units are validated on ingest.

libwebrtc/src/native/rtp_sender.rs

@@ -95,6 +95,7 @@ impl From<VideoEncoderBackend> for sys_webrtc::ffi::VideoEncoderBackend {
             VideoEncoderBackend::Nvenc => Self::Nvenc,
             VideoEncoderBackend::Vaapi => Self::Vaapi,
             VideoEncoderBackend::VideoToolbox => Self::VideoToolbox,
+            VideoEncoderBackend::PreEncoded => Self::PreEncoded,
         }
     }
 }
@@ -108,6 +109,7 @@ impl From<sys_webrtc::ffi::VideoEncoderBackend> for VideoEncoderBackend {
             sys_webrtc::ffi::VideoEncoderBackend::Nvenc => Self::Nvenc,
             sys_webrtc::ffi::VideoEncoderBackend::Vaapi => Self::Vaapi,
             sys_webrtc::ffi::VideoEncoderBackend::VideoToolbox => Self::VideoToolbox,
+            sys_webrtc::ffi::VideoEncoderBackend::PreEncoded => Self::PreEncoded,
             _ => panic!("unknown VideoEncoderBackend"),
         }
     }
@@ -130,6 +132,7 @@ mod tests {
             (VideoEncoderBackend::Nvenc, sys_webrtc::ffi::VideoEncoderBackend::Nvenc),
             (VideoEncoderBackend::Vaapi, sys_webrtc::ffi::VideoEncoderBackend::Vaapi),
             (VideoEncoderBackend::VideoToolbox, sys_webrtc::ffi::VideoEncoderBackend::VideoToolbox),
+            (VideoEncoderBackend::PreEncoded, sys_webrtc::ffi::VideoEncoderBackend::PreEncoded),
         ];
 
         for (backend, expected) in cases {

libwebrtc/src/native/video_frame.rs

@@ -51,7 +51,9 @@ pub fn new_video_frame_buffer(
             vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer {
                 handle: NV12Buffer { sys_handle: sys_handle.pin_mut().get_nv12() },
             }),
-            _ => unreachable!(),
+            _ => {
+                Box::new(vf::I420Buffer { handle: I420Buffer { sys_handle: sys_handle.to_i420() } })
+            }
         }
     }
 }

libwebrtc/src/native/video_source.rs

@@ -26,8 +26,8 @@ use webrtc_sys::{video_frame as vf_sys, video_frame::ffi::VideoRotation, video_t
 use crate::video_frame::FrameMetadata;
 use crate::{
     native::packet_trailer::PacketTrailerHandler,
-    video_frame::{I420Buffer, VideoBuffer, VideoFrame},
-    video_source::VideoResolution,
+    video_frame::{EncodedVideoFrame, I420Buffer, VideoBuffer, VideoFrame},
+    video_source::{EncodedRateControl, VideoResolution},
 };
 
 impl From<vt_sys::ffi::VideoResolution> for VideoResolution {
@@ -54,6 +54,24 @@ struct VideoSourceInner {
 
 impl NativeVideoSource {
     pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource {
+        Self::new_inner(resolution, is_screencast, true)
+    }
+
+    /// Creates a source for pre-encoded access units.
+    ///
+    /// Unlike [`NativeVideoSource::new`], no raw black-frame keepalive is
+    /// injected before the first capture: raw frames would start a real
+    /// encoder on a sender meant for the pass-through encoder and corrupt
+    /// the encoded stream.
+    pub fn new_encoded(resolution: VideoResolution) -> NativeVideoSource {
+        Self::new_inner(resolution, false, false)
+    }
+
+    fn new_inner(
+        resolution: VideoResolution,
+        is_screencast: bool,
+        raw_keepalive: bool,
+    ) -> NativeVideoSource {
         let source = Self {
             sys_handle: vt_sys::ffi::new_video_track_source(
                 &vt_sys::ffi::VideoResolution::from(resolution.clone()),
@@ -62,39 +80,41 @@ impl NativeVideoSource {
             inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })),
         };
 
-        livekit_runtime::spawn({
-            let source = source.clone();
-            let i420 = I420Buffer::new(resolution.width, resolution.height);
-            async move {
-                let mut interval = interval(Duration::from_millis(100)); // 10 fps
+        if raw_keepalive {
+            livekit_runtime::spawn({
+                let source = source.clone();
+                let i420 = I420Buffer::new(resolution.width, resolution.height);
+                async move {
+                    let mut interval = interval(Duration::from_millis(100)); // 10 fps
 
-                loop {
-                    interval.tick().await;
+                    loop {
+                        interval.tick().await;
 
-                    let inner = source.inner.lock();
-                    if inner.captured_frames > 0 {
-                        break;
-                    }
+                        let inner = source.inner.lock();
+                        if inner.captured_frames > 0 {
+                            break;
+                        }
 
-                    let mut builder = vf_sys::ffi::new_video_frame_builder();
-                    builder.pin_mut().set_rotation(VideoRotation::VideoRotation0);
-                    builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle());
-
-                    let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
-                    builder.pin_mut().set_timestamp_us(now.as_micros() as i64);
-
-                    source.sys_handle.on_captured_frame(
-                        &builder.pin_mut().build(),
-                        &vt_sys::ffi::FrameMetadata {
-                            has_packet_trailer: false,
-                            user_timestamp: 0,
-                            frame_id: 0,
-                            user_data: Vec::new(),
-                        },
-                    );
+                        let mut builder = vf_sys::ffi::new_video_frame_builder();
+                        builder.pin_mut().set_rotation(VideoRotation::VideoRotation0);
+                        builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle());
+
+                        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+                        builder.pin_mut().set_timestamp_us(now.as_micros() as i64);
+
+                        source.sys_handle.on_captured_frame(
+                            &builder.pin_mut().build(),
+                            &vt_sys::ffi::FrameMetadata {
+                                has_packet_trailer: false,
+                                user_timestamp: 0,
+                                frame_id: 0,
+                                user_data: Vec::new(),
+                            },
+                        );
+                    }
                 }
-            }
-        });
+            });
+        }
 
         source
     }
@@ -139,6 +159,60 @@ impl NativeVideoSource {
         );
     }
 
+    pub fn capture_encoded_frame(&self, frame: &EncodedVideoFrame<'_>) -> bool {
+        let (has_trailer, user_ts, fid, user_data) = match &frame.frame_metadata {
+            Some(meta) => (
+                true,
+                meta.user_timestamp.unwrap_or(0),
+                meta.frame_id.unwrap_or(0),
+                meta.user_data.clone().unwrap_or_default(),
+            ),
+            None => (false, 0, 0, Vec::new()),
+        };
+
+        let capture_ts = if frame.timestamp_us == 0 {
+            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+            now.as_micros() as i64
+        } else {
+            frame.timestamp_us
+        };
+
+        self.inner.lock().captured_frames += 1;
+        self.sys_handle.capture_encoded_frame(
+            frame.width as i32,
+            frame.height as i32,
+            &vt_sys::ffi::EncodedVideoFrameData {
+                codec: frame.codec.into(),
+                frame_type: frame.frame_type.into(),
+                timestamp_us: capture_ts,
+            },
+            frame.payload,
+            &vt_sys::ffi::FrameMetadata {
+                has_packet_trailer: has_trailer,
+                user_timestamp: user_ts,
+                frame_id: fid,
+                user_data,
+            },
+        )
+    }
+
+    /// Returns and clears the pending keyframe request raised by the
+    /// pass-through encoder (PLI/FIR or reconfiguration). Poll from the
+    /// capture loop and forward the request to the upstream encoder.
+    pub fn take_keyframe_request(&self) -> bool {
+        self.sys_handle.take_keyframe_request()
+    }
+
+    /// Returns and clears the pending rate-control target raised by the
+    /// pass-through encoder.
+    pub fn take_rate_control_request(&self) -> Option<EncodedRateControl> {
+        let request = self.sys_handle.take_rate_control_request();
+        request.has_request.then_some(EncodedRateControl {
+            target_bitrate_bps: request.target_bitrate_bps,
+            framerate_fps: request.framerate_fps,
+        })
+    }
+
     /// Captures a Jetson DMA-buffer backed video frame.
     ///
     /// `pixel_format` is `0` for NV12 and `1` for YUV420M.

libwebrtc/src/rtp_sender.rs

@@ -36,6 +36,8 @@ pub enum VideoEncoderBackend {
     Vaapi,
     /// Prefer VideoToolbox on Apple platforms when available.
     VideoToolbox,
+    /// Pass pre-encoded frames through without encoding raw video frames.
+    PreEncoded,
 }
 
 impl VideoEncoderBackend {

libwebrtc/src/video_frame.rs

@@ -71,6 +71,73 @@ pub struct FrameMetadata {
     pub user_data: Option<Vec<u8>>,
 }
 
+/// Codec carried by a pre-encoded video access unit.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[non_exhaustive]
+pub enum EncodedVideoCodec {
+    /// H.264/AVC video.
+    H264,
+    /// H.265/HEVC video.
+    H265,
+    /// VP8 video.
+    VP8,
+    /// VP9 video.
+    VP9,
+    /// AV1 video.
+    AV1,
+}
+
+/// Frame type of a pre-encoded video access unit.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EncodedFrameType {
+    /// A key frame.
+    Key,
+    /// A delta frame.
+    Delta,
+}
+
+/// A pre-encoded video access unit ready for passthrough publishing.
+#[derive(Debug, Clone)]
+pub struct EncodedVideoFrame<'a> {
+    /// Encoded video codec.
+    pub codec: EncodedVideoCodec,
+    /// Encoded access-unit payload.
+    pub payload: &'a [u8],
+    /// Capture timestamp in microseconds.
+    pub timestamp_us: i64,
+    /// Encoded frame type.
+    pub frame_type: EncodedFrameType,
+    /// Encoded frame width in pixels.
+    pub width: u32,
+    /// Encoded frame height in pixels.
+    pub height: u32,
+    /// Optional metadata to attach through packet trailers.
+    pub frame_metadata: Option<FrameMetadata>,
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl From<EncodedVideoCodec> for webrtc_sys::video_track::ffi::EncodedVideoCodec {
+    fn from(value: EncodedVideoCodec) -> Self {
+        match value {
+            EncodedVideoCodec::H264 => Self::H264,
+            EncodedVideoCodec::H265 => Self::H265,
+            EncodedVideoCodec::VP8 => Self::VP8,
+            EncodedVideoCodec::VP9 => Self::VP9,
+            EncodedVideoCodec::AV1 => Self::AV1,
+        }
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+impl From<EncodedFrameType> for webrtc_sys::video_track::ffi::EncodedFrameType {
+    fn from(value: EncodedFrameType) -> Self {
+        match value {
+            EncodedFrameType::Key => Self::Key,
+            EncodedFrameType::Delta => Self::Delta,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct VideoFrame<T>
 where

libwebrtc/src/video_source.rs

@@ -20,6 +20,15 @@ pub struct VideoResolution {
     pub height: u32,
 }
 
+/// Encoder rate-control target requested by WebRTC for a pre-encoded source.
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct EncodedRateControl {
+    /// Target bitrate in bits per second.
+    pub target_bitrate_bps: u64,
+    /// Target frame rate in frames per second.
+    pub framerate_fps: f64,
+}
+
 impl Default for VideoResolution {
     // Default to 720p
     fn default() -> Self {
@@ -51,7 +60,7 @@ pub mod native {
     use crate::native::packet_trailer::PacketTrailerHandler;
     #[cfg(target_os = "linux")]
     use crate::video_frame::FrameMetadata;
-    use crate::video_frame::{VideoBuffer, VideoFrame};
+    use crate::video_frame::{EncodedVideoFrame, VideoBuffer, VideoFrame};
 
     #[derive(Clone)]
     pub struct NativeVideoSource {
@@ -75,10 +84,33 @@ pub mod native {
             Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) }
         }
 
+        /// Creates a source for pre-encoded access units: no raw black-frame
+        /// keepalive is injected before the first capture.
+        pub fn new_encoded(resolution: VideoResolution) -> Self {
+            Self { handle: vs_imp::NativeVideoSource::new_encoded(resolution) }
+        }
+
         pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {
             self.handle.capture_frame(frame)
         }
 
+        /// Captures one pre-encoded video access unit.
+        pub fn capture_encoded_frame(&self, frame: &EncodedVideoFrame<'_>) -> bool {
+            self.handle.capture_encoded_frame(frame)
+        }
+
+        /// Returns and clears the pending keyframe request raised by the
+        /// pass-through encoder (PLI/FIR or reconfiguration).
+        pub fn take_keyframe_request(&self) -> bool {
+            self.handle.take_keyframe_request()
+        }
+
+        /// Returns and clears the pending rate-control target raised by the
+        /// pass-through encoder.
+        pub fn take_rate_control_request(&self) -> Option<EncodedRateControl> {
+            self.handle.take_rate_control_request()
+        }
+
         /// Captures a Jetson DMA-buffer backed video frame.
         ///
         /// `pixel_format` is `0` for NV12 and `1` for YUV420M.

webrtc-sys/build.rs

@@ -84,7 +84,9 @@ fn main() {
         "src/video_frame.cpp",
         "src/video_frame_buffer.cpp",
         "src/dmabuf_video_frame_buffer.cpp",
+        "src/encoded_video_frame_buffer.cpp",
         "src/video_encoder_factory.cpp",
+        "src/passthrough_video_encoder.cpp",
         "src/video_decoder_factory.cpp",
         "src/synthetic_audio_device.cpp",
         "src/adm_proxy.cpp",
@@ -96,6 +98,7 @@ fn main() {
         "src/audio_mixer.cpp",
         "src/packet_trailer.cpp",
         "src/packet_trailer_av1.cpp",
+        "src/jetson/jetson_av1_bitstream.cpp",
     ]);
 
     if is_desktop {
@@ -231,7 +234,6 @@ fn main() {
                         .file("src/jetson/h264_encoder_impl.cpp")
                         .file("src/jetson/h265_encoder_impl.cpp")
                         .file("src/jetson/av1_encoder_impl.cpp")
-                        .file("src/jetson/jetson_av1_bitstream.cpp")
                         .file("src/jetson/jetson_encoder_factory.cpp")
                         .flag("-DUSE_JETSON_VIDEO_CODEC=1");