From 88a543fb57d00232125083cf49fa150d8677ae5c Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 6 Jul 2026 15:50:11 -0700 Subject: [PATCH] Add pre-encoded video passthrough publish path to webrtc-sys and libwebrtc webrtc-sys gains a passthrough video encoder and an encoded video frame buffer so pre-encoded H264/H265/VP8/VP9/AV1 access units can flow through the WebRTC pipeline without re-encoding, with rate-control targets and keyframe requests forwarded back to the encoded source. libwebrtc exposes the corresponding public API: EncodedVideoFrame, EncodedVideoCodec, EncodedFrameType, EncodedRateControl, and VideoEncoderBackend::PreEncoded. The Jetson AV1 bitstream helpers are now compiled on all platforms for AV1 ingest validation. Co-Authored-By: Claude Fable 5 --- .changeset/preencoded-passthrough-core.md | 7 + libwebrtc/src/native/rtp_sender.rs | 3 + libwebrtc/src/native/video_frame.rs | 4 +- libwebrtc/src/native/video_source.rs | 136 ++++-- libwebrtc/src/rtp_sender.rs | 2 + libwebrtc/src/video_frame.rs | 67 +++ libwebrtc/src/video_source.rs | 34 +- webrtc-sys/build.rs | 4 +- .../livekit/encoded_video_frame_buffer.h | 119 +++++ .../livekit/passthrough_video_encoder.h | 47 ++ webrtc-sys/include/livekit/video_track.h | 28 ++ webrtc-sys/src/encoded_video_frame_buffer.cpp | 123 +++++ webrtc-sys/src/jetson/av1_encoder_impl.cpp | 8 +- .../src/jetson/jetson_av1_bitstream.cpp | 47 +- webrtc-sys/src/jetson/jetson_av1_bitstream.h | 9 + webrtc-sys/src/passthrough_video_encoder.cpp | 426 ++++++++++++++++++ webrtc-sys/src/rtp_sender.cpp | 16 +- webrtc-sys/src/video_encoder_factory.cpp | 106 ++++- webrtc-sys/src/video_track.cpp | 83 ++++ webrtc-sys/src/video_track.rs | 41 ++ webrtc-sys/src/webrtc.rs | 1 + 21 files changed, 1267 insertions(+), 44 deletions(-) create mode 100644 .changeset/preencoded-passthrough-core.md create mode 100644 webrtc-sys/include/livekit/encoded_video_frame_buffer.h create mode 100644 webrtc-sys/include/livekit/passthrough_video_encoder.h create mode 100644 webrtc-sys/src/encoded_video_frame_buffer.cpp create mode 100644 webrtc-sys/src/passthrough_video_encoder.cpp diff --git a/.changeset/preencoded-passthrough-core.md b/.changeset/preencoded-passthrough-core.md new file mode 100644 index 000000000..0782e59bf --- /dev/null +++ b/.changeset/preencoded-passthrough-core.md @@ -0,0 +1,7 @@ +--- +"webrtc-sys": patch +"libwebrtc": patch +"livekit": patch +--- + +Add a pre-encoded video publish path: a passthrough video encoder and encoded video frame buffer in webrtc-sys, and `EncodedVideoFrame`/`EncodedVideoCodec`/`EncodedFrameType` publish APIs with a `VideoEncoderBackend::PreEncoded` backend in libwebrtc. WebRTC rate-control targets and keyframe requests are forwarded to encoded sources, and pre-encoded AV1 and H265 access units are validated on ingest. diff --git a/libwebrtc/src/native/rtp_sender.rs b/libwebrtc/src/native/rtp_sender.rs index f58b08484..e90b43a75 100644 --- a/libwebrtc/src/native/rtp_sender.rs +++ b/libwebrtc/src/native/rtp_sender.rs @@ -95,6 +95,7 @@ impl From for sys_webrtc::ffi::VideoEncoderBackend { VideoEncoderBackend::Nvenc => Self::Nvenc, VideoEncoderBackend::Vaapi => Self::Vaapi, VideoEncoderBackend::VideoToolbox => Self::VideoToolbox, + VideoEncoderBackend::PreEncoded => Self::PreEncoded, } } } @@ -108,6 +109,7 @@ impl From for VideoEncoderBackend { sys_webrtc::ffi::VideoEncoderBackend::Nvenc => Self::Nvenc, sys_webrtc::ffi::VideoEncoderBackend::Vaapi => Self::Vaapi, sys_webrtc::ffi::VideoEncoderBackend::VideoToolbox => Self::VideoToolbox, + sys_webrtc::ffi::VideoEncoderBackend::PreEncoded => Self::PreEncoded, _ => panic!("unknown VideoEncoderBackend"), } } @@ -130,6 +132,7 @@ mod tests { (VideoEncoderBackend::Nvenc, sys_webrtc::ffi::VideoEncoderBackend::Nvenc), (VideoEncoderBackend::Vaapi, sys_webrtc::ffi::VideoEncoderBackend::Vaapi), (VideoEncoderBackend::VideoToolbox, sys_webrtc::ffi::VideoEncoderBackend::VideoToolbox), + (VideoEncoderBackend::PreEncoded, sys_webrtc::ffi::VideoEncoderBackend::PreEncoded), ]; for (backend, expected) in cases { diff --git a/libwebrtc/src/native/video_frame.rs b/libwebrtc/src/native/video_frame.rs index aaa5491ff..dc3d5fbb8 100644 --- a/libwebrtc/src/native/video_frame.rs +++ b/libwebrtc/src/native/video_frame.rs @@ -51,7 +51,9 @@ pub fn new_video_frame_buffer( vfb_sys::ffi::VideoFrameBufferType::NV12 => Box::new(vf::NV12Buffer { handle: NV12Buffer { sys_handle: sys_handle.pin_mut().get_nv12() }, }), - _ => unreachable!(), + _ => { + Box::new(vf::I420Buffer { handle: I420Buffer { sys_handle: sys_handle.to_i420() } }) + } } } } diff --git a/libwebrtc/src/native/video_source.rs b/libwebrtc/src/native/video_source.rs index 323e61c88..47b2ff2b3 100644 --- a/libwebrtc/src/native/video_source.rs +++ b/libwebrtc/src/native/video_source.rs @@ -26,8 +26,8 @@ use webrtc_sys::{video_frame as vf_sys, video_frame::ffi::VideoRotation, video_t use crate::video_frame::FrameMetadata; use crate::{ native::packet_trailer::PacketTrailerHandler, - video_frame::{I420Buffer, VideoBuffer, VideoFrame}, - video_source::VideoResolution, + video_frame::{EncodedVideoFrame, I420Buffer, VideoBuffer, VideoFrame}, + video_source::{EncodedRateControl, VideoResolution}, }; impl From for VideoResolution { @@ -54,6 +54,24 @@ struct VideoSourceInner { impl NativeVideoSource { pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource { + Self::new_inner(resolution, is_screencast, true) + } + + /// Creates a source for pre-encoded access units. + /// + /// Unlike [`NativeVideoSource::new`], no raw black-frame keepalive is + /// injected before the first capture: raw frames would start a real + /// encoder on a sender meant for the pass-through encoder and corrupt + /// the encoded stream. + pub fn new_encoded(resolution: VideoResolution) -> NativeVideoSource { + Self::new_inner(resolution, false, false) + } + + fn new_inner( + resolution: VideoResolution, + is_screencast: bool, + raw_keepalive: bool, + ) -> NativeVideoSource { let source = Self { sys_handle: vt_sys::ffi::new_video_track_source( &vt_sys::ffi::VideoResolution::from(resolution.clone()), @@ -62,39 +80,41 @@ impl NativeVideoSource { inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })), }; - livekit_runtime::spawn({ - let source = source.clone(); - let i420 = I420Buffer::new(resolution.width, resolution.height); - async move { - let mut interval = interval(Duration::from_millis(100)); // 10 fps + if raw_keepalive { + livekit_runtime::spawn({ + let source = source.clone(); + let i420 = I420Buffer::new(resolution.width, resolution.height); + async move { + let mut interval = interval(Duration::from_millis(100)); // 10 fps - loop { - interval.tick().await; + loop { + interval.tick().await; - let inner = source.inner.lock(); - if inner.captured_frames > 0 { - break; - } + let inner = source.inner.lock(); + if inner.captured_frames > 0 { + break; + } - let mut builder = vf_sys::ffi::new_video_frame_builder(); - builder.pin_mut().set_rotation(VideoRotation::VideoRotation0); - builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle()); - - let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); - builder.pin_mut().set_timestamp_us(now.as_micros() as i64); - - source.sys_handle.on_captured_frame( - &builder.pin_mut().build(), - &vt_sys::ffi::FrameMetadata { - has_packet_trailer: false, - user_timestamp: 0, - frame_id: 0, - user_data: Vec::new(), - }, - ); + let mut builder = vf_sys::ffi::new_video_frame_builder(); + builder.pin_mut().set_rotation(VideoRotation::VideoRotation0); + builder.pin_mut().set_video_frame_buffer(i420.as_ref().sys_handle()); + + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + builder.pin_mut().set_timestamp_us(now.as_micros() as i64); + + source.sys_handle.on_captured_frame( + &builder.pin_mut().build(), + &vt_sys::ffi::FrameMetadata { + has_packet_trailer: false, + user_timestamp: 0, + frame_id: 0, + user_data: Vec::new(), + }, + ); + } } - } - }); + }); + } source } @@ -139,6 +159,60 @@ impl NativeVideoSource { ); } + pub fn capture_encoded_frame(&self, frame: &EncodedVideoFrame<'_>) -> bool { + let (has_trailer, user_ts, fid, user_data) = match &frame.frame_metadata { + Some(meta) => ( + true, + meta.user_timestamp.unwrap_or(0), + meta.frame_id.unwrap_or(0), + meta.user_data.clone().unwrap_or_default(), + ), + None => (false, 0, 0, Vec::new()), + }; + + let capture_ts = if frame.timestamp_us == 0 { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + now.as_micros() as i64 + } else { + frame.timestamp_us + }; + + self.inner.lock().captured_frames += 1; + self.sys_handle.capture_encoded_frame( + frame.width as i32, + frame.height as i32, + &vt_sys::ffi::EncodedVideoFrameData { + codec: frame.codec.into(), + frame_type: frame.frame_type.into(), + timestamp_us: capture_ts, + }, + frame.payload, + &vt_sys::ffi::FrameMetadata { + has_packet_trailer: has_trailer, + user_timestamp: user_ts, + frame_id: fid, + user_data, + }, + ) + } + + /// Returns and clears the pending keyframe request raised by the + /// pass-through encoder (PLI/FIR or reconfiguration). Poll from the + /// capture loop and forward the request to the upstream encoder. + pub fn take_keyframe_request(&self) -> bool { + self.sys_handle.take_keyframe_request() + } + + /// Returns and clears the pending rate-control target raised by the + /// pass-through encoder. + pub fn take_rate_control_request(&self) -> Option { + let request = self.sys_handle.take_rate_control_request(); + request.has_request.then_some(EncodedRateControl { + target_bitrate_bps: request.target_bitrate_bps, + framerate_fps: request.framerate_fps, + }) + } + /// Captures a Jetson DMA-buffer backed video frame. /// /// `pixel_format` is `0` for NV12 and `1` for YUV420M. diff --git a/libwebrtc/src/rtp_sender.rs b/libwebrtc/src/rtp_sender.rs index b16c008c2..1bdf96add 100644 --- a/libwebrtc/src/rtp_sender.rs +++ b/libwebrtc/src/rtp_sender.rs @@ -36,6 +36,8 @@ pub enum VideoEncoderBackend { Vaapi, /// Prefer VideoToolbox on Apple platforms when available. VideoToolbox, + /// Pass pre-encoded frames through without encoding raw video frames. + PreEncoded, } impl VideoEncoderBackend { diff --git a/libwebrtc/src/video_frame.rs b/libwebrtc/src/video_frame.rs index 1c728c3e9..d89095f13 100644 --- a/libwebrtc/src/video_frame.rs +++ b/libwebrtc/src/video_frame.rs @@ -71,6 +71,73 @@ pub struct FrameMetadata { pub user_data: Option>, } +/// Codec carried by a pre-encoded video access unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum EncodedVideoCodec { + /// H.264/AVC video. + H264, + /// H.265/HEVC video. + H265, + /// VP8 video. + VP8, + /// VP9 video. + VP9, + /// AV1 video. + AV1, +} + +/// Frame type of a pre-encoded video access unit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodedFrameType { + /// A key frame. + Key, + /// A delta frame. + Delta, +} + +/// A pre-encoded video access unit ready for passthrough publishing. +#[derive(Debug, Clone)] +pub struct EncodedVideoFrame<'a> { + /// Encoded video codec. + pub codec: EncodedVideoCodec, + /// Encoded access-unit payload. + pub payload: &'a [u8], + /// Capture timestamp in microseconds. + pub timestamp_us: i64, + /// Encoded frame type. + pub frame_type: EncodedFrameType, + /// Encoded frame width in pixels. + pub width: u32, + /// Encoded frame height in pixels. + pub height: u32, + /// Optional metadata to attach through packet trailers. + pub frame_metadata: Option, +} + +#[cfg(not(target_arch = "wasm32"))] +impl From for webrtc_sys::video_track::ffi::EncodedVideoCodec { + fn from(value: EncodedVideoCodec) -> Self { + match value { + EncodedVideoCodec::H264 => Self::H264, + EncodedVideoCodec::H265 => Self::H265, + EncodedVideoCodec::VP8 => Self::VP8, + EncodedVideoCodec::VP9 => Self::VP9, + EncodedVideoCodec::AV1 => Self::AV1, + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl From for webrtc_sys::video_track::ffi::EncodedFrameType { + fn from(value: EncodedFrameType) -> Self { + match value { + EncodedFrameType::Key => Self::Key, + EncodedFrameType::Delta => Self::Delta, + } + } +} + #[derive(Debug)] pub struct VideoFrame where diff --git a/libwebrtc/src/video_source.rs b/libwebrtc/src/video_source.rs index e88fd73de..ac3294f7e 100644 --- a/libwebrtc/src/video_source.rs +++ b/libwebrtc/src/video_source.rs @@ -20,6 +20,15 @@ pub struct VideoResolution { pub height: u32, } +/// Encoder rate-control target requested by WebRTC for a pre-encoded source. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct EncodedRateControl { + /// Target bitrate in bits per second. + pub target_bitrate_bps: u64, + /// Target frame rate in frames per second. + pub framerate_fps: f64, +} + impl Default for VideoResolution { // Default to 720p fn default() -> Self { @@ -51,7 +60,7 @@ pub mod native { use crate::native::packet_trailer::PacketTrailerHandler; #[cfg(target_os = "linux")] use crate::video_frame::FrameMetadata; - use crate::video_frame::{VideoBuffer, VideoFrame}; + use crate::video_frame::{EncodedVideoFrame, VideoBuffer, VideoFrame}; #[derive(Clone)] pub struct NativeVideoSource { @@ -75,10 +84,33 @@ pub mod native { Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) } } + /// Creates a source for pre-encoded access units: no raw black-frame + /// keepalive is injected before the first capture. + pub fn new_encoded(resolution: VideoResolution) -> Self { + Self { handle: vs_imp::NativeVideoSource::new_encoded(resolution) } + } + pub fn capture_frame>(&self, frame: &VideoFrame) { self.handle.capture_frame(frame) } + /// Captures one pre-encoded video access unit. + pub fn capture_encoded_frame(&self, frame: &EncodedVideoFrame<'_>) -> bool { + self.handle.capture_encoded_frame(frame) + } + + /// Returns and clears the pending keyframe request raised by the + /// pass-through encoder (PLI/FIR or reconfiguration). + pub fn take_keyframe_request(&self) -> bool { + self.handle.take_keyframe_request() + } + + /// Returns and clears the pending rate-control target raised by the + /// pass-through encoder. + pub fn take_rate_control_request(&self) -> Option { + self.handle.take_rate_control_request() + } + /// Captures a Jetson DMA-buffer backed video frame. /// /// `pixel_format` is `0` for NV12 and `1` for YUV420M. diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index a7e73a6ef..b69c5a877 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -84,7 +84,9 @@ fn main() { "src/video_frame.cpp", "src/video_frame_buffer.cpp", "src/dmabuf_video_frame_buffer.cpp", + "src/encoded_video_frame_buffer.cpp", "src/video_encoder_factory.cpp", + "src/passthrough_video_encoder.cpp", "src/video_decoder_factory.cpp", "src/synthetic_audio_device.cpp", "src/adm_proxy.cpp", @@ -96,6 +98,7 @@ fn main() { "src/audio_mixer.cpp", "src/packet_trailer.cpp", "src/packet_trailer_av1.cpp", + "src/jetson/jetson_av1_bitstream.cpp", ]); if is_desktop { @@ -231,7 +234,6 @@ fn main() { .file("src/jetson/h264_encoder_impl.cpp") .file("src/jetson/h265_encoder_impl.cpp") .file("src/jetson/av1_encoder_impl.cpp") - .file("src/jetson/jetson_av1_bitstream.cpp") .file("src/jetson/jetson_encoder_factory.cpp") .flag("-DUSE_JETSON_VIDEO_CODEC=1"); diff --git a/webrtc-sys/include/livekit/encoded_video_frame_buffer.h b/webrtc-sys/include/livekit/encoded_video_frame_buffer.h new file mode 100644 index 000000000..40e5fece0 --- /dev/null +++ b/webrtc-sys/include/livekit/encoded_video_frame_buffer.h @@ -0,0 +1,119 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "api/video/encoded_image.h" +#include "api/video/video_frame_buffer.h" + +namespace livekit { + +enum class EncodedVideoCodec { + kH264, + kH265, + kVP8, + kVP9, + kAV1, +}; + +enum class EncodedFrameType { + kKey, + kDelta, +}; + +struct EncodedRateControlRequest { + bool has_request = false; + uint64_t target_bitrate_bps = 0; + double framerate_fps = 0.0; +}; + +// Latest-wins rate-control mailbox shared between the pass-through encoder and +// the Rust capture side. +class EncodedRateControlState { + public: + void Store(uint64_t target_bitrate_bps, double framerate_fps); + EncodedRateControlRequest Take(); + + private: + std::mutex mutex_; + EncodedRateControlRequest request_; +}; + +// A native WebRTC frame buffer carrying one encoded video access unit. +class EncodedVideoFrameBuffer : public webrtc::VideoFrameBuffer { + public: + // `keyframe_request_flag` is shared with the owning video source: the + // pass-through encoder sets it when the RTP layer asks for a keyframe the + // pending frame cannot satisfy, and the capture side polls it to forward + // the request upstream. + EncodedVideoFrameBuffer( + int width, + int height, + EncodedVideoCodec codec, + EncodedFrameType frame_type, + webrtc::scoped_refptr payload, + std::shared_ptr> keyframe_request_flag = nullptr, + std::shared_ptr rate_control_state = nullptr); + ~EncodedVideoFrameBuffer() override = default; + + Type type() const override; + int width() const override; + int height() const override; + webrtc::scoped_refptr ToI420() override; + webrtc::scoped_refptr CropAndScale( + int offset_x, + int offset_y, + int crop_width, + int crop_height, + int scaled_width, + int scaled_height) override; + + EncodedVideoCodec codec() const { return codec_; } + EncodedFrameType frame_type() const { return frame_type_; } + + // The encoded access unit. Shared with the pass-through encoder so the + // payload is not copied again on the send path. + webrtc::scoped_refptr encoded_data() const { + return payload_; + } + const uint8_t* payload_data() const { return payload_->data(); } + size_t payload_size() const { return payload_->size(); } + + // Asks the capture side to produce a keyframe (e.g. on PLI/FIR). + void request_keyframe() const; + + // Updates the capture side with the latest encoder rate-control target. + void set_rate_control_request(uint64_t target_bitrate_bps, + double framerate_fps) const; + + static EncodedVideoFrameBuffer* FromNative(webrtc::VideoFrameBuffer* buffer); + + private: + int width_; + int height_; + EncodedVideoCodec codec_; + EncodedFrameType frame_type_; + webrtc::scoped_refptr payload_; + std::shared_ptr> keyframe_request_flag_; + std::shared_ptr rate_control_state_; +}; + +} // namespace livekit diff --git a/webrtc-sys/include/livekit/passthrough_video_encoder.h b/webrtc-sys/include/livekit/passthrough_video_encoder.h new file mode 100644 index 000000000..a50f9f63a --- /dev/null +++ b/webrtc-sys/include/livekit/passthrough_video_encoder.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" + +namespace livekit_ffi { + +class PassthroughVideoEncoderFactory : public webrtc::VideoEncoderFactory { + public: + PassthroughVideoEncoderFactory(); + ~PassthroughVideoEncoderFactory() override = default; + + std::vector GetSupportedFormats() const override; + std::vector GetImplementations() const override; + CodecSupport QueryCodecSupport( + const webrtc::SdpVideoFormat& format, + std::optional scalability_mode) const override; + std::unique_ptr Create( + const webrtc::Environment& env, + const webrtc::SdpVideoFormat& format) override; + + private: + std::vector supported_formats_; +}; + +} // namespace livekit_ffi diff --git a/webrtc-sys/include/livekit/video_track.h b/webrtc-sys/include/livekit/video_track.h index 5520cf2ec..213377946 100644 --- a/webrtc-sys/include/livekit/video_track.h +++ b/webrtc-sys/include/livekit/video_track.h @@ -16,11 +16,13 @@ #pragma once +#include #include #include "api/media_stream_interface.h" #include "api/video/video_frame.h" #include "livekit/helper.h" +#include "livekit/encoded_video_frame_buffer.h" #include "livekit/media_stream_track.h" #include "livekit/video_frame.h" #include "livekit/webrtc.h" @@ -105,11 +107,25 @@ class VideoTrackSource { void set_packet_trailer_handler( std::shared_ptr handler); + // Shared with every EncodedVideoFrameBuffer this source emits; the + // pass-through encoder raises it on unsatisfied keyframe requests. + std::shared_ptr> keyframe_request_flag() const { + return keyframe_request_flag_; + } + std::shared_ptr rate_control_state() + const { + return rate_control_state_; + } + private: mutable webrtc::Mutex mutex_; webrtc::TimestampAligner timestamp_aligner_; VideoResolution resolution_; std::shared_ptr packet_trailer_handler_; + std::shared_ptr> keyframe_request_flag_ = + std::make_shared>(false); + std::shared_ptr rate_control_state_ = + std::make_shared(); bool is_screencast_; }; @@ -132,6 +148,18 @@ class VideoTrackSource { int64_t timestamp_us, const FrameMetadata& frame_metadata) const; + bool capture_encoded_frame(int width, + int height, + const EncodedVideoFrameData& frame, + rust::Slice payload, + const FrameMetadata& frame_metadata) const; + + // Returns and clears the pending upstream keyframe request raised by the + // pass-through encoder (PLI/FIR or post-reconfigure). Poll from the + // capture loop. + bool take_keyframe_request() const; + EncodedRateControlRequest take_rate_control_request() const; + void set_packet_trailer_handler( std::shared_ptr handler) const; diff --git a/webrtc-sys/src/encoded_video_frame_buffer.cpp b/webrtc-sys/src/encoded_video_frame_buffer.cpp new file mode 100644 index 000000000..67314ffe6 --- /dev/null +++ b/webrtc-sys/src/encoded_video_frame_buffer.cpp @@ -0,0 +1,123 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/encoded_video_frame_buffer.h" + +#include + +#include "api/video/i420_buffer.h" +#include "rtc_base/logging.h" + +namespace livekit { + +void EncodedRateControlState::Store(uint64_t target_bitrate_bps, + double framerate_fps) { + std::lock_guard lock(mutex_); + request_.has_request = true; + request_.target_bitrate_bps = target_bitrate_bps; + request_.framerate_fps = framerate_fps; +} + +EncodedRateControlRequest EncodedRateControlState::Take() { + std::lock_guard lock(mutex_); + EncodedRateControlRequest request = request_; + request_ = EncodedRateControlRequest(); + return request; +} + +EncodedVideoFrameBuffer::EncodedVideoFrameBuffer( + int width, + int height, + EncodedVideoCodec codec, + EncodedFrameType frame_type, + webrtc::scoped_refptr payload, + std::shared_ptr> keyframe_request_flag, + std::shared_ptr rate_control_state) + : width_(width), + height_(height), + codec_(codec), + frame_type_(frame_type), + payload_(std::move(payload)), + keyframe_request_flag_(std::move(keyframe_request_flag)), + rate_control_state_(std::move(rate_control_state)) {} + +webrtc::VideoFrameBuffer::Type EncodedVideoFrameBuffer::type() const { + return Type::kNative; +} + +int EncodedVideoFrameBuffer::width() const { + return width_; +} + +int EncodedVideoFrameBuffer::height() const { + return height_; +} + +webrtc::scoped_refptr +EncodedVideoFrameBuffer::ToI420() { + // Sinks attached to a pre-encoded track (local preview, FFI color + // conversion) convert whatever buffer they receive; the encoded payload + // cannot be decoded here, so hand back a black frame instead of a null + // buffer that would crash the caller. + static std::atomic logged{false}; + if (!logged.exchange(true)) { + RTC_LOG(LS_WARNING) << "EncodedVideoFrameBuffer::ToI420 cannot decode an " + "encoded access unit; returning black frames"; + } + webrtc::scoped_refptr buffer = + webrtc::I420Buffer::Create(width_, height_); + webrtc::I420Buffer::SetBlack(buffer.get()); + return buffer; +} + +webrtc::scoped_refptr +EncodedVideoFrameBuffer::CropAndScale(int /* offset_x */, + int /* offset_y */, + int /* crop_width */, + int /* crop_height */, + int /* scaled_width */, + int /* scaled_height */) { + // Encoded payloads cannot be rescaled; returning the buffer unchanged + // keeps misbehaving callers alive (the capture path never scales encoded + // frames). + RTC_LOG(LS_WARNING) << "EncodedVideoFrameBuffer::CropAndScale is " + "unsupported; returning the frame unscaled"; + return webrtc::scoped_refptr(this); +} + +void EncodedVideoFrameBuffer::request_keyframe() const { + if (keyframe_request_flag_) { + keyframe_request_flag_->store(true, std::memory_order_relaxed); + } +} + +void EncodedVideoFrameBuffer::set_rate_control_request( + uint64_t target_bitrate_bps, + double framerate_fps) const { + if (rate_control_state_) { + rate_control_state_->Store(target_bitrate_bps, framerate_fps); + } +} + +EncodedVideoFrameBuffer* EncodedVideoFrameBuffer::FromNative( + webrtc::VideoFrameBuffer* buffer) { + if (!buffer || buffer->type() != webrtc::VideoFrameBuffer::Type::kNative) { + return nullptr; + } + return dynamic_cast(buffer); +} + +} // namespace livekit diff --git a/webrtc-sys/src/jetson/av1_encoder_impl.cpp b/webrtc-sys/src/jetson/av1_encoder_impl.cpp index 9626de4b4..679328d64 100644 --- a/webrtc-sys/src/jetson/av1_encoder_impl.cpp +++ b/webrtc-sys/src/jetson/av1_encoder_impl.cpp @@ -283,13 +283,12 @@ int32_t JetsonAV1EncoderImpl::Encode( return WEBRTC_VIDEO_CODEC_NO_OUTPUT; } - livekit::av1::StripIvfFrameHeaderIfPresent(&packet); + livekit::av1::NormalizeForRtp(&packet); if (packet.empty()) { - RTC_LOG(LS_ERROR) - << "Jetson MMAPI AV1 packet contained only IVF framing; skipping."; + RTC_LOG(LS_ERROR) << "Jetson MMAPI AV1 packet contained no transferable " + "OBUs after RTP normalization; skipping."; return WEBRTC_VIDEO_CODEC_NO_OUTPUT; } - livekit::av1::ConvertAnnexBToLowOverheadIfPresent(&packet); std::vector sequence_header; if (livekit::av1::ExtractSequenceHeaderObu(packet.data(), packet.size(), @@ -353,6 +352,7 @@ int32_t JetsonAV1EncoderImpl::ProcessEncodedFrame( encoded_image_.qp_ = -1; CodecSpecificInfo codecInfo; + codecInfo.codecSpecific = {}; codecInfo.codecType = kVideoCodecAV1; codecInfo.end_of_picture = true; codecInfo.scalability_mode = ScalabilityMode::kL1T1; diff --git a/webrtc-sys/src/jetson/jetson_av1_bitstream.cpp b/webrtc-sys/src/jetson/jetson_av1_bitstream.cpp index bc3ab2206..aba7373c9 100644 --- a/webrtc-sys/src/jetson/jetson_av1_bitstream.cpp +++ b/webrtc-sys/src/jetson/jetson_av1_bitstream.cpp @@ -82,7 +82,9 @@ bool StripIvfFrameHeader(std::vector* packet) { } const uint32_t declared_size = ReadLittleEndianUint32(*packet); - if (declared_size == 0 || declared_size > packet->size() - 12) { + // A standalone IVF frame header must describe exactly the remaining buffer. + // Otherwise valid low-overhead OBUs can be mistaken for a length prefix. + if (declared_size == 0 || declared_size != packet->size() - 12) { return false; } @@ -200,6 +202,39 @@ bool ConvertAnnexBToLowOverhead(std::vector* packet) { return true; } +bool StripNonTransferObus(std::vector* packet) { + if (!packet || packet->empty()) { + return false; + } + + const std::vector obus = ParseObus(packet->data(), packet->size()); + if (obus.empty()) { + return false; + } + + size_t transfer_size = 0; + bool already_contiguous = true; + size_t next_offset = 0; + for (const ObuSpan& obu : obus) { + transfer_size += obu.total_size; + already_contiguous = already_contiguous && obu.offset == next_offset; + next_offset = obu.offset + obu.total_size; + } + + if (transfer_size == packet->size() && already_contiguous) { + return false; + } + + std::vector filtered; + filtered.reserve(transfer_size); + for (const ObuSpan& obu : obus) { + filtered.insert(filtered.end(), packet->begin() + obu.offset, + packet->begin() + obu.offset + obu.total_size); + } + packet->swap(filtered); + return true; +} + } // namespace std::vector ParseObus(const uint8_t* data, size_t len) { @@ -308,6 +343,16 @@ void ConvertAnnexBToLowOverheadIfPresent(std::vector* packet) { ConvertAnnexBToLowOverhead(packet); } +void StripNonTransferObusIfPresent(std::vector* packet) { + StripNonTransferObus(packet); +} + +void NormalizeForRtp(std::vector* packet) { + StripIvfFrameHeaderIfPresent(packet); + ConvertAnnexBToLowOverheadIfPresent(packet); + StripNonTransferObusIfPresent(packet); +} + bool IsWebRtcParseable(const uint8_t* data, size_t len) { if (!data || len == 0) { return false; diff --git a/webrtc-sys/src/jetson/jetson_av1_bitstream.h b/webrtc-sys/src/jetson/jetson_av1_bitstream.h index 0af9614e0..2a7605661 100644 --- a/webrtc-sys/src/jetson/jetson_av1_bitstream.h +++ b/webrtc-sys/src/jetson/jetson_av1_bitstream.h @@ -55,6 +55,15 @@ void StripIvfFrameHeaderIfPresent(std::vector* packet); /// present. void ConvertAnnexBToLowOverheadIfPresent(std::vector* packet); +/// Strip OBUs that should not be transferred in WebRTC RTP payloads when present. +void StripNonTransferObusIfPresent(std::vector* packet); + +/// Normalizes an AV1 temporal unit for WebRTC RTP packetization: strips IVF +/// framing, converts Annex-B units to low-overhead OBUs, and strips +/// non-transfer OBUs. Shared by every encoder that emits AV1 into the RTP +/// pipeline so the steps cannot drift apart. +void NormalizeForRtp(std::vector* packet); + /// Basic validation that WebRTC's AV1 packetizer can parse the bitstream. bool IsWebRtcParseable(const uint8_t* data, size_t len); diff --git a/webrtc-sys/src/passthrough_video_encoder.cpp b/webrtc-sys/src/passthrough_video_encoder.cpp new file mode 100644 index 000000000..8faa32378 --- /dev/null +++ b/webrtc-sys/src/passthrough_video_encoder.cpp @@ -0,0 +1,426 @@ +/* + * Copyright 2026 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "livekit/passthrough_video_encoder.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/inlined_vector.h" +#include "api/video/encoded_image.h" +#include "api/video/video_frame.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" +#include "common_video/h264/h264_common.h" +#include "jetson/jetson_av1_bitstream.h" +#include "livekit/encoded_video_frame_buffer.h" +#include "media/base/media_constants.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/scalable_video_controller_no_layering.h" +#include "rtc_base/logging.h" +#include "rtc_base/synchronization/mutex.h" + +namespace livekit_ffi { +namespace { + +using livekit::EncodedVideoFrameBuffer; +using webrtc::CodecSpecificInfo; +using webrtc::EncodedImage; +using webrtc::EncodedImageBuffer; +using webrtc::EncodedImageCallback; +using webrtc::Environment; +using webrtc::H264PacketizationMode; +using webrtc::ScalabilityMode; +using webrtc::ScalableVideoController; +using webrtc::ScalableVideoControllerNoLayering; +using webrtc::SdpVideoFormat; +using webrtc::VideoCodec; +using webrtc::VideoCodecType; +using webrtc::VideoEncoder; +using webrtc::VideoFrame; +using webrtc::VideoFrameBuffer; +using webrtc::VideoFrameType; + +VideoCodecType CodecTypeFromFormat(const SdpVideoFormat& format) { + if (format.name == "H264") { + return webrtc::kVideoCodecH264; + } + if (format.name == "H265" || format.name == "HEVC") { + return webrtc::kVideoCodecH265; + } + if (format.name == "VP8") { + return webrtc::kVideoCodecVP8; + } + if (format.name == "VP9") { + return webrtc::kVideoCodecVP9; + } + if (format.name == "AV1") { + return webrtc::kVideoCodecAV1; + } + return webrtc::kVideoCodecGeneric; +} + +VideoCodecType CodecTypeFromBuffer(livekit::EncodedVideoCodec codec) { + switch (codec) { + case livekit::EncodedVideoCodec::kH264: + return webrtc::kVideoCodecH264; + case livekit::EncodedVideoCodec::kH265: + return webrtc::kVideoCodecH265; + case livekit::EncodedVideoCodec::kVP8: + return webrtc::kVideoCodecVP8; + case livekit::EncodedVideoCodec::kVP9: + return webrtc::kVideoCodecVP9; + case livekit::EncodedVideoCodec::kAV1: + return webrtc::kVideoCodecAV1; + } +} + +VideoFrameType FrameTypeFromBuffer(livekit::EncodedFrameType frame_type) { + switch (frame_type) { + case livekit::EncodedFrameType::kKey: + return VideoFrameType::kVideoFrameKey; + case livekit::EncodedFrameType::kDelta: + return VideoFrameType::kVideoFrameDelta; + } +} + +bool IsAv1Codec(VideoCodecType codec_type) { + return codec_type == webrtc::kVideoCodecAV1; +} + +bool IsKeyframe(livekit::EncodedFrameType frame_type) { + return frame_type == livekit::EncodedFrameType::kKey; +} + +// SDP profile parameters constrain real encoders, not a pass-through: the +// forwarded bytes are whatever the upstream encoder produced. Match formats +// by codec only (H265/HEVC are aliases via CodecTypeFromFormat). +bool IsSameCodecType(const SdpVideoFormat& a, const SdpVideoFormat& b) { + VideoCodecType type_a = CodecTypeFromFormat(a); + return type_a != webrtc::kVideoCodecGeneric && + type_a == CodecTypeFromFormat(b); +} + +void FillSingleLayerCodecSpecific( + CodecSpecificInfo* codec_info, + VideoCodecType codec_type, + int width, + int height, + bool keyframe, + ScalableVideoControllerNoLayering* av1_svc_controller) { + codec_info->codecType = codec_type; + codec_info->end_of_picture = true; + + switch (codec_type) { + case webrtc::kVideoCodecH264: + codec_info->codecSpecific.H264.packetization_mode = + H264PacketizationMode::NonInterleaved; + break; + case webrtc::kVideoCodecVP8: + codec_info->codecSpecific.VP8.nonReference = false; + codec_info->codecSpecific.VP8.temporalIdx = 0; + codec_info->codecSpecific.VP8.layerSync = false; + codec_info->codecSpecific.VP8.keyIdx = -1; + break; + case webrtc::kVideoCodecVP9: + codec_info->codecSpecific.VP9.first_frame_in_picture = true; + codec_info->codecSpecific.VP9.inter_pic_predicted = !keyframe; + codec_info->codecSpecific.VP9.flexible_mode = false; + codec_info->codecSpecific.VP9.ss_data_available = keyframe; + codec_info->codecSpecific.VP9.temporal_idx = 0; + codec_info->codecSpecific.VP9.temporal_up_switch = true; + codec_info->codecSpecific.VP9.inter_layer_predicted = false; + codec_info->codecSpecific.VP9.gof_idx = 0; + codec_info->codecSpecific.VP9.num_spatial_layers = 1; + codec_info->codecSpecific.VP9.first_active_layer = 0; + codec_info->codecSpecific.VP9.spatial_layer_resolution_present = keyframe; + codec_info->codecSpecific.VP9.width[0] = width; + codec_info->codecSpecific.VP9.height[0] = height; + codec_info->codecSpecific.VP9.gof.SetGofInfoVP9( + webrtc::kTemporalStructureMode1); + codec_info->codecSpecific.VP9.num_ref_pics = keyframe ? 0 : 1; + codec_info->codecSpecific.VP9.p_diff[0] = 1; + break; + case webrtc::kVideoCodecAV1: { + codec_info->scalability_mode = ScalabilityMode::kL1T1; + std::vector layer_frames = + av1_svc_controller->NextFrameConfig(/*restart=*/keyframe); + if (!layer_frames.empty()) { + const ScalableVideoController::LayerFrameConfig& layer_frame = + layer_frames.front(); + codec_info->generic_frame_info = + av1_svc_controller->OnEncodeDone(layer_frame); + if (layer_frame.IsKeyframe()) { + codec_info->template_structure = + av1_svc_controller->DependencyStructure(); + } + } + break; + } + default: + break; + } +} + +class PassthroughVideoEncoder final : public VideoEncoder { + public: + PassthroughVideoEncoder(const Environment& env, const SdpVideoFormat& format) + : env_(env), format_(format), codec_type_(CodecTypeFromFormat(format)) {} + + int32_t InitEncode(const VideoCodec* codec_settings, + const Settings& /* settings */) override { + if (!codec_settings || codec_settings->codecType != codec_type_) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + codec_ = *codec_settings; + cached_sequence_header_obu_.clear(); + av1_svc_controller_ = ScalableVideoControllerNoLayering(); + if (IsAv1Codec(codec_type_) && !codec_.GetScalabilityMode().has_value()) { + codec_.SetScalabilityMode(ScalabilityMode::kL1T1); + } + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override { + encoded_image_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { + encoded_image_callback_ = nullptr; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Encode(const VideoFrame& frame, + const std::vector* frame_types) override { + if (!encoded_image_callback_) { + RTC_LOG(LS_ERROR) + << "PassthroughVideoEncoder callback is not registered"; + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + webrtc::scoped_refptr frame_buffer = + frame.video_frame_buffer(); + EncodedVideoFrameBuffer* encoded_buffer = + EncodedVideoFrameBuffer::FromNative(frame_buffer.get()); + if (!encoded_buffer) { + RTC_LOG(LS_ERROR) + << "PassthroughVideoEncoder received a non-encoded frame buffer"; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + if (CodecTypeFromBuffer(encoded_buffer->codec()) != codec_type_) { + RTC_LOG(LS_ERROR) + << "PassthroughVideoEncoder frame codec does not match sender codec"; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + ForwardPendingRateControl(encoded_buffer); + + const bool is_keyframe = IsKeyframe(encoded_buffer->frame_type()); + + // A pass-through cannot synthesize the keyframe the RTP layer wants + // (PLI/FIR, late subscriber, reconfiguration); forward the request to + // the capture source so the upstream encoder can produce an IDR. + const bool keyframe_requested = + frame_types != nullptr && + std::any_of(frame_types->begin(), frame_types->end(), + [](VideoFrameType type) { + return type == VideoFrameType::kVideoFrameKey; + }); + if (keyframe_requested && !is_keyframe) { + encoded_buffer->request_keyframe(); + } + + if (encoded_buffer->payload_size() == 0) { + RTC_LOG(LS_ERROR) << "PassthroughVideoEncoder received an empty frame"; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + // Non-AV1 payloads are forwarded without copying: the buffer already + // owns a webrtc::EncodedImageBuffer. AV1 needs RTP normalization, which + // may rewrite the bytes, so it works on a copy. + webrtc::scoped_refptr encoded_data; + if (IsAv1Codec(codec_type_)) { + std::vector payload( + encoded_buffer->payload_data(), + encoded_buffer->payload_data() + encoded_buffer->payload_size()); + livekit::av1::NormalizeForRtp(&payload); + + std::vector sequence_header; + if (livekit::av1::ExtractSequenceHeaderObu( + payload.data(), payload.size(), &sequence_header)) { + cached_sequence_header_obu_ = std::move(sequence_header); + } else if (is_keyframe && !cached_sequence_header_obu_.empty()) { + livekit::av1::EnsureSequenceHeaderOnKeyframe( + &payload, cached_sequence_header_obu_); + } + if (payload.empty() || + !livekit::av1::IsWebRtcParseable(payload.data(), payload.size())) { + RTC_LOG(LS_ERROR) + << "PassthroughVideoEncoder received an AV1 frame that WebRTC " + "cannot packetize"; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + encoded_data = EncodedImageBuffer::Create(payload.data(), payload.size()); + } else { + encoded_data = encoded_buffer->encoded_data(); + } + + EncodedImage encoded_image; + encoded_image._encodedWidth = encoded_buffer->width(); + encoded_image._encodedHeight = encoded_buffer->height(); + encoded_image.SetRtpTimestamp(frame.rtp_timestamp()); + encoded_image.SetSimulcastIndex(0); + encoded_image.ntp_time_ms_ = frame.ntp_time_ms(); + encoded_image.capture_time_ms_ = frame.render_time_ms(); + encoded_image.rotation_ = frame.rotation(); + encoded_image.content_type_ = webrtc::VideoContentType::UNSPECIFIED; + encoded_image.timing_.flags = webrtc::VideoSendTiming::kInvalid; + encoded_image._frameType = FrameTypeFromBuffer(encoded_buffer->frame_type()); + encoded_image.SetColorSpace(frame.color_space()); + const size_t encoded_size = encoded_data->size(); + encoded_image.SetEncodedData(std::move(encoded_data)); + encoded_image.set_size(encoded_size); + encoded_image.qp_ = -1; + + CodecSpecificInfo codec_info; + codec_info.codecSpecific = {}; + FillSingleLayerCodecSpecific(&codec_info, codec_type_, encoded_buffer->width(), + encoded_buffer->height(), is_keyframe, + &av1_svc_controller_); + + const auto result = + encoded_image_callback_->OnEncodedImage(encoded_image, &codec_info); + if (result.error != EncodedImageCallback::Result::OK) { + RTC_LOG(LS_ERROR) << "PassthroughVideoEncoder callback failed " + << result.error; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; + } + + void SetRates(const RateControlParameters& parameters) override { + webrtc::MutexLock lock(&rate_control_mutex_); + latest_rate_control_request_ = livekit::EncodedRateControlRequest{ + true, parameters.bitrate.get_sum_bps(), parameters.framerate_fps}; + } + + EncoderInfo GetEncoderInfo() const override { + EncoderInfo info; + info.supports_native_handle = true; + info.implementation_name = "LiveKit pre-encoded passthrough"; + info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + info.is_hardware_accelerated = false; + info.supports_simulcast = false; + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kNative}; + return info; + } + + private: + void ForwardPendingRateControl( + EncodedVideoFrameBuffer* encoded_buffer) { + std::optional request; + { + webrtc::MutexLock lock(&rate_control_mutex_); + request = latest_rate_control_request_; + latest_rate_control_request_.reset(); + } + if (request.has_value()) { + encoded_buffer->set_rate_control_request(request->target_bitrate_bps, + request->framerate_fps); + } + } + + Environment env_; + SdpVideoFormat format_; + VideoCodecType codec_type_; + VideoCodec codec_; + EncodedImageCallback* encoded_image_callback_ = nullptr; + ScalableVideoControllerNoLayering av1_svc_controller_; + std::vector cached_sequence_header_obu_; + webrtc::Mutex rate_control_mutex_; + std::optional latest_rate_control_request_; +}; + +} // namespace + +PassthroughVideoEncoderFactory::PassthroughVideoEncoderFactory() { + std::map h264_parameters = { + {"profile-level-id", "42e01f"}, + {"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}, + }; + absl::InlinedVector + scalability_modes; + scalability_modes.push_back(ScalabilityMode::kL1T1); + supported_formats_.push_back(SdpVideoFormat::VP8()); + supported_formats_.push_back(SdpVideoFormat::VP9Profile0()); + supported_formats_.push_back( + SdpVideoFormat(SdpVideoFormat::AV1Profile0(), scalability_modes)); + supported_formats_.push_back(SdpVideoFormat("H264", h264_parameters)); + supported_formats_.push_back(SdpVideoFormat("H265")); + supported_formats_.push_back(SdpVideoFormat("HEVC")); +} + +std::vector +PassthroughVideoEncoderFactory::GetSupportedFormats() const { + return supported_formats_; +} + +std::vector +PassthroughVideoEncoderFactory::GetImplementations() const { + return supported_formats_; +} + +PassthroughVideoEncoderFactory::CodecSupport +PassthroughVideoEncoderFactory::QueryCodecSupport( + const SdpVideoFormat& format, + std::optional scalability_mode) const { + for (const auto& supported_format : supported_formats_) { + if (IsSameCodecType(format, supported_format)) { + if (format.name == "AV1" && scalability_mode.has_value() && + *scalability_mode != "L1T1") { + return {.is_supported = false, .is_power_efficient = false}; + } + return {.is_supported = true, .is_power_efficient = true}; + } + } + return {.is_supported = false, .is_power_efficient = false}; +} + +std::unique_ptr PassthroughVideoEncoderFactory::Create( + const Environment& env, + const SdpVideoFormat& format) { + // Match by codec, not by exact profile: rejecting e.g. a High-profile + // H264 negotiation here would hand the session to a real encoder that + // cannot consume pre-encoded frames. + for (const auto& supported_format : supported_formats_) { + if (IsSameCodecType(format, supported_format)) { + return std::make_unique(env, format); + } + } + return nullptr; +} + +} // namespace livekit_ffi diff --git a/webrtc-sys/src/rtp_sender.cpp b/webrtc-sys/src/rtp_sender.cpp index cb7809d4a..e97bed2d6 100644 --- a/webrtc-sys/src/rtp_sender.cpp +++ b/webrtc-sys/src/rtp_sender.cpp @@ -46,6 +46,8 @@ const char* BackendName(VideoEncoderBackend backend) { return "vaapi"; case VideoEncoderBackend::VideoToolbox: return "videotoolbox"; + case VideoEncoderBackend::PreEncoded: + return "preencoded"; } } @@ -71,6 +73,9 @@ std::optional BackendFromFormat( if (it->second == BackendName(VideoEncoderBackend::VideoToolbox)) { return VideoEncoderBackend::VideoToolbox; } + if (it->second == BackendName(VideoEncoderBackend::PreEncoded)) { + return VideoEncoderBackend::PreEncoded; + } return std::nullopt; } @@ -105,7 +110,16 @@ class FixedVideoEncoderSelector final } std::optional OnEncoderBroken() override { - return std::nullopt; + // The preferred backend is a hard requirement for this sender (e.g. + // pre-encoded pass-through). When the active encoder breaks — including + // when the initial untagged encoder could not even be created — request + // the preferred backend explicitly instead of giving up, so the sender + // recovers onto the right encoder. + if (!current_encoder_) { + return std::nullopt; + } + requested_ = true; + return WithBackend(*current_encoder_, backend_); } private: diff --git a/webrtc-sys/src/video_encoder_factory.cpp b/webrtc-sys/src/video_encoder_factory.cpp index 98505813e..d8b7fd454 100644 --- a/webrtc-sys/src/video_encoder_factory.cpp +++ b/webrtc-sys/src/video_encoder_factory.cpp @@ -16,6 +16,8 @@ #include "livekit/video_encoder_factory.h" +#include +#include #include #include #include @@ -26,6 +28,7 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder_factory_template.h" #include "livekit/objc_video_factory.h" +#include "livekit/passthrough_video_encoder.h" #include "livekit/webrtc.h" #include "media/base/media_constants.h" #include "media/engine/simulcast_encoder_adapter.h" @@ -107,6 +110,8 @@ const char* BackendName(VideoEncoderBackend backend) { return "vaapi"; case VideoEncoderBackend::VideoToolbox: return "videotoolbox"; + case VideoEncoderBackend::PreEncoded: + return "preencoded"; } } @@ -132,6 +137,9 @@ std::optional BackendFromFormat( if (it->second == BackendName(VideoEncoderBackend::VideoToolbox)) { return VideoEncoderBackend::VideoToolbox; } + if (it->second == BackendName(VideoEncoderBackend::PreEncoded)) { + return VideoEncoderBackend::PreEncoded; + } return std::nullopt; } @@ -160,8 +168,43 @@ bool IsSpecificHardwareBackend(VideoEncoderBackend backend) { bool BackendMatches(VideoEncoderBackend requested, VideoEncoderBackend actual) { return requested == actual || (requested == VideoEncoderBackend::Hardware && - actual != VideoEncoderBackend::Software && - actual != VideoEncoderBackend::Auto); + (actual == VideoEncoderBackend::Hardware || + IsSpecificHardwareBackend(actual))); +} + +bool IsAutomaticFallbackBackend(VideoEncoderBackend backend) { + return backend != VideoEncoderBackend::PreEncoded; +} + +bool EqualsIgnoreAsciiCase(std::string_view a, std::string_view b) { + return a.size() == b.size() && + std::equal(a.begin(), a.end(), b.begin(), [](char x, char y) { + return std::tolower(static_cast(x)) == + std::tolower(static_cast(y)); + }); +} + +bool IsSameCodecName(std::string_view a, std::string_view b) { + if (EqualsIgnoreAsciiCase(a, b)) { + return true; + } + auto is_h265 = [](std::string_view name) { + return EqualsIgnoreAsciiCase(name, "H265") || + EqualsIgnoreAsciiCase(name, "HEVC"); + }; + return is_h265(a) && is_h265(b); +} + +// The pass-through backend forwards pre-encoded bytes, so SDP profile +// parameters do not constrain it: match it by codec name only. Real +// encoder backends keep exact profile matching. +bool FormatSupportedByBackendFactory(VideoEncoderBackend backend, + const webrtc::SdpVideoFormat& supported, + const webrtc::SdpVideoFormat& requested) { + if (backend == VideoEncoderBackend::PreEncoded) { + return IsSameCodecName(supported.name, requested.name); + } + return supported.IsSameCodec(requested); } void AddBackendFactory( @@ -256,6 +299,7 @@ rust::Vec video_encoder_backend_list() { rust::Vec backends; backends.push_back(VideoEncoderBackend::Auto); backends.push_back(VideoEncoderBackend::Software); + backends.push_back(VideoEncoderBackend::PreEncoded); bool has_hardware_backend = false; bool hardware_backend_listed = false; @@ -299,6 +343,11 @@ rust::Vec video_encoder_backend_list() { } VideoEncoderFactory::InternalFactory::InternalFactory() { + AddBackendFactory( + factories_, + VideoEncoderBackend::PreEncoded, + std::make_unique()); + #ifdef __APPLE__ AddBackendFactory( factories_, @@ -331,10 +380,35 @@ VideoEncoderFactory::InternalFactory::GetSupportedFormats() const { std::vector formats = Factory().GetSupportedFormats(); for (const auto& backend_factory : factories_) { + if (backend_factory.backend == VideoEncoderBackend::PreEncoded) { + continue; + } auto supported_formats = backend_factory.factory->GetSupportedFormats(); formats.insert(formats.end(), supported_formats.begin(), supported_formats.end()); } + + // The pass-through factory would otherwise advertise codecs no real + // encoder implements (e.g. H265 on desktops); a normal session + // negotiating such a codec would end up with a sender that cannot create + // an encoder. Only advertise pass-through formats for codecs some real + // encoder already supports. + const size_t real_format_count = formats.size(); + for (const auto& backend_factory : factories_) { + if (backend_factory.backend != VideoEncoderBackend::PreEncoded) { + continue; + } + for (const auto& format : backend_factory.factory->GetSupportedFormats()) { + const bool codec_available = std::any_of( + formats.begin(), formats.begin() + real_format_count, + [&](const webrtc::SdpVideoFormat& existing) { + return IsSameCodecName(existing.name, format.name); + }); + if (codec_available) { + formats.push_back(format); + } + } + } return formats; } @@ -342,6 +416,9 @@ std::vector VideoEncoderFactory::InternalFactory::GetImplementations() const { std::vector formats; for (const auto& backend_factory : factories_) { + if (backend_factory.backend == VideoEncoderBackend::PreEncoded) { + continue; + } for (const auto& format : backend_factory.factory->GetImplementations()) { formats.push_back(WithBackend(format, backend_factory.backend)); if (IsSpecificHardwareBackend(backend_factory.backend)) { @@ -384,7 +461,9 @@ VideoEncoderFactory::InternalFactory::QueryCodecSupport( for (const auto& supported_format : backend_factory.factory->GetSupportedFormats()) { - if (stripped_format.IsSameCodec(supported_format)) { + if (FormatSupportedByBackendFactory(backend_factory.backend, + supported_format, + stripped_format)) { return webrtc::VideoEncoderFactory::CodecSupport{ .is_supported = true, .is_power_efficient = true, @@ -406,6 +485,9 @@ VideoEncoderFactory::InternalFactory::QueryCodecSupport( } for (const auto& backend_factory : factories_) { + if (!IsAutomaticFallbackBackend(backend_factory.backend)) { + continue; + } for (const auto& supported_format : backend_factory.factory->GetSupportedFormats()) { if (stripped_format.IsSameCodec(supported_format)) { @@ -448,7 +530,9 @@ VideoEncoderFactory::InternalFactory::Create( for (const auto& supported_format : backend_factory.factory->GetSupportedFormats()) { - if (supported_format.IsSameCodec(stripped_format)) { + if (FormatSupportedByBackendFactory(backend_factory.backend, + supported_format, + stripped_format)) { auto encoder = backend_factory.factory->Create(env, stripped_format); if (encoder) { return encoder; @@ -457,6 +541,17 @@ VideoEncoderFactory::InternalFactory::Create( } } + // A real encoder cannot consume the pre-encoded native frame buffers + // this session produces, so falling back would yield a silently broken + // sender. Fail loudly instead. + if (*requested_backend == VideoEncoderBackend::PreEncoded) { + RTC_LOG(LS_ERROR) + << "Pre-encoded pass-through encoder is unavailable for " + << stripped_format.name + << "; refusing to fall back to a real encoder."; + return nullptr; + } + requested_backend_unavailable = true; } @@ -468,6 +563,9 @@ VideoEncoderFactory::InternalFactory::Create( } for (const auto& backend_factory : factories_) { + if (!IsAutomaticFallbackBackend(backend_factory.backend)) { + continue; + } for (const auto& supported_format : backend_factory.factory->GetSupportedFormats()) { if (supported_format.IsSameCodec(stripped_format)) diff --git a/webrtc-sys/src/video_track.cpp b/webrtc-sys/src/video_track.cpp index 44d1351fa..ba89c298b 100644 --- a/webrtc-sys/src/video_track.cpp +++ b/webrtc-sys/src/video_track.cpp @@ -27,6 +27,7 @@ #include "audio/remix_resample.h" #include "common_audio/include/audio_util.h" #include "livekit/dmabuf_video_frame_buffer.h" +#include "livekit/encoded_video_frame_buffer.h" #include "livekit/media_stream.h" #include "livekit/packet_trailer.h" #include "livekit/video_track.h" @@ -38,6 +39,33 @@ #include "webrtc-sys/src/video_track.rs.h" namespace livekit_ffi { +namespace { + +livekit::EncodedVideoCodec ToNativeEncodedCodec(EncodedVideoCodec codec) { + switch (codec) { + case EncodedVideoCodec::H264: + return livekit::EncodedVideoCodec::kH264; + case EncodedVideoCodec::H265: + return livekit::EncodedVideoCodec::kH265; + case EncodedVideoCodec::VP8: + return livekit::EncodedVideoCodec::kVP8; + case EncodedVideoCodec::VP9: + return livekit::EncodedVideoCodec::kVP9; + case EncodedVideoCodec::AV1: + return livekit::EncodedVideoCodec::kAV1; + } +} + +livekit::EncodedFrameType ToNativeEncodedFrameType(EncodedFrameType frame_type) { + switch (frame_type) { + case EncodedFrameType::Key: + return livekit::EncodedFrameType::kKey; + case EncodedFrameType::Delta: + return livekit::EncodedFrameType::kDelta; + } +} + +} // namespace VideoTrack::VideoTrack(std::shared_ptr rtc_runtime, webrtc::scoped_refptr track) @@ -164,6 +192,25 @@ bool VideoTrackSource::InternalSource::on_captured_frame( static_cast(buffer->height())}; } + // Pre-encoded access units bypass the adapter entirely: frame-rate and + // resolution adaptation operate on raw frames, and dropping or scaling an + // encoded delta frame would corrupt the bitstream for every receiver. + if (livekit::EncodedVideoFrameBuffer::FromNative(buffer.get())) { + if (packet_trailer_handler_) { + packet_trailer_handler_->emit_publish_timing( + VideoPublishTimingStage::EncoderUpload, + frame_metadata.has_packet_trailer ? frame_metadata.user_timestamp + : 0, + frame_metadata.has_packet_trailer ? frame_metadata.frame_id : 0); + } + OnFrame(webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(frame.rotation()) + .set_timestamp_us(aligned_timestamp_us) + .build()); + return true; + } + int adapted_width, adapted_height, crop_width, crop_height, crop_x, crop_y; if (!AdaptFrame(buffer->width(), buffer->height(), aligned_timestamp_us, &adapted_width, &adapted_height, &crop_width, &crop_height, @@ -246,6 +293,42 @@ bool VideoTrackSource::capture_dmabuf_frame(int dmabuf_fd, return source_->on_captured_frame(frame, frame_metadata); } +bool VideoTrackSource::capture_encoded_frame( + int width, + int height, + const EncodedVideoFrameData& encoded_frame, + rust::Slice payload, + const FrameMetadata& frame_metadata) const { + // The single unavoidable copy on this path: the Rust payload only lives + // for the duration of this call, while the EncodedImageBuffer is shared + // (uncopied) with the pass-through encoder downstream. + auto buffer = webrtc::make_ref_counted( + width, height, ToNativeEncodedCodec(encoded_frame.codec), + ToNativeEncodedFrameType(encoded_frame.frame_type), + webrtc::EncodedImageBuffer::Create(payload.data(), payload.size()), + source_->keyframe_request_flag(), source_->rate_control_state()); + + auto frame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(std::move(buffer)) + .set_rotation(webrtc::kVideoRotation_0) + .set_timestamp_us(encoded_frame.timestamp_us) + .build(); + + return source_->on_captured_frame(frame, frame_metadata); +} + +bool VideoTrackSource::take_keyframe_request() const { + return source_->keyframe_request_flag()->exchange(false, + std::memory_order_relaxed); +} + +EncodedRateControlRequest VideoTrackSource::take_rate_control_request() const { + auto request = source_->rate_control_state()->Take(); + return EncodedRateControlRequest{request.has_request, + request.target_bitrate_bps, + request.framerate_fps}; +} + void VideoTrackSource::set_packet_trailer_handler( std::shared_ptr handler) const { source_->set_packet_trailer_handler(std::move(handler)); diff --git a/webrtc-sys/src/video_track.rs b/webrtc-sys/src/video_track.rs index 6c0a584e9..8fe4cae99 100644 --- a/webrtc-sys/src/video_track.rs +++ b/webrtc-sys/src/video_track.rs @@ -50,6 +50,37 @@ pub mod ffi { pub user_data: Vec, } + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[repr(i32)] + pub enum EncodedVideoCodec { + H264, + H265, + VP8, + VP9, + AV1, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[repr(i32)] + pub enum EncodedFrameType { + Key, + Delta, + } + + #[derive(Debug)] + pub struct EncodedVideoFrameData { + pub codec: EncodedVideoCodec, + pub frame_type: EncodedFrameType, + pub timestamp_us: i64, + } + + #[derive(Debug)] + pub struct EncodedRateControlRequest { + pub has_request: bool, + pub target_bitrate_bps: u64, + pub framerate_fps: f64, + } + extern "C++" { include!("livekit/video_frame.h"); include!("livekit/media_stream_track.h"); @@ -94,6 +125,16 @@ pub mod ffi { timestamp_us: i64, frame_metadata: &FrameMetadata, ) -> bool; + fn capture_encoded_frame( + self: &VideoTrackSource, + width: i32, + height: i32, + frame: &EncodedVideoFrameData, + payload: &[u8], + frame_metadata: &FrameMetadata, + ) -> bool; + fn take_keyframe_request(self: &VideoTrackSource) -> bool; + fn take_rate_control_request(self: &VideoTrackSource) -> EncodedRateControlRequest; fn set_packet_trailer_handler( self: &VideoTrackSource, handler: SharedPtr, diff --git a/webrtc-sys/src/webrtc.rs b/webrtc-sys/src/webrtc.rs index b7878341c..fc7b7d099 100644 --- a/webrtc-sys/src/webrtc.rs +++ b/webrtc-sys/src/webrtc.rs @@ -63,6 +63,7 @@ pub mod ffi { Nvenc, Vaapi, VideoToolbox, + PreEncoded, } unsafe extern "C++" {