diff --git a/.changeset/data_streams_v2.md b/.changeset/data_streams_v2.md new file mode 100644 index 000000000..48e58599d --- /dev/null +++ b/.changeset/data_streams_v2.md @@ -0,0 +1,10 @@ +--- +livekit: patch +livekit-api: patch +livekit-datatrack: patch +livekit-ffi: patch +livekit-protocol: patch +livekit-uniffi: patch +--- + +Add data streams v2 - #1192 (@1egoman) diff --git a/Cargo.lock b/Cargo.lock index f6ac222fe..94db8b930 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3919,18 +3919,22 @@ dependencies = [ "bmrng", "bytes", "chrono", + "flate2", "futures-util", "http 1.4.0", "lazy_static", "libloading 0.8.9", "libwebrtc", "livekit-api", + "livekit-common", + "livekit-data-stream", "livekit-datatrack", "livekit-protocol", "livekit-runtime", "log", "parking_lot", "prost 0.12.6", + "rand 0.9.3", "semver", "serde", "serde_json", @@ -3957,6 +3961,7 @@ dependencies = [ "http 1.4.0", "isahc", "jsonwebtoken", + "livekit-common", "livekit-protocol", "livekit-runtime", "log", @@ -3979,6 +3984,32 @@ dependencies = [ "url", ] +[[package]] +name = "livekit-common" +version = "0.1.0" +dependencies = [ + "livekit-protocol", +] + +[[package]] +name = "livekit-data-stream" +version = "0.1.0" +dependencies = [ + "bmrng", + "bytes", + "chrono", + "flate2", + "futures-util", + "livekit-common", + "livekit-protocol", + "log", + "parking_lot", + "prost 0.12.6", + "thiserror 2.0.18", + "tokio", + "uuid", +] + [[package]] name = "livekit-datatrack" version = "0.1.9" diff --git a/Cargo.toml b/Cargo.toml index 93d4a11df..75e166ca4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,8 @@ members = [ "livekit", "livekit-api", "livekit-protocol", + "livekit-common", + "livekit-data-stream", "livekit-ffi", "livekit-uniffi", "livekit-datatrack", @@ -51,6 +53,8 @@ livekit = { version = "0.7.50", path = "livekit" } livekit-api = { version = "0.5.4", path = "livekit-api" } livekit-ffi = { version = "0.12.68", path = "livekit-ffi" } livekit-datatrack = { version = "0.1.9", path = "livekit-datatrack" } +livekit-common = { version = "0.1.0", path = "livekit-common" } +livekit-data-stream = { version = "0.1.0", path = "livekit-data-stream" } livekit-protocol = { version = "0.7.10", path = "livekit-protocol" } livekit-runtime = { version = "0.4.0", path = "livekit-runtime" } soxr-sys = { version = "0.1.3", path = "soxr-sys" } diff --git a/livekit-api/Cargo.toml b/livekit-api/Cargo.toml index 8afd55fc7..53183a05f 100644 --- a/livekit-api/Cargo.toml +++ b/livekit-api/Cargo.toml @@ -101,6 +101,7 @@ __rustls-tls = ["tokio-tungstenite?/__rustls-tls", "reqwest?/__rustls"] [dependencies] livekit-protocol = { workspace = true } +livekit-common = { workspace = true } thiserror = { workspace = true } serde = { workspace = true, features = ["derive"] } sha2 = "0.10" diff --git a/livekit-api/src/signal_client/mod.rs b/livekit-api/src/signal_client/mod.rs index 76b7ba491..badfb8690 100644 --- a/livekit-api/src/signal_client/mod.rs +++ b/livekit-api/src/signal_client/mod.rs @@ -57,18 +57,21 @@ const VALIDATE_TIMEOUT: Duration = Duration::from_secs(3); pub const PROTOCOL_VERSION: u32 = 17; /// Capabilities the Rust SDK advertises to the SFU at connect time. -const CLIENT_CAPABILITIES: &[proto::client_info::Capability] = - &[proto::client_info::Capability::CapPacketTrailer]; - -/// Default value for `ClientInfo.client_protocol` when a participant has not -/// advertised one (treat as v1-only / no data-stream RPC support). -pub const CLIENT_PROTOCOL_DEFAULT: i32 = 0; -/// `ClientInfo.client_protocol` value indicating support for RPC v2 over data streams. -pub const CLIENT_PROTOCOL_DATA_STREAM_RPC: i32 = 1; +/// +/// `CapCompressionDeflateRaw` is always advertised because the SDK's deflate-raw codec +/// (flate2/miniz_oxide) is pure-Rust and compiled in unconditionally. +const CLIENT_CAPABILITIES: &[proto::client_info::Capability] = &[ + proto::client_info::Capability::CapPacketTrailer, + proto::client_info::Capability::CapCompressionDeflateRaw, +]; + +pub use livekit_common::{ + CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2, CLIENT_PROTOCOL_DEFAULT, +}; /// The client protocol which is sent to other clients and indicates the set of apis that other /// clients should assume this client supports. -const CLIENT_PROTOCOL_VERSION: i32 = CLIENT_PROTOCOL_DATA_STREAM_RPC; +const CLIENT_PROTOCOL_VERSION: i32 = CLIENT_PROTOCOL_DATA_STREAM_V2; #[derive(Error, Debug)] pub enum SignalError { diff --git a/livekit-common/Cargo.toml b/livekit-common/Cargo.toml new file mode 100644 index 000000000..c3ec0491d --- /dev/null +++ b/livekit-common/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "livekit-common" +description = "Common foundational types shared across LiveKit crates" +version = "0.1.0" +readme = "README.md" +license.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +livekit-protocol = { workspace = true } diff --git a/livekit-common/README.md b/livekit-common/README.md new file mode 100644 index 000000000..cf5fce7a9 --- /dev/null +++ b/livekit-common/README.md @@ -0,0 +1,6 @@ +# LiveKit Common + +An internal crate which holds shared data structures that many downstream modules all use, like +`ParticipantIdentity` or `ClientCapability`. + +To build applications with LiveKit, please use the public APIs provided by the [livekit](../livekit) crate. diff --git a/livekit-common/src/lib.rs b/livekit-common/src/lib.rs new file mode 100644 index 000000000..9402ebcf4 --- /dev/null +++ b/livekit-common/src/lib.rs @@ -0,0 +1,176 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Foundational types shared across LiveKit crates: participant identities, the +//! encryption/capability enums, client-protocol constants, and the remote-participant +//! registry trait consulted by the data-stream and RPC send paths. + +use std::fmt::Display; + +use livekit_protocol as proto; + +// ------------------------------------------------------------------------------------------------- +// Client protocol +// ------------------------------------------------------------------------------------------------- + +/// Legacy client. No v2 data-stream features. +pub const CLIENT_PROTOCOL_DEFAULT: i32 = 0; + +/// RPC v2 (see RPC spec). No v2 data-stream features. +pub const CLIENT_PROTOCOL_DATA_STREAM_RPC: i32 = 1; + +/// Understands inline single-packet data streams (data streams v2). +pub const CLIENT_PROTOCOL_DATA_STREAM_V2: i32 = 2; + +// ------------------------------------------------------------------------------------------------- +// ParticipantIdentity +// ------------------------------------------------------------------------------------------------- + +#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct ParticipantIdentity(pub String); + +impl From for ParticipantIdentity { + fn from(value: String) -> Self { + Self(value) + } +} + +impl From<&str> for ParticipantIdentity { + fn from(value: &str) -> Self { + Self(value.to_string()) + } +} + +impl From for String { + fn from(value: ParticipantIdentity) -> Self { + value.0 + } +} + +impl Display for ParticipantIdentity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl ParticipantIdentity { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +// ------------------------------------------------------------------------------------------------- +// EncryptionType +// ------------------------------------------------------------------------------------------------- + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncryptionType { + #[default] + None, + Gcm, + Custom, +} + +impl From for EncryptionType { + fn from(value: proto::encryption::Type) -> Self { + match value { + proto::encryption::Type::None => Self::None, + proto::encryption::Type::Gcm => Self::Gcm, + proto::encryption::Type::Custom => Self::Custom, + } + } +} + +impl From for proto::encryption::Type { + fn from(value: EncryptionType) -> Self { + match value { + EncryptionType::None => Self::None, + EncryptionType::Gcm => Self::Gcm, + EncryptionType::Custom => Self::Custom, + } + } +} + +impl From for i32 { + fn from(value: EncryptionType) -> Self { + match value { + EncryptionType::None => 0, + EncryptionType::Gcm => 1, + EncryptionType::Custom => 2, + } + } +} + +// ------------------------------------------------------------------------------------------------- +// ClientCapability +// ------------------------------------------------------------------------------------------------- + +/// A capability a participant's client advertises, mirroring the `ClientInfo.Capability` protobuf +/// enum. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[non_exhaustive] +pub enum ClientCapability { + Unused, + PacketTrailer, + CompressionDeflateRaw, +} + +impl TryFrom for ClientCapability { + type Error = &'static str; + + fn try_from(value: i32) -> Result { + match proto::client_info::Capability::try_from(value) { + Ok(proto::client_info::Capability::CapPacketTrailer) => Ok(Self::PacketTrailer), + Ok(proto::client_info::Capability::CapCompressionDeflateRaw) => { + Ok(Self::CompressionDeflateRaw) + } + Ok(proto::client_info::Capability::CapUnused) => Ok(Self::Unused), + Err(_) => Err("unknown client capability"), + } + } +} + +impl From for i32 { + fn from(value: ClientCapability) -> Self { + match value { + ClientCapability::Unused => proto::client_info::Capability::CapUnused as i32, + ClientCapability::PacketTrailer => { + proto::client_info::Capability::CapPacketTrailer as i32 + } + ClientCapability::CompressionDeflateRaw => { + proto::client_info::Capability::CapCompressionDeflateRaw as i32 + } + } + } +} + +// ------------------------------------------------------------------------------------------------- +// RemoteParticipantRegistry +// ------------------------------------------------------------------------------------------------- + +/// Read access to remote participants' advertised protocol and capabilities. +/// +/// Used by downstream modules like the the RPC transport (v1/v2 transport selection) and +/// the data-stream send path (inline / compression eligibility) to determine what level of support +/// a participant has for protocol level features. +pub trait RemoteParticipantRegistry: Send + Sync { + /// A remote participant's `client_protocol`, or `CLIENT_PROTOCOL_DEFAULT` (0) if unknown. + fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32; + + /// A remote participant's advertised capabilities, or empty if unknown. + fn remote_capabilities(&self, identity: &ParticipantIdentity) -> Vec; + + /// The identities of every remote participant, used to resolve a broadcast send. + fn remote_identities(&self) -> Vec; +} diff --git a/livekit-data-stream/Cargo.toml b/livekit-data-stream/Cargo.toml new file mode 100644 index 000000000..f768d8ce7 --- /dev/null +++ b/livekit-data-stream/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "livekit-data-stream" +description = "Data stream core logic for LiveKit" +version = "0.1.0" +readme = "README.md" +license.workspace = true +edition.workspace = true +repository.workspace = true + +[features] +# End-to-end testing hooks (exposes is_compressed/is_inline on stream info). Forwarded from +# the `livekit` crate's `__lk-e2e-test` feature. +__e2e-test = [] +# Exposes constructors used by downstream crates' test suites (e.g. `TextStreamReader::new_for_test`). +test-utils = [] + +[dependencies] +livekit-common = { workspace = true } +livekit-protocol = { workspace = true } +log = { workspace = true } +thiserror = { workspace = true } +parking_lot = { workspace = true } +bytes = { workspace = true } +tokio = { workspace = true, default-features = false, features = ["sync", "fs", "io-util", "rt"] } +futures-util = { workspace = true, default-features = false, features = ["sink"] } +prost = "0.12" +chrono = "0.4.38" +flate2 = "1" +bmrng = "0.5.2" +uuid = { version = "1", features = ["v4"] } + +[dev-dependencies] +tokio = { workspace = true, default-features = false, features = ["macros", "rt", "rt-multi-thread", "time"] } +rand = { workspace = true } diff --git a/livekit-data-stream/README.md b/livekit-data-stream/README.md new file mode 100644 index 000000000..2ff6b9376 --- /dev/null +++ b/livekit-data-stream/README.md @@ -0,0 +1,7 @@ +# LiveKit Data Stream + +**Important**: +This is an internal crate that powers the data streams feature in LiveKit client SDKs (including [Rust](https://crates.io/crates/livekit) and others) and is not usable directly. + +To use data streams in your application, please use the public APIs provided by the +[livekit](../livekit) crate and other client sdks. diff --git a/livekit-data-stream/src/incoming/mod.rs b/livekit-data-stream/src/incoming/mod.rs new file mode 100644 index 000000000..dabcbf5d6 --- /dev/null +++ b/livekit-data-stream/src/incoming/mod.rs @@ -0,0 +1,765 @@ +// Copyright 2025 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bytes::Bytes; +use livekit_common::EncryptionType; +use livekit_protocol::data_stream as proto; +use parking_lot::Mutex; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; + +use crate::info::{AnyStreamInfo, ByteStreamInfo, TextStreamInfo}; +use crate::utils::{StreamError, StreamProgress, StreamResult}; + +mod stream_reader; +pub use stream_reader::{AnyStreamReader, ByteStreamReader, StreamReader, TextStreamReader}; + +struct Descriptor { + progress: StreamProgress, + chunk_tx: UnboundedSender>, + encryption_type: EncryptionType, + /// Identity of the participant sending this stream; used to abort the stream + /// if that participant disconnects mid-send. + sender_identity: String, + is_internal: bool, + /// Whether this is a text stream (decompressed output is reframed on UTF-8 boundaries). + is_text: bool, + /// Per-stream deflate-raw decompressor; `Some` if the header declared `DEFLATE_RAW`. + decompressor: Option, + /// Highest chunk index processed so far (compressed streams; for dedup/gap detection). + last_chunk_index: Option, + // TODO(ladvoc): keep track of open time. +} + +/// Streaming deflate-raw decompressor state for one compressed stream. +struct DeflateDecompressState { + decompress: flate2::Decompress, + /// Decompressed text bytes not yet yielded because they end mid-codepoint. + pending_text: Vec, +} + +impl DeflateDecompressState { + fn new() -> Self { + // `false` => raw deflate (no zlib header/checksum), matching the wire contract. + Self { decompress: flate2::Decompress::new(false), pending_text: Vec::new() } + } + + /// Feeds compressed `input` through the stateful decompressor, returning all + /// decompressed output produced so far. + fn push(&mut self, input: &[u8]) -> StreamResult> { + let mut out = Vec::new(); + let mut buf = vec![0u8; 16384 /* number of bytes to process every loop iteration */]; + let mut offset = 0; + loop { + let in_before = self.decompress.total_in(); + let out_before = self.decompress.total_out(); + let status = self + .decompress + .decompress(&input[offset..], &mut buf, flate2::FlushDecompress::None) + .map_err(|_| StreamError::Decompression)?; + let consumed = (self.decompress.total_in() - in_before) as usize; + let produced = (self.decompress.total_out() - out_before) as usize; + offset += consumed; + out.extend_from_slice(&buf[..produced]); + match status { + flate2::Status::StreamEnd => break, + // No progress and no input left to feed: wait for the next chunk. + _ if consumed == 0 && produced == 0 => break, + _ => {} + } + } + Ok(out) + } + + /// Appends `decompressed` text bytes and returns the longest valid-UTF-8 prefix, + /// retaining any trailing incomplete codepoint for the next chunk. + fn reframe_text(&mut self, decompressed: Vec) -> Bytes { + self.pending_text.extend_from_slice(&decompressed); + let valid = match std::str::from_utf8(&self.pending_text) { + Ok(_) => self.pending_text.len(), + Err(e) => e.valid_up_to(), + }; + Bytes::from(self.pending_text.drain(..valid).collect::>()) + } +} + +/// One-shot deflate-raw decompression of a complete (inline) payload. +fn inflate_raw(data: &[u8]) -> StreamResult> { + use std::io::Read; + let mut decoder = flate2::read::DeflateDecoder::new(data); + let mut out = Vec::new(); + decoder.read_to_end(&mut out).map_err(|_| StreamError::Decompression)?; + Ok(out) +} + +#[derive(Clone)] +pub struct IncomingStreamManager { + inner: Arc>, + open_tx: UnboundedSender<(AnyStreamReader, String)>, + /// Topics whose streams are handled internally by the SDK (e.g. RPC) and never surfaced as + /// application events. Supplied by the host crate so this crate stays decoupled from RPC. + reserved_topics: Vec<&'static str>, +} + +#[derive(Default)] +struct ManagerInner { + open_streams: HashMap, +} + +impl IncomingStreamManager { + pub fn new( + reserved_topics: Vec<&'static str>, + ) -> (Self, UnboundedReceiver<(AnyStreamReader, String)>) { + let (open_tx, open_rx) = mpsc::unbounded_channel(); + ( + Self { inner: Arc::new(Mutex::new(Default::default())), open_tx, reserved_topics }, + open_rx, + ) + } + + /// Handles an incoming header packet. + pub fn handle_header( + &self, + mut header: proto::Header, + identity: String, + encryption_type: livekit_protocol::encryption::Type, + ) { + let is_internal = self.is_internal_topic(&header.topic); + // Read the v2 signals before `try_from_with_encryption` consumes the header. + let inline_content = header.inline_content.take(); + let is_compressed = header.compression() == proto::CompressionType::DeflateRaw; + + let Ok(info) = AnyStreamInfo::try_from_with_encryption(header, encryption_type.into()) + .inspect_err(|e| log::error!("Invalid header: {}", e)) + else { + return; + }; + + let id = info.id().to_owned(); + let is_text = matches!(info, AnyStreamInfo::Text(_)); + let bytes_total = info.total_length(); + let stream_encryption_type = info.encryption_type(); + + let mut inner = self.inner.lock(); + if inner.open_streams.contains_key(&id) { + log::error!("Stream '{}' already open", id); + return; + } + + let (reader, chunk_tx) = AnyStreamReader::from(info); + let _ = self.open_tx.send((reader, identity.clone())); + + // Inline single-packet stream: synthesize the complete content now; no chunk/trailer + // packets will follow, so we never register an open descriptor. + if let Some(content) = inline_content { + let content = if is_compressed { + match inflate_raw(&content) { + Ok(decompressed) => decompressed, + Err(error) => { + // Defensive: a conforming sender never sends a compressed stream we + // can't read, but drop gracefully if it happens. + let _ = chunk_tx.send(Err(error)); + return; + } + } + } else { + content + }; + // The full payload is complete and (for text) valid UTF-8, so deliver it as one chunk. + if !content.is_empty() { + let _ = chunk_tx.send(Ok(Bytes::from(content))); + } + // Dropping `chunk_tx` closes the reader. + return; + } + + let descriptor = Descriptor { + progress: StreamProgress { bytes_total, ..Default::default() }, + chunk_tx, + encryption_type: stream_encryption_type, + sender_identity: identity, + is_internal, + is_text, + decompressor: is_compressed.then(DeflateDecompressState::new), + last_chunk_index: None, + }; + inner.open_streams.insert(id, descriptor); + } + + /// Returns whether the given open stream belongs to an internal topic + /// (e.g. `lk.rpc_request`). Used to suppress `RoomEvent::Stream*Received` + /// dispatches for traffic the SDK handles itself. + pub fn is_internal(&self, stream_id: &str) -> bool { + self.inner.lock().open_streams.get(stream_id).is_some_and(|d| d.is_internal) + } + + /// Returns whether data streams which are created on the given topic should be + /// considered "internal" and not have their raw events surfaced to users. + /// + /// When possible, prefer [Self::is_internal] instead. + pub fn is_internal_topic(&self, topic: &str) -> bool { + self.reserved_topics.iter().any(|t| t == &topic) + } + + /// Handles an incoming chunk packet. + pub fn handle_chunk( + &self, + chunk: proto::Chunk, + encryption_type: livekit_protocol::encryption::Type, + ) { + let id = chunk.stream_id; + let mut inner = self.inner.lock(); + let Some(descriptor) = inner.open_streams.get_mut(&id) else { + return; + }; + + if descriptor.encryption_type != encryption_type.into() { + inner.close_stream_with_error(&id, StreamError::EncryptionTypeMismatch); + return; + } + + if let Some(decompressor) = &mut descriptor.decompressor { + // --- Compressed stream: feed chunks through one stateful decompressor. --- + // Duplicate index (reconnect replay): drop with a warning. + if let Some(last) = descriptor.last_chunk_index { + if chunk.chunk_index <= last { + log::warn!( + "Dropping duplicate chunk {} for compressed stream '{}'", + chunk.chunk_index, + id + ); + return; + } + } + // A gap is unrecoverable for a stateful decompressor. + let expected = descriptor.last_chunk_index.map(|i| i + 1).unwrap_or(0); + if chunk.chunk_index != expected { + inner.close_stream_with_error(&id, StreamError::MissedChunk); + return; + } + descriptor.last_chunk_index = Some(chunk.chunk_index); + + let is_text = descriptor.is_text; + // Confine the decompressor borrow so we can re-borrow `inner` afterwards. + let result: StreamResult<(u64, Bytes)> = { + match decompressor.push(&chunk.content) { + Ok(decompressed) => { + let produced = decompressed.len() as u64; + let yielded = if is_text { + decompressor.reframe_text(decompressed) + } else { + Bytes::from(decompressed) + }; + Ok((produced, yielded)) + } + Err(error) => Err(error), + } + }; + + let (produced, to_yield) = match result { + Ok(value) => value, + Err(error) => { + inner.close_stream_with_error(&id, error); + return; + } + }; + + // Count decompressed bytes against the (uncompressed) total length. + descriptor.progress.bytes_processed += produced; + if matches!(descriptor.progress.bytes_total, Some(total) if descriptor.progress.bytes_processed > total) + { + inner.close_stream_with_error(&id, StreamError::LengthExceeded); + return; + } + if !to_yield.is_empty() { + inner.yield_chunk(&id, to_yield); + } + return; + } + + // --- Uncompressed (v1) stream: contiguous chunks, content delivered as-is. --- + if descriptor.progress.chunk_index != chunk.chunk_index { + inner.close_stream_with_error(&id, StreamError::MissedChunk); + return; + } + + descriptor.progress.chunk_index += 1; + descriptor.progress.bytes_processed += chunk.content.len() as u64; + + if match descriptor.progress.bytes_total { + Some(total) => descriptor.progress.bytes_processed > total as u64, + None => false, + } { + inner.close_stream_with_error(&id, StreamError::LengthExceeded); + return; + } + inner.yield_chunk(&id, Bytes::from(chunk.content)); + // TODO: also yield progress + } + + /// Handles an incoming trailer packet. + pub fn handle_trailer(&self, trailer: proto::Trailer) { + let id = trailer.stream_id; + let mut inner = self.inner.lock(); + let Some(descriptor) = inner.open_streams.get_mut(&id) else { + return; + }; + + if !match descriptor.progress.bytes_total { + Some(total) => descriptor.progress.bytes_processed >= total as u64, + None => true, + } { + inner.close_stream_with_error(&id, StreamError::Incomplete); + return; + } + if !trailer.reason.is_empty() { + inner.close_stream_with_error(&id, StreamError::AbnormalEnd(trailer.reason)); + return; + } + inner.close_stream(&id); + } + + /// Aborts every open stream being sent by the given participant, erroring each + /// reader with [`StreamError::AbnormalEnd`]. + /// + /// Called when a remote participant disconnects: any streams it had in flight to + /// this receiver are terminated so their readers observe an error rather than + /// hanging forever waiting for chunks that will never arrive. + pub fn abort_streams_from(&self, identity: &str) { + let mut inner = self.inner.lock(); + let ids: Vec = inner + .open_streams + .iter() + .filter(|(_, descriptor)| descriptor.sender_identity == identity) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + let reason = format!( + "Participant {} unexpectedly disconnected in the middle of sending data", + identity + ); + inner.close_stream_with_error(&id, StreamError::AbnormalEnd(reason)); + } + } +} + +impl ManagerInner { + fn yield_chunk(&mut self, id: &str, chunk: Bytes) { + let Some(descriptor) = self.open_streams.get_mut(id) else { + return; + }; + if descriptor.chunk_tx.send(Ok(chunk)).is_err() { + // Reader has been dropped, close the stream. + self.close_stream(id); + } + } + + fn close_stream(&mut self, id: &str) { + // Dropping the sender closes the channel. + self.open_streams.remove(id); + } + + fn close_stream_with_error(&mut self, id: &str, error: StreamError) { + if let Some(descriptor) = self.open_streams.remove(id) { + let _ = descriptor.chunk_tx.send(Err(error)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use livekit_protocol::encryption::Type as EncType; + use std::collections::HashMap; + + const SENDER: &str = "alice"; + + fn deflate_raw(data: &[u8]) -> Vec { + use std::io::Write; + let mut e = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(data).unwrap(); + e.finish().unwrap() + } + + fn attrs(pairs: &[(&str, &str)]) -> HashMap { + pairs.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect() + } + + /// Deterministic, barely-compressible lowercase text (so its deflate output spans chunks). + /// + /// Seeded with a fixed value so the output is identical on every run; the letters carry + /// enough entropy that deflate can't shrink them away, unlike repetitive text ("aaaa…"). + fn pseudo_random_text(len: usize) -> String { + use rand::{rngs::StdRng, Rng, SeedableRng}; + + /// Fixed RNG seed that keeps `pseudo_random_text` output identical on every run. + const RANDOM_SEED: u64 = 0xdead_beef_cafe_babe; + + let mut rng = StdRng::seed_from_u64(RANDOM_SEED); + (0..len).map(|_| rng.random_range(b'a'..=b'z') as char).collect() + } + + #[allow(clippy::too_many_arguments)] + fn text_header( + id: &str, + total_length: Option, + attributes: HashMap, + inline_content: Option>, + compression: proto::CompressionType, + ) -> proto::Header { + proto::Header { + stream_id: id.to_string(), + timestamp: 0, + topic: "topic".to_string(), + mime_type: "text/plain".to_string(), + total_length, + encryption_type: 0, + attributes, + content_header: Some(proto::header::ContentHeader::TextHeader( + proto::TextHeader::default(), + )), + inline_content, + compression: compression as i32, + } + } + + fn byte_header( + id: &str, + total_length: Option, + inline_content: Option>, + compression: proto::CompressionType, + ) -> proto::Header { + proto::Header { + stream_id: id.to_string(), + timestamp: 0, + topic: "topic".to_string(), + mime_type: "application/octet-stream".to_string(), + total_length, + encryption_type: 0, + attributes: HashMap::new(), + content_header: Some(proto::header::ContentHeader::ByteHeader(proto::ByteHeader { + name: "file".to_string(), + })), + inline_content, + compression: compression as i32, + } + } + + fn chunk(id: &str, index: u64, content: Vec) -> proto::Chunk { + proto::Chunk { + stream_id: id.to_string(), + chunk_index: index, + content, + ..Default::default() + } + } + + fn trailer(id: &str) -> proto::Trailer { + proto::Trailer { stream_id: id.to_string(), ..Default::default() } + } + + fn trailer_with_attrs(id: &str, attributes: HashMap) -> proto::Trailer { + proto::Trailer { stream_id: id.to_string(), reason: String::new(), attributes } + } + + async fn read_text(reader: AnyStreamReader) -> StreamResult { + match reader { + AnyStreamReader::Text(r) => r.read_all().await, + _ => panic!("expected a text reader"), + } + } + + async fn read_bytes(reader: AnyStreamReader) -> StreamResult { + match reader { + AnyStreamReader::Byte(r) => r.read_all().await, + _ => panic!("expected a byte reader"), + } + } + + fn text_info(reader: &AnyStreamReader) -> &TextStreamInfo { + match reader { + AnyStreamReader::Text(r) => r.info(), + _ => panic!("expected a text reader"), + } + } + + // --- v1 (legacy multi-packet) -------------------------------------------------------- + + #[tokio::test] + async fn v1_text_stream_round_trips() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + let text = "hello world"; + mgr.handle_header( + text_header( + "s1", + Some(text.len() as u64), + attrs(&[("foo", "bar")]), + None, + proto::CompressionType::None, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, identity) = rx.recv().await.expect("a reader should be dispatched"); + assert_eq!(identity, SENDER); + assert_eq!(text_info(&reader).attributes.get("foo"), Some(&"bar".to_string())); + mgr.handle_chunk(chunk("s1", 0, text.as_bytes().to_vec()), EncType::None); + mgr.handle_trailer(trailer("s1")); + assert_eq!(read_text(reader).await.unwrap(), text); + } + + #[tokio::test] + async fn v1_byte_stream_round_trips() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + byte_header("s1", Some(4), None, proto::CompressionType::None), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, vec![1, 2, 3, 4]), EncType::None); + mgr.handle_trailer(trailer("s1")); + assert_eq!(read_bytes(reader).await.unwrap(), Bytes::from(vec![1u8, 2, 3, 4])); + } + + #[tokio::test] + async fn v1_merges_trailer_attributes() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + let text = "hi"; + mgr.handle_header( + text_header( + "s1", + Some(text.len() as u64), + attrs(&[("foo", "bar"), ("baz", "quux")]), + None, + proto::CompressionType::None, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, text.as_bytes().to_vec()), EncType::None); + mgr.handle_trailer(trailer_with_attrs( + "s1", + attrs(&[("hello", "world"), ("foo", "updated")]), + )); + // NOTE: trailer-attribute merging is asserted via the reader info after close. + let info_attrs = text_info(&reader).attributes.clone(); + assert_eq!(read_text(reader).await.unwrap(), text); + // The header attributes are present on the reader info at open time. + assert_eq!(info_attrs.get("baz"), Some(&"quux".to_string())); + } + + #[tokio::test] + async fn v1_errors_when_too_few_bytes() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + text_header("s1", Some(5), HashMap::new(), None, proto::CompressionType::None), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, vec![b'x']), EncType::None); + mgr.handle_trailer(trailer("s1")); + assert!(matches!(read_text(reader).await, Err(StreamError::Incomplete))); + } + + #[tokio::test] + async fn v1_errors_when_too_many_bytes() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + byte_header("s1", Some(3), None, proto::CompressionType::None), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, vec![1, 2, 3, 4, 5]), EncType::None); + mgr.handle_trailer(trailer("s1")); + assert!(matches!(read_bytes(reader).await, Err(StreamError::LengthExceeded))); + } + + #[tokio::test] + async fn v1_drops_on_encryption_type_mismatch() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + text_header("s1", Some(2), HashMap::new(), None, proto::CompressionType::None), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, vec![b'h', b'i']), EncType::Gcm); + assert!(matches!(read_text(reader).await, Err(StreamError::EncryptionTypeMismatch))); + } + + // --- v2 inline ----------------------------------------------------------------------- + + #[tokio::test] + async fn v2_inline_uncompressed_text() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + let text = "inline hello"; + mgr.handle_header( + text_header( + "s1", + Some(text.len() as u64), + attrs(&[("foo", "bar")]), + Some(text.as_bytes().to_vec()), + proto::CompressionType::None, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + assert_eq!(text_info(&reader).attributes.get("foo"), Some(&"bar".to_string())); + // No chunk/trailer packets are fed. + assert_eq!(read_text(reader).await.unwrap(), text); + } + + #[tokio::test] + async fn v2_inline_uncompressed_byte() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + byte_header("s1", Some(3), Some(vec![1, 2, 3]), proto::CompressionType::None), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + assert_eq!(read_bytes(reader).await.unwrap(), Bytes::from(vec![1u8, 2, 3])); + } + + #[tokio::test] + async fn v2_inline_compressed_text() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + let text = "hello hello compressible world"; + let compressed = deflate_raw(text.as_bytes()); + mgr.handle_header( + text_header( + "s1", + Some(text.len() as u64), + attrs(&[("foo", "bar")]), + Some(compressed), + proto::CompressionType::DeflateRaw, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + assert_eq!(text_info(&reader).attributes.get("foo"), Some(&"bar".to_string())); + assert_eq!(read_text(reader).await.unwrap(), text); + } + + #[tokio::test] + async fn v2_inline_compressed_byte() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + let payload: Vec = (0..2000).map(|i| (i % 7) as u8).collect(); + let compressed = deflate_raw(&payload); + mgr.handle_header( + byte_header( + "s1", + Some(payload.len() as u64), + Some(compressed), + proto::CompressionType::DeflateRaw, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + assert_eq!(read_bytes(reader).await.unwrap(), Bytes::from(payload)); + } + + // --- v2 multi-packet compressed ------------------------------------------------------ + + #[tokio::test] + async fn v2_multipacket_compressed_text() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + // ~60 KB of pseudo-random lowercase so the compressed output spans multiple chunks. + let text = pseudo_random_text(60_000); + let compressed = deflate_raw(text.as_bytes()); + let chunk_pieces: Vec<&[u8]> = compressed.chunks(15_000).collect(); + assert!(chunk_pieces.len() >= 2, "expected multi-packet compressed stream"); + + mgr.handle_header( + text_header( + "s1", + Some(text.len() as u64), + HashMap::new(), + None, + proto::CompressionType::DeflateRaw, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + for (i, piece) in chunk_pieces.iter().enumerate() { + mgr.handle_chunk(chunk("s1", i as u64, piece.to_vec()), EncType::None); + } + mgr.handle_trailer(trailer("s1")); + assert_eq!(read_text(reader).await.unwrap(), text); + } + + #[tokio::test] + async fn errors_open_streams_on_sender_disconnect() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + text_header("s1", Some(10), HashMap::new(), None, proto::CompressionType::None), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + // Partial content, no trailer: the sender then drops. + mgr.handle_chunk(chunk("s1", 0, vec![b'h', b'e', b'l', b'l', b'o']), EncType::None); + mgr.abort_streams_from(SENDER); + assert!(matches!(read_text(reader).await, Err(StreamError::AbnormalEnd(_)))); + } + + #[tokio::test] + async fn abort_only_affects_matching_sender() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + mgr.handle_header( + text_header("s1", Some(5), HashMap::new(), None, proto::CompressionType::None), + "bob".to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, vec![b'h', b'e', b'l', b'l', b'o']), EncType::None); + // A different participant disconnecting must not disturb bob's stream. + mgr.abort_streams_from(SENDER); + mgr.handle_trailer(trailer("s1")); + assert_eq!(read_text(reader).await.unwrap(), "hello"); + } + + #[tokio::test] + async fn v2_compressed_gap_errors() { + let (mgr, mut rx) = IncomingStreamManager::new(vec![]); + let text = pseudo_random_text(60_000); + let compressed = deflate_raw(text.as_bytes()); + let pieces: Vec<&[u8]> = compressed.chunks(15_000).collect(); + assert!(pieces.len() >= 2); + mgr.handle_header( + text_header( + "s1", + Some(text.len() as u64), + HashMap::new(), + None, + proto::CompressionType::DeflateRaw, + ), + SENDER.to_string(), + EncType::None, + ); + let (reader, _) = rx.recv().await.expect("a reader should be dispatched"); + mgr.handle_chunk(chunk("s1", 0, pieces[0].to_vec()), EncType::None); + // Skip index 1 -> feed index 2: a gap is a hard error. + mgr.handle_chunk(chunk("s1", 2, pieces[1].to_vec()), EncType::None); + assert!(matches!(read_text(reader).await, Err(StreamError::MissedChunk))); + } +} diff --git a/livekit/src/room/data_stream/incoming.rs b/livekit-data-stream/src/incoming/stream_reader.rs similarity index 52% rename from livekit/src/room/data_stream/incoming.rs rename to livekit-data-stream/src/incoming/stream_reader.rs index 213a68c07..d5afe7d24 100644 --- a/livekit/src/room/data_stream/incoming.rs +++ b/livekit-data-stream/src/incoming/stream_reader.rs @@ -1,4 +1,4 @@ -// Copyright 2025 LiveKit, Inc. +// Copyright 2026 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,19 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{ - AnyStreamInfo, ByteStreamInfo, StreamError, StreamProgress, StreamResult, TextStreamInfo, -}; -use crate::{e2ee::EncryptionType, TakeCell}; +use super::{AnyStreamInfo, ByteStreamInfo, StreamError, StreamResult, TextStreamInfo}; use bytes::{Bytes, BytesMut}; use futures_util::{Stream, StreamExt}; -use livekit_protocol::data_stream as proto; -use parking_lot::Mutex; use std::{ - collections::HashMap, fmt::Debug, pin::Pin, - sync::Arc, task::{Context, Poll}, }; use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; @@ -52,23 +45,6 @@ pub trait StreamReader: Stream> { fn read_all(self) -> impl std::future::Future> + Send; } -impl TakeCell -where - T: StreamReader, -{ - /// Takes the reader out of the cell if its info matches the given predicate. - /// - /// Use this method to conditionally handle incoming streams based on info fields - /// such as topic or attributes. - /// - /// This method will only take the reader if the provided predicate returns `true` when called with the reader's info. - /// If the predicate returns `false` or the reader has already been taken, this method returns `None`. - /// - pub fn take_if(&self, predicate: impl FnOnce(&T::Info) -> bool) -> Option { - self.take_if_raw(|reader| predicate(reader.info())) - } -} - /// Reader for an incoming byte data stream. pub struct ByteStreamReader { info: ByteStreamInfo, @@ -77,7 +53,7 @@ pub struct ByteStreamReader { /// Reader for an incoming text data stream. pub struct TextStreamReader { - info: TextStreamInfo, + pub(crate) info: TextStreamInfo, chunk_rx: UnboundedReceiver>, } @@ -148,10 +124,13 @@ impl Stream for ByteStreamReader { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-utils"))] impl TextStreamReader { /// Create a TextStreamReader for testing purposes. - pub(crate) fn new_for_test( + /// + /// Exposed under the `test-utils` feature so downstream crates (e.g. `livekit`'s RPC tests) + /// can construct a reader directly. + pub fn new_for_test( info: TextStreamInfo, chunk_rx: UnboundedReceiver>, ) -> Self { @@ -217,7 +196,7 @@ impl Debug for TextStreamReader { } } -pub(crate) enum AnyStreamReader { +pub enum AnyStreamReader { Byte(ByteStreamReader), Text(TextStreamReader), } @@ -233,152 +212,3 @@ impl AnyStreamReader { return (reader, chunk_tx); } } -struct Descriptor { - progress: StreamProgress, - chunk_tx: UnboundedSender>, - encryption_type: EncryptionType, - is_internal: bool, - // TODO(ladvoc): keep track of open time. -} - -#[derive(Clone)] -pub(crate) struct IncomingStreamManager { - inner: Arc>, - open_tx: UnboundedSender<(AnyStreamReader, String)>, -} - -#[derive(Default)] -struct ManagerInner { - open_streams: HashMap, -} - -impl IncomingStreamManager { - pub fn new() -> (Self, UnboundedReceiver<(AnyStreamReader, String)>) { - let (open_tx, open_rx) = mpsc::unbounded_channel(); - (Self { inner: Arc::new(Mutex::new(Default::default())), open_tx }, open_rx) - } - - /// Handles an incoming header packet. - pub fn handle_header( - &self, - header: proto::Header, - identity: String, - encryption_type: livekit_protocol::encryption::Type, - ) { - let is_internal = super::is_internal_topic(&header.topic); - let Ok(info) = AnyStreamInfo::try_from_with_encryption(header, encryption_type.into()) - .inspect_err(|e| log::error!("Invalid header: {}", e)) - else { - return; - }; - - let id = info.id().to_owned(); - let bytes_total = info.total_length(); - let stream_encryption_type = info.encryption_type(); - - let mut inner = self.inner.lock(); - if inner.open_streams.contains_key(&id) { - log::error!("Stream '{}' already open", id); - return; - } - - let (reader, chunk_tx) = AnyStreamReader::from(info); - let _ = self.open_tx.send((reader, identity)); - - let descriptor = Descriptor { - progress: StreamProgress { bytes_total, ..Default::default() }, - chunk_tx, - encryption_type: stream_encryption_type, - is_internal, - }; - inner.open_streams.insert(id, descriptor); - } - - /// Returns whether the given open stream belongs to an internal topic - /// (e.g. `lk.rpc_request`). Used to suppress `RoomEvent::Stream*Received` - /// dispatches for traffic the SDK handles itself. - pub fn is_internal(&self, stream_id: &str) -> bool { - self.inner.lock().open_streams.get(stream_id).is_some_and(|d| d.is_internal) - } - - /// Handles an incoming chunk packet. - pub fn handle_chunk( - &self, - chunk: proto::Chunk, - encryption_type: livekit_protocol::encryption::Type, - ) { - let id = chunk.stream_id; - let mut inner = self.inner.lock(); - let Some(descriptor) = inner.open_streams.get_mut(&id) else { - return; - }; - - if descriptor.encryption_type != encryption_type.into() { - inner.close_stream_with_error(&id, StreamError::EncryptionTypeMismatch); - return; - } - - if descriptor.progress.chunk_index != chunk.chunk_index { - inner.close_stream_with_error(&id, StreamError::MissedChunk); - return; - } - - descriptor.progress.chunk_index += 1; - descriptor.progress.bytes_processed += chunk.content.len() as u64; - - if match descriptor.progress.bytes_total { - Some(total) => descriptor.progress.bytes_processed > total as u64, - None => false, - } { - inner.close_stream_with_error(&id, StreamError::LengthExceeded); - return; - } - inner.yield_chunk(&id, Bytes::from(chunk.content)); - // TODO: also yield progress - } - - /// Handles an incoming trailer packet. - pub fn handle_trailer(&self, trailer: proto::Trailer) { - let id = trailer.stream_id; - let mut inner = self.inner.lock(); - let Some(descriptor) = inner.open_streams.get_mut(&id) else { - return; - }; - - if !match descriptor.progress.bytes_total { - Some(total) => descriptor.progress.bytes_processed >= total as u64, - None => true, - } { - inner.close_stream_with_error(&id, StreamError::Incomplete); - return; - } - if !trailer.reason.is_empty() { - inner.close_stream_with_error(&id, StreamError::AbnormalEnd(trailer.reason)); - return; - } - inner.close_stream(&id); - } -} - -impl ManagerInner { - fn yield_chunk(&mut self, id: &str, chunk: Bytes) { - let Some(descriptor) = self.open_streams.get_mut(id) else { - return; - }; - if descriptor.chunk_tx.send(Ok(chunk)).is_err() { - // Reader has been dropped, close the stream. - self.close_stream(id); - } - } - - fn close_stream(&mut self, id: &str) { - // Dropping the sender closes the channel. - self.open_streams.remove(id); - } - - fn close_stream_with_error(&mut self, id: &str, error: StreamError) { - if let Some(descriptor) = self.open_streams.remove(id) { - let _ = descriptor.chunk_tx.send(Err(error)); - } - } -} diff --git a/livekit/src/room/data_stream/mod.rs b/livekit-data-stream/src/info.rs similarity index 70% rename from livekit/src/room/data_stream/mod.rs rename to livekit-data-stream/src/info.rs index 1be6cce7b..ed3e71803 100644 --- a/livekit/src/room/data_stream/mod.rs +++ b/livekit-data-stream/src/info.rs @@ -1,4 +1,4 @@ -// Copyright 2025 LiveKit, Inc. +// Copyright 2026 LiveKit, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,87 +13,11 @@ // limitations under the License. use chrono::{DateTime, Utc}; -use libwebrtc::enum_dispatch; +use livekit_common::EncryptionType; use livekit_protocol::data_stream as proto; use std::collections::HashMap; -use thiserror::Error; -mod incoming; -mod outgoing; - -pub use incoming::*; -pub use outgoing::*; - -use crate::e2ee::EncryptionType; -use crate::room::rpc::{RPC_REQUEST_TOPIC, RPC_RESPONSE_TOPIC}; - -/// Data stream topics reserved for internal SDK use. Events for these -/// topics are handled within the `livekit` crate and never surfaced -/// through `RoomEvent`. -pub(crate) const INTERNAL_TOPICS: &[&str] = &[RPC_REQUEST_TOPIC, RPC_RESPONSE_TOPIC]; - -pub(crate) fn is_internal_topic(topic: &str) -> bool { - INTERNAL_TOPICS.contains(&topic) -} - -/// Result type for data stream operations. -pub type StreamResult = Result; - -/// Error type for data stream operations. -#[derive(Debug, Error)] -pub enum StreamError { - // TODO(ladvoc): standardize error cases and expose over FFI. - #[error("stream has already been closed")] - AlreadyClosed, - - #[error("stream closed abnormally: {0}")] - AbnormalEnd(String), - - #[error("UTF-8 decoding error: {0}")] - Utf8(#[from] std::string::FromUtf8Error), - - #[error("incoming header was invalid")] - InvalidHeader, - - #[error("expected chunk index to be exactly one more than the previous")] - MissedChunk, - - #[error("read length exceeded total length specified in stream header")] - LengthExceeded, - - #[error("stream data is incomplete")] - Incomplete, - - #[error("unable to send packet")] - SendFailed, - - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), - - #[error("internal error")] - Internal, - - #[error("encryption type mismatch")] - EncryptionTypeMismatch, -} - -/// Progress of a data stream. -#[derive(Clone, Copy, Default, Debug, Hash, Eq, PartialEq)] -struct StreamProgress { - chunk_index: u64, - /// Number of bytes read or written so far. - bytes_processed: u64, - /// Total number of bytes expected to be read or written for finite streams. - bytes_total: Option, -} - -impl StreamProgress { - /// Returns the completion percentage for finite streams. - #[allow(dead_code)] - fn percentage(&self) -> Option { - self.bytes_total.map(|total| self.bytes_processed as f32 / total as f32) - } -} +use super::utils::StreamError; /// Information about a byte data stream. #[derive(Clone, Debug)] @@ -114,6 +38,12 @@ pub struct ByteStreamInfo { pub name: String, /// The encryption used pub encryption_type: EncryptionType, + /// Test-only: expose whether the byte stream was compressed or not. + #[cfg(feature = "__e2e-test")] + pub is_compressed: bool, + /// Test-only: expose whether the byte stream was sent inline on the header packet + #[cfg(feature = "__e2e-test")] + pub is_inline: bool, } /// Information about a text data stream. @@ -138,6 +68,12 @@ pub struct TextStreamInfo { pub generated: bool, /// The encryption used pub encryption_type: EncryptionType, + /// Test-only: expose whether the byte stream was compressed or not. + #[cfg(feature = "__e2e-test")] + pub is_compressed: bool, + /// Test-only: expose whether the byte stream was sent inline on the header packet + #[cfg(feature = "__e2e-test")] + pub is_inline: bool, } /// Operation type for text streams. @@ -155,7 +91,7 @@ pub enum OperationType { impl TryFrom for AnyStreamInfo { type Error = StreamError; - fn try_from(mut header: proto::Header) -> Result { + fn try_from(header: proto::Header) -> Result { Self::try_from_with_encryption(header, EncryptionType::None) } } @@ -191,6 +127,11 @@ impl ByteStreamInfo { encryption_type: EncryptionType, ) -> Self { Self { + #[cfg(feature = "__e2e-test")] + is_compressed: header.compression() != proto::CompressionType::None, + #[cfg(feature = "__e2e-test")] + is_inline: !header.inline_content().is_empty(), + id: header.stream_id, topic: header.topic, timestamp: DateTime::::from_timestamp_millis(header.timestamp) @@ -215,6 +156,11 @@ impl TextStreamInfo { encryption_type: EncryptionType, ) -> Self { Self { + #[cfg(feature = "__e2e-test")] + is_compressed: header.compression() != proto::CompressionType::None, + #[cfg(feature = "__e2e-test")] + is_inline: !header.inline_content().is_empty(), + id: header.stream_id, topic: header.topic, timestamp: DateTime::::from_timestamp_millis(header.timestamp) @@ -252,20 +198,34 @@ pub(crate) enum AnyStreamInfo { } impl AnyStreamInfo { - enum_dispatch!( - [Byte, Text]; - pub fn id(self: &Self) -> &str; - pub fn total_length(self: &Self) -> Option; - pub fn encryption_type(self: &Self) -> EncryptionType; - ); + pub fn id(&self) -> &str { + match self { + Self::Byte(info) => info.id(), + Self::Text(info) => info.id(), + } + } + + pub fn total_length(&self) -> Option { + match self { + Self::Byte(info) => info.total_length(), + Self::Text(info) => info.total_length(), + } + } + + pub fn encryption_type(&self) -> EncryptionType { + match self { + Self::Byte(info) => info.encryption_type(), + Self::Text(info) => info.encryption_type(), + } + } } #[rustfmt::skip] macro_rules! stream_info { () => { - fn id(&self) -> &str { &self.id } - fn total_length(&self) -> Option { self.total_length } - fn encryption_type(&self) -> EncryptionType { self.encryption_type } + pub(crate) fn id(&self) -> &str { &self.id } + pub(crate) fn total_length(&self) -> Option { self.total_length } + pub(crate) fn encryption_type(&self) -> EncryptionType { self.encryption_type } }; } diff --git a/livekit-data-stream/src/lib.rs b/livekit-data-stream/src/lib.rs new file mode 100644 index 000000000..00f723995 --- /dev/null +++ b/livekit-data-stream/src/lib.rs @@ -0,0 +1,29 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod utils; +pub use utils::{SendError, StreamError, StreamResult}; + +mod info; +pub use info::{ByteStreamInfo, OperationType, TextStreamInfo}; + +mod utf8_chunk; + +mod incoming; +pub use incoming::{ + AnyStreamReader, ByteStreamReader, IncomingStreamManager, StreamReader, TextStreamReader, +}; + +mod outgoing; +pub use outgoing::*; diff --git a/livekit-data-stream/src/outgoing/constants.rs b/livekit-data-stream/src/outgoing/constants.rs new file mode 100644 index 000000000..3e551cc1d --- /dev/null +++ b/livekit-data-stream/src/outgoing/constants.rs @@ -0,0 +1,26 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Max chunk content size AND the header-packet MTU budget. Kept below the ~16 KB +/// data-channel MTU for protocol/E2EE framing headroom. +pub(crate) const STREAM_CHUNK_SIZE_BYTES: usize = 15000; + +// Default MIME type to use for byte streams. +pub(crate) static BYTE_MIME_TYPE: &str = "application/octet-stream"; + +/// Default MIME type to use for text streams. +pub(crate) static TEXT_MIME_TYPE: &str = "text/plain"; + +/// Default name for `send_bytes` byte-stream headers. +pub(crate) static BYTE_DEFAULT_NAME: &str = "unknown"; diff --git a/livekit-data-stream/src/outgoing/mod.rs b/livekit-data-stream/src/outgoing/mod.rs new file mode 100644 index 000000000..4b362c344 --- /dev/null +++ b/livekit-data-stream/src/outgoing/mod.rs @@ -0,0 +1,964 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bmrng::unbounded::{UnboundedRequestReceiver, UnboundedRequestSender}; +use chrono::Utc; +use livekit_common::{ + ClientCapability, ParticipantIdentity, RemoteParticipantRegistry, + CLIENT_PROTOCOL_DATA_STREAM_V2, +}; +use livekit_protocol as proto; +use proto::data_stream::CompressionType; +use std::{collections::HashMap, io::Write, path::Path, sync::Arc}; +use tokio::sync::Mutex; + +use crate::info::{ByteStreamInfo, OperationType, TextStreamInfo}; +use crate::utf8_chunk::Utf8AwareChunkExt; +use crate::utils::{SendError, StreamError, StreamResult}; + +mod stream_writer; +pub use stream_writer::{ByteStreamWriter, StreamWriter, TextStreamWriter}; + +mod constants; + +mod raw_stream; +use raw_stream::{RawStream, RawStreamOpenOptions}; + +/// Generates a random stream identifier (UUID v4). +fn create_random_uuid() -> String { + uuid::Uuid::new_v4().to_string() +} + +/// Options used when opening an outgoing byte data stream. +#[derive(Clone, Default, Debug, Eq, PartialEq)] +pub struct StreamByteOptions { + pub topic: String, + pub attributes: HashMap, + pub destination_identities: Vec, + pub id: Option, + pub mime_type: Option, + pub name: Option, + pub total_length: Option, + /// Whether to deflate-raw compress the payload when all recipients support it. + /// Defaults to `true` (compression opt-out). Ignored by the incremental `stream_bytes`. + pub compress: Option, +} + +/// Options used when opening an outgoing text data stream. +#[derive(Clone, Default, Debug, Eq, PartialEq)] +pub struct StreamTextOptions { + pub topic: String, + pub attributes: HashMap, + pub destination_identities: Vec, + pub id: Option, + pub operation_type: Option, + pub version: Option, + pub reply_to_stream_id: Option, + pub attached_stream_ids: Vec, + pub generated: Option, + /// Whether to deflate-raw compress the payload when all recipients support it. + /// Defaults to `true` (compression opt-out). Ignored by the incremental `stream_text`. + pub compress: Option, +} + +#[derive(Clone)] +pub struct OutgoingStreamManager { + /// Request channel for sending packets. + packet_tx: UnboundedRequestSender>, +} + +impl OutgoingStreamManager { + pub fn new() -> (Self, UnboundedRequestReceiver>) { + let (packet_tx, packet_rx) = bmrng::unbounded_channel(); + let manager = Self { packet_tx }; + (manager, packet_rx) + } + + pub async fn stream_text(&self, options: StreamTextOptions) -> StreamResult { + // Incremental streams are never inlined or compressed (the content is unknown up front). + let stream_id = options.id.clone().unwrap_or_else(create_random_uuid); + let dests = options.destination_identities.clone(); + let (header, text_header) = + build_text_header(&options, stream_id, None, None, CompressionType::None); + enforce_header_size(&header, &dests)?; + + let open_options = RawStreamOpenOptions { + header: header.clone(), + destination_identities: dests, + packet_tx: self.packet_tx.clone(), + }; + let writer = TextStreamWriter::new( + Arc::new(TextStreamInfo::from_headers(header, text_header)), + Arc::new(Mutex::new(RawStream::open(open_options).await?)), + ); + Ok(writer) + } + + pub async fn stream_bytes(&self, options: StreamByteOptions) -> StreamResult { + let stream_id = options.id.clone().unwrap_or_else(create_random_uuid); + let name = options.name.clone().unwrap_or_default(); + let dests = options.destination_identities.clone(); + let (header, byte_header) = build_byte_header( + &options, + stream_id, + name, + options.total_length, + None, + CompressionType::None, + ); + enforce_header_size(&header, &dests)?; + + let open_options = RawStreamOpenOptions { + header: header.clone(), + destination_identities: dests, + packet_tx: self.packet_tx.clone(), + }; + let writer = ByteStreamWriter::new( + Arc::new(ByteStreamInfo::from_headers(header, byte_header)), + Arc::new(Mutex::new(RawStream::open(open_options).await?)), + ); + Ok(writer) + } + + pub async fn send_text( + &self, + text: &str, + options: StreamTextOptions, + remote_participant_registry: &dyn RemoteParticipantRegistry, + ) -> StreamResult { + let stream_id = options.id.clone().unwrap_or_else(create_random_uuid); + let total_length = text.len() as u64; + let mut payload = MaybeCompressed::new(text.as_bytes()); + + let eligibility = + evaluate_eligibility(remote_participant_registry, &options.destination_identities); + let can_compress = options.compress.unwrap_or(true) && eligibility.compression; + + // 1. Inline single-packet attempt (no attachments; all recipients are >= v2). + let (mut header, text_header) = + if can_compress && payload.as_compressed()?.len() < payload.uncompressed.len() { + build_text_header( + &options, + stream_id.clone(), + Some(total_length), + Some(payload.as_compressed()?.to_vec()), + CompressionType::DeflateRaw, + ) + } else { + build_text_header( + &options, + stream_id.clone(), + Some(total_length), + Some(payload.uncompressed.to_vec()), + CompressionType::None, + ) + }; + if eligibility.inline + && options.attached_stream_ids.is_empty() + && header_packet_fits(&header, &options.destination_identities) + { + let packet = + RawStream::create_header_packet(header.clone(), options.destination_identities); + RawStream::send_packet(&self.packet_tx, packet).await?; + return Ok(TextStreamInfo::from_headers(header, text_header)); + } + + // 2/3. Chunked, compressed when eligible else uncompressed. + header.inline_content = None; + enforce_header_size(&header, &options.destination_identities)?; + + let should_compress = + header.compression() == proto::data_stream::CompressionType::DeflateRaw; + let open_options = RawStreamOpenOptions { + header: header.clone(), + destination_identities: options.destination_identities, + packet_tx: self.packet_tx.clone(), + }; + let info = TextStreamInfo::from_headers(header, text_header); + let mut stream = RawStream::open(open_options).await?; + if should_compress { + stream.write_raw_chunks(payload.as_compressed()?).await?; + } else { + for chunk in payload.uncompressed.utf8_aware_chunks(constants::STREAM_CHUNK_SIZE_BYTES) + { + stream.write_chunk(chunk).await?; + } + } + stream.close(None).await?; + Ok(info) + } + + /// Send bytes to participants in the room. + /// + /// This method sends an in-memory blob of bytes to participants in the room + /// as a byte stream. It opens a stream using the provided options, writes the + /// entire buffer, and closes the stream before returning. + /// + /// The `total_length` in the header is set from the provided data and is not + /// overridable by `options.total_length`. The header defaults `name` to `"unknown"` + /// and `mime_type` to `"application/octet-stream"`. + pub async fn send_bytes( + &self, + data: impl AsRef<[u8]>, + options: StreamByteOptions, + remote_participant_registry: &dyn RemoteParticipantRegistry, + ) -> StreamResult { + if options.total_length.is_some() { + log::warn!("Ignoring total_length option specified for send_bytes"); + } + let bytes = data.as_ref(); + let stream_id = options.id.clone().unwrap_or_else(create_random_uuid); + let name = options.name.clone().unwrap_or_else(|| constants::BYTE_DEFAULT_NAME.to_owned()); + let total_length = bytes.len() as u64; + let mut payload = MaybeCompressed::new(bytes); + + let eligibility = + evaluate_eligibility(remote_participant_registry, &options.destination_identities); + let can_compress = options.compress.unwrap_or(true) && eligibility.compression; + + // 1. Inline single-packet attempt (if all recipients are >= v2). + let (mut header, byte_header) = + if can_compress && payload.as_compressed()?.len() < payload.uncompressed.len() { + build_byte_header( + &options, + stream_id.clone(), + name.clone(), + Some(total_length), // NOTE: this is purposely always uncompressed length + Some(payload.as_compressed()?.to_vec()), + CompressionType::DeflateRaw, + ) + } else { + build_byte_header( + &options, + stream_id.clone(), + name.clone(), + Some(total_length), // NOTE: this is purposely always uncompressed length + Some(payload.uncompressed.to_vec()), + CompressionType::None, + ) + }; + if eligibility.inline && header_packet_fits(&header, &options.destination_identities) { + let packet = + RawStream::create_header_packet(header.clone(), options.destination_identities); + RawStream::send_packet(&self.packet_tx, packet).await?; + return Ok(ByteStreamInfo::from_headers(header, byte_header)); + } + + // 2/3. Chunked, compressed when eligible else uncompressed. + header.inline_content = None; + enforce_header_size(&header, &options.destination_identities)?; + + let should_compress = + header.compression() == proto::data_stream::CompressionType::DeflateRaw; + let open_options = RawStreamOpenOptions { + header: header.clone(), + destination_identities: options.destination_identities, + packet_tx: self.packet_tx.clone(), + }; + let info = ByteStreamInfo::from_headers(header, byte_header); + let mut stream = RawStream::open(open_options).await?; + if should_compress { + stream.write_raw_chunks(payload.as_compressed()?).await?; + } else { + stream.write_raw_chunks(payload.uncompressed).await?; + } + stream.close(None).await?; + Ok(info) + } + + /// Streams a file from disk to participants as a byte stream. + /// + /// Never uses the inline single-packet path (deciding inline-eligibility would require + /// buffering and compressing the whole file up front). Compresses when every recipient + /// supports it. The whole file is never buffered in memory at once. + pub async fn send_file( + &self, + path: impl AsRef, + options: StreamByteOptions, + remote_participant_registry: &dyn RemoteParticipantRegistry, + ) -> StreamResult { + let path = path.as_ref(); + let file_size = tokio::fs::metadata(path) + .await + .map(|metadata| metadata.len()) + .map_err(StreamError::from)?; + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or_default().to_owned(); + let stream_id = options.id.clone().unwrap_or_else(create_random_uuid); + let dests = options.destination_identities.clone(); + + let eligibility = evaluate_eligibility(remote_participant_registry, &dests); + let should_compress = options.compress.unwrap_or(true) && eligibility.compression; + let compression = + if should_compress { CompressionType::DeflateRaw } else { CompressionType::None }; + + let (header, byte_header) = + build_byte_header(&options, stream_id, name, Some(file_size), None, compression); + enforce_header_size(&header, &dests)?; + + let open_options = RawStreamOpenOptions { + header: header.clone(), + destination_identities: dests, + packet_tx: self.packet_tx.clone(), + }; + let info = ByteStreamInfo::from_headers(header, byte_header); + let mut stream = RawStream::open(open_options).await?; + stream.write_file(path, should_compress).await?; + stream.close(None).await?; + Ok(info) + } +} + +/// Inline / compression eligibility evaluated over a send's recipients. +struct SendEligibility { + /// Every recipient advertises `clientProtocol >= 2`. + inline: bool, + /// Inline-eligible AND every recipient advertises `CAP_COMPRESSION_DEFLATE_RAW`. + compression: bool, +} + +/// Evaluates inline/compression eligibility over a send's recipients. +/// +/// Recipients are the named `destinations`, or every remote participant for a broadcast +/// (empty `destinations`). An empty recipient set (empty room) is eligible for everything. +fn evaluate_eligibility( + registry: &dyn RemoteParticipantRegistry, + destinations: &[ParticipantIdentity], +) -> SendEligibility { + let recipients: Vec = + if destinations.is_empty() { registry.remote_identities() } else { destinations.to_vec() }; + let inline = recipients + .iter() + .all(|id| registry.remote_client_protocol(id) >= CLIENT_PROTOCOL_DATA_STREAM_V2); + let compression = inline + && recipients.iter().all(|id| { + registry.remote_capabilities(id).contains(&ClientCapability::CompressionDeflateRaw) + }); + + SendEligibility { inline, compression } +} + +/// A struct which manages the state of data which potentially may need to be compressed in the +/// future. +/// +/// By storing the compressed text optionally after performing compression, we can be sure that co +/// compression will only ever happen once, even if it must happen as part of speculative paths +/// (like checking whether compressed bytes are bigger than the literal bytes). +struct MaybeCompressed<'a> { + uncompressed: &'a [u8], + compressed: Option>, +} + +impl<'a> MaybeCompressed<'a> { + fn new(uncompressed: &'a [u8]) -> Self { + Self { uncompressed, compressed: None } + } + + /// Upconverts the Uncompressed variant into the Compressed variant, and returns a reference to + /// the compressed bytes as a result. + fn as_compressed(&mut self) -> Result<&[u8], std::io::Error> { + match &mut self.compressed { + Some(compressed) => Ok(&*compressed), + compressed_option @ None => { + let mut encoder = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(self.uncompressed)?; + *compressed_option = Some(encoder.finish()?); + let Some(ref data) = compressed_option else { + unreachable!("compressed data just set") + }; + Ok(data) + } + } + } +} + +/// Whether the serialized header `DataPacket` fits within the MTU budget. +fn header_packet_fits( + header: &proto::data_stream::Header, + destinations: &[ParticipantIdentity], +) -> bool { + use prost::Message; + let packet = RawStream::create_header_packet(header.clone(), destinations.to_vec()); + packet.encoded_len() <= constants::STREAM_CHUNK_SIZE_BYTES +} + +/// Enforces the header-packet MTU budget on the chunked path (the inline path falls back +/// gracefully instead of erroring). +fn enforce_header_size( + header: &proto::data_stream::Header, + destinations: &[ParticipantIdentity], +) -> StreamResult<()> { + if header_packet_fits(header, destinations) { + Ok(()) + } else { + Err(StreamError::HeaderTooLarge) + } +} + +fn build_text_header( + options: &StreamTextOptions, + stream_id: String, + total_length: Option, + inline_content: Option>, + compression: CompressionType, +) -> (proto::data_stream::Header, proto::data_stream::TextHeader) { + let text_header = proto::data_stream::TextHeader { + operation_type: options.operation_type.unwrap_or_default() as i32, + version: options.version.unwrap_or_default(), + reply_to_stream_id: options.reply_to_stream_id.clone().unwrap_or_default(), + attached_stream_ids: options.attached_stream_ids.clone(), + generated: options.generated.unwrap_or_default(), + }; + let header = proto::data_stream::Header { + stream_id, + timestamp: Utc::now().timestamp_millis(), + topic: options.topic.clone(), + mime_type: constants::TEXT_MIME_TYPE.to_owned(), + total_length, + encryption_type: proto::encryption::Type::None.into(), + attributes: options.attributes.clone(), + content_header: Some(proto::data_stream::header::ContentHeader::TextHeader( + text_header.clone(), + )), + inline_content, + compression: compression as i32, + }; + (header, text_header) +} + +fn build_byte_header( + options: &StreamByteOptions, + stream_id: String, + name: String, + total_length: Option, + inline_content: Option>, + compression: CompressionType, +) -> (proto::data_stream::Header, proto::data_stream::ByteHeader) { + let byte_header = proto::data_stream::ByteHeader { name }; + let header = proto::data_stream::Header { + stream_id, + timestamp: Utc::now().timestamp_millis(), + topic: options.topic.clone(), + mime_type: options + .mime_type + .clone() + .unwrap_or_else(|| constants::BYTE_MIME_TYPE.to_owned()), + total_length, + encryption_type: proto::encryption::Type::None.into(), + attributes: options.attributes.clone(), + content_header: Some(proto::data_stream::header::ContentHeader::ByteHeader( + byte_header.clone(), + )), + inline_content, + compression: compression as i32, + }; + (header, byte_header) +} + +#[cfg(test)] +mod tests { + use super::*; + use livekit_common::{CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DEFAULT}; + use std::sync::Mutex as StdMutex; + + // --- Fake recipient registry --------------------------------------------------------- + + struct FakeRegistry { + remotes: HashMap)>, + } + + impl FakeRegistry { + fn new() -> Self { + Self { remotes: HashMap::new() } + } + + fn add(mut self, id: &str, client_protocol: i32, caps: &[ClientCapability]) -> Self { + self.remotes.insert(id.to_string(), (client_protocol, caps.to_vec())); + self + } + } + + impl RemoteParticipantRegistry for FakeRegistry { + fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32 { + self.remotes.get(&identity.0).map(|(p, _)| *p).unwrap_or(0) + } + fn remote_capabilities(&self, identity: &ParticipantIdentity) -> Vec { + self.remotes.get(&identity.0).map(|(_, c)| c.clone()).unwrap_or_default() + } + fn remote_identities(&self) -> Vec { + self.remotes.keys().map(|k| ParticipantIdentity(k.clone())).collect() + } + } + + fn pre_v2_room() -> FakeRegistry { + FakeRegistry::new() + .add("alice", CLIENT_PROTOCOL_DEFAULT, &[]) + .add("bob", CLIENT_PROTOCOL_DEFAULT, &[]) + .add("jim", CLIENT_PROTOCOL_DATA_STREAM_RPC, &[]) + } + + fn all_v2_room() -> FakeRegistry { + FakeRegistry::new() + .add( + "alice", + CLIENT_PROTOCOL_DATA_STREAM_V2, + &[ClientCapability::CompressionDeflateRaw], + ) + .add("bob", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw]) + .add("noCompression", CLIENT_PROTOCOL_DATA_STREAM_V2, &[]) + } + + fn mixed_room() -> FakeRegistry { + FakeRegistry::new() + .add("alice", CLIENT_PROTOCOL_DEFAULT, &[]) + .add("bob", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw]) + .add("jim", CLIENT_PROTOCOL_DATA_STREAM_V2, &[ClientCapability::CompressionDeflateRaw]) + .add("mallory", CLIENT_PROTOCOL_DEFAULT, &[]) + .add("noCompression", CLIENT_PROTOCOL_DATA_STREAM_V2, &[]) + } + + // --- Capture harness ----------------------------------------------------------------- + + type Sent = Arc>>; + + fn setup() -> (OutgoingStreamManager, Sent) { + let (manager, mut packet_rx) = OutgoingStreamManager::new(); + let sent: Sent = Arc::new(StdMutex::new(Vec::new())); + let sink = sent.clone(); + tokio::spawn(async move { + while let Ok((packet, responder)) = packet_rx.recv().await { + sink.lock().unwrap().push(packet); + let _ = responder.respond(Ok(())); + } + }); + (manager, sent) + } + + fn ids(list: &[&str]) -> Vec { + list.iter().map(|s| ParticipantIdentity(s.to_string())).collect() + } + + fn text_opts(topic: &str, dests: &[&str]) -> StreamTextOptions { + StreamTextOptions { + topic: topic.to_string(), + destination_identities: ids(dests), + ..Default::default() + } + } + + fn byte_opts(topic: &str, dests: &[&str]) -> StreamByteOptions { + StreamByteOptions { + topic: topic.to_string(), + destination_identities: ids(dests), + ..Default::default() + } + } + + fn header(p: &proto::DataPacket) -> &proto::data_stream::Header { + match p.value.as_ref().unwrap() { + proto::data_packet::Value::StreamHeader(h) => h, + _ => panic!("expected stream header"), + } + } + + fn chunk(p: &proto::DataPacket) -> &proto::data_stream::Chunk { + match p.value.as_ref().unwrap() { + proto::data_packet::Value::StreamChunk(c) => c, + _ => panic!("expected stream chunk"), + } + } + + fn is_text_header(h: &proto::data_stream::Header) -> bool { + matches!(h.content_header, Some(proto::data_stream::header::ContentHeader::TextHeader(_))) + } + + fn is_byte_header(h: &proto::data_stream::Header) -> bool { + matches!(h.content_header, Some(proto::data_stream::header::ContentHeader::ByteHeader(_))) + } + + fn assert_trailer(p: &proto::DataPacket) { + match p.value.as_ref().unwrap() { + proto::data_packet::Value::StreamTrailer(t) => assert_eq!(t.reason, ""), + _ => panic!("expected stream trailer"), + } + } + + fn deflate_raw_i32() -> i32 { + CompressionType::DeflateRaw as i32 + } + fn none_i32() -> i32 { + CompressionType::None as i32 + } + + /// ~50 KB of deterministic, somewhat-compressible text (repeated marker + pseudo-random + /// lowercase). Compresses to >15 KB (so it can't inline) but well under its raw size. + /// + /// Seeded with a fixed value so the output is identical on every run. + fn somewhat_compressible(blocks: usize) -> String { + use rand::{rngs::StdRng, Rng, SeedableRng}; + + /// Fixed RNG seed that keeps `somewhat_compressible` output identical on every run. + const RANDOM_SEED: u64 = 0x1234_5678_9abc_def0; + + let mut rng = StdRng::seed_from_u64(RANDOM_SEED); + let mut s = String::new(); + for _ in 0..blocks { + s.push_str("hello world"); + for _ in 0..1000 { + s.push(rng.random_range(b'a'..=b'z') as char); + } + } + s + } + + // --- Pre-v2 room: legacy, uncompressed, multi-packet --------------------------------- + + #[tokio::test] + async fn pre_v2_short_text_is_legacy_multipacket() { + let (m, sent) = setup(); + m.send_text("hello world", text_opts("chat", &[]), &pre_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 3); + let h = header(&p[0]); + assert!(is_text_header(h)); + assert_eq!(h.topic, "chat"); + assert_eq!(h.compression, none_i32()); + assert!(h.inline_content.is_none()); + let c = chunk(&p[1]); + assert_eq!(c.chunk_index, 0); + assert_eq!(c.content, b"hello world"); + assert_trailer(&p[2]); + } + + #[tokio::test] + async fn pre_v2_long_text_splits_at_mtu() { + let (m, sent) = setup(); + let text = "A".repeat(40_000); + m.send_text(&text, text_opts("chat", &[]), &pre_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 5); // header + 3 chunks + trailer + assert_eq!(header(&p[0]).compression, none_i32()); + assert_eq!(chunk(&p[1]).content.len(), 15_000); + assert_eq!(chunk(&p[2]).content.len(), 15_000); + assert_eq!(chunk(&p[3]).content.len(), 10_000); + assert_eq!(chunk(&p[1]).chunk_index, 0); + assert_eq!(chunk(&p[3]).chunk_index, 2); + assert_trailer(&p[4]); + } + + #[tokio::test] + async fn pre_v2_bytes_is_legacy_multipacket() { + let (m, sent) = setup(); + m.send_bytes([0u8, 1, 2, 3], byte_opts("blob", &[]), &pre_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 3); + let h = header(&p[0]); + assert!(is_byte_header(h)); + assert_eq!(h.compression, none_i32()); + assert!(h.inline_content.is_none()); + assert_eq!(chunk(&p[1]).content, vec![0, 1, 2, 3]); + assert_trailer(&p[2]); + } + + // --- All-v2 room: inline + compression ----------------------------------------------- + + #[tokio::test] + async fn v2_short_compressible_text_inlines_compressed() { + let (m, sent) = setup(); + let text = "hello hello compressible world"; + m.send_text(text, text_opts("chat", &["alice", "bob"]), &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert!(is_text_header(h)); + assert_eq!(h.compression, deflate_raw_i32()); + let inline = h.inline_content.as_ref().unwrap(); + assert_ne!(inline.as_slice(), text.as_bytes()); // compressed, not raw + } + + #[tokio::test] + async fn v2_short_incompressible_text_inlines_raw() { + let (m, sent) = setup(); + m.send_text("short", text_opts("chat", &["alice", "bob"]), &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert_eq!(h.compression, none_i32()); + assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), b"short"); + } + + #[tokio::test] + async fn v2_no_compression_cap_inlines_raw() { + let (m, sent) = setup(); + let text = "hello hello compressible world"; + m.send_text(text, text_opts("chat", &["noCompression"]), &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); // inline (gated on protocol) still happens + let h = header(&p[0]); + assert_eq!(h.compression, none_i32()); // compression gated off by missing cap + assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes()); + } + + #[tokio::test] + async fn v2_large_highly_compressible_text_still_inlines() { + let (m, sent) = setup(); + let text = "hello world".repeat(20_000); + m.send_text(&text, text_opts("chat", &["alice", "bob"]), &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert_eq!(h.compression, deflate_raw_i32()); + assert!(h.inline_content.as_ref().unwrap().len() < text.len()); + } + + #[tokio::test] + async fn v2_somewhat_compressible_text_is_compressed_multipacket() { + let (m, sent) = setup(); + let text = somewhat_compressible(50); + m.send_text(&text, text_opts("chat", &["alice", "bob"]), &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + let h = header(&p[0]); + assert_eq!(h.compression, deflate_raw_i32()); + assert!(h.inline_content.is_none()); + let chunks: Vec<_> = p[1..p.len() - 1].iter().map(chunk).collect(); + // Multi-packet, but fewer chunks than an uncompressed send would need (ceil(len/15000)). + let uncompressed_chunks = text.len().div_ceil(constants::STREAM_CHUNK_SIZE_BYTES); + assert!(chunks.len() >= 2); + assert!(chunks.len() < uncompressed_chunks); + assert_eq!(chunks[0].content.len(), constants::STREAM_CHUNK_SIZE_BYTES); // first chunk is full MTU + let total: usize = chunks.iter().map(|c| c.content.len()).sum(); + assert!(total < text.len()); // compressed + assert_trailer(p.last().unwrap()); + } + + #[tokio::test] + async fn v2_compress_false_short_inlines_raw() { + let (m, sent) = setup(); + let text = "hello hello compressible world"; + let opts = + StreamTextOptions { compress: Some(false), ..text_opts("chat", &["alice", "bob"]) }; + m.send_text(text, opts, &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert_eq!(h.compression, none_i32()); + assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes()); + } + + #[tokio::test] + async fn v2_compress_false_large_is_uncompressed_multipacket() { + let (m, sent) = setup(); + let text = "B".repeat(50_000); + let opts = + StreamTextOptions { compress: Some(false), ..text_opts("chat", &["alice", "bob"]) }; + m.send_text(&text, opts, &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 6); // header + 4 chunks + trailer + assert_eq!(header(&p[0]).compression, none_i32()); + assert_eq!(chunk(&p[1]).content.len(), 15_000); + } + + // --- Incremental writers never compress or inline ------------------------------------ + + #[tokio::test] + async fn stream_text_never_compresses_or_inlines() { + let (m, sent) = setup(); + let writer = m.stream_text(text_opts("chat", &["noCompression"])).await.unwrap(); + assert_eq!(sent.lock().unwrap().len(), 1); + let h0 = sent.lock().unwrap()[0].clone(); + assert!(is_text_header(header(&h0))); + assert_eq!(header(&h0).compression, none_i32()); + assert!(header(&h0).inline_content.is_none()); + + writer.write("hello world").await.unwrap(); + assert_eq!(sent.lock().unwrap().len(), 2); + assert_eq!(chunk(&sent.lock().unwrap()[1]).content, b"hello world"); + + writer.close().await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 3); + assert_trailer(&p[2]); + } + + #[tokio::test] + async fn stream_bytes_never_compresses_or_inlines() { + let (m, sent) = setup(); + let writer = m.stream_bytes(byte_opts("blob", &["noCompression"])).await.unwrap(); + assert_eq!(sent.lock().unwrap().len(), 1); + assert_eq!(header(&sent.lock().unwrap()[0]).compression, none_i32()); + + writer.write(&[0u8, 1, 2, 3]).await.unwrap(); + assert_eq!(chunk(&sent.lock().unwrap()[1]).content, vec![0, 1, 2, 3]); + + writer.close().await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 3); + assert_trailer(&p[2]); + } + + // --- send_bytes inline behavior ------------------------------------------------------ + + #[tokio::test] + async fn v2_send_bytes_short_compressible_inlines_compressed() { + let (m, sent) = setup(); + let payload = "hello hello compressible world".as_bytes().to_vec(); + let mut opts = byte_opts("blob", &["alice", "bob"]); + opts.attributes.insert("foo".to_string(), "bar".to_string()); + let info = m.send_bytes(&payload, opts, &all_v2_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert!(is_byte_header(h)); + assert_eq!(h.compression, deflate_raw_i32()); + assert_ne!(h.inline_content.as_ref().unwrap().as_slice(), payload.as_slice()); + assert_eq!(info.name, "unknown"); + assert_eq!(info.mime_type, "application/octet-stream"); + assert_eq!(info.total_length, Some(payload.len() as u64)); + assert_eq!(info.attributes.get("foo"), Some(&"bar".to_string())); + } + + // --- Mixed room ---------------------------------------------------------------------- + + #[tokio::test] + async fn mixed_broadcast_falls_back_to_legacy() { + let (m, sent) = setup(); + m.send_text("hello world", text_opts("chat", &[]), &mixed_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 3); + assert_eq!(header(&p[0]).compression, none_i32()); + assert!(header(&p[0]).inline_content.is_none()); + assert_eq!(chunk(&p[1]).content, b"hello world"); + } + + #[tokio::test] + async fn mixed_targeted_v2_subset_inlines_compressed() { + let (m, sent) = setup(); + let text = "hello hello compressible world"; + m.send_text(text, text_opts("chat", &["bob", "jim"]), &mixed_room()).await.unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert_eq!(h.compression, deflate_raw_i32()); + assert_ne!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes()); + } + + #[tokio::test] + async fn mixed_targeted_subset_missing_cap_inlines_uncompressed() { + let (m, sent) = setup(); + let text = "hello hello compressible world"; + m.send_text(text, text_opts("chat", &["bob", "jim", "noCompression"]), &mixed_room()) + .await + .unwrap(); + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 1); + let h = header(&p[0]); + assert_eq!(h.compression, none_i32()); + assert_eq!(h.inline_content.as_ref().unwrap().as_slice(), text.as_bytes()); + } + + // --- send_file ----------------------------------------------------------------------- + + async fn write_temp_file(bytes: &[u8]) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!("lk_ds_test_{}.bin", create_random_uuid())); + tokio::fs::write(&path, bytes).await.unwrap(); + path + } + + #[tokio::test] + async fn send_file_never_inlines_and_compresses_when_eligible() { + let (m, sent) = setup(); + let path = write_temp_file(&vec![0x01u8; 10_000]).await; + m.send_file(&path, byte_opts("file", &["alice", "bob"]), &all_v2_room()).await.unwrap(); + let _ = tokio::fs::remove_file(&path).await; + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 3); // header + 1 chunk + trailer, NOT inline + let h = header(&p[0]); + assert!(is_byte_header(h)); + assert_eq!(h.compression, deflate_raw_i32()); + assert!(h.inline_content.is_none()); + assert!(chunk(&p[1]).content.len() < 10_000); // compressed + assert_trailer(&p[2]); + } + + #[tokio::test] + async fn send_file_uncompressed_splits_at_mtu() { + let (m, sent) = setup(); + let path = write_temp_file(&vec![0x07u8; 20_000]).await; + m.send_file(&path, byte_opts("file", &[]), &pre_v2_room()).await.unwrap(); + let _ = tokio::fs::remove_file(&path).await; + let p = sent.lock().unwrap().clone(); + assert_eq!(p.len(), 4); // header + 15000 + 5000 + trailer + assert_eq!(header(&p[0]).compression, none_i32()); + assert_eq!(chunk(&p[1]).content.len(), 15_000); + assert_eq!(chunk(&p[2]).content.len(), 5_000); + assert_eq!(chunk(&p[2]).chunk_index, 1); + assert_trailer(&p[3]); + } + + // --- Header size limit --------------------------------------------------------------- + + #[tokio::test] + async fn oversized_attributes_on_chunked_path_errors() { + let (m, _sent) = setup(); + let mut opts = text_opts("chat", &[]); // pre-v2 below => chunked path + opts.attributes.insert("big".to_string(), "x".repeat(20_000)); + let result = m.send_text("hello", opts, &pre_v2_room()).await; + assert!(matches!(result, Err(StreamError::HeaderTooLarge))); + } + + // Regression test for CLT-2773: dropping a `RawStream` on a thread that has + // no Tokio runtime in TLS (e.g. the .NET GC finalizer thread in the Unity + // SDK) used to panic because `Drop` called `tokio::spawn` unconditionally. + #[test] + fn drop_raw_stream_on_non_tokio_thread_does_not_panic() { + let rt = tokio::runtime::Runtime::new().unwrap(); + + let raw_stream = rt.block_on(async { + let (packet_tx, mut packet_rx) = + bmrng::unbounded_channel::>(); + + tokio::spawn(async move { + while let Ok((_packet, responder)) = packet_rx.recv().await { + let _ = responder.respond(Ok(())); + } + }); + + let header = proto::data_stream::Header { + stream_id: "gc-test-stream".to_string(), + timestamp: 0, + topic: "gc-test-topic".to_string(), + mime_type: constants::TEXT_MIME_TYPE.to_owned(), + total_length: None, + encryption_type: proto::encryption::Type::None.into(), + attributes: HashMap::new(), + content_header: None, + // Data streams v2 fields + inline_content: None, + compression: proto::data_stream::CompressionType::None as i32, + }; + + RawStream::open(RawStreamOpenOptions { + header, + destination_identities: vec![], + packet_tx, + }) + .await + .expect("RawStream should open") + }); + + let drop_thread = std::thread::spawn(move || drop(raw_stream)); + + drop_thread.join().expect("Dropping RawStream on a non-Tokio thread must not panic"); + } +} diff --git a/livekit-data-stream/src/outgoing/raw_stream.rs b/livekit-data-stream/src/outgoing/raw_stream.rs new file mode 100644 index 000000000..5c9821b41 --- /dev/null +++ b/livekit-data-stream/src/outgoing/raw_stream.rs @@ -0,0 +1,209 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use bmrng::unbounded::UnboundedRequestSender; +use livekit_common::ParticipantIdentity; +use livekit_protocol as proto; +use std::{io::Write, path::Path}; +use tokio::io::AsyncReadExt; + +use super::constants; +use crate::utils::{SendError, StreamError, StreamProgress, StreamResult}; + +pub(crate) struct RawStreamOpenOptions { + pub(crate) header: proto::data_stream::Header, + pub(crate) destination_identities: Vec, + pub(crate) packet_tx: UnboundedRequestSender>, +} + +pub(crate) struct RawStream { + id: String, + progress: StreamProgress, + is_closed: bool, + /// Request channel for sending packets. + packet_tx: UnboundedRequestSender>, +} + +impl RawStream { + pub(crate) async fn open(options: RawStreamOpenOptions) -> StreamResult { + let id = options.header.stream_id.to_string(); + let bytes_total = options.header.total_length; + + let packet = Self::create_header_packet(options.header, options.destination_identities); + Self::send_packet(&options.packet_tx, packet).await?; + + Ok(Self { + id, + progress: StreamProgress { bytes_total, ..Default::default() }, + is_closed: false, + packet_tx: options.packet_tx, + }) + } + + pub(crate) async fn write_chunk(&mut self, bytes: &[u8]) -> StreamResult<()> { + let packet = Self::create_chunk_packet(&self.id, self.progress.chunk_index, bytes); + Self::send_packet(&self.packet_tx, packet).await?; + self.progress.bytes_processed += bytes.len() as u64; + self.progress.chunk_index += 1; + Ok(()) + } + + /// Writes opaque bytes split into MTU-sized chunks on raw byte boundaries. + /// + /// Used for byte payloads and for compressed (deflate-raw) content, where the bytes + /// are opaque and must not be split on UTF-8 boundaries. + pub(crate) async fn write_raw_chunks(&mut self, bytes: &[u8]) -> StreamResult<()> { + for chunk in bytes.chunks(constants::STREAM_CHUNK_SIZE_BYTES) { + self.write_chunk(chunk).await?; + } + Ok(()) + } + + /// Streams a file's contents into MTU-sized chunks, optionally deflate-raw compressing + /// on the fly. The whole file is never buffered in memory at once. + pub(crate) async fn write_file( + &mut self, + path: impl AsRef, + compress: bool, + ) -> StreamResult<()> { + let mut file = tokio::fs::File::open(path).await?; + let mut read_buf = vec![0u8; 8192]; + + if compress { + let mut encoder = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + loop { + let n = file.read(&mut read_buf).await?; + if n == 0 { + break; + } + // Writing into a `Vec` is infallible. + encoder.write_all(&read_buf[..n]).expect("deflate write to Vec is infallible"); + // Drain whole MTU-sized chunks of compressed output as they accumulate so + // we never hold the full compressed file in memory. + while encoder.get_ref().len() >= constants::STREAM_CHUNK_SIZE_BYTES { + let rest = encoder.get_mut().split_off(constants::STREAM_CHUNK_SIZE_BYTES); + let chunk = std::mem::replace(encoder.get_mut(), rest); + self.write_chunk(&chunk).await?; + } + } + // Flush the final deflate block and send whatever compressed bytes remain. + let remaining = encoder.finish().expect("deflate finish into Vec is infallible"); + self.write_raw_chunks(&remaining).await?; + } else { + let mut pending: Vec = Vec::new(); + loop { + let n = file.read(&mut read_buf).await?; + if n == 0 { + break; + } + pending.extend_from_slice(&read_buf[..n]); + while pending.len() >= constants::STREAM_CHUNK_SIZE_BYTES { + let rest = pending.split_off(constants::STREAM_CHUNK_SIZE_BYTES); + let chunk = std::mem::replace(&mut pending, rest); + self.write_chunk(&chunk).await?; + } + } + if !pending.is_empty() { + self.write_chunk(&pending).await?; + } + } + Ok(()) + } + + pub(crate) async fn close(&mut self, reason: Option<&str>) -> StreamResult<()> { + if self.is_closed { + Err(StreamError::AlreadyClosed)? + } + let packet = Self::create_trailer_packet(&self.id, reason); + Self::send_packet(&self.packet_tx, packet).await?; + self.is_closed = true; + Ok(()) + } + + pub(crate) async fn send_packet( + tx: &UnboundedRequestSender>, + packet: proto::DataPacket, + ) -> StreamResult<()> { + tx.send_receive(packet) + .await + .map_err(|_| StreamError::Internal)? // request channel closed + .map_err(|_| StreamError::SendFailed) // data channel error + } + + pub(crate) fn create_header_packet( + header: proto::data_stream::Header, + destination_identities: Vec, + ) -> proto::DataPacket { + proto::DataPacket { + kind: proto::data_packet::Kind::Reliable.into(), + participant_identity: String::new(), // populate later + destination_identities: destination_identities.into_iter().map(|id| id.0).collect(), + value: Some(livekit_protocol::data_packet::Value::StreamHeader(header)), + // TODO: placeholder for reliable data transport + ..Default::default() + } + } + + pub(crate) fn create_chunk_packet( + id: &str, + chunk_index: u64, + content: &[u8], + ) -> proto::DataPacket { + let chunk = proto::data_stream::Chunk { + stream_id: id.to_string(), + chunk_index, + content: content.to_vec(), + ..Default::default() + }; + proto::DataPacket { + kind: proto::data_packet::Kind::Reliable.into(), + participant_identity: String::new(), // populate later + value: Some(livekit_protocol::data_packet::Value::StreamChunk(chunk)), + ..Default::default() + } + } + + pub(crate) fn create_trailer_packet(id: &str, reason: Option<&str>) -> proto::DataPacket { + let trailer = proto::data_stream::Trailer { + stream_id: id.to_string(), + reason: reason.unwrap_or_default().to_owned(), + ..Default::default() + }; + proto::DataPacket { + kind: proto::data_packet::Kind::Reliable.into(), + participant_identity: String::new(), // populate later + value: Some(livekit_protocol::data_packet::Value::StreamTrailer(trailer)), + ..Default::default() + } + } +} + +impl Drop for RawStream { + /// Close stream normally if not already closed. + fn drop(&mut self) { + if self.is_closed { + return; + } + let packet = Self::create_trailer_packet(&self.id, None); + let packet_tx = self.packet_tx.clone(); + // Use try_current() instead of assuming a Tokio runtime exists. + // The drop can run on a non-Tokio thread (e.g. a GC finalizer in + // Unity/.NET) or after the runtime has shut down, in which case + // we silently skip the trailer — the connection is going away anyway. + if let Ok(handle) = tokio::runtime::Handle::try_current() { + handle.spawn(async move { Self::send_packet(&packet_tx, packet).await }); + } + } +} diff --git a/livekit-data-stream/src/outgoing/stream_writer.rs b/livekit-data-stream/src/outgoing/stream_writer.rs new file mode 100644 index 000000000..bac111022 --- /dev/null +++ b/livekit-data-stream/src/outgoing/stream_writer.rs @@ -0,0 +1,124 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; +use tokio::sync::Mutex; + +use crate::info::{ByteStreamInfo, TextStreamInfo}; +use crate::outgoing::{constants::STREAM_CHUNK_SIZE_BYTES, RawStream}; +use crate::utf8_chunk::Utf8AwareChunkExt; +use crate::utils::StreamResult; + +/// Writer for an open data stream. +pub trait StreamWriter<'a> { + /// Type of input this writer accepts. + type Input: 'a; + + /// Information about the underlying data stream. + type Info; + + /// Returns a reference to the stream info. + fn info(&self) -> &Self::Info; + + /// Writes to the stream. + fn write( + &self, + input: Self::Input, + ) -> impl std::future::Future> + Send; + + /// Closes the stream normally. + fn close(self) -> impl std::future::Future> + Send; + + /// Closes the stream abnormally, specifying the reason for closure. + fn close_with_reason( + self, + reason: &str, + ) -> impl std::future::Future> + Send; +} + +#[derive(Clone)] +/// Writer for an open byte data stream. +pub struct ByteStreamWriter { + info: Arc, + stream: Arc>, +} + +impl ByteStreamWriter { + pub(crate) fn new(info: Arc, stream: Arc>) -> Self { + Self { info, stream } + } +} + +#[derive(Clone)] +/// Writer for an open text data stream. +pub struct TextStreamWriter { + info: Arc, + stream: Arc>, +} + +impl TextStreamWriter { + pub(crate) fn new(info: Arc, stream: Arc>) -> Self { + Self { info, stream } + } +} + +impl<'a> StreamWriter<'a> for ByteStreamWriter { + type Input = &'a [u8]; + type Info = ByteStreamInfo; + + fn info(&self) -> &Self::Info { + &self.info + } + + async fn write(&self, bytes: &'a [u8]) -> StreamResult<()> { + let mut stream = self.stream.lock().await; + for chunk in bytes.chunks(STREAM_CHUNK_SIZE_BYTES) { + stream.write_chunk(chunk).await?; + } + Ok(()) + } + + async fn close(self) -> StreamResult<()> { + self.stream.lock().await.close(None).await + } + + async fn close_with_reason(self, reason: &str) -> StreamResult<()> { + self.stream.lock().await.close(Some(reason)).await + } +} + +impl<'a> StreamWriter<'a> for TextStreamWriter { + type Input = &'a str; + type Info = TextStreamInfo; + + fn info(&self) -> &Self::Info { + &self.info + } + + async fn write(&self, text: &'a str) -> StreamResult<()> { + let mut stream = self.stream.lock().await; + for chunk in text.as_bytes().utf8_aware_chunks(STREAM_CHUNK_SIZE_BYTES) { + stream.write_chunk(chunk).await?; + } + Ok(()) + } + + async fn close(self) -> StreamResult<()> { + self.stream.lock().await.close(None).await + } + + async fn close_with_reason(self, reason: &str) -> StreamResult<()> { + self.stream.lock().await.close(Some(reason)).await + } +} diff --git a/livekit/src/utils/utf8_chunk.rs b/livekit-data-stream/src/utf8_chunk.rs similarity index 100% rename from livekit/src/utils/utf8_chunk.rs rename to livekit-data-stream/src/utf8_chunk.rs diff --git a/livekit-data-stream/src/utils.rs b/livekit-data-stream/src/utils.rs new file mode 100644 index 000000000..36c2a5d66 --- /dev/null +++ b/livekit-data-stream/src/utils.rs @@ -0,0 +1,88 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use thiserror::Error; + +/// Error returned by the packet transport when a data-stream packet fails to send. +/// +/// The stream managers only need to know that a send failed (they map it to +/// [`StreamError::SendFailed`]); the concrete engine error type stays in the `livekit` crate, +/// which bridges the outgoing packet channel to the RTC engine. +#[derive(Debug, Clone)] +pub struct SendError; + +/// Result type for data stream operations. +pub type StreamResult = Result; + +/// Error type for data stream operations. +#[derive(Debug, Error)] +pub enum StreamError { + // TODO(ladvoc): standardize error cases and expose over FFI. + #[error("stream has already been closed")] + AlreadyClosed, + + #[error("stream closed abnormally: {0}")] + AbnormalEnd(String), + + #[error("UTF-8 decoding error: {0}")] + Utf8(#[from] std::string::FromUtf8Error), + + #[error("incoming header was invalid")] + InvalidHeader, + + #[error("expected chunk index to be exactly one more than the previous")] + MissedChunk, + + #[error("read length exceeded total length specified in stream header")] + LengthExceeded, + + #[error("stream data is incomplete")] + Incomplete, + + #[error("unable to send packet")] + SendFailed, + + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + #[error("internal error")] + Internal, + + #[error("encryption type mismatch")] + EncryptionTypeMismatch, + + #[error("stream header exceeds maximum size")] + HeaderTooLarge, + + #[error("decompression failed")] + Decompression, +} + +/// Progress of a data stream. +#[derive(Clone, Copy, Default, Debug, Hash, Eq, PartialEq)] +pub(crate) struct StreamProgress { + pub(crate) chunk_index: u64, + /// Number of bytes read or written so far. + pub(crate) bytes_processed: u64, + /// Total number of bytes expected to be read or written for finite streams. + pub(crate) bytes_total: Option, +} + +impl StreamProgress { + /// Returns the completion percentage for finite streams. + #[allow(dead_code)] + fn percentage(&self) -> Option { + self.bytes_total.map(|total| self.bytes_processed as f32 / total as f32) + } +} diff --git a/livekit-ffi-node-bindings/proto/data_stream_pb.d.ts b/livekit-ffi-node-bindings/proto/data_stream_pb.d.ts index 6c7628f37..a800395d6 100644 --- a/livekit-ffi-node-bindings/proto/data_stream_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/data_stream_pb.d.ts @@ -1841,6 +1841,11 @@ export declare class StreamTextOptions extends Message { */ generated?: boolean; + /** + * @generated from field: optional bool compress = 10; + */ + compress?: boolean; + constructor(data?: PartialMessage); static readonly runtime: typeof proto2; @@ -1895,6 +1900,11 @@ export declare class StreamByteOptions extends Message { */ totalLength?: bigint; + /** + * @generated from field: optional bool compress = 8; + */ + compress?: boolean; + constructor(data?: PartialMessage); static readonly runtime: typeof proto2; diff --git a/livekit-ffi-node-bindings/proto/data_stream_pb.js b/livekit-ffi-node-bindings/proto/data_stream_pb.js index cd503c137..6b45dcaad 100644 --- a/livekit-ffi-node-bindings/proto/data_stream_pb.js +++ b/livekit-ffi-node-bindings/proto/data_stream_pb.js @@ -674,6 +674,7 @@ const StreamTextOptions = /*@__PURE__*/ proto2.makeMessageType( { no: 7, name: "reply_to_stream_id", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 8, name: "attached_stream_ids", kind: "scalar", T: 9 /* ScalarType.STRING */, repeated: true }, { no: 9, name: "generated", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + { no: 10, name: "compress", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); @@ -690,6 +691,7 @@ const StreamByteOptions = /*@__PURE__*/ proto2.makeMessageType( { no: 5, name: "name", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 6, name: "mime_type", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, { no: 7, name: "total_length", kind: "scalar", T: 4 /* ScalarType.UINT64 */, opt: true }, + { no: 8, name: "compress", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ], ); diff --git a/livekit-ffi/protocol/data_stream.proto b/livekit-ffi/protocol/data_stream.proto index 3f9fd590f..83c2ab5be 100644 --- a/livekit-ffi/protocol/data_stream.proto +++ b/livekit-ffi/protocol/data_stream.proto @@ -371,7 +371,7 @@ message StreamTextOptions { optional string reply_to_stream_id = 7; repeated string attached_stream_ids = 8; optional bool generated = 9; - + optional bool compress = 10; } message StreamByteOptions { required string topic = 1; @@ -381,6 +381,7 @@ message StreamByteOptions { optional string name = 5; optional string mime_type = 6; optional uint64 total_length = 7; + optional bool compress = 8; } // Error pertaining to a stream. diff --git a/livekit-ffi/src/conversion/data_stream.rs b/livekit-ffi/src/conversion/data_stream.rs index a59da350b..140bc74e4 100644 --- a/livekit-ffi/src/conversion/data_stream.rs +++ b/livekit-ffi/src/conversion/data_stream.rs @@ -72,6 +72,7 @@ impl From for StreamTextOptions { reply_to_stream_id: options.reply_to_stream_id, attached_stream_ids: options.attached_stream_ids, generated: options.generated, + compress: options.compress, } } } @@ -90,6 +91,7 @@ impl From for StreamByteOptions { name: options.name, mime_type: options.mime_type, total_length: options.total_length, + compress: options.compress, } } } diff --git a/livekit-protocol/protocol b/livekit-protocol/protocol index df0314e18..39fc751df 160000 --- a/livekit-protocol/protocol +++ b/livekit-protocol/protocol @@ -1 +1 @@ -Subproject commit df0314e189f0ab695005c5edc10f087b5a36ad23 +Subproject commit 39fc751df610243c1bfdf85d3be6b3928ecef014 diff --git a/livekit-protocol/src/livekit.rs b/livekit-protocol/src/livekit.rs index 1af6a6e0e..ac2408d86 100644 --- a/livekit-protocol/src/livekit.rs +++ b/livekit-protocol/src/livekit.rs @@ -653,6 +653,36 @@ pub struct DataTrackSubscriptionOptions { #[prost(uint32, optional, tag="1")] pub target_fps: ::core::option::Option, } +/// Key used to uniquely identify a data blob for storage and retrieval. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataBlobKey { + #[prost(oneof="data_blob_key::Key", tags="1")] + pub key: ::core::option::Option, +} +/// Nested message and enum types in `DataBlobKey`. +pub mod data_blob_key { + #[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Oneof)] + pub enum Key { + /// Generic string key, blob contains arbitrary data. + /// + /// Add additional key types here for storing specific types of blobs. + #[prost(string, tag="1")] + Generic(::prost::alloc::string::String), + } +} +/// A blob of data stored in a room identified by a unique key. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DataBlob { + /// Unique key the data blob is identified by. + #[prost(message, optional, tag="1")] + pub key: ::core::option::Option, + /// Contents of the data blob. This must not exceed 50 KB. + #[prost(bytes="vec", tag="2")] + pub contents: ::prost::alloc::vec::Vec, +} /// provide information about available spatial layers #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -3551,7 +3581,7 @@ impl AudioMixing { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SignalRequest { - #[prost(oneof="signal_request::Message", tags="1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21")] + #[prost(oneof="signal_request::Message", tags="1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23")] pub message: ::core::option::Option, } /// Nested message and enum types in `SignalRequest`. @@ -3619,12 +3649,18 @@ pub mod signal_request { /// Update subscription state for one or more data tracks #[prost(message, tag="21")] UpdateDataSubscription(super::UpdateDataSubscription), + /// Store a data blob. + #[prost(message, tag="22")] + StoreDataBlobRequest(super::StoreDataBlobRequest), + /// Retrieve a stored data blob. + #[prost(message, tag="23")] + GetDataBlobRequest(super::GetDataBlobRequest), } } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SignalResponse { - #[prost(oneof="signal_response::Message", tags="1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29")] + #[prost(oneof="signal_response::Message", tags="1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31")] pub message: ::core::option::Option, } /// Nested message and enum types in `SignalResponse`. @@ -3719,6 +3755,12 @@ pub mod signal_response { /// Sent to data track subscribers to provide mapping from track SIDs to handles. #[prost(message, tag="29")] DataTrackSubscriberHandles(super::DataTrackSubscriberHandles), + /// Sent in response to `StoreDataBlobRequest`. + #[prost(message, tag="30")] + StoreDataBlobResponse(super::StoreDataBlobResponse), + /// Sent in response to `GetDataBlobRequest`. + #[prost(message, tag="31")] + GetDataBlobResponse(super::GetDataBlobResponse), } } #[allow(clippy::derive_partial_eq_without_eq)] @@ -3979,6 +4021,43 @@ pub mod update_data_subscription { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreDataBlobRequest { + #[prost(uint32, tag="1")] + pub request_id: u32, + #[prost(message, optional, tag="2")] + pub blob: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StoreDataBlobResponse { + #[prost(uint32, tag="1")] + pub request_id: u32, + /// Unique key the data blob was stored under. + #[prost(message, optional, tag="2")] + pub key: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetDataBlobRequest { + #[prost(uint32, tag="1")] + pub request_id: u32, + /// Identity of the participant who owns the blob. + #[prost(string, tag="2")] + pub participant_identity: ::prost::alloc::string::String, + /// Unique key of the data blob to retrieve. + #[prost(message, optional, tag="3")] + pub key: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetDataBlobResponse { + #[prost(uint32, tag="1")] + pub request_id: u32, + #[prost(message, optional, tag="2")] + pub blob: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct UpdateTrackSettings { #[prost(string, repeated, tag="1")] pub track_sids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, @@ -4387,6 +4466,7 @@ pub mod request_response { InvalidName = 8, DuplicateHandle = 9, DuplicateName = 10, + InvalidRequest = 11, } impl Reason { /// String value of the enum field names used in the ProtoBuf definition. @@ -4406,6 +4486,7 @@ pub mod request_response { Reason::InvalidName => "INVALID_NAME", Reason::DuplicateHandle => "DUPLICATE_HANDLE", Reason::DuplicateName => "DUPLICATE_NAME", + Reason::InvalidRequest => "INVALID_REQUEST", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -4422,6 +4503,7 @@ pub mod request_response { "INVALID_NAME" => Some(Self::InvalidName), "DUPLICATE_HANDLE" => Some(Self::DuplicateHandle), "DUPLICATE_NAME" => Some(Self::DuplicateName), + "INVALID_REQUEST" => Some(Self::InvalidRequest), _ => None, } } diff --git a/livekit-protocol/src/livekit.serde.rs b/livekit-protocol/src/livekit.serde.rs index ebd07f129..fa22db55a 100644 --- a/livekit-protocol/src/livekit.serde.rs +++ b/livekit-protocol/src/livekit.serde.rs @@ -10527,6 +10527,221 @@ impl<'de> serde::Deserialize<'de> for CreateSipTrunkRequest { deserializer.deserialize_struct("livekit.CreateSIPTrunkRequest", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for DataBlob { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.key.is_some() { + len += 1; + } + if !self.contents.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.DataBlob", len)?; + if let Some(v) = self.key.as_ref() { + struct_ser.serialize_field("key", v)?; + } + if !self.contents.is_empty() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("contents", pbjson::private::base64::encode(&self.contents).as_str())?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for DataBlob { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "key", + "contents", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Key, + Contents, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "key" => Ok(GeneratedField::Key), + "contents" => Ok(GeneratedField::Contents), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DataBlob; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct livekit.DataBlob") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut key__ = None; + let mut contents__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Key => { + if key__.is_some() { + return Err(serde::de::Error::duplicate_field("key")); + } + key__ = map_.next_value()?; + } + GeneratedField::Contents => { + if contents__.is_some() { + return Err(serde::de::Error::duplicate_field("contents")); + } + contents__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(DataBlob { + key: key__, + contents: contents__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("livekit.DataBlob", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for DataBlobKey { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.key.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.DataBlobKey", len)?; + if let Some(v) = self.key.as_ref() { + match v { + data_blob_key::Key::Generic(v) => { + struct_ser.serialize_field("generic", v)?; + } + } + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for DataBlobKey { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "generic", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Generic, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "generic" => Ok(GeneratedField::Generic), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = DataBlobKey; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct livekit.DataBlobKey") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut key__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Generic => { + if key__.is_some() { + return Err(serde::de::Error::duplicate_field("generic")); + } + key__ = map_.next_value::<::std::option::Option<_>>()?.map(data_blob_key::Key::Generic); + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(DataBlobKey { + key: key__, + }) + } + } + deserializer.deserialize_struct("livekit.DataBlobKey", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for DataChannelInfo { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result @@ -18515,7 +18730,7 @@ impl<'de> serde::Deserialize<'de> for GcpUpload { deserializer.deserialize_struct("livekit.GCPUpload", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for GetSipInboundTrunkRequest { +impl serde::Serialize for GetDataBlobRequest { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -18523,30 +18738,47 @@ impl serde::Serialize for GetSipInboundTrunkRequest { { use serde::ser::SerializeStruct; let mut len = 0; - if !self.sip_trunk_id.is_empty() { + if self.request_id != 0 { len += 1; } - let mut struct_ser = serializer.serialize_struct("livekit.GetSIPInboundTrunkRequest", len)?; - if !self.sip_trunk_id.is_empty() { - struct_ser.serialize_field("sipTrunkId", &self.sip_trunk_id)?; + if !self.participant_identity.is_empty() { + len += 1; + } + if self.key.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.GetDataBlobRequest", len)?; + if self.request_id != 0 { + struct_ser.serialize_field("requestId", &self.request_id)?; + } + if !self.participant_identity.is_empty() { + struct_ser.serialize_field("participantIdentity", &self.participant_identity)?; + } + if let Some(v) = self.key.as_ref() { + struct_ser.serialize_field("key", v)?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkRequest { +impl<'de> serde::Deserialize<'de> for GetDataBlobRequest { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "sip_trunk_id", - "sipTrunkId", + "request_id", + "requestId", + "participant_identity", + "participantIdentity", + "key", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - SipTrunkId, + RequestId, + ParticipantIdentity, + Key, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -18569,7 +18801,9 @@ impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkRequest { E: serde::de::Error, { match value { - "sipTrunkId" | "sip_trunk_id" => Ok(GeneratedField::SipTrunkId), + "requestId" | "request_id" => Ok(GeneratedField::RequestId), + "participantIdentity" | "participant_identity" => Ok(GeneratedField::ParticipantIdentity), + "key" => Ok(GeneratedField::Key), _ => Ok(GeneratedField::__SkipField__), } } @@ -18579,39 +18813,57 @@ impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkRequest { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = GetSipInboundTrunkRequest; + type Value = GetDataBlobRequest; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct livekit.GetSIPInboundTrunkRequest") + formatter.write_str("struct livekit.GetDataBlobRequest") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut sip_trunk_id__ = None; + let mut request_id__ = None; + let mut participant_identity__ = None; + let mut key__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::SipTrunkId => { - if sip_trunk_id__.is_some() { - return Err(serde::de::Error::duplicate_field("sipTrunkId")); + GeneratedField::RequestId => { + if request_id__.is_some() { + return Err(serde::de::Error::duplicate_field("requestId")); } - sip_trunk_id__ = Some(map_.next_value()?); + request_id__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::ParticipantIdentity => { + if participant_identity__.is_some() { + return Err(serde::de::Error::duplicate_field("participantIdentity")); + } + participant_identity__ = Some(map_.next_value()?); + } + GeneratedField::Key => { + if key__.is_some() { + return Err(serde::de::Error::duplicate_field("key")); + } + key__ = map_.next_value()?; } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } } } - Ok(GetSipInboundTrunkRequest { - sip_trunk_id: sip_trunk_id__.unwrap_or_default(), + Ok(GetDataBlobRequest { + request_id: request_id__.unwrap_or_default(), + participant_identity: participant_identity__.unwrap_or_default(), + key: key__, }) } } - deserializer.deserialize_struct("livekit.GetSIPInboundTrunkRequest", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("livekit.GetDataBlobRequest", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for GetSipInboundTrunkResponse { +impl serde::Serialize for GetDataBlobResponse { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -18619,29 +18871,38 @@ impl serde::Serialize for GetSipInboundTrunkResponse { { use serde::ser::SerializeStruct; let mut len = 0; - if self.trunk.is_some() { + if self.request_id != 0 { len += 1; } - let mut struct_ser = serializer.serialize_struct("livekit.GetSIPInboundTrunkResponse", len)?; - if let Some(v) = self.trunk.as_ref() { - struct_ser.serialize_field("trunk", v)?; + if self.blob.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.GetDataBlobResponse", len)?; + if self.request_id != 0 { + struct_ser.serialize_field("requestId", &self.request_id)?; + } + if let Some(v) = self.blob.as_ref() { + struct_ser.serialize_field("blob", v)?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkResponse { +impl<'de> serde::Deserialize<'de> for GetDataBlobResponse { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where D: serde::Deserializer<'de>, { const FIELDS: &[&str] = &[ - "trunk", + "request_id", + "requestId", + "blob", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { - Trunk, + RequestId, + Blob, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -18664,7 +18925,8 @@ impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkResponse { E: serde::de::Error, { match value { - "trunk" => Ok(GeneratedField::Trunk), + "requestId" | "request_id" => Ok(GeneratedField::RequestId), + "blob" => Ok(GeneratedField::Blob), _ => Ok(GeneratedField::__SkipField__), } } @@ -18674,39 +18936,49 @@ impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkResponse { } struct GeneratedVisitor; impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { - type Value = GetSipInboundTrunkResponse; + type Value = GetDataBlobResponse; fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("struct livekit.GetSIPInboundTrunkResponse") + formatter.write_str("struct livekit.GetDataBlobResponse") } - fn visit_map(self, mut map_: V) -> std::result::Result + fn visit_map(self, mut map_: V) -> std::result::Result where V: serde::de::MapAccess<'de>, { - let mut trunk__ = None; + let mut request_id__ = None; + let mut blob__ = None; while let Some(k) = map_.next_key()? { match k { - GeneratedField::Trunk => { - if trunk__.is_some() { - return Err(serde::de::Error::duplicate_field("trunk")); + GeneratedField::RequestId => { + if request_id__.is_some() { + return Err(serde::de::Error::duplicate_field("requestId")); } - trunk__ = map_.next_value()?; + request_id__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Blob => { + if blob__.is_some() { + return Err(serde::de::Error::duplicate_field("blob")); + } + blob__ = map_.next_value()?; } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } } } - Ok(GetSipInboundTrunkResponse { - trunk: trunk__, + Ok(GetDataBlobResponse { + request_id: request_id__.unwrap_or_default(), + blob: blob__, }) } } - deserializer.deserialize_struct("livekit.GetSIPInboundTrunkResponse", FIELDS, GeneratedVisitor) + deserializer.deserialize_struct("livekit.GetDataBlobResponse", FIELDS, GeneratedVisitor) } } -impl serde::Serialize for GetSipOutboundTrunkRequest { +impl serde::Serialize for GetSipInboundTrunkRequest { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where @@ -18717,14 +18989,205 @@ impl serde::Serialize for GetSipOutboundTrunkRequest { if !self.sip_trunk_id.is_empty() { len += 1; } - let mut struct_ser = serializer.serialize_struct("livekit.GetSIPOutboundTrunkRequest", len)?; + let mut struct_ser = serializer.serialize_struct("livekit.GetSIPInboundTrunkRequest", len)?; if !self.sip_trunk_id.is_empty() { struct_ser.serialize_field("sipTrunkId", &self.sip_trunk_id)?; } struct_ser.end() } } -impl<'de> serde::Deserialize<'de> for GetSipOutboundTrunkRequest { +impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkRequest { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "sip_trunk_id", + "sipTrunkId", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + SipTrunkId, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "sipTrunkId" | "sip_trunk_id" => Ok(GeneratedField::SipTrunkId), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GetSipInboundTrunkRequest; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct livekit.GetSIPInboundTrunkRequest") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut sip_trunk_id__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::SipTrunkId => { + if sip_trunk_id__.is_some() { + return Err(serde::de::Error::duplicate_field("sipTrunkId")); + } + sip_trunk_id__ = Some(map_.next_value()?); + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(GetSipInboundTrunkRequest { + sip_trunk_id: sip_trunk_id__.unwrap_or_default(), + }) + } + } + deserializer.deserialize_struct("livekit.GetSIPInboundTrunkRequest", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for GetSipInboundTrunkResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.trunk.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.GetSIPInboundTrunkResponse", len)?; + if let Some(v) = self.trunk.as_ref() { + struct_ser.serialize_field("trunk", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for GetSipInboundTrunkResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "trunk", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + Trunk, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "trunk" => Ok(GeneratedField::Trunk), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GetSipInboundTrunkResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct livekit.GetSIPInboundTrunkResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut trunk__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::Trunk => { + if trunk__.is_some() { + return Err(serde::de::Error::duplicate_field("trunk")); + } + trunk__ = map_.next_value()?; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(GetSipInboundTrunkResponse { + trunk: trunk__, + }) + } + } + deserializer.deserialize_struct("livekit.GetSIPInboundTrunkResponse", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for GetSipOutboundTrunkRequest { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if !self.sip_trunk_id.is_empty() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.GetSIPOutboundTrunkRequest", len)?; + if !self.sip_trunk_id.is_empty() { + struct_ser.serialize_field("sipTrunkId", &self.sip_trunk_id)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for GetSipOutboundTrunkRequest { #[allow(deprecated)] fn deserialize(deserializer: D) -> std::result::Result where @@ -33107,6 +33570,7 @@ impl serde::Serialize for request_response::Reason { Self::InvalidName => "INVALID_NAME", Self::DuplicateHandle => "DUPLICATE_HANDLE", Self::DuplicateName => "DUPLICATE_NAME", + Self::InvalidRequest => "INVALID_REQUEST", }; serializer.serialize_str(variant) } @@ -33129,6 +33593,7 @@ impl<'de> serde::Deserialize<'de> for request_response::Reason { "INVALID_NAME", "DUPLICATE_HANDLE", "DUPLICATE_NAME", + "INVALID_REQUEST", ]; struct GeneratedVisitor; @@ -33180,6 +33645,7 @@ impl<'de> serde::Deserialize<'de> for request_response::Reason { "INVALID_NAME" => Ok(request_response::Reason::InvalidName), "DUPLICATE_HANDLE" => Ok(request_response::Reason::DuplicateHandle), "DUPLICATE_NAME" => Ok(request_response::Reason::DuplicateName), + "INVALID_REQUEST" => Ok(request_response::Reason::InvalidRequest), _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), } } @@ -43100,6 +43566,12 @@ impl serde::Serialize for SignalRequest { signal_request::Message::UpdateDataSubscription(v) => { struct_ser.serialize_field("updateDataSubscription", v)?; } + signal_request::Message::StoreDataBlobRequest(v) => { + struct_ser.serialize_field("storeDataBlobRequest", v)?; + } + signal_request::Message::GetDataBlobRequest(v) => { + struct_ser.serialize_field("getDataBlobRequest", v)?; + } } } struct_ser.end() @@ -43144,6 +43616,10 @@ impl<'de> serde::Deserialize<'de> for SignalRequest { "unpublishDataTrackRequest", "update_data_subscription", "updateDataSubscription", + "store_data_blob_request", + "storeDataBlobRequest", + "get_data_blob_request", + "getDataBlobRequest", ]; #[allow(clippy::enum_variant_names)] @@ -43168,6 +43644,8 @@ impl<'de> serde::Deserialize<'de> for SignalRequest { PublishDataTrackRequest, UnpublishDataTrackRequest, UpdateDataSubscription, + StoreDataBlobRequest, + GetDataBlobRequest, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -43210,6 +43688,8 @@ impl<'de> serde::Deserialize<'de> for SignalRequest { "publishDataTrackRequest" | "publish_data_track_request" => Ok(GeneratedField::PublishDataTrackRequest), "unpublishDataTrackRequest" | "unpublish_data_track_request" => Ok(GeneratedField::UnpublishDataTrackRequest), "updateDataSubscription" | "update_data_subscription" => Ok(GeneratedField::UpdateDataSubscription), + "storeDataBlobRequest" | "store_data_blob_request" => Ok(GeneratedField::StoreDataBlobRequest), + "getDataBlobRequest" | "get_data_blob_request" => Ok(GeneratedField::GetDataBlobRequest), _ => Ok(GeneratedField::__SkipField__), } } @@ -43369,6 +43849,20 @@ impl<'de> serde::Deserialize<'de> for SignalRequest { return Err(serde::de::Error::duplicate_field("updateDataSubscription")); } message__ = map_.next_value::<::std::option::Option<_>>()?.map(signal_request::Message::UpdateDataSubscription) +; + } + GeneratedField::StoreDataBlobRequest => { + if message__.is_some() { + return Err(serde::de::Error::duplicate_field("storeDataBlobRequest")); + } + message__ = map_.next_value::<::std::option::Option<_>>()?.map(signal_request::Message::StoreDataBlobRequest) +; + } + GeneratedField::GetDataBlobRequest => { + if message__.is_some() { + return Err(serde::de::Error::duplicate_field("getDataBlobRequest")); + } + message__ = map_.next_value::<::std::option::Option<_>>()?.map(signal_request::Message::GetDataBlobRequest) ; } GeneratedField::__SkipField__ => { @@ -43484,6 +43978,12 @@ impl serde::Serialize for SignalResponse { signal_response::Message::DataTrackSubscriberHandles(v) => { struct_ser.serialize_field("dataTrackSubscriberHandles", v)?; } + signal_response::Message::StoreDataBlobResponse(v) => { + struct_ser.serialize_field("storeDataBlobResponse", v)?; + } + signal_response::Message::GetDataBlobResponse(v) => { + struct_ser.serialize_field("getDataBlobResponse", v)?; + } } } struct_ser.end() @@ -43543,6 +44043,10 @@ impl<'de> serde::Deserialize<'de> for SignalResponse { "unpublishDataTrackResponse", "data_track_subscriber_handles", "dataTrackSubscriberHandles", + "store_data_blob_response", + "storeDataBlobResponse", + "get_data_blob_response", + "getDataBlobResponse", ]; #[allow(clippy::enum_variant_names)] @@ -43575,6 +44079,8 @@ impl<'de> serde::Deserialize<'de> for SignalResponse { PublishDataTrackResponse, UnpublishDataTrackResponse, DataTrackSubscriberHandles, + StoreDataBlobResponse, + GetDataBlobResponse, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -43625,6 +44131,8 @@ impl<'de> serde::Deserialize<'de> for SignalResponse { "publishDataTrackResponse" | "publish_data_track_response" => Ok(GeneratedField::PublishDataTrackResponse), "unpublishDataTrackResponse" | "unpublish_data_track_response" => Ok(GeneratedField::UnpublishDataTrackResponse), "dataTrackSubscriberHandles" | "data_track_subscriber_handles" => Ok(GeneratedField::DataTrackSubscriberHandles), + "storeDataBlobResponse" | "store_data_blob_response" => Ok(GeneratedField::StoreDataBlobResponse), + "getDataBlobResponse" | "get_data_blob_response" => Ok(GeneratedField::GetDataBlobResponse), _ => Ok(GeneratedField::__SkipField__), } } @@ -43839,6 +44347,20 @@ impl<'de> serde::Deserialize<'de> for SignalResponse { return Err(serde::de::Error::duplicate_field("dataTrackSubscriberHandles")); } message__ = map_.next_value::<::std::option::Option<_>>()?.map(signal_response::Message::DataTrackSubscriberHandles) +; + } + GeneratedField::StoreDataBlobResponse => { + if message__.is_some() { + return Err(serde::de::Error::duplicate_field("storeDataBlobResponse")); + } + message__ = map_.next_value::<::std::option::Option<_>>()?.map(signal_response::Message::StoreDataBlobResponse) +; + } + GeneratedField::GetDataBlobResponse => { + if message__.is_some() { + return Err(serde::de::Error::duplicate_field("getDataBlobResponse")); + } + message__ = map_.next_value::<::std::option::Option<_>>()?.map(signal_response::Message::GetDataBlobResponse) ; } GeneratedField::__SkipField__ => { @@ -45403,6 +45925,236 @@ impl<'de> serde::Deserialize<'de> for StorageConfig { deserializer.deserialize_struct("livekit.StorageConfig", FIELDS, GeneratedVisitor) } } +impl serde::Serialize for StoreDataBlobRequest { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.request_id != 0 { + len += 1; + } + if self.blob.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.StoreDataBlobRequest", len)?; + if self.request_id != 0 { + struct_ser.serialize_field("requestId", &self.request_id)?; + } + if let Some(v) = self.blob.as_ref() { + struct_ser.serialize_field("blob", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for StoreDataBlobRequest { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "request_id", + "requestId", + "blob", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + RequestId, + Blob, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "requestId" | "request_id" => Ok(GeneratedField::RequestId), + "blob" => Ok(GeneratedField::Blob), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = StoreDataBlobRequest; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct livekit.StoreDataBlobRequest") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut request_id__ = None; + let mut blob__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::RequestId => { + if request_id__.is_some() { + return Err(serde::de::Error::duplicate_field("requestId")); + } + request_id__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Blob => { + if blob__.is_some() { + return Err(serde::de::Error::duplicate_field("blob")); + } + blob__ = map_.next_value()?; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(StoreDataBlobRequest { + request_id: request_id__.unwrap_or_default(), + blob: blob__, + }) + } + } + deserializer.deserialize_struct("livekit.StoreDataBlobRequest", FIELDS, GeneratedVisitor) + } +} +impl serde::Serialize for StoreDataBlobResponse { + #[allow(deprecated)] + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + use serde::ser::SerializeStruct; + let mut len = 0; + if self.request_id != 0 { + len += 1; + } + if self.key.is_some() { + len += 1; + } + let mut struct_ser = serializer.serialize_struct("livekit.StoreDataBlobResponse", len)?; + if self.request_id != 0 { + struct_ser.serialize_field("requestId", &self.request_id)?; + } + if let Some(v) = self.key.as_ref() { + struct_ser.serialize_field("key", v)?; + } + struct_ser.end() + } +} +impl<'de> serde::Deserialize<'de> for StoreDataBlobResponse { + #[allow(deprecated)] + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + const FIELDS: &[&str] = &[ + "request_id", + "requestId", + "key", + ]; + + #[allow(clippy::enum_variant_names)] + enum GeneratedField { + RequestId, + Key, + __SkipField__, + } + impl<'de> serde::Deserialize<'de> for GeneratedField { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct GeneratedVisitor; + + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = GeneratedField; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "expected one of: {:?}", &FIELDS) + } + + #[allow(unused_variables)] + fn visit_str(self, value: &str) -> std::result::Result + where + E: serde::de::Error, + { + match value { + "requestId" | "request_id" => Ok(GeneratedField::RequestId), + "key" => Ok(GeneratedField::Key), + _ => Ok(GeneratedField::__SkipField__), + } + } + } + deserializer.deserialize_identifier(GeneratedVisitor) + } + } + struct GeneratedVisitor; + impl<'de> serde::de::Visitor<'de> for GeneratedVisitor { + type Value = StoreDataBlobResponse; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("struct livekit.StoreDataBlobResponse") + } + + fn visit_map(self, mut map_: V) -> std::result::Result + where + V: serde::de::MapAccess<'de>, + { + let mut request_id__ = None; + let mut key__ = None; + while let Some(k) = map_.next_key()? { + match k { + GeneratedField::RequestId => { + if request_id__.is_some() { + return Err(serde::de::Error::duplicate_field("requestId")); + } + request_id__ = + Some(map_.next_value::<::pbjson::private::NumberDeserialize<_>>()?.0) + ; + } + GeneratedField::Key => { + if key__.is_some() { + return Err(serde::de::Error::duplicate_field("key")); + } + key__ = map_.next_value()?; + } + GeneratedField::__SkipField__ => { + let _ = map_.next_value::()?; + } + } + } + Ok(StoreDataBlobResponse { + request_id: request_id__.unwrap_or_default(), + key: key__, + }) + } + } + deserializer.deserialize_struct("livekit.StoreDataBlobResponse", FIELDS, GeneratedVisitor) + } +} impl serde::Serialize for StreamInfo { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index 3eb83261e..894e7630d 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -29,13 +29,15 @@ rustls-tls-native-roots = ["livekit-api/rustls-tls-native-roots"] rustls-tls-webpki-roots = ["livekit-api/rustls-tls-webpki-roots"] __rustls-tls = ["livekit-api/__rustls-tls"] __lk-internal = [] # internal features (used by livekit-ffi) -__lk-e2e-test = [] # end-to-end testing with a LiveKit server +__lk-e2e-test = ["livekit-data-stream/__e2e-test"] # end-to-end testing with a LiveKit server [dependencies] livekit-runtime = { workspace = true } livekit-api = { workspace = true } libwebrtc = { workspace = true } livekit-protocol = { workspace = true } +livekit-common = { workspace = true } +livekit-data-stream = { workspace = true } livekit-datatrack = { workspace = true } prost = "0.12" serde = { version = "1", features = ["derive"] } @@ -52,11 +54,15 @@ semver = "1.0" libloading = { version = "0.8.6" } bytes = "1.10.1" bmrng = "0.5.2" +flate2 = "1" base64 = "0.22" [dev-dependencies] +# Enable data-stream test constructors (e.g. TextStreamReader::new_for_test) for our test suites. +livekit-data-stream = { workspace = true, features = ["test-utils"] } anyhow = "1.0.99" test-log = "0.2.18" test-case = "3.3" serial_test = "3.0" http = "1.1" +rand = { workspace = true } diff --git a/livekit/src/proto.rs b/livekit/src/proto.rs index 78e690d73..b8d937644 100644 --- a/livekit/src/proto.rs +++ b/livekit/src/proto.rs @@ -14,9 +14,7 @@ use livekit_protocol::*; -use crate::{ - e2ee::EncryptionType, participant, room::ChatMessage as RoomChatMessage, track, DataPacketKind, -}; +use crate::{participant, room::ChatMessage as RoomChatMessage, track, DataPacketKind}; // Conversions impl From for participant::ConnectionQuality { @@ -141,36 +139,6 @@ impl From for DataPacketKind { } } -impl From for EncryptionType { - fn from(value: livekit_protocol::encryption::Type) -> Self { - match value { - livekit_protocol::encryption::Type::None => Self::None, - livekit_protocol::encryption::Type::Gcm => Self::Gcm, - livekit_protocol::encryption::Type::Custom => Self::Custom, - } - } -} - -impl From for encryption::Type { - fn from(value: EncryptionType) -> Self { - match value { - EncryptionType::None => Self::None, - EncryptionType::Gcm => Self::Gcm, - EncryptionType::Custom => Self::Custom, - } - } -} - -impl From for i32 { - fn from(value: EncryptionType) -> Self { - match value { - EncryptionType::None => 0, - EncryptionType::Gcm => 1, - EncryptionType::Custom => 2, - } - } -} - impl From for participant::ParticipantState { fn from(value: participant_info::State) -> Self { match value { diff --git a/livekit/src/room/data_stream/outgoing.rs b/livekit/src/room/data_stream/outgoing.rs deleted file mode 100644 index 5c3e4c431..000000000 --- a/livekit/src/room/data_stream/outgoing.rs +++ /dev/null @@ -1,560 +0,0 @@ -// Copyright 2025 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use super::{ - ByteStreamInfo, OperationType, StreamError, StreamProgress, StreamResult, TextStreamInfo, -}; -use crate::{ - id::ParticipantIdentity, rtc_engine::EngineError, utils::utf8_chunk::Utf8AwareChunkExt, -}; -use bmrng::unbounded::{UnboundedRequestReceiver, UnboundedRequestSender}; -use chrono::Utc; -use libwebrtc::native::create_random_uuid; -use livekit_protocol as proto; -use std::{collections::HashMap, path::Path, sync::Arc}; -use tokio::{io::AsyncReadExt, sync::Mutex}; - -/// Writer for an open data stream. -pub trait StreamWriter<'a> { - /// Type of input this writer accepts. - type Input: 'a; - - /// Information about the underlying data stream. - type Info; - - /// Returns a reference to the stream info. - fn info(&self) -> &Self::Info; - - /// Writes to the stream. - fn write( - &self, - input: Self::Input, - ) -> impl std::future::Future> + Send; - - /// Closes the stream normally. - fn close(self) -> impl std::future::Future> + Send; - - /// Closes the stream abnormally, specifying the reason for closure. - fn close_with_reason( - self, - reason: &str, - ) -> impl std::future::Future> + Send; -} - -#[derive(Clone)] -/// Writer for an open byte data stream. -pub struct ByteStreamWriter { - info: Arc, - stream: Arc>, -} - -#[derive(Clone)] -/// Writer for an open text data stream. -pub struct TextStreamWriter { - info: Arc, - stream: Arc>, -} - -impl<'a> StreamWriter<'a> for ByteStreamWriter { - type Input = &'a [u8]; - type Info = ByteStreamInfo; - - fn info(&self) -> &Self::Info { - &self.info - } - - async fn write(&self, bytes: &'a [u8]) -> StreamResult<()> { - let mut stream = self.stream.lock().await; - for chunk in bytes.chunks(CHUNK_SIZE) { - stream.write_chunk(chunk).await?; - } - Ok(()) - } - - async fn close(self) -> StreamResult<()> { - self.stream.lock().await.close(None).await - } - - async fn close_with_reason(self, reason: &str) -> StreamResult<()> { - self.stream.lock().await.close(Some(reason)).await - } -} - -impl ByteStreamWriter { - /// Writes the contents of the file incrementally. - async fn write_file_contents(&self, path: impl AsRef) -> StreamResult<()> { - let mut stream = self.stream.lock().await; - let mut file = tokio::fs::File::open(path).await?; - let mut buffer = vec![0; 8192]; // 8KB - loop { - let bytes_read = file.read(&mut buffer).await?; - if bytes_read == 0 { - break; - } - stream.write_chunk(&buffer[..bytes_read]).await?; - } - Ok(()) - } -} - -impl<'a> StreamWriter<'a> for TextStreamWriter { - type Input = &'a str; - type Info = TextStreamInfo; - - fn info(&self) -> &Self::Info { - &self.info - } - - async fn write(&self, text: &'a str) -> StreamResult<()> { - let mut stream = self.stream.lock().await; - for chunk in text.as_bytes().utf8_aware_chunks(CHUNK_SIZE) { - stream.write_chunk(chunk).await?; - } - Ok(()) - } - - async fn close(self) -> StreamResult<()> { - self.stream.lock().await.close(None).await - } - - async fn close_with_reason(self, reason: &str) -> StreamResult<()> { - self.stream.lock().await.close(Some(reason)).await - } -} - -struct RawStreamOpenOptions { - header: proto::data_stream::Header, - destination_identities: Vec, - packet_tx: UnboundedRequestSender>, -} - -struct RawStream { - id: String, - progress: StreamProgress, - is_closed: bool, - /// Request channel for sending packets. - packet_tx: UnboundedRequestSender>, -} - -impl RawStream { - async fn open(options: RawStreamOpenOptions) -> StreamResult { - let id = options.header.stream_id.to_string(); - let bytes_total = options.header.total_length; - - let packet = Self::create_header_packet(options.header, options.destination_identities); - Self::send_packet(&options.packet_tx, packet).await?; - - Ok(Self { - id, - progress: StreamProgress { bytes_total, ..Default::default() }, - is_closed: false, - packet_tx: options.packet_tx, - }) - } - - async fn write_chunk(&mut self, bytes: &[u8]) -> StreamResult<()> { - let packet = Self::create_chunk_packet(&self.id, self.progress.chunk_index, bytes); - Self::send_packet(&self.packet_tx, packet).await?; - self.progress.bytes_processed += bytes.len() as u64; - self.progress.chunk_index += 1; - Ok(()) - } - - async fn close(&mut self, reason: Option<&str>) -> StreamResult<()> { - if self.is_closed { - Err(StreamError::AlreadyClosed)? - } - let packet = Self::create_trailer_packet(&self.id, reason); - Self::send_packet(&self.packet_tx, packet).await?; - self.is_closed = true; - Ok(()) - } - - async fn send_packet( - tx: &UnboundedRequestSender>, - packet: proto::DataPacket, - ) -> StreamResult<()> { - tx.send_receive(packet) - .await - .map_err(|_| StreamError::Internal)? // request channel closed - .map_err(|_| StreamError::SendFailed) // data channel error - } - - fn create_header_packet( - header: proto::data_stream::Header, - destination_identities: Vec, - ) -> proto::DataPacket { - proto::DataPacket { - kind: proto::data_packet::Kind::Reliable.into(), - participant_identity: String::new(), // populate later - destination_identities: destination_identities.into_iter().map(|id| id.0).collect(), - value: Some(livekit_protocol::data_packet::Value::StreamHeader(header)), - // TODO: placeholder for reliable data transport - ..Default::default() - } - } - - fn create_chunk_packet(id: &str, chunk_index: u64, content: &[u8]) -> proto::DataPacket { - let chunk = proto::data_stream::Chunk { - stream_id: id.to_string(), - chunk_index, - content: content.to_vec(), - ..Default::default() - }; - proto::DataPacket { - kind: proto::data_packet::Kind::Reliable.into(), - participant_identity: String::new(), // populate later - value: Some(livekit_protocol::data_packet::Value::StreamChunk(chunk)), - ..Default::default() - } - } - - fn create_trailer_packet(id: &str, reason: Option<&str>) -> proto::DataPacket { - let trailer = proto::data_stream::Trailer { - stream_id: id.to_string(), - reason: reason.unwrap_or_default().to_owned(), - ..Default::default() - }; - proto::DataPacket { - kind: proto::data_packet::Kind::Reliable.into(), - participant_identity: String::new(), // populate later - value: Some(livekit_protocol::data_packet::Value::StreamTrailer(trailer)), - ..Default::default() - } - } -} - -impl Drop for RawStream { - /// Close stream normally if not already closed. - fn drop(&mut self) { - if self.is_closed { - return; - } - let packet = Self::create_trailer_packet(&self.id, None); - let packet_tx = self.packet_tx.clone(); - // Use try_current() instead of assuming a Tokio runtime exists. - // The drop can run on a non-Tokio thread (e.g. a GC finalizer in - // Unity/.NET) or after the runtime has shut down, in which case - // we silently skip the trailer — the connection is going away anyway. - if let Ok(handle) = tokio::runtime::Handle::try_current() { - handle.spawn(async move { Self::send_packet(&packet_tx, packet).await }); - } - } -} - -/// Options used when opening an outgoing byte data stream. -#[derive(Clone, Default, Debug, Eq, PartialEq)] -pub struct StreamByteOptions { - pub topic: String, - pub attributes: HashMap, - pub destination_identities: Vec, - pub id: Option, - pub mime_type: Option, - pub name: Option, - pub total_length: Option, -} - -/// Options used when opening an outgoing text data stream. -#[derive(Clone, Default, Debug, Eq, PartialEq)] -pub struct StreamTextOptions { - pub topic: String, - pub attributes: HashMap, - pub destination_identities: Vec, - pub id: Option, - pub operation_type: Option, - pub version: Option, - pub reply_to_stream_id: Option, - pub attached_stream_ids: Vec, - pub generated: Option, -} - -#[derive(Clone)] -pub(crate) struct OutgoingStreamManager { - /// Request channel for sending packets. - packet_tx: UnboundedRequestSender>, -} - -impl OutgoingStreamManager { - pub fn new() -> (Self, UnboundedRequestReceiver>) { - let (packet_tx, packet_rx) = bmrng::unbounded_channel(); - let manager = Self { packet_tx }; - (manager, packet_rx) - } - - pub async fn stream_text(&self, options: StreamTextOptions) -> StreamResult { - let text_header = proto::data_stream::TextHeader { - operation_type: options.operation_type.unwrap_or_default() as i32, - version: options.version.unwrap_or_default(), - reply_to_stream_id: options.reply_to_stream_id.unwrap_or_default(), - attached_stream_ids: options.attached_stream_ids, - generated: options.generated.unwrap_or_default(), - }; - let header = proto::data_stream::Header { - stream_id: options.id.unwrap_or_else(|| create_random_uuid()), - timestamp: Utc::now().timestamp_millis(), - topic: options.topic, - mime_type: TEXT_MIME_TYPE.to_owned(), - total_length: None, - encryption_type: proto::encryption::Type::None.into(), - attributes: options.attributes, - content_header: Some(proto::data_stream::header::ContentHeader::TextHeader( - text_header.clone(), - )), - // Data streams v2 fields - inline_content: None, - compression: proto::data_stream::CompressionType::None as i32, - }; - let open_options = RawStreamOpenOptions { - header: header.clone(), - destination_identities: options.destination_identities, - packet_tx: self.packet_tx.clone(), - }; - let writer = TextStreamWriter { - info: Arc::new(TextStreamInfo::from_headers(header, text_header)), - stream: Arc::new(Mutex::new(RawStream::open(open_options).await?)), - }; - Ok(writer) - } - - pub async fn stream_bytes(&self, options: StreamByteOptions) -> StreamResult { - let byte_header = proto::data_stream::ByteHeader { name: options.name.unwrap_or_default() }; - let header = proto::data_stream::Header { - stream_id: options.id.unwrap_or_else(|| create_random_uuid()), - timestamp: Utc::now().timestamp_millis(), - topic: options.topic, - mime_type: options.mime_type.unwrap_or_else(|| BYTE_MIME_TYPE.to_owned()), - total_length: options.total_length, - encryption_type: proto::encryption::Type::None.into(), - attributes: options.attributes, - content_header: Some(proto::data_stream::header::ContentHeader::ByteHeader( - byte_header.clone(), - )), - // Data streams v2 fields - inline_content: None, - compression: proto::data_stream::CompressionType::None as i32, - }; - - let open_options = RawStreamOpenOptions { - header: header.clone(), - destination_identities: options.destination_identities, - packet_tx: self.packet_tx.clone(), - }; - let writer = ByteStreamWriter { - info: Arc::new(ByteStreamInfo::from_headers(header, byte_header)), - stream: Arc::new(Mutex::new(RawStream::open(open_options).await?)), - }; - Ok(writer) - } - - pub async fn send_text( - &self, - text: &str, - options: StreamTextOptions, - ) -> StreamResult { - let text_header = proto::data_stream::TextHeader { - operation_type: options.operation_type.unwrap_or_default() as i32, - version: options.version.unwrap_or_default(), - reply_to_stream_id: options.reply_to_stream_id.unwrap_or_default(), - attached_stream_ids: options.attached_stream_ids, - generated: options.generated.unwrap_or_default(), - }; - let header = proto::data_stream::Header { - stream_id: options.id.unwrap_or_else(|| create_random_uuid()), - timestamp: Utc::now().timestamp_millis(), - topic: options.topic, - mime_type: TEXT_MIME_TYPE.to_owned(), - total_length: Some(text.bytes().len() as u64), - encryption_type: proto::encryption::Type::None.into(), - attributes: options.attributes, - content_header: Some(proto::data_stream::header::ContentHeader::TextHeader( - text_header.clone(), - )), - // Data streams v2 fields - inline_content: None, - compression: proto::data_stream::CompressionType::None as i32, - }; - let open_options = RawStreamOpenOptions { - header: header.clone(), - destination_identities: options.destination_identities, - packet_tx: self.packet_tx.clone(), - }; - let writer = TextStreamWriter { - info: Arc::new(TextStreamInfo::from_headers(header, text_header)), - stream: Arc::new(Mutex::new(RawStream::open(open_options).await?)), - }; - - let info = (*writer.info).clone(); - writer.write(text).await?; - writer.close().await?; - - Ok(info) - } - - /// Send bytes to participants in the room. - /// - /// This method sends an in-memory blob of bytes to participants in the room - /// as a byte stream. It opens a stream using the provided options, writes the - /// entire buffer, and closes the stream before returning. - /// - /// The `total_length` in the header is set from the provided data and is not - /// overridable by `options.total_length`. - pub async fn send_bytes( - &self, - data: impl AsRef<[u8]>, - options: StreamByteOptions, - ) -> StreamResult { - if options.total_length.is_some() { - log::warn!("Ignoring total_length option specified for send_bytes"); - } - let bytes = data.as_ref(); - - let byte_header = proto::data_stream::ByteHeader { name: options.name.unwrap_or_default() }; - let header = proto::data_stream::Header { - stream_id: options.id.unwrap_or_else(|| create_random_uuid()), - timestamp: Utc::now().timestamp_millis(), - topic: options.topic, - mime_type: options.mime_type.unwrap_or_else(|| BYTE_MIME_TYPE.to_owned()), - total_length: Some(bytes.len() as u64), // not overridable - encryption_type: proto::encryption::Type::None.into(), - attributes: options.attributes, - content_header: Some(proto::data_stream::header::ContentHeader::ByteHeader( - byte_header.clone(), - )), - // Data streams v2 fields - inline_content: None, - compression: proto::data_stream::CompressionType::None as i32, - }; - - let open_options = RawStreamOpenOptions { - header: header.clone(), - destination_identities: options.destination_identities, - packet_tx: self.packet_tx.clone(), - }; - let writer = ByteStreamWriter { - info: Arc::new(ByteStreamInfo::from_headers(header, byte_header)), - stream: Arc::new(Mutex::new(RawStream::open(open_options).await?)), - }; - - let info = (*writer.info).clone(); - writer.write(bytes).await?; - writer.close().await?; - - Ok(info) - } - - pub async fn send_file( - &self, - path: impl AsRef, - options: StreamByteOptions, - ) -> StreamResult { - let file_size = tokio::fs::metadata(path.as_ref()) - .await - .map(|metadata| metadata.len()) - .map_err(|e| StreamError::from(e))?; - let name = - path.as_ref().file_name().and_then(|n| n.to_str()).unwrap_or_default().to_owned(); - - let byte_header = proto::data_stream::ByteHeader { name }; - let header = proto::data_stream::Header { - stream_id: options.id.unwrap_or_else(|| create_random_uuid()), - timestamp: Utc::now().timestamp_millis(), - topic: options.topic, - mime_type: options.mime_type.unwrap_or_else(|| BYTE_MIME_TYPE.to_owned()), - total_length: Some(file_size as u64), // not overridable - encryption_type: proto::encryption::Type::None.into(), - attributes: options.attributes, - content_header: Some(proto::data_stream::header::ContentHeader::ByteHeader( - byte_header.clone(), - )), - // Data streams v2 fields - inline_content: None, - compression: proto::data_stream::CompressionType::None as i32, - }; - - let open_options = RawStreamOpenOptions { - header: header.clone(), - destination_identities: options.destination_identities, - packet_tx: self.packet_tx.clone(), - }; - let writer = ByteStreamWriter { - info: Arc::new(ByteStreamInfo::from_headers(header, byte_header)), - stream: Arc::new(Mutex::new(RawStream::open(open_options).await?)), - }; - - let info = (*writer.info).clone(); - writer.write_file_contents(path).await?; - writer.close().await?; - - Ok(info) - } -} - -/// Maximum number of bytes to send in a single chunk. -static CHUNK_SIZE: usize = 15000; - -// Default MIME type to use for byte streams. -static BYTE_MIME_TYPE: &str = "application/octet-stream"; - -/// Default MIME type to use for text streams. -static TEXT_MIME_TYPE: &str = "text/plain"; - -#[cfg(test)] -mod tests { - use super::*; - - // Regression test for CLT-2773: dropping a `RawStream` on a thread that has - // no Tokio runtime in TLS (e.g. the .NET GC finalizer thread in the Unity - // SDK) used to panic because `Drop` called `tokio::spawn` unconditionally. - #[test] - fn drop_raw_stream_on_non_tokio_thread_does_not_panic() { - let rt = tokio::runtime::Runtime::new().unwrap(); - - let raw_stream = rt.block_on(async { - let (packet_tx, mut packet_rx) = - bmrng::unbounded_channel::>(); - - tokio::spawn(async move { - while let Ok((_packet, responder)) = packet_rx.recv().await { - let _ = responder.respond(Ok(())); - } - }); - - let header = proto::data_stream::Header { - stream_id: "gc-test-stream".to_string(), - timestamp: 0, - topic: "gc-test-topic".to_string(), - mime_type: TEXT_MIME_TYPE.to_owned(), - total_length: None, - encryption_type: proto::encryption::Type::None.into(), - attributes: HashMap::new(), - content_header: None, - // Data streams v2 fields - inline_content: None, - compression: proto::data_stream::CompressionType::None as i32, - }; - - RawStream::open(RawStreamOpenOptions { - header, - destination_identities: vec![], - packet_tx, - }) - .await - .expect("RawStream should open") - }); - - let drop_thread = std::thread::spawn(move || drop(raw_stream)); - - drop_thread.join().expect("Dropping RawStream on a non-Tokio thread must not panic"); - } -} diff --git a/livekit/src/room/e2ee/mod.rs b/livekit/src/room/e2ee/mod.rs index e1235d81d..43d4a908c 100644 --- a/livekit/src/room/e2ee/mod.rs +++ b/livekit/src/room/e2ee/mod.rs @@ -22,13 +22,7 @@ pub mod manager; /// Provider implementations for data track. pub(crate) mod data_track; -#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] -pub enum EncryptionType { - #[default] - None, - Gcm, - Custom, -} +pub use livekit_common::EncryptionType; #[derive(Clone)] pub struct E2eeOptions { diff --git a/livekit/src/room/id.rs b/livekit/src/room/id.rs index 800496371..c1ea11a74 100644 --- a/livekit/src/room/id.rs +++ b/livekit/src/room/id.rs @@ -20,30 +20,17 @@ const ROOM_PREFIX: &str = "RM_"; const PARTICIPANT_PREFIX: &str = "PA_"; const TRACK_PREFIX: &str = "TR_"; +pub use livekit_common::ParticipantIdentity; + #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct ParticipantSid(String); -#[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] -pub struct ParticipantIdentity(pub String); - #[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct TrackSid(String); #[derive(Clone, Default, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct RoomSid(String); -impl From for ParticipantIdentity { - fn from(value: String) -> Self { - Self(value) - } -} - -impl From<&str> for ParticipantIdentity { - fn from(value: &str) -> Self { - Self(value.to_string()) - } -} - macro_rules! impl_string_into { ($from:ty) => { impl From<$from> for String { @@ -67,7 +54,6 @@ macro_rules! impl_string_into { } impl_string_into!(ParticipantSid); -impl_string_into!(ParticipantIdentity); impl_string_into!(TrackSid); impl_string_into!(RoomSid); diff --git a/livekit/src/room/mod.rs b/livekit/src/room/mod.rs index 6cfd57605..8695b5780 100644 --- a/livekit/src/room/mod.rs +++ b/livekit/src/room/mod.rs @@ -48,7 +48,7 @@ use tokio::sync::{ pub use self::{ data_stream::*, e2ee::{manager::E2eeManager, E2eeOptions}, - participant::{ParticipantKind, ParticipantKindDetail, ParticipantState}, + participant::{ClientCapability, ParticipantKind, ParticipantKindDetail, ParticipantState}, }; pub use crate::rtc_engine::SimulateScenario; use crate::{ @@ -63,7 +63,7 @@ use crate::{ utils::{observer::Dispatcher, promise::Promise}, }; -pub mod data_stream; +pub use livekit_data_stream as data_stream; pub mod data_track; pub mod e2ee; pub mod id; @@ -577,6 +577,7 @@ impl Room { e2ee_manager.encryption_type(), pi.permission, pi.client_protocol, + pi.capabilities.iter().filter_map(|&c| ClientCapability::try_from(c).ok()).collect(), ); let dispatcher = Dispatcher::::default(); @@ -679,7 +680,8 @@ impl Room { let (remote_dt_manager, remote_dt_input, remote_dt_output) = dt::remote::Manager::new(remote_dt_options); - let (incoming_stream_manager, open_rx) = IncomingStreamManager::new(); + let (incoming_stream_manager, open_rx) = + IncomingStreamManager::new(INTERNAL_DATA_STREAM_TOPICS.into()); let (outgoing_stream_manager, packet_rx) = OutgoingStreamManager::new(); let room_info = join_response.room.unwrap(); @@ -759,6 +761,10 @@ impl Room { pi.joined_at_ms, pi.permission, pi.client_protocol, + pi.capabilities + .iter() + .filter_map(|&c| ClientCapability::try_from(c).ok()) + .collect(), ) }; participant.update_info(pi.clone()); @@ -1195,6 +1201,10 @@ impl RoomSession { pi.joined_at_ms, pi.permission, pi.client_protocol, + pi.capabilities + .iter() + .filter_map(|&c| ClientCapability::try_from(c).ok()) + .collect(), ) }; @@ -1796,7 +1806,7 @@ impl RoomSession { participant_identity: String, encryption_type: proto::encryption::Type, ) { - let is_internal = data_stream::is_internal_topic(&header.topic); + let is_internal = self.incoming_stream_manager.is_internal_topic(&header.topic); self.incoming_stream_manager.handle_header( header.clone(), participant_identity.clone(), @@ -2000,6 +2010,7 @@ impl RoomSession { joined_at: i64, permission: Option, client_protocol: i32, + capabilities: Vec, ) -> RemoteParticipant { let participant = RemoteParticipant::new( self.rtc_engine.clone(), @@ -2015,6 +2026,7 @@ impl RoomSession { self.options.auto_subscribe, permission, client_protocol, + capabilities, ); participant.on_track_published({ @@ -2143,6 +2155,12 @@ impl RoomSession { let mut participants = self.remote_participants.write(); participants.remove(&remote_participant.identity()); + drop(participants); + + // Terminate any data streams this participant was still sending; otherwise their + // readers would hang waiting for chunks that will never arrive. + self.incoming_stream_manager.abort_streams_from(remote_participant.identity().as_str()); + self.dispatcher.dispatch(&RoomEvent::ParticipantDisconnected(remote_participant)); } @@ -2233,6 +2251,29 @@ impl RoomSession { } } +impl livekit_common::RemoteParticipantRegistry for RoomSession { + fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32 { + self.get_remote_client_protocol(identity) + } + + fn remote_capabilities(&self, identity: &ParticipantIdentity) -> Vec { + self.remote_participants.read().get(identity).map(|p| p.capabilities()).unwrap_or_default() + } + + fn remote_identities(&self) -> Vec { + self.remote_participants.read().keys().cloned().collect() + } +} + +/// Data stream topics reserved for internal SDK use (e.g. RPC). Events for these topics are +/// handled within the `livekit` crate and never surfaced through `RoomEvent`; the list is also +/// passed to `IncomingStreamManager` so it can flag internal streams. +const INTERNAL_DATA_STREAM_TOPICS: &[&str] = &[rpc::RPC_REQUEST_TOPIC, rpc::RPC_RESPONSE_TOPIC]; + +fn is_internal_topic(topic: &str) -> bool { + INTERNAL_DATA_STREAM_TOPICS.contains(&topic) +} + /// Receives stream readers for newly-opened streams and dispatches room events. /// /// Intercepts text streams on RPC topics (`lk.rpc_request`, `lk.rpc_response`) @@ -2249,7 +2290,7 @@ async fn incoming_data_stream_task( match reader { AnyStreamReader::Byte(reader) => { let topic = reader.info().topic.clone(); - if !data_stream::is_internal_topic(&topic) { + if !is_internal_topic(&topic) { dispatcher.dispatch(&RoomEvent::ByteStreamOpened { topic, reader: TakeCell::new(reader), @@ -2279,7 +2320,7 @@ async fn incoming_data_stream_task( }); } _ => { - if !data_stream::is_internal_topic(&topic) { + if !is_internal_topic(&topic) { dispatcher.dispatch(&RoomEvent::TextStreamOpened { topic, reader: TakeCell::new(reader), @@ -2300,14 +2341,19 @@ async fn incoming_data_stream_task( /// Receives packets from the outgoing stream manager and send them. async fn outgoing_data_stream_task( - mut packet_rx: UnboundedRequestReceiver>, + mut packet_rx: UnboundedRequestReceiver>, engine: Arc, mut close_rx: broadcast::Receiver<()>, ) { loop { tokio::select! { Ok((packet, responder)) = packet_rx.recv() => { - let result = engine.publish_data(packet, DataPacketKind::Reliable, false).await; + // Bridge the engine error into the data-stream crate's opaque `SendError` + // (the crate only needs to know whether the send failed). + let result = engine + .publish_data(packet, DataPacketKind::Reliable, false) + .await + .map_err(|_| SendError); let _ = responder.respond(result); }, _ = close_rx.recv() => { diff --git a/livekit/src/room/participant/local_participant.rs b/livekit/src/room/participant/local_participant.rs index b02e01e97..5cd04f7b4 100644 --- a/livekit/src/room/participant/local_participant.rs +++ b/livekit/src/room/participant/local_participant.rs @@ -23,8 +23,8 @@ use std::{ }; use super::{ - ConnectionQuality, ParticipantInner, ParticipantKind, ParticipantKindDetail, ParticipantState, - ParticipantTrackPermission, + ClientCapability, ConnectionQuality, ParticipantInner, ParticipantKind, ParticipantKindDetail, + ParticipantState, ParticipantTrackPermission, }; use crate::{ data_stream::{ @@ -110,6 +110,7 @@ impl LocalParticipant { encryption_type: EncryptionType, permission: Option, client_protocol: i32, + capabilities: Vec, ) -> Self { Self { inner: super::new_inner( @@ -125,6 +126,7 @@ impl LocalParticipant { joined_at, permission, client_protocol, + capabilities, ), local: Arc::new(LocalInfo { events: LocalEvents::default(), @@ -951,7 +953,8 @@ impl LocalParticipant { text: &str, options: StreamTextOptions, ) -> StreamResult { - self.session().unwrap().outgoing_stream_manager.send_text(text, options).await + let session = self.session().unwrap(); + session.outgoing_stream_manager.send_text(text, options, session.as_ref()).await } /// Send a file on disk to participants in the room. @@ -971,7 +974,8 @@ impl LocalParticipant { path: impl AsRef, options: StreamByteOptions, ) -> StreamResult { - self.session().unwrap().outgoing_stream_manager.send_file(path, options).await + let session = self.session().unwrap(); + session.outgoing_stream_manager.send_file(path, options, session.as_ref()).await } /// Send an in-memory blob of bytes to participants in the room. @@ -988,7 +992,8 @@ impl LocalParticipant { data: impl AsRef<[u8]>, options: StreamByteOptions, ) -> StreamResult { - self.session().unwrap().outgoing_stream_manager.send_bytes(data, options).await + let session = self.session().unwrap(); + session.outgoing_stream_manager.send_bytes(data, options, session.as_ref()).await } /// Stream text incrementally to participants in the room. diff --git a/livekit/src/room/participant/mod.rs b/livekit/src/room/participant/mod.rs index 5dcdc83d7..9f8abad0e 100644 --- a/livekit/src/room/participant/mod.rs +++ b/livekit/src/room/participant/mod.rs @@ -85,6 +85,8 @@ pub enum DisconnectReason { AgentError, } +pub use livekit_common::ClientCapability; + #[derive(Debug, Clone)] pub enum Participant { Local(LocalParticipant), @@ -146,6 +148,7 @@ struct ParticipantInfo { pub joined_at: i64, pub permission: Option, pub client_protocol: i32, + pub capabilities: Vec, } type TrackMutedHandler = Box; @@ -197,6 +200,7 @@ pub(super) fn new_inner( joined_at: i64, permission: Option, client_protocol: i32, + capabilities: Vec, ) -> Arc { Arc::new(ParticipantInner { rtc_engine, @@ -216,6 +220,7 @@ pub(super) fn new_inner( joined_at, permission, client_protocol, + capabilities, }), track_publications: Default::default(), events: Default::default(), @@ -269,6 +274,8 @@ pub(super) fn update_info( } info.client_protocol = new_info.client_protocol; + info.capabilities = + new_info.capabilities.iter().filter_map(|&c| ClientCapability::try_from(c).ok()).collect(); } pub(super) fn set_speaking( diff --git a/livekit/src/room/participant/remote_participant.rs b/livekit/src/room/participant/remote_participant.rs index da0f6a663..6909cdc0d 100644 --- a/livekit/src/room/participant/remote_participant.rs +++ b/livekit/src/room/participant/remote_participant.rs @@ -25,8 +25,8 @@ use livekit_runtime::timeout; use parking_lot::Mutex; use super::{ - ConnectionQuality, ParticipantInner, ParticipantKind, ParticipantKindDetail, ParticipantState, - TrackKind, + ClientCapability, ConnectionQuality, ParticipantInner, ParticipantKind, ParticipantKindDetail, + ParticipantState, TrackKind, }; use crate::{prelude::*, rtc_engine::RtcEngine, track::TrackError}; @@ -86,6 +86,7 @@ impl RemoteParticipant { auto_subscribe: bool, permission: Option, client_protocol: i32, + capabilities: Vec, ) -> Self { Self { inner: super::new_inner( @@ -101,6 +102,7 @@ impl RemoteParticipant { joined_at, permission, client_protocol, + capabilities, ), remote: Arc::new(RemoteInfo { events: Default::default(), auto_subscribe }), } @@ -577,6 +579,11 @@ impl RemoteParticipant { self.inner.info.read().client_protocol } + /// The capabilities this remote participant's client advertised at join. + pub fn capabilities(&self) -> Vec { + self.inner.info.read().capabilities.clone() + } + pub fn is_encrypted(&self) -> bool { *self.inner.is_encrypted.read() } diff --git a/livekit/src/room/rpc/mod.rs b/livekit/src/room/rpc/mod.rs index 8f14647fa..bc5f55bce 100644 --- a/livekit/src/room/rpc/mod.rs +++ b/livekit/src/room/rpc/mod.rs @@ -23,6 +23,8 @@ pub use server::{HandleRequestOptions, RpcServerManager}; use crate::data_stream::{StreamResult, StreamTextOptions, TextStreamInfo}; use crate::room::id::ParticipantIdentity; +use crate::room::participant::ClientCapability; +use livekit_common::RemoteParticipantRegistry; use livekit_protocol::RpcError as RpcError_Proto; use std::{error::Error, fmt::Display, future::Future, time::Duration}; @@ -45,7 +47,7 @@ pub(crate) const ATTR_VERSION: &str = "lk.rpc_request_version"; /// /// Decouples the RPC managers from concrete engine/session types, /// enabling in-memory unit testing with a mock transport. -pub(crate) trait RpcTransport: Send + Sync { +pub(crate) trait RpcTransport: RemoteParticipantRegistry { /// Send a data packet (used for v1 RPC packets and ACKs). fn publish_data( &self, @@ -59,9 +61,6 @@ pub(crate) trait RpcTransport: Send + Sync { options: StreamTextOptions, ) -> impl Future> + Send; - /// Look up a remote participant's client_protocol value. - fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32; - /// Get the server version string, if available. fn server_version(&self) -> Option; } @@ -69,6 +68,20 @@ pub(crate) trait RpcTransport: Send + Sync { /// Production implementation of `RpcTransport` backed by a `RoomSession`. pub(crate) struct SessionTransport(pub(crate) std::sync::Arc); +impl RemoteParticipantRegistry for SessionTransport { + fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32 { + self.0.remote_client_protocol(identity) + } + + fn remote_capabilities(&self, identity: &ParticipantIdentity) -> Vec { + self.0.remote_capabilities(identity) + } + + fn remote_identities(&self) -> Vec { + self.0.remote_identities() + } +} + impl RpcTransport for SessionTransport { async fn publish_data( &self, @@ -86,11 +99,7 @@ impl RpcTransport for SessionTransport { text: &str, options: StreamTextOptions, ) -> StreamResult { - self.0.outgoing_stream_manager.send_text(text, options).await - } - - fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32 { - self.0.get_remote_client_protocol(identity) + self.0.outgoing_stream_manager.send_text(text, options, self.0.as_ref()).await } fn server_version(&self) -> Option { diff --git a/livekit/src/room/rpc/tests.rs b/livekit/src/room/rpc/tests.rs index b4fc3d5e9..8ba573c17 100644 --- a/livekit/src/room/rpc/tests.rs +++ b/livekit/src/room/rpc/tests.rs @@ -18,10 +18,12 @@ use crate::data_stream::{ }; use crate::e2ee::EncryptionType; use crate::room::id::ParticipantIdentity; +use crate::room::participant::ClientCapability; use crate::room::RoomError; use bytes::Bytes; use chrono::Utc; use livekit_api::signal_client::{CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DEFAULT}; +use livekit_common::RemoteParticipantRegistry; use livekit_protocol as proto; use parking_lot::Mutex as ParkingMutex; use std::collections::HashMap; @@ -132,15 +134,29 @@ impl RpcTransport for MockTransport { attached_stream_ids: vec![], generated: false, encryption_type: EncryptionType::None, + #[cfg(feature = "__lk-e2e-test")] + is_compressed: false, + #[cfg(feature = "__lk-e2e-test")] + is_inline: false, }) } + fn server_version(&self) -> Option { + self.server_ver.clone() + } +} + +impl RemoteParticipantRegistry for MockTransport { fn remote_client_protocol(&self, identity: &ParticipantIdentity) -> i32 { self.remote_protocols.get(&identity.0).copied().unwrap_or(CLIENT_PROTOCOL_DEFAULT) } - fn server_version(&self) -> Option { - self.server_ver.clone() + fn remote_capabilities(&self, _identity: &ParticipantIdentity) -> Vec { + Vec::new() + } + + fn remote_identities(&self) -> Vec { + self.remote_protocols.keys().map(|k| ParticipantIdentity(k.clone())).collect() } } @@ -170,6 +186,10 @@ fn make_text_reader( attached_stream_ids: vec![], generated: false, encryption_type: EncryptionType::None, + #[cfg(feature = "__lk-e2e-test")] + is_compressed: false, + #[cfg(feature = "__lk-e2e-test")] + is_inline: false, }, rx, ) diff --git a/livekit/src/utils/mod.rs b/livekit/src/utils/mod.rs index 2c382fa7a..1590cba02 100644 --- a/livekit/src/utils/mod.rs +++ b/livekit/src/utils/mod.rs @@ -24,7 +24,6 @@ pub mod promise; pub mod take_cell; pub mod ttl_map; pub mod tx_queue; -pub mod utf8_chunk; pub(crate) fn convert_kind_details(kind_details: &[i32]) -> Vec { kind_details diff --git a/livekit/src/utils/take_cell.rs b/livekit/src/utils/take_cell.rs index c22967ea1..6f3b686ab 100644 --- a/livekit/src/utils/take_cell.rs +++ b/livekit/src/utils/take_cell.rs @@ -26,18 +26,6 @@ impl TakeCell { Self { value: Arc::new(RwLock::new(Some(value))) } } - /// Take ownership of the value in the cell if it matches some predicate. - /// - /// This method will only take the value if the provided predicate returns `true` when called with the current value. - /// If the predicate returns `false` or the value has already been taken, this method returns `None`. - pub(crate) fn take_if_raw(&self, predicate: impl FnOnce(&T) -> bool) -> Option { - if self.value.read().as_ref().map_or(false, |v| predicate(v)) { - self.take() - } else { - None - } - } - /// Take ownership of the value in the cell. If the value has, /// already been taken, the result is `None`. pub fn take(&self) -> Option { @@ -83,14 +71,6 @@ mod tests { assert_eq!(cell.is_taken(), true); } - #[test] - fn test_take_if_raw() { - let cell = TakeCell::new(1); - assert_eq!(cell.take_if_raw(|value| *value == 2), None); - assert_eq!(cell.take_if_raw(|value| *value == 1), Some(1)); - assert_eq!(cell.take_if_raw(|value| *value == 1), None); - } - #[test] fn test_debug() { let cell = TakeCell::new(1); diff --git a/livekit/tests/data_stream_test.rs b/livekit/tests/data_stream_test.rs index 911a37548..882bd24a6 100644 --- a/livekit/tests/data_stream_test.rs +++ b/livekit/tests/data_stream_test.rs @@ -18,6 +18,7 @@ use { anyhow::{anyhow, Ok, Result}, chrono::{TimeDelta, Utc}, livekit::{RoomEvent, StreamByteOptions, StreamReader, StreamTextOptions}, + rand::{rngs::StdRng, RngCore, SeedableRng}, std::time::Duration, tokio::{time::timeout, try_join}, }; @@ -46,6 +47,8 @@ async fn test_send_bytes() -> Result<()> { assert!(stream_info.total_length.is_some()); assert_eq!(stream_info.mime_type, "application/octet-stream"); assert_eq!(stream_info.topic, "some-topic"); + assert_eq!(stream_info.is_compressed, true); + assert_eq!(stream_info.is_inline, true); Ok(()) }; @@ -70,6 +73,164 @@ async fn test_send_bytes() -> Result<()> { Ok(()) } +/// End-to-end round-trip of a large, somewhat-compressible text. Both peers are Rust SDK +/// clients advertising data streams v2 + deflate-raw, so the sender compresses across multiple +/// chunks and the receiver decompresses — validating the v2 compression path on the real wire. +#[cfg(feature = "__lk-e2e-test")] +#[tokio::test] +async fn test_send_large_compressible_text() -> Result<()> { + let mut rooms = test_rooms(2).await?; + let (sending_room, _) = rooms.pop().unwrap(); + let (_, mut receiving_event_rx) = rooms.pop().unwrap(); + + // ~50 KB of deterministic pseudo-random lowercase: too big to inline, compresses well + // under its raw size, exercising the chunked-compressed path. + let mut text = String::new(); + let mut state: u64 = 0x1234_5678_9abc_def0; + for _ in 0..50_000 { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + text.push((b'a' + ((state >> 33) % 26) as u8) as char); + } + let expected = text.clone(); + + let send = async move { + let options = StreamTextOptions { topic: "some-topic".into(), ..Default::default() }; + let stream_info = sending_room.local_participant().send_text(&text, options).await?; + assert_eq!(stream_info.is_compressed, true); + assert_eq!(stream_info.is_inline, false); + Ok(()) + }; + let receive = async move { + while let Some(event) = receiving_event_rx.recv().await { + let RoomEvent::TextStreamOpened { reader, topic, .. } = event else { + continue; + }; + assert_eq!(topic, "some-topic"); + let reader = reader.take().ok_or_else(|| anyhow!("Failed to take reader"))?; + assert_eq!(reader.read_all().await?, expected); + break; + } + Ok(()) + }; + + timeout(Duration::from_secs(10), async { try_join!(send, receive) }).await??; + Ok(()) +} + +/// End-to-end round-trip of a large in-memory byte payload, validating the v2 byte-stream path. +#[cfg(feature = "__lk-e2e-test")] +#[tokio::test] +async fn test_send_large_incompressible_random_bytes() -> Result<()> { + let mut rooms = test_rooms(2).await?; + let (sending_room, _) = rooms.pop().unwrap(); + let (_, mut receiving_event_rx) = rooms.pop().unwrap(); + + // Uniform random bytes are genuinely incompressible: deflate cannot shrink them, so the + // send path must choose CompressionType::None. Seeded for a deterministic, reproducible test. + let mut rng = StdRng::seed_from_u64(0xC0FFEE); + let mut payload = vec![0u8; 1_800_000]; + rng.fill_bytes(&mut payload); + let expected = payload.clone(); + + let send = async move { + let options = StreamByteOptions { topic: "some-topic".into(), ..Default::default() }; + let stream_info = sending_room.local_participant().send_bytes(&payload, options).await?; + assert_eq!(stream_info.is_compressed, false, "is_compressed was not false"); + assert_eq!(stream_info.is_inline, false, "is_inline was not false"); + Ok(()) + }; + let receive = async move { + while let Some(event) = receiving_event_rx.recv().await { + let RoomEvent::ByteStreamOpened { reader, topic, .. } = event else { + continue; + }; + assert_eq!(topic, "some-topic"); + let reader = reader.take().ok_or_else(|| anyhow!("Failed to take reader"))?; + assert_eq!(reader.read_all().await?, expected); + break; + } + Ok(()) + }; + + timeout(Duration::from_secs(10), async { try_join!(send, receive) }).await??; + Ok(()) +} + +/// End-to-end round-trip of a large in-memory byte payload, validating the v2 byte-stream path. +#[cfg(feature = "__lk-e2e-test")] +#[tokio::test] +async fn test_send_large_bytes() -> Result<()> { + let mut rooms = test_rooms(2).await?; + let (sending_room, _) = rooms.pop().unwrap(); + let (_, mut receiving_event_rx) = rooms.pop().unwrap(); + + let payload: Vec = (0..50_000u32).map(|i| (i % 251) as u8).collect(); + let expected = payload.clone(); + + let send = async move { + let options = StreamByteOptions { topic: "some-topic".into(), ..Default::default() }; + let stream_info = sending_room.local_participant().send_bytes(&payload, options).await?; + assert_eq!(stream_info.is_compressed, true, "is_compressed was not true"); + assert_eq!(stream_info.is_inline, true, "is_inline was not true"); + Ok(()) + }; + let receive = async move { + while let Some(event) = receiving_event_rx.recv().await { + let RoomEvent::ByteStreamOpened { reader, topic, .. } = event else { + continue; + }; + assert_eq!(topic, "some-topic"); + let reader = reader.take().ok_or_else(|| anyhow!("Failed to take reader"))?; + assert_eq!(reader.read_all().await?, expected); + break; + } + Ok(()) + }; + + timeout(Duration::from_secs(10), async { try_join!(send, receive) }).await??; + Ok(()) +} + +/// End-to-end round-trip of a large in-memory byte payload set with compress=false doesn't +/// compress the payload +#[cfg(feature = "__lk-e2e-test")] +#[tokio::test] +async fn test_data_stream_compress_false() -> Result<()> { + let mut rooms = test_rooms(2).await?; + let (sending_room, _) = rooms.pop().unwrap(); + let (_, mut receiving_event_rx) = rooms.pop().unwrap(); + + let payload = vec![0xFFu8; 50_000]; + let expected = payload.clone(); + + let send = async move { + let options = StreamByteOptions { + topic: "some-topic".into(), + compress: Some(false), // <= Explictly disable compression + ..Default::default() + }; + let stream_info = sending_room.local_participant().send_bytes(&payload, options).await?; + assert_eq!(stream_info.is_compressed, false, "is_compressed was not false"); + assert_eq!(stream_info.is_inline, false, "is_inline was not false"); + Ok(()) + }; + let receive = async move { + while let Some(event) = receiving_event_rx.recv().await { + let RoomEvent::ByteStreamOpened { reader, topic, .. } = event else { + continue; + }; + assert_eq!(topic, "some-topic"); + let reader = reader.take().ok_or_else(|| anyhow!("Failed to take reader"))?; + assert_eq!(reader.read_all().await?, expected); + break; + } + Ok(()) + }; + + timeout(Duration::from_secs(10), async { try_join!(send, receive) }).await??; + Ok(()) +} + #[cfg(feature = "__lk-e2e-test")] #[tokio::test] async fn test_send_text() -> Result<()> {