From d08073920de13f241ab6baac507196ae7dba12e1 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Wed, 3 Dec 2025 11:40:34 -0800 Subject: [PATCH 1/2] Turbopack: Add a custom serde_self_describing Serializer/Deserializer --- turbopack/crates/turbo-bincode/src/lib.rs | 1 + .../src/serde_self_describing/de.rs | 393 ++++++++++++++++ .../src/serde_self_describing/mod.rs | 327 ++++++++++++++ .../src/serde_self_describing/ser.rs | 422 ++++++++++++++++++ 4 files changed, 1143 insertions(+) create mode 100644 turbopack/crates/turbo-bincode/src/serde_self_describing/de.rs create mode 100644 turbopack/crates/turbo-bincode/src/serde_self_describing/mod.rs create mode 100644 turbopack/crates/turbo-bincode/src/serde_self_describing/ser.rs diff --git a/turbopack/crates/turbo-bincode/src/lib.rs b/turbopack/crates/turbo-bincode/src/lib.rs index af6b399e8f59..8fe7600c9392 100644 --- a/turbopack/crates/turbo-bincode/src/lib.rs +++ b/turbopack/crates/turbo-bincode/src/lib.rs @@ -1,5 +1,6 @@ #[doc(hidden)] pub mod macro_helpers; +pub mod serde_self_describing; use std::{any::Any, ptr::copy_nonoverlapping}; diff --git a/turbopack/crates/turbo-bincode/src/serde_self_describing/de.rs b/turbopack/crates/turbo-bincode/src/serde_self_describing/de.rs new file mode 100644 index 000000000000..18acd4b1ddce --- /dev/null +++ b/turbopack/crates/turbo-bincode/src/serde_self_describing/de.rs @@ -0,0 +1,393 @@ +use std::{ + error::Error as StdError, + fmt::{self, Display}, +}; + +use bincode::{Decode, de::Decoder, error::DecodeError}; +use serde::{ + Deserializer, + de::{DeserializeSeed, EnumAccess, MapAccess, SeqAccess, VariantAccess, Visitor}, +}; + +use crate::serde_self_describing::TypeTag; + +#[derive(Debug)] +pub struct Error(pub DecodeError); + +impl serde::de::Error for Error { + fn custom(msg: T) -> Self + where + T: Display, + { + Self(DecodeError::OtherString(msg.to_string())) + } +} + +impl StdError for Error { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + Some(&self.0) + } +} + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl From for Error { + fn from(err: DecodeError) -> Self { + Self(err) + } +} + +type Result = std::result::Result; + +fn decode_tag(decoder: &mut impl Decoder) -> Result { + let tag_byte: u8 = Decode::decode(decoder)?; + Ok(TypeTag::try_from(tag_byte)?) +} + +pub struct BincodeDeserializer<'a, D> { + decoder: &'a mut D, +} + +impl<'a, D> BincodeDeserializer<'a, D> { + pub fn new(decoder: &'a mut D) -> Self { + Self { decoder } + } +} + +impl<'a, 'de, D: Decoder> Deserializer<'de> for BincodeDeserializer<'a, D> { + type Error = Error; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + let tag = decode_tag(self.decoder)?; + BincodeTaggedDeserializer { + tag, + decoder: self.decoder, + } + .deserialize_any(visitor) + } + + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string bytes + byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map struct enum identifier ignored_any + } +} + +/// Helper type used when we have already consumed the type tag +struct BincodeTaggedDeserializer<'a, D> { + tag: TypeTag, + decoder: &'a mut D, +} + +impl<'a, D> BincodeTaggedDeserializer<'a, D> { + fn new(tag: TypeTag, decoder: &'a mut D) -> Self { + Self { tag, decoder } + } +} + +impl<'a, 'de, D: Decoder> Deserializer<'de> for BincodeTaggedDeserializer<'a, D> { + type Error = Error; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + match self.tag { + TypeTag::BoolTrue => visitor.visit_bool(true), + TypeTag::BoolFalse => visitor.visit_bool(false), + TypeTag::U8 => visitor.visit_u8(Decode::decode(self.decoder)?), + TypeTag::U16 => visitor.visit_u16(Decode::decode(self.decoder)?), + TypeTag::U32 => visitor.visit_u32(Decode::decode(self.decoder)?), + TypeTag::U64 => visitor.visit_u64(Decode::decode(self.decoder)?), + TypeTag::I8 => visitor.visit_i8(Decode::decode(self.decoder)?), + TypeTag::I16 => visitor.visit_i16(Decode::decode(self.decoder)?), + TypeTag::I32 => visitor.visit_i32(Decode::decode(self.decoder)?), + TypeTag::I64 => visitor.visit_i64(Decode::decode(self.decoder)?), + TypeTag::F32 => visitor.visit_f32(Decode::decode(self.decoder)?), + TypeTag::F64 => visitor.visit_f64(Decode::decode(self.decoder)?), + TypeTag::Char => visitor.visit_char(Decode::decode(self.decoder)?), + TypeTag::String => visitor.visit_string(Decode::decode(self.decoder)?), + TypeTag::Bytes => visitor.visit_byte_buf(Decode::decode(self.decoder)?), + TypeTag::OptionNone => visitor.visit_none(), + TypeTag::OptionSome => visitor.visit_some(BincodeDeserializer::new(self.decoder)), + TypeTag::Unit | TypeTag::UnitStruct => visitor.visit_unit(), + TypeTag::NewtypeStruct => { + visitor.visit_newtype_struct(BincodeDeserializer::new(self.decoder)) + } + TypeTag::SeqSized | TypeTag::Tuple | TypeTag::TupleStruct => { + let len = Decode::decode(self.decoder)?; + visitor.visit_seq(BincodeSizedAccess::new(self.decoder, len)) + } + TypeTag::SeqUnsizedStart => visitor.visit_seq(BincodeUnsizedAccess::new(self.decoder)), + TypeTag::MapSized => { + let len = Decode::decode(self.decoder)?; + visitor.visit_map(BincodeSizedAccess::new(self.decoder, len)) + } + TypeTag::MapUnsizedStart => visitor.visit_map(BincodeUnsizedAccess::new(self.decoder)), + TypeTag::Struct => { + let len = Decode::decode(self.decoder)?; + visitor.visit_map(BincodeStructAccess::new(self.decoder, len)) + } + TypeTag::UnitVariant + | TypeTag::NewtypeVariant + | TypeTag::TupleVariant + | TypeTag::StructVariant => { + visitor.visit_enum(BincodeEnumAccess::new(self.decoder, self.tag)) + } + TypeTag::CollectionEnd => { + Err(DecodeError::Other("unexpected CollectionEnd tag").into()) + } + } + } + + serde::forward_to_deserialize_any! { + bool i8 i16 i32 i64 u8 u16 u32 u64 f32 f64 char str string bytes + byte_buf option unit unit_struct newtype_struct seq tuple + tuple_struct map struct enum identifier ignored_any + } +} + +struct BincodeSizedAccess<'a, D> { + decoder: &'a mut D, + remaining: usize, +} + +impl<'a, D> BincodeSizedAccess<'a, D> { + fn new(decoder: &'a mut D, len: usize) -> Self { + Self { + decoder, + remaining: len, + } + } +} + +impl<'a, 'de, D: Decoder> SeqAccess<'de> for BincodeSizedAccess<'a, D> { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: DeserializeSeed<'de>, + { + if self.remaining > 0 { + self.remaining -= 1; + Ok(Some( + seed.deserialize(BincodeDeserializer::new(self.decoder))?, + )) + } else { + Ok(None) + } + } + + fn size_hint(&self) -> Option { + Some(self.remaining) + } +} + +impl<'a, 'de, D: Decoder> MapAccess<'de> for BincodeSizedAccess<'a, D> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: DeserializeSeed<'de>, + { + // behaves the same as `SeqAccess` + SeqAccess::next_element_seed(self, seed) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + // we already decremented `remaining` in `next_key_seed`, just decode the value + seed.deserialize(BincodeDeserializer::new(self.decoder)) + } + + fn size_hint(&self) -> Option { + Some(self.remaining) + } +} + +struct BincodeUnsizedAccess<'a, D> { + decoder: &'a mut D, +} + +impl<'a, D> BincodeUnsizedAccess<'a, D> { + fn new(decoder: &'a mut D) -> Self { + Self { decoder } + } +} + +impl<'a, 'de, D: Decoder> SeqAccess<'de> for BincodeUnsizedAccess<'a, D> { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: DeserializeSeed<'de>, + { + let tag = decode_tag(self.decoder)?; + if tag == TypeTag::CollectionEnd { + return Ok(None); + } + Ok(Some(seed.deserialize(BincodeTaggedDeserializer::new( + tag, + self.decoder, + ))?)) + } +} + +impl<'a, 'de, D: Decoder> MapAccess<'de> for BincodeUnsizedAccess<'a, D> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: DeserializeSeed<'de>, + { + // behaves the same as `SeqAccess` + SeqAccess::next_element_seed(self, seed) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + seed.deserialize(BincodeDeserializer::new(self.decoder)) + } +} + +struct BincodeStructAccess<'a, D> { + decoder: &'a mut D, + remaining: usize, +} + +impl<'a, D> BincodeStructAccess<'a, D> { + fn new(decoder: &'a mut D, len: usize) -> Self { + Self { + decoder, + remaining: len, + } + } +} + +impl<'a, 'de, D: Decoder> MapAccess<'de> for BincodeStructAccess<'a, D> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: DeserializeSeed<'de>, + { + if self.remaining > 0 { + self.remaining -= 1; + Ok(Some(seed.deserialize(BincodeTaggedDeserializer::new( + TypeTag::String, + self.decoder, + ))?)) + } else { + Ok(None) + } + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + seed.deserialize(BincodeDeserializer::new(self.decoder)) + } + + fn size_hint(&self) -> Option { + Some(self.remaining) + } +} + +struct BincodeEnumAccess<'a, D> { + decoder: &'a mut D, + tag: TypeTag, +} + +impl<'a, D> BincodeEnumAccess<'a, D> { + fn new(decoder: &'a mut D, tag: TypeTag) -> Self { + Self { decoder, tag } + } +} + +impl<'a, 'de, D: Decoder> EnumAccess<'de> for BincodeEnumAccess<'a, D> { + type Error = Error; + type Variant = Self; + + fn variant_seed(self, seed: V) -> Result<(V::Value, Self::Variant)> + where + V: DeserializeSeed<'de>, + { + let variant_name = seed.deserialize(BincodeTaggedDeserializer::new( + TypeTag::String, + self.decoder, + ))?; + Ok((variant_name, self)) + } +} + +impl<'a, 'de, D: Decoder> VariantAccess<'de> for BincodeEnumAccess<'a, D> { + type Error = Error; + + fn unit_variant(self) -> Result<()> { + match self.tag { + TypeTag::UnitVariant => Ok(()), + _ => Err(DecodeError::Other("expected unit variant").into()), + } + } + + fn newtype_variant_seed(self, seed: T) -> Result + where + T: DeserializeSeed<'de>, + { + match self.tag { + TypeTag::NewtypeVariant => seed.deserialize(BincodeDeserializer::new(self.decoder)), + _ => Err(DecodeError::Other("expected newtype variant").into()), + } + } + + fn tuple_variant(self, expected_len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + match self.tag { + TypeTag::TupleVariant => { + let len: usize = Decode::decode(self.decoder)?; + if len != expected_len { + return Err(DecodeError::OtherString(format!( + "tuple variant length mismatch: expected {expected_len}, got {len}" + )) + .into()); + } + visitor.visit_seq(BincodeSizedAccess::new(self.decoder, len)) + } + _ => Err(DecodeError::Other("expected tuple variant").into()), + } + } + + fn struct_variant(self, fields: &'static [&'static str], visitor: V) -> Result + where + V: Visitor<'de>, + { + match self.tag { + TypeTag::StructVariant => { + let len: usize = Decode::decode(self.decoder)?; + if len != fields.len() { + return Err(DecodeError::OtherString(format!( + "struct variant field count mismatch: expected {}, got {len}", + fields.len() + )) + .into()); + } + visitor.visit_map(BincodeStructAccess::new(self.decoder, len)) + } + _ => Err(DecodeError::Other("expected struct variant").into()), + } + } +} diff --git a/turbopack/crates/turbo-bincode/src/serde_self_describing/mod.rs b/turbopack/crates/turbo-bincode/src/serde_self_describing/mod.rs new file mode 100644 index 000000000000..a8486a10eb37 --- /dev/null +++ b/turbopack/crates/turbo-bincode/src/serde_self_describing/mod.rs @@ -0,0 +1,327 @@ +//! Helpers for serializing serde-compatible types inside of a bincode [`Encode`] or [`Decode`] +//! implementation using a self-describing format. This works with [types that +//! `#[bincode(serde)]` does not support][bincode::serde#known-issues]. +//! +//! These helper functions can be used in the [`Encode`] and [`Decode`] derive macros with the +//! `#[bincode(with = "turbo_bincode]` attribute. +//! +//! [`Encode`]: bincode::Encode +//! [`Decode`]: bincode::Decode + +use bincode::{ + de::{BorrowDecoder, Decoder}, + enc::Encoder, + error::{DecodeError, EncodeError}, +}; +use serde::{Serialize, de::DeserializeOwned}; + +mod de; +mod ser; + +/// Uses a u8 representation, which is slightly more efficient than bincode's default u32 varint +/// approach for enum discriminants: +/// https://docs.rs/bincode/latest/bincode/spec/index.html#discriminant-representation +#[derive(Copy, Clone, PartialEq, Eq)] +#[repr(u8)] +enum TypeTag { + BoolTrue = 1, + BoolFalse, + U8, + U16, + U32, + U64, + I8, + I16, + I32, + I64, + F32, + F64, + Char, + String, + Bytes, + OptionNone, + OptionSome, + Unit, + UnitStruct, + UnitVariant, + NewtypeStruct, + NewtypeVariant, + SeqSized, + SeqUnsizedStart, + Tuple, + TupleStruct, + TupleVariant, + MapSized, + MapUnsizedStart, + Struct, + StructVariant, + CollectionEnd, +} + +impl TryFrom for TypeTag { + type Error = DecodeError; + + fn try_from(num: u8) -> Result { + let tag = match num { + 1 => TypeTag::BoolTrue, + 2 => TypeTag::BoolFalse, + 3 => TypeTag::U8, + 4 => TypeTag::U16, + 5 => TypeTag::U32, + 6 => TypeTag::U64, + 7 => TypeTag::I8, + 8 => TypeTag::I16, + 9 => TypeTag::I32, + 10 => TypeTag::I64, + 11 => TypeTag::F32, + 12 => TypeTag::F64, + 13 => TypeTag::Char, + 14 => TypeTag::String, + 15 => TypeTag::Bytes, + 16 => TypeTag::OptionNone, + 17 => TypeTag::OptionSome, + 18 => TypeTag::Unit, + 19 => TypeTag::UnitStruct, + 20 => TypeTag::UnitVariant, + 21 => TypeTag::NewtypeStruct, + 22 => TypeTag::NewtypeVariant, + 23 => TypeTag::SeqSized, + 24 => TypeTag::SeqUnsizedStart, + 25 => TypeTag::Tuple, + 26 => TypeTag::TupleStruct, + 27 => TypeTag::TupleVariant, + 28 => TypeTag::MapSized, + 29 => TypeTag::MapUnsizedStart, + 30 => TypeTag::Struct, + 31 => TypeTag::StructVariant, + 32 => TypeTag::CollectionEnd, + _ => { + return Err(DecodeError::OtherString(format!("invalid type tag: {num}"))); + } + }; + debug_assert_eq!(tag as u8, num); + Ok(tag) + } +} + +pub fn encode(value: &T, encoder: &mut E) -> Result<(), EncodeError> { + value + .serialize(&mut ser::BincodeSerializer::new(encoder)) + .map_err(|e| e.0) +} + +pub fn decode, T: DeserializeOwned>( + decoder: &mut D, +) -> Result { + T::deserialize(de::BincodeDeserializer::new(decoder)).map_err(|e| e.0) +} + +pub fn borrow_decode< + 'de, + Context, + D: BorrowDecoder<'de, Context = Context>, + T: serde::de::Deserialize<'de>, +>( + decoder: &mut D, +) -> Result { + T::deserialize(de::BincodeDeserializer::new(decoder)).map_err(|e| e.0) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, fmt::Debug}; + + use bincode::{Decode, Encode, decode_from_slice, encode_to_vec}; + use serde::{Deserialize, Serialize, de::DeserializeOwned}; + + fn round_trip(value: T) -> T { + #[derive(Encode)] + #[bincode(encode_bounds = "T: Serialize")] + struct EncodeWrapper<'a, T>(#[bincode(with = "crate::serde_self_describing")] &'a T); + + #[derive(Decode)] + #[bincode( + decode_bounds = "T: DeserializeOwned", + borrow_decode_bounds = "T: Deserialize<'__de>" + )] + struct DecodeWrapper(#[bincode(with = "crate::serde_self_describing")] T); + + let config = bincode::config::standard(); + + let encoded = encode_to_vec(EncodeWrapper(&value), config).unwrap(); + let (DecodeWrapper(decoded), len) = decode_from_slice(&encoded, config).unwrap(); + assert_eq!(value, decoded); + assert_eq!(len, encoded.len(), "the entire buffer must be decoded"); + decoded + } + + #[test] + fn test_primitives() { + round_trip(true); + round_trip(false); + round_trip(42u8); + round_trip(42u16); + round_trip(42u32); + round_trip(42u64); + round_trip(-42i8); + round_trip(-42i16); + round_trip(-42i32); + round_trip(-42i64); + round_trip(1.23f32); + round_trip(1.23f64); + round_trip('a'); + } + + #[test] + fn test_string() { + round_trip(String::new()); + round_trip(String::from("hello world")); + } + + #[test] + fn test_option() { + round_trip(Option::::None); + round_trip(Some(42)); + round_trip(Some(String::from("hello"))); + round_trip(Some(Some(42))); + } + + #[test] + fn test_vec() { + round_trip(Vec::::new()); + round_trip(vec![String::from("a"), String::from("b")]); + round_trip(vec![vec![1, 2], vec![3, 4]]); + round_trip(b"abc\0def".to_vec()); + } + + #[test] + fn test_tuple() { + round_trip(()); + round_trip((vec![1, 2], "hello".to_string(), Some(42), false, true)); + } + + #[test] + fn test_hashmap() { + let mut map = HashMap::new(); + map.insert("key1".to_string(), 1); + map.insert("key2".to_string(), 2); + round_trip(map); + + let empty: HashMap = HashMap::new(); + round_trip(empty); + } + + #[test] + fn test_struct() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Struct { + a: String, + b: Option, + #[serde(flatten)] + flattened: Flattened, + } + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct Flattened { + d: i32, + } + + round_trip(Struct { + a: "hello".to_string(), + b: None, + flattened: Flattened { d: 42 }, + }); + round_trip(Struct { + a: "hello".to_string(), + b: Some(42), + flattened: Flattened { d: 43 }, + }); + } + + #[test] + fn test_unit_struct() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct UnitStruct; + + round_trip(UnitStruct); + } + + #[test] + fn test_newtype_struct() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct NewtypeStruct(i32); + + round_trip(NewtypeStruct(42)); + } + + #[test] + fn test_tuple_struct() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + struct TupleStruct(i32, String, bool); + + round_trip(TupleStruct(42, "hello".to_string(), true)); + } + + #[test] + fn test_enum_unit_variants() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + enum Color { + Red, + Green, + Blue, + } + + round_trip(Color::Red); + round_trip(Color::Green); + round_trip(Color::Blue); + } + + #[test] + fn test_enum_newtype_variants() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + enum Value { + #[allow(unused)] + #[serde(skip)] + Empty(()), + Int(i32), + Text(String), + } + + round_trip(Value::Int(42)); + round_trip(Value::Text("hello".to_string())); + } + + #[test] + fn test_enum_tuple_variants() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + enum Point { + TwoD(i32, i32), + ThreeD(i32, i32, i32), + } + + round_trip(Point::TwoD(1, 2)); + round_trip(Point::ThreeD(1, 2, 3)); + } + + #[test] + fn test_enum_struct_variants() { + #[derive(Debug, PartialEq, Serialize, Deserialize)] + enum Message { + Request { id: u32, method: String }, + Response { id: u32, result: Option }, + } + + round_trip(Message::Request { + id: 1, + method: "get".to_string(), + }); + round_trip(Message::Response { + id: 1, + result: Some("ok".to_string()), + }); + round_trip(Message::Response { + id: 2, + result: None, + }); + } +} diff --git a/turbopack/crates/turbo-bincode/src/serde_self_describing/ser.rs b/turbopack/crates/turbo-bincode/src/serde_self_describing/ser.rs new file mode 100644 index 000000000000..8fcb7375d0a5 --- /dev/null +++ b/turbopack/crates/turbo-bincode/src/serde_self_describing/ser.rs @@ -0,0 +1,422 @@ +use std::{ + error::Error as StdError, + fmt::{self, Display}, +}; + +use bincode::{Encode, enc::Encoder, error::EncodeError}; +use serde::{ + Serialize, Serializer, + ser::{ + self, SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, + SerializeTupleStruct, + }, +}; + +use crate::serde_self_describing::TypeTag; + +#[derive(Debug)] +pub struct Error(pub EncodeError); + +impl serde::ser::Error for Error { + fn custom(msg: T) -> Self + where + T: Display, + { + Self(EncodeError::OtherString(msg.to_string())) + } +} + +impl StdError for Error { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + Some(&self.0) + } +} + +impl Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl From for Error { + fn from(err: EncodeError) -> Self { + Self(err) + } +} + +type Result = std::result::Result; + +pub struct BincodeSerializer { + encoder: E, +} + +impl BincodeSerializer { + pub fn new(encoder: E) -> Self { + Self { encoder } + } +} + +impl BincodeSerializer { + fn encode_tag(&mut self, tag: TypeTag) -> Result<()> { + Ok(Encode::encode(&(tag as u8), &mut self.encoder)?) + } + + fn encode_primitive(&mut self, value: T) -> Result<()> { + Ok(Encode::encode(&value, &mut self.encoder)?) + } +} + +impl<'a, E: Encoder> Serializer for &'a mut BincodeSerializer { + type Ok = (); + type Error = Error; + + type SerializeSeq = BincodeCollectionSerializer<'a, E>; + type SerializeTuple = BincodeCollectionSerializer<'a, E>; + type SerializeTupleStruct = BincodeCollectionSerializer<'a, E>; + type SerializeTupleVariant = BincodeCollectionSerializer<'a, E>; + type SerializeMap = BincodeCollectionSerializer<'a, E>; + type SerializeStruct = BincodeCollectionSerializer<'a, E>; + type SerializeStructVariant = BincodeCollectionSerializer<'a, E>; + + fn serialize_bool(self, v: bool) -> Result<()> { + if v { + self.encode_tag(TypeTag::BoolTrue) + } else { + self.encode_tag(TypeTag::BoolFalse) + } + } + + fn serialize_u8(self, v: u8) -> Result<()> { + self.encode_tag(TypeTag::U8)?; + self.encode_primitive(v) + } + + fn serialize_u16(self, v: u16) -> Result<()> { + self.encode_tag(TypeTag::U16)?; + self.encode_primitive(v) + } + + fn serialize_u32(self, v: u32) -> Result<()> { + self.encode_tag(TypeTag::U32)?; + self.encode_primitive(v) + } + + fn serialize_u64(self, v: u64) -> Result<()> { + self.encode_tag(TypeTag::U64)?; + self.encode_primitive(v) + } + + fn serialize_i8(self, v: i8) -> Result<()> { + self.encode_tag(TypeTag::I8)?; + self.encode_primitive(v) + } + + fn serialize_i16(self, v: i16) -> Result<()> { + self.encode_tag(TypeTag::I16)?; + self.encode_primitive(v) + } + + fn serialize_i32(self, v: i32) -> Result<()> { + self.encode_tag(TypeTag::I32)?; + self.encode_primitive(v) + } + + fn serialize_i64(self, v: i64) -> Result<()> { + self.encode_tag(TypeTag::I64)?; + self.encode_primitive(v) + } + + fn serialize_f32(self, v: f32) -> Result<()> { + self.encode_tag(TypeTag::F32)?; + self.encode_primitive(v) + } + + fn serialize_f64(self, v: f64) -> Result<()> { + self.encode_tag(TypeTag::F64)?; + self.encode_primitive(v) + } + + fn serialize_char(self, v: char) -> Result<()> { + self.encode_tag(TypeTag::Char)?; + self.encode_primitive(v) + } + + fn serialize_str(self, v: &str) -> Result<()> { + self.encode_tag(TypeTag::String)?; + self.encode_primitive(v) + } + + fn serialize_bytes(self, v: &[u8]) -> Result<()> { + self.encode_tag(TypeTag::Bytes)?; + self.encode_primitive(v) + } + + fn serialize_none(self) -> Result<()> { + self.encode_tag(TypeTag::OptionNone) + } + + fn serialize_some(self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.encode_tag(TypeTag::OptionSome)?; + value.serialize(self) + } + + fn serialize_unit(self) -> Result<()> { + self.encode_tag(TypeTag::Unit) + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + self.encode_tag(TypeTag::UnitStruct) + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<()> { + // We must store enum variants by name, otherwise there's a bug in serde that skipped + // variants are deserialized with the wrong index: + // - https://github.com/serde-rs/serde/issues/2614 + // - https://github.com/bincode-org/bincode/issues/184 + self.encode_tag(TypeTag::UnitVariant)?; + self.encode_primitive(variant) + } + + fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.encode_tag(TypeTag::NewtypeStruct)?; + value.serialize(self) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result<()> + where + T: ?Sized + Serialize, + { + self.encode_tag(TypeTag::NewtypeVariant)?; + self.encode_primitive(variant)?; + value.serialize(self) + } + + fn serialize_seq(self, len: Option) -> Result { + if let Some(len) = len { + self.encode_tag(TypeTag::SeqSized)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } else { + self.encode_tag(TypeTag::SeqUnsizedStart)?; + Ok(BincodeCollectionSerializer::new(self, true)) + } + } + + fn serialize_tuple(self, len: usize) -> Result { + self.encode_tag(TypeTag::Tuple)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + len: usize, + ) -> Result { + self.encode_tag(TypeTag::TupleStruct)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.encode_tag(TypeTag::TupleVariant)?; + self.encode_primitive(variant)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } + + fn serialize_map(self, len: Option) -> Result { + if let Some(len) = len { + self.encode_tag(TypeTag::MapSized)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } else { + self.encode_tag(TypeTag::MapUnsizedStart)?; + Ok(BincodeCollectionSerializer::new(self, true)) + } + } + + fn serialize_struct(self, _name: &'static str, len: usize) -> Result { + self.encode_tag(TypeTag::Struct)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + self.encode_tag(TypeTag::StructVariant)?; + self.encode_primitive(variant)?; + self.encode_primitive(len)?; + Ok(BincodeCollectionSerializer::new(self, false)) + } +} + +pub struct BincodeCollectionSerializer<'a, E> { + inner: &'a mut BincodeSerializer, + emit_end_tag: bool, +} + +impl<'a, E: Encoder> BincodeCollectionSerializer<'a, E> { + fn new(inner: &'a mut BincodeSerializer, emit_end_tag: bool) -> Self { + Self { + inner, + emit_end_tag, + } + } + + fn maybe_emit_end(&mut self) -> Result<()> { + if self.emit_end_tag { + self.inner.encode_tag(TypeTag::CollectionEnd)?; + } + Ok(()) + } +} + +impl SerializeSeq for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} + +impl SerializeTuple for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} + +impl SerializeTupleStruct for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} + +impl ser::SerializeTupleVariant for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} + +impl ser::SerializeMap for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + key.serialize(&mut *self.inner) + } + + fn serialize_value(&mut self, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} + +impl SerializeStruct for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.inner.encode_primitive(key)?; + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} + +impl SerializeStructVariant for BincodeCollectionSerializer<'_, E> { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + where + T: ?Sized + Serialize, + { + self.inner.encode_primitive(key)?; + value.serialize(&mut *self.inner) + } + + fn end(mut self) -> Result<()> { + self.maybe_emit_end() + } +} From d54eae290b598280f03f2d7723c9c6acd1a6e585 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Fri, 5 Dec 2025 13:48:23 -0800 Subject: [PATCH 2/2] Use the new self-describing format, remove serde_json wrapper --- Cargo.lock | 4 +- crates/next-core/src/next_config.rs | 38 +++++----- turbopack/crates/turbo-bincode/Cargo.toml | 1 - turbopack/crates/turbo-bincode/src/lib.rs | 70 ------------------- turbopack/crates/turbo-persistence/Cargo.toml | 6 +- .../crates/turbo-persistence/src/meta_file.rs | 26 +++++-- .../src/static_sorted_file_builder.rs | 9 +-- turbopack/crates/turbo-tasks/Cargo.toml | 1 - .../crates/turbo-tasks/src/primitives.rs | 4 +- .../turbopack-node/src/transforms/util.rs | 2 +- .../turbopack-node/src/transforms/webpack.rs | 4 +- 11 files changed, 54 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71dfff27ac0f..30e1e534bdd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9108,7 +9108,6 @@ dependencies = [ "mime", "ringmap", "serde", - "serde_json", "smallvec", "unty", ] @@ -9135,6 +9134,7 @@ name = "turbo-persistence" version = "0.1.0" dependencies = [ "anyhow", + "bincode 2.0.1", "bitfield", "byteorder", "dashmap 6.1.0", @@ -9154,6 +9154,7 @@ dependencies = [ "tempfile", "thread_local", "tracing", + "turbo-bincode", "twox-hash 2.1.0", "zstd", ] @@ -9227,7 +9228,6 @@ dependencies = [ "once_cell", "parking_lot", "pin-project-lite", - "pot", "rayon", "regex", "rustc-hash 2.1.1", diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs index 4b6a14e7d108..936e8fabf357 100644 --- a/crates/next-core/src/next_config.rs +++ b/crates/next-core/src/next_config.rs @@ -92,7 +92,7 @@ pub struct NextConfig { cache_handler: Option, #[bincode(with_serde)] cache_handlers: Option>, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] env: FxIndexMap, experimental: ExperimentalConfig, images: ImageConfig, @@ -101,12 +101,12 @@ pub struct NextConfig { react_production_profiling: Option, react_strict_mode: Option, transpile_packages: Option>, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] modularize_imports: Option>, dist_dir: RcStr, dist_dir_root: RcStr, deployment_id: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] sass_options: Option, trailing_slash: Option, asset_prefix: Option, @@ -119,9 +119,9 @@ pub struct NextConfig { output: Option, turbopack: Option, production_browser_source_maps: bool, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] output_file_tracing_includes: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] output_file_tracing_excludes: Option, // TODO: This option is not respected, it uses Turbopack's root instead. output_file_tracing_root: Option, @@ -154,9 +154,9 @@ pub struct NextConfig { http_agent_options: HttpAgentConfig, on_demand_entries: OnDemandEntriesConfig, powered_by_header: bool, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] public_runtime_config: FxIndexMap, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] server_runtime_config: FxIndexMap, static_page_generation_timeout: f64, target: Option, @@ -686,9 +686,9 @@ pub enum RemotePatternProtocol { )] #[serde(rename_all = "camelCase")] pub struct TurbopackConfig { - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] pub rules: Option>, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] pub resolve_alias: Option>, pub resolve_extensions: Option>, pub debug_ids: Option, @@ -955,7 +955,7 @@ pub struct ExperimentalConfig { /// @see [api reference](https://nextjs.org/docs/app/api-reference/next-config-js/mdxRs) mdx_rs: Option, strict_next_head: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] swc_plugins: Option>, external_middleware_rewrites_resolve: Option, scroll_restoration: Option, @@ -964,7 +964,7 @@ pub struct ExperimentalConfig { middleware_prefetch: Option, /// optimizeCss can be boolean or critters' option object /// Use Record as critters doesn't export its Option type ([link](https://github.com/GoogleChromeLabs/critters/blob/a590c05f9197b656d2aeaae9369df2483c26b072/packages/critters/src/index.d.ts)) - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] optimize_css: Option, next_script_workers: Option, web_vitals_attribution: Option>, @@ -982,7 +982,7 @@ pub struct ExperimentalConfig { adjust_font_fallbacks_with_size_adjust: Option, after: Option, app_document_preloading: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] cache_life: Option>, case_sensitive_routes: Option, cpus: Option, @@ -990,7 +990,7 @@ pub struct ExperimentalConfig { disable_optimized_loading: Option, disable_postcss_preset_env: Option, esm_externals: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] extension_alias: Option, external_dir: Option, /// If set to `false`, webpack won't fall back to polyfill Node.js modules @@ -1005,7 +1005,7 @@ pub struct ExperimentalConfig { instrumentation_hook: Option, client_trace_metadata: Option>, large_page_data_bytes: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] logging: Option, memory_based_workers_count: Option, /// Optimize React APIs for server builds. @@ -1024,7 +1024,7 @@ pub struct ExperimentalConfig { /// @internal Used by the Next.js internals only. trust_host_header: Option, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] url_imports: Option, /// This option is to enable running the Webpack build in a worker thread /// (doesn't apply to Turbopack). @@ -1395,7 +1395,7 @@ pub struct ResolveExtensions(Option>); #[turbo_tasks::value(transparent)] pub struct SwcPlugins( - #[bincode(with = "turbo_bincode::serde_json")] Vec<(RcStr, serde_json::Value)>, + #[bincode(with = "turbo_bincode::serde_self_describing")] Vec<(RcStr, serde_json::Value)>, ); #[turbo_tasks::value(transparent)] @@ -1413,7 +1413,7 @@ pub struct OptionServerActions(Option); #[turbo_tasks::value(transparent)] pub struct OptionJsonValue( - #[bincode(with = "turbo_bincode::serde_json")] pub Option, + #[bincode(with = "turbo_bincode::serde_self_describing")] pub Option, ); fn turbopack_config_documentation_link() -> RcStr { @@ -1466,7 +1466,7 @@ impl Issue for InvalidLoaderRuleRenameAsIssue { #[turbo_tasks::value(shared)] struct InvalidLoaderRuleConditionIssue { - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] condition: ConfigConditionItem, config_file_path: FileSystemPath, } @@ -2207,7 +2207,7 @@ impl NextConfig { #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Encode, Decode)] #[serde(rename_all = "camelCase")] pub struct JsConfig { - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] compiler_options: Option, } diff --git a/turbopack/crates/turbo-bincode/Cargo.toml b/turbopack/crates/turbo-bincode/Cargo.toml index 30ebc25e0a61..d556b5500993 100644 --- a/turbopack/crates/turbo-bincode/Cargo.toml +++ b/turbopack/crates/turbo-bincode/Cargo.toml @@ -17,6 +17,5 @@ indexmap = { workspace = true } mime = { workspace = true } ringmap = { workspace = true } serde = { workspace = true } -serde_json = { workspace = true } smallvec = { workspace = true } unty = { workspace = true } diff --git a/turbopack/crates/turbo-bincode/src/lib.rs b/turbopack/crates/turbo-bincode/src/lib.rs index 8fe7600c9392..a70a3a71f02d 100644 --- a/turbopack/crates/turbo-bincode/src/lib.rs +++ b/turbopack/crates/turbo-bincode/src/lib.rs @@ -473,76 +473,6 @@ pub mod mime_option { } } -/// Encode/decode as a serialized string encoded using `serde_json`. -/// -/// This encodes less efficiently than `#[bincode(with_serde)]` would, but avoids [bincode's known -/// compatibility issues][serde-issues]. Use this for infrequently-serialized types and when you're -/// unsure if the underlying type may trigger a serde compatibility issue. -/// -/// In the future this could be replaced with a more efficient serde-compatible self-describing -/// format with a compact binary representation (e.g. pot or MessagePack), but `serde_json` is -/// convenient because it avoids introducing additional dependencies. -/// -/// [serde-issues]: https://docs.rs/bincode/latest/bincode/serde/index.html#known-issues -pub mod serde_json { - use super::*; - - pub fn encode( - value: &T, - encoder: &mut E, - ) -> Result<(), EncodeError> { - let json_str = - ::serde_json::to_string(value).map_err(|e| EncodeError::OtherString(e.to_string()))?; - Encode::encode(&json_str, encoder) - } - - pub fn decode, T: serde::de::DeserializeOwned>( - decoder: &mut D, - ) -> Result { - let json_str: String = Decode::decode(decoder)?; - ::serde_json::from_str(&json_str).map_err(|e| DecodeError::OtherString(e.to_string())) - } - - pub fn borrow_decode< - 'de, - Context, - D: BorrowDecoder<'de, Context = Context>, - T: serde::de::Deserialize<'de>, - >( - decoder: &mut D, - ) -> Result { - let json_str: &str = BorrowDecode::borrow_decode(decoder)?; - ::serde_json::from_str(json_str).map_err(|e| DecodeError::OtherString(e.to_string())) - } - - #[cfg(test)] - mod tests { - use ::serde_json::{Value, json}; - use bincode::{decode_from_slice, encode_to_vec}; - - use super::*; - - #[test] - fn test_roundtrip() { - let cfg = bincode::config::standard(); - - #[derive(Encode, Decode)] - struct Wrapper(#[bincode(with = "crate::serde_json")] Value); - - let value1 = Wrapper(json!({ - "key1": [1, 2, 3], - "key2": [4, 5, 6] - })); - - let value2: Wrapper = decode_from_slice(&encode_to_vec(&value1, cfg).unwrap(), cfg) - .unwrap() - .0; - - assert_eq!(value1.0, value2.0); - } - } -} - pub mod either { use ::either::Either; diff --git a/turbopack/crates/turbo-persistence/Cargo.toml b/turbopack/crates/turbo-persistence/Cargo.toml index e25a637f0ce0..f7da512015cd 100644 --- a/turbopack/crates/turbo-persistence/Cargo.toml +++ b/turbopack/crates/turbo-persistence/Cargo.toml @@ -14,22 +14,24 @@ verbose_log = [] [dependencies] anyhow = { workspace = true } +bincode = { workspace = true } bitfield = { workspace = true } +byteorder = { workspace = true } dashmap = { workspace = true} either = { workspace = true } -pot = "3.0.0" -byteorder = { workspace = true } jiff = "0.2.10" lzzzz = "1.1.0" memmap2 = "0.9.5" nohash-hasher = { workspace = true } parking_lot = { workspace = true } +pot = "3.0.0" qfilter = { version = "0.2.4", features = ["serde"] } quick_cache = { workspace = true } rustc-hash = { workspace = true } smallvec = { workspace = true } thread_local = { workspace = true } tracing = { workspace = true } +turbo-bincode= { workspace = true } twox-hash = { workspace = true } zstd = { version = "0.13.2", features = ["zdict_builder"] } diff --git a/turbopack/crates/turbo-persistence/src/meta_file.rs b/turbopack/crates/turbo-persistence/src/meta_file.rs index 7f279c0aec61..f75f4ffca491 100644 --- a/turbopack/crates/turbo-persistence/src/meta_file.rs +++ b/turbopack/crates/turbo-persistence/src/meta_file.rs @@ -9,12 +9,14 @@ use std::{ }; use anyhow::{Context, Result, bail}; +use bincode::{Decode, Encode}; use bitfield::bitfield; use byteorder::{BE, ReadBytesExt}; use either::Either; use memmap2::{Mmap, MmapOptions}; use quick_cache::sync::GuardResult; use rustc_hash::FxHasher; +use turbo_bincode::turbo_bincode_decode; use crate::{ QueryKey, @@ -62,6 +64,14 @@ impl Display for MetaEntryFlags { } } +/// A wrapper around [`qfilter::Filter`] that implements [`Encode`] and [`Decode`]. +#[derive(Encode, Decode)] +pub struct AmqfBincodeWrapper( + // this annotation can be replaced with `#[bincode(serde)]` once + // is resolved + #[bincode(with = "turbo_bincode::serde_self_describing")] pub qfilter::Filter, +); + pub struct MetaEntry { /// The metadata for the static sorted file. sst_data: StaticSortedFileMetaData, @@ -113,13 +123,15 @@ impl MetaEntry { pub fn deserialize_amqf(&self, meta: &MetaFile) -> Result { let amqf = self.raw_amqf(meta.amqf_data()); - pot::from_slice(amqf).with_context(|| { - format!( - "Failed to deserialize AMQF from {:08}.meta for {:08}.sst", - meta.sequence_number, - self.sequence_number() - ) - }) + Ok(turbo_bincode_decode::(amqf) + .with_context(|| { + format!( + "Failed to deserialize AMQF from {:08}.meta for {:08}.sst", + meta.sequence_number, + self.sequence_number() + ) + })? + .0) } pub fn amqf( diff --git a/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs b/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs index 89278ed5e323..c105c8484c32 100644 --- a/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs +++ b/turbopack/crates/turbo-persistence/src/static_sorted_file_builder.rs @@ -8,10 +8,11 @@ use std::{ use anyhow::{Context, Result}; use byteorder::{BE, ByteOrder, WriteBytesExt}; +use turbo_bincode::{TurboBincodeBuffer, turbo_bincode_encode}; use crate::{ compression::compress_into_buffer, - meta_file::MetaEntryFlags, + meta_file::{AmqfBincodeWrapper, MetaEntryFlags}, static_sorted_file::{ BLOCK_TYPE_INDEX, BLOCK_TYPE_KEY, KEY_BLOCK_ENTRY_TYPE_BLOB, KEY_BLOCK_ENTRY_TYPE_DELETED, KEY_BLOCK_ENTRY_TYPE_MEDIUM, KEY_BLOCK_ENTRY_TYPE_SMALL, @@ -141,7 +142,7 @@ pub fn write_static_stored_file( let meta = StaticSortedFileBuilderMeta { min_hash, max_hash, - amqf: Cow::Owned(amqf), + amqf: Cow::Owned(amqf.into_vec()), key_compression_dictionary_length: key_dict.len().try_into().unwrap(), block_count, size: file.stream_position()?, @@ -395,7 +396,7 @@ fn write_key_blocks_and_compute_amqf( key_compression_dictionary: &[u8], writer: &mut BlockWriter<'_>, buffer: &mut Vec, -) -> Result> { +) -> Result { let mut filter = qfilter::Filter::new(entries.len() as u64, AMQF_FALSE_POSITIVE_RATE) // This won't fail as we limit the number of entries per SST file .expect("Filter can't be constructed"); @@ -502,7 +503,7 @@ fn write_key_blocks_and_compute_amqf( writer.write_index_block(buffer, key_compression_dictionary)?; buffer.clear(); - Ok(pot::to_vec(&filter).expect("AMQF serialization failed")) + Ok(turbo_bincode_encode(&AmqfBincodeWrapper(filter)).expect("AMQF serialization failed")) } /// Builder for a single key block diff --git a/turbopack/crates/turbo-tasks/Cargo.toml b/turbopack/crates/turbo-tasks/Cargo.toml index 5697a6b28b57..b48e6a0bd6e8 100644 --- a/turbopack/crates/turbo-tasks/Cargo.toml +++ b/turbopack/crates/turbo-tasks/Cargo.toml @@ -56,7 +56,6 @@ turbo-tasks-hash = { workspace = true } turbo-tasks-macros = { workspace = true } turbo-tasks-malloc = { workspace = true } unsize = { workspace = true } -pot = "3.0.0" [dev-dependencies] criterion = { workspace = true, features = ["async_tokio"] } diff --git a/turbopack/crates/turbo-tasks/src/primitives.rs b/turbopack/crates/turbo-tasks/src/primitives.rs index 9557d20efaac..123d456792b6 100644 --- a/turbopack/crates/turbo-tasks/src/primitives.rs +++ b/turbopack/crates/turbo-tasks/src/primitives.rs @@ -56,7 +56,7 @@ impl ManualEncodeWrapper for JsonValueEncodeWrapper<'_> { impl Encode for JsonValueEncodeWrapper<'_> { fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { - turbo_bincode::serde_json::encode(self.0, encoder) + turbo_bincode::serde_self_describing::encode(self.0, encoder) } } @@ -72,6 +72,6 @@ impl ManualDecodeWrapper for JsonValueDecodeWrapper { impl Decode for JsonValueDecodeWrapper { fn decode>(decoder: &mut D) -> Result { - Ok(Self(turbo_bincode::serde_json::decode(decoder)?)) + Ok(Self(turbo_bincode::serde_self_describing::decode(decoder)?)) } } diff --git a/turbopack/crates/turbopack-node/src/transforms/util.rs b/turbopack/crates/turbopack-node/src/transforms/util.rs index c6e555a8789c..d58d555a2964 100644 --- a/turbopack/crates/turbopack-node/src/transforms/util.rs +++ b/turbopack/crates/turbopack-node/src/transforms/util.rs @@ -18,7 +18,7 @@ use turbopack_core::{ pub struct EmittedAsset { file: RcStr, content: RcStr, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] source_map: Option, } diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index 34047af1d32e..13e4516c2c24 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -98,7 +98,7 @@ struct WebpackLoadersProcessingResult { pub struct WebpackLoaderItem { pub loader: RcStr, #[serde(default)] - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] pub options: serde_json::Map, } @@ -364,7 +364,7 @@ enum LogType { pub struct LogInfo { time: u64, log_type: LogType, - #[bincode(with = "turbo_bincode::serde_json")] + #[bincode(with = "turbo_bincode::serde_self_describing")] args: Vec, trace: Option>>, }