diff --git a/CHANGELOG.md b/CHANGELOG.md index bb339cf..c6446be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## wasm_split_cli vfuture + +- Experimental support to emit debug sections. By default enabled via environment variable + `WASM_SPLIT_CLI_ENABLE_DWARF`. The current setup duplicates the DWARF information into + all output modules, which can lead to large split files. + ## wasm_split_helpers v0.2.3 - Propagate `debug_assertions` to the CLI with a new marker in the hidden `#[link_section]`. diff --git a/Cargo.lock b/Cargo.lock index 93b0501..007e608 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -207,6 +207,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -242,6 +248,18 @@ dependencies = [ "wasip3", ] +[[package]] +name = "gimli" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033caf0b349c518623b5396bfb2cf0bddf44f0306d543a250e5743297aafd10" +dependencies = [ + "fnv", + "hashbrown 0.17.1", + "indexmap", + "stable_deref_trait", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -517,6 +535,12 @@ version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "strsim" version = "0.11.1" @@ -693,6 +717,7 @@ version = "0.2.2" dependencies = [ "clap", "eyre", + "gimli", "lazy_static", "regex", "tempfile", diff --git a/crates/wasm_split_cli/Cargo.toml b/crates/wasm_split_cli/Cargo.toml index 7fe0990..da81748 100644 --- a/crates/wasm_split_cli/Cargo.toml +++ b/crates/wasm_split_cli/Cargo.toml @@ -19,6 +19,7 @@ wasmparser = "0.252" # bin-only depdencies clap = { version = "4.5", features = ["derive"], optional = true } tracing-subscriber = { version = "0.3", features = ["fmt"], optional = true } +gimli = { version = "0.34.0", default-features = false, features = ["std", "read", "write"] } [dev-dependencies] tempfile = "3.27.0" diff --git a/crates/wasm_split_cli/src/dep_graph.rs b/crates/wasm_split_cli/src/dep_graph.rs index 46680d3..2974e48 100644 --- a/crates/wasm_split_cli/src/dep_graph.rs +++ b/crates/wasm_split_cli/src/dep_graph.rs @@ -47,6 +47,8 @@ pub fn get_dependencies(module: &InputModule) -> Result { RelocDetails::TableNumber(details) => Some(DepNode::Table(details.index)), RelocDetails::GlobalIndex(details) => Some(DepNode::Global(details.index)), RelocDetails::TagIndex(details) => Some(DepNode::Tag(details.index)), + RelocDetails::FunctionOffset(details) => Some(DepNode::Function(details.index)), + RelocDetails::SectionOffset(_) => None, }; if let Some(target) = target { self.add_dep(a, target); @@ -146,7 +148,7 @@ fn iter_functions_with_relocs<'m>( module: &'m InputModule, ) -> impl Iterator> { let code_relocs = module.reloc_info.iter_code_relocs(); - let code_section_offset = module.reloc_info.code_section_offset(); + let code_section_offset = module.reloc_info.code_section_reloc_base(); let mut function_index = 0; code_relocs.map(move |entry| { let reloc_file_range = shift_range(entry.relocation_range()?, code_section_offset); @@ -275,7 +277,7 @@ macro_rules! emit_iter_err { fn iter_data_dependencies<'m>( module: &'m InputModule, ) -> impl Iterator>> { - let data_section_offset = module.reloc_info.data_section_offset(); + let data_section_offset = module.reloc_info.data_section_reloc_base(); let mut data_relocs = module.reloc_info.iter_data_relocs().peekable(); let mut data_symbols = module.reloc_info.data_symbols.iter().peekable(); diff --git a/crates/wasm_split_cli/src/emit.rs b/crates/wasm_split_cli/src/emit.rs index def690f..d6b0a73 100644 --- a/crates/wasm_split_cli/src/emit.rs +++ b/crates/wasm_split_cli/src/emit.rs @@ -15,8 +15,8 @@ use eyre::{anyhow, bail, Context, Result}; use tracing::{trace, warn}; use wasm_encoder::{reencode::Reencode, ConstExpr, EntityType, ProducersField, ProducersSection}; use wasmparser::{ - BinaryReader, Data, DataKind, DefinedDataSymbol, ExternalKind, Operator, - ProducersSectionReader, RelocationType, SegmentFlags, SymbolInfo, TypeRef, + Data, DataKind, DefinedDataSymbol, ExternalKind, Operator, RelocationType, SegmentFlags, + SymbolInfo, TypeRef, }; pub(crate) struct EmitState<'a> { @@ -536,10 +536,10 @@ impl DataEmitInfo { &self, symbol_index: usize, data: &DefinedDataSymbol, - ) -> Option { + ) -> Result, ()> { if data.size == 0 { // zero-sized symbols are not relocated - return None; + return Ok(None); } let segment_idx = data.index as usize; let DataSegmentEmitInfo::Ranges { @@ -551,17 +551,17 @@ impl DataEmitInfo { } = &self.per_segment[segment_idx] else { // If just copied, then its not relocated - return None; + return Ok(None); + }; + let Some(&(range_index, offset_in_range)) = range_lookup.get(&symbol_index) else { + return Err(()); }; - let &(range_index, offset_in_range) = range_lookup - .get(&symbol_index) - .expect("to find a data relocation index"); let range = &ranges[range_index]; let mut address = *base_address; address += per_output_offset[&range.in_module]; address += range.in_module_offset; address += offset_in_range; - Some(address) + Ok(Some(address)) } } @@ -579,19 +579,21 @@ pub struct OutputExport<'a> { pub index: u32, } +type ModuleFuncId = usize; + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] enum OutputFunction { DefinedFromInput { input_func_id: InputFuncId, - local_index: usize, + local_index: ModuleFuncId, }, IndirectCallShim { input_func_id: InputFuncId, - shim_index: usize, + shim_index: ModuleFuncId, }, LocalShim { input_func_id: InputFuncId, - shim_index: usize, + shim_index: ModuleFuncId, }, } @@ -607,7 +609,10 @@ struct ModuleEmitState<'a> { defined_functions: Vec, dep_to_local_index: HashMap, - local_shims: HashMap, + local_shims: HashMap, + + function_header_len: usize, + function_offset_hint: HashMap, } impl RelocTarget for ModuleEmitState<'_> { @@ -622,10 +627,18 @@ impl RelocTarget for ModuleEmitState<'_> { let Some(symbol) = details.definition else { return Ok(None); }; - Ok(self + let Ok(address) = self .emit_state .data_relocations - .find_relocated_address(details.symbol_index, symbol)) + .find_relocated_address(details.symbol_index, symbol) + else { + bail!( + "Dependency analysis error: \ + No output address for data symbol {symbol:?} \ + referenced by relocation." + ); + }; + Ok(address) } RelocDetails::TableIndex(details) => { let input_func_id = details.index; @@ -656,8 +669,8 @@ impl RelocTarget for ModuleEmitState<'_> { else { bail!( "Dependency analysis error: \ - No output function for input function {input_func_id} \ - referenced by relocation." + No output function for input function {input_func_id} \ + referenced by relocation." ); }; Ok(Some(output_func_id)) @@ -682,10 +695,13 @@ impl RelocTarget for ModuleEmitState<'_> { } Ok(None) } + _ => bail!("unexpected relocation {reloc:?} in code/data section"), } } } +mod dwarf; + impl<'a> ModuleEmitState<'a> { fn new( emit_state: &'a EmitState, @@ -879,6 +895,8 @@ impl<'a> ModuleEmitState<'a> { exports, dep_to_local_index, local_shims, + function_header_len: 0, + function_offset_hint: Default::default(), } } @@ -891,7 +909,15 @@ impl<'a> ModuleEmitState<'a> { } fn generate(&mut self) -> Result<()> { - // Encode type section + // Encode sections. These must occur in order + // most custom sections at the end. The wasm standard doesn't specify a (partial) order, but llvm does + // reference: https://github.com/llvm/llvm-project/blob/b5fa9eee6798b678fc7cb5f2b42a977932b708f9/llvm/lib/Object/WasmObjectFile.cpp#L2212-L2220 + // type < dylink + // data < linking + // linking < reloc, name + // name < producers + // producers < target_features + // we don't emit dylink, linking, reloc at this point. self.generate_type_section()?; self.generate_import_section(); self.generate_function_section(); @@ -904,10 +930,12 @@ impl<'a> ModuleEmitState<'a> { self.generate_data_count_section(); self.generate_code_section()?; self.generate_data_section()?; - self.generate_wasm_bindgen_sections(); + // our chosen order thus is: name -> producers -> target_features self.generate_name_section()?; - self.generate_target_features_section(); self.generate_producers_section()?; + self.generate_target_features_section(); + self.generate_debug_sections()?; + self.generate_wasm_bindgen_sections(); Ok(()) } @@ -1210,7 +1238,10 @@ impl<'a> ModuleEmitState<'a> { .parse_function_body(&mut section, body) .with_context(|| format!("re-encoding no-reloc func {}", input_func_id))?; } - OutputFunction::DefinedFromInput { input_func_id, .. } => { + OutputFunction::DefinedFromInput { + input_func_id, + local_index, + } => { let input_func = &self.input_module.defined_funcs [input_func_id - self.input_module.imported_funcs.len()]; let relocated_def = self @@ -1222,6 +1253,10 @@ impl<'a> ModuleEmitState<'a> { ) })?; section.raw(&relocated_def); + let offset_after = section.byte_len(); + let func_offset = offset_after - relocated_def.len(); + tracing::trace!("FunctionOffset[{}] = {}", local_index, func_offset); + self.function_offset_hint.insert(local_index, func_offset); } OutputFunction::IndirectCallShim { input_func_id, .. } => { let indirect_index = self @@ -1247,7 +1282,26 @@ impl<'a> ModuleEmitState<'a> { } } } + // We need to adjust the offsets by the header of the section. + // The section gets written as + // struct { + // id: const(10u8), + // len: uleb(_), + // // ---- offsets must be relative to this address + // count: uleb(function_count), + // // ---- offsets above are relative to this address, since we don't predict function_count + // bytes: [u8; section.byte_len()], + // } + let offset_before_section = self.output_module.len(); + let content_len = section.byte_len(); self.output_module.section(§ion); + let offset_before_content = self.output_module.len() - content_len; + + let mut target_offset = offset_before_section; + target_offset += 1; // skip id byte (constant 0xA) + target_offset += encoded_uleb_len(&self.output_module.as_slice()[target_offset..]); + self.function_header_len = offset_before_content - target_offset; + tracing::trace!("function_header_len = {}", self.function_header_len); Ok(()) } @@ -1421,46 +1475,35 @@ impl<'a> ModuleEmitState<'a> { } fn generate_wasm_bindgen_sections(&mut self) { - for custom in self.input_module.custom_sections.iter() { - if self.is_main() && custom.name == "__wasm_bindgen_unstable" { - self.output_module.section(&wasm_encoder::CustomSection { - name: custom.name.into(), - data: custom.data.into(), - }); - } + if !self.is_main() { + return; + } + for custom in &self.input_module.wasm_bindgen_unstable { + self.output_module.section(&wasm_encoder::CustomSection { + name: custom.name().into(), + data: custom.data().into(), + }); } } fn generate_target_features_section(&mut self) { - for custom in self.input_module.custom_sections.iter() { - if custom.name == "target_features" { - self.output_module.section(&wasm_encoder::CustomSection { - name: custom.name.into(), - data: custom.data.into(), - }); - } + if let Some(custom) = self.input_module.target_features.as_ref() { + self.output_module.section(&wasm_encoder::CustomSection { + name: custom.name().into(), + data: custom.data().into(), + }); } } fn generate_producers_section(&mut self) -> Result<()> { let mut producers = ProducersSection::new(); let mut produced_by = ProducersField::new(); - const PRODUCERS_NAME: &str = "producers"; const PROCESSED_BY_FIELD_NAME: &str = "processed-by"; if self.is_main() { // copy the section from input wasm, but insert ourselves - if let Some(input_producers) = self - .input_module - .custom_sections - .iter() - .find(|section| section.name == PRODUCERS_NAME) - { - let fields = ProducersSectionReader::new(BinaryReader::new( - input_producers.data, - input_producers.data_offset, - ))?; - for input_field in fields.into_iter() { + if let Some(fields) = self.input_module.producers.as_ref() { + for input_field in fields.clone().into_iter() { let input_field = input_field?; let mut field = ProducersField::new(); for entry in input_field.values.into_iter() { @@ -1480,6 +1523,25 @@ impl<'a> ModuleEmitState<'a> { self.output_module.section(&producers); Ok(()) } + + fn generate_debug_sections(&mut self) -> Result<()> { + if !self.emit_state.input_options.emit_dwarf { + return Ok(()); + } + dwarf::emit_debug_info(self) + } +} + +/// Seek the length of a uleb that is encoded at the start of the passed buffer +fn encoded_uleb_len(bytes: &[u8]) -> usize { + let mut pos = 0; + loop { + let last_byte = (bytes[pos] & 0x80) == 0; + pos += 1; + if last_byte { + return pos; + } + } } #[derive(Debug)] diff --git a/crates/wasm_split_cli/src/emit/dwarf.rs b/crates/wasm_split_cli/src/emit/dwarf.rs new file mode 100644 index 0000000..1690824 --- /dev/null +++ b/crates/wasm_split_cli/src/emit/dwarf.rs @@ -0,0 +1,406 @@ +use eyre::bail; +use eyre::ensure; +use gimli::Section; +use wasm_encoder::CustomSection; + +use crate::dep_graph::DepNode; +use crate::read::DwarfReader; +use crate::reloc; +use crate::reloc::DataDetails; +use crate::reloc::RelocDetails; +use crate::reloc::RelocInfo; +use crate::reloc::RelocTarget; + +use super::ModuleEmitState; +use super::Result; + +struct DwarfRelocTarget<'m, 'a> { + module: &'m ModuleEmitState<'a>, +} + +const RELOC_TO_TOMBSTONE_ADDRESS: Option = Some(reloc::SENTINEL_UNDEF); + +impl RelocTarget for DwarfRelocTarget<'_, '_> { + const SENTINEL_UNDEF: bool = true; + + fn fixup_reloc_entry(&self, entry: &wasmparser::RelocationEntry) -> Result> { + // Should we try and recover the function offset from some internal code map? Would be + // more effort to compute and keep up to date. We also need to read the current value + // from `data` and use that to recover the function index. + ensure!( + matches!(entry.ty, wasmparser::RelocationType::FunctionOffsetI32 | wasmparser::RelocationType::MemoryAddrI32), + "expected a FUNCTION_OFFSET or MEMORY_ADDR relocation against a (private) function or data which had its symbol scrubbed, got {entry:?}", + ); + Ok(RELOC_TO_TOMBSTONE_ADDRESS) + } + fn reloc_value(&self, reloc: RelocDetails<'_>) -> Result> { + let reloc = match reloc { + RelocDetails::GlobalIndex(_) => return self.module.reloc_value(reloc), + RelocDetails::FunctionOffset(details) => { + let local_def = self + .module + .dep_to_local_index + .get(&DepNode::Function(details.index)); + let local_offset = + local_def.and_then(|local_def| self.module.function_offset_hint.get(local_def)); + match local_offset { + Some(offset) => Some(self.module.function_header_len + offset), + None => RELOC_TO_TOMBSTONE_ADDRESS, + } + } + RelocDetails::MemoryAddr(DataDetails { + definition: None, .. + }) => None, // undefined symbols don't get relocated + RelocDetails::MemoryAddr(DataDetails { + definition: Some(symbol), + symbol_index, + .. + }) => match self + .module + .emit_state + .data_relocations + .find_relocated_address(symbol_index, symbol) + { + Ok(address) => address, + _ => RELOC_TO_TOMBSTONE_ADDRESS, + }, + // We do not move data in custom sections around. + // TODO: assert that the addressed section is indeed one of our debug sections? + RelocDetails::SectionOffset(details) => { + let _ = details; + None + } + _ => bail!("unexpected reloc in debug section: {:?}", reloc), + }; + Ok(reloc) + } +} + +fn write_relocate_dwarf_section<'a, S: Section>>( + module: &mut ModuleEmitState<'a>, + section: &S, +) -> Result> { + let reader = section.reader(); + let input_range = reader.range(); + if input_range.is_empty() { + // We emit empty sections + return Ok(vec![]); + } + let target = DwarfRelocTarget { module }; + let reloc_data = RelocInfo::get_relocated_data(module.input_module, input_range, &target)?; + module.output_module.section(&CustomSection { + name: S::section_name().into(), + data: (&reloc_data).into(), + }); + Ok(reloc_data) +} + +pub fn emit_debug_info(module: &mut ModuleEmitState<'_>) -> Result<()> { + let crate::read::DwarfState::Inline(input_dwarf) = &module.input_module.dwarf else { + return Ok(()); + }; + let mut error_writer = ErrorWriter::new(std::io::BufWriter::new(std::io::stderr())); + if module.emit_state.input_options.strict_tests && module.is_main() { + validate_info(&mut error_writer, input_dwarf.borrow(|v| *v)); + validate_line_progs(&mut error_writer, input_dwarf.borrow(|v| *v)); + if !error_writer.check_valid_and_reset() { + tracing::warn!("original debug info didn't pass validation!"); + } + } + + let mut validate = gimli::DwarfSections::default(); + // There is no strict order defined for these, but let's try to go from common to less useful + // Further, some sections are needed to successfully parse other ones, define those in order of dependency + // All section names (included commented out ones) have been taken from the table of section version numbers + // of the current DWARF 6 draft. + macro_rules! write_relocatable_section { + ($name:ident in $input:expr) => { + validate.$name = write_relocate_dwarf_section(module, &$input.$name)?.into() + }; + } + write_relocatable_section!(debug_abbrev in input_dwarf); + write_relocatable_section!(debug_info in input_dwarf); + + write_relocatable_section!(debug_line in input_dwarf); + write_relocatable_section!(debug_loc in input_dwarf); + write_relocatable_section!(debug_ranges in input_dwarf); + write_relocatable_section!(debug_str in input_dwarf); + + write_relocatable_section!(debug_addr in input_dwarf); + write_relocatable_section!(debug_aranges in input_dwarf); + // write_relocatable_section!(debug_frame in input_dwarf); // as of now not supported, no exception handling in wasm + write_relocatable_section!(debug_line_str in input_dwarf); + write_relocatable_section!(debug_loclists in input_dwarf); + write_relocatable_section!(debug_macinfo in input_dwarf); + write_relocatable_section!(debug_macro in input_dwarf); + write_relocatable_section!(debug_names in input_dwarf); + // write_relocatable_section!(debug_pubnames in input_dwarf); // old, currently unused and not present in dwarf 5+ + // write_relocatable_section!(debug_pubtypes in input_dwarf); // same as above + write_relocatable_section!(debug_rnglists in input_dwarf); + write_relocatable_section!(debug_str_offsets in input_dwarf); + // write_relocatable_section!(debug_sup in input_dwarf); // supplemental files not supported at the moment + write_relocatable_section!(debug_types in input_dwarf); + + // You can dump the contained dwarf sections with `llvm-dwarfdump` which can read wasm object files + // TODO: the debugging information is currently NOT stripped. + // Downstream tools are required to understand tombstone markers in lineprogs and we also copy all bytes into all modules + // With more processing, we could garbage collect the output and strip it further. + // This would require more work though, and seems only worth for release builds with debugging information, or misbehaving + // tools. At least one issue https://github.com/emscripten-core/emscripten/issues/23710 points out that support might + // not be universal and we could put in more effort to clean up the relocated information. + + if module.emit_state.input_options.strict_tests { + validate_info( + &mut error_writer, + validate.borrow(|v| gimli::EndianSlice::new(&v[..], gimli::LittleEndian)), + ); + validate_line_progs( + &mut error_writer, + validate.borrow(|v| gimli::EndianSlice::new(&v[..], gimli::LittleEndian)), + ); + if !error_writer.check_valid_and_reset() { + bail!("transformed debug info didn't pass validation!"); + } + } + Ok(()) +} + +struct UnitSummary { + // True if we successfully parsed all the DIEs and attributes in the compilation unit + internally_valid: bool, + offset: gimli::DebugInfoOffset, + die_offsets: Vec, + global_die_references: Vec<(gimli::UnitOffset, gimli::DebugInfoOffset)>, +} + +struct ErrorWriter { + inner: std::sync::Mutex<(W, usize)>, +} +impl ErrorWriter { + fn new(w: W) -> Self { + Self { + inner: std::sync::Mutex::new((w, 0)), + } + } + fn check_valid_and_reset(&mut self) -> bool { + let mut lock = self.inner.lock().unwrap(); + std::mem::take(&mut lock.1) == 0 + } +} + +impl std::io::Write for ErrorWriter { + fn write_fmt(&mut self, args: std::fmt::Arguments<'_>) -> std::io::Result<()> { + let mut lock = self.inner.lock().unwrap(); + writeln!(&mut lock.0, "DWARF error: {}", args)?; + lock.1 += 1; + Ok(()) + } + + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let mut lock = self.inner.lock().unwrap(); + let len = lock.0.write(buf)?; + lock.1 += 1; + Ok(len) + } + + fn flush(&mut self) -> std::io::Result<()> { + let mut lock = self.inner.lock().unwrap(); + lock.0.flush() + } +} + +fn validate_line_progs(w: &mut ErrorWriter, dwarf: gimli::Dwarf) +where + W: std::io::Write + Send, + R: gimli::Reader, +{ + use std::io::Write; + let mut line_prog_offsets = vec![]; + for unit in dwarf.units() { + let unit = unit.and_then(|unit| dwarf.unit(unit)); + let unit = match unit { + Ok(unit) => unit, + Err(err) => { + let _ = writeln!(w, "error when reading unit: {err}"); + continue; + } + }; + let address_size = unit.address_size(); + let comp_dir = &unit.comp_dir; + let comp_name = &unit.name; + let mut entries = unit.entries(); + while let Some(die) = entries.next_dfs().transpose() { + let die = match die { + Ok(die) => die, + Err(err) => { + let _ = writeln!(w, "error reading die: {err}"); + continue; + } + }; + for attr in die.attrs() { + if let gimli::AttributeValue::DebugLineRef(dlp_offset) = attr.value() { + line_prog_offsets.push(( + dlp_offset, + address_size, + comp_dir.clone(), + comp_name.clone(), + )); + } + } + } + } + for (offset, address_size, comp_dir, comp_name) in line_prog_offsets { + let program = dwarf + .debug_line + .program(offset, address_size, comp_dir, comp_name); + let program = match program { + Ok(program) => program, + Err(err) => { + let _ = writeln!(w, "error reading program 0x{:x}: {err}", offset.0); + continue; + } + }; + let mut rows = program.rows(); + while let Some(row) = rows.next_row().transpose() { + if let Err(err) = row { + let _ = writeln!(w, "invalid line in program 0x{:x}: {err}", offset.0); + } + } + } +} + +// This is almost verbatim from gimli examples, copied with slight modifications +// under Apache License. Their MIT license quotes "Copyright (c) 2015 The Rust Project Developers" +// though that is a bit questionable attribution. The Apache license has not been filled in. +fn validate_info(w: &mut ErrorWriter, dwarf: gimli::Dwarf) +where + W: std::io::Write + Send, + R: gimli::Reader, +{ + use std::io::Write; + let debug_info = &dwarf.debug_info; + let debug_abbrev = &dwarf.debug_abbrev; + + let mut units = Vec::new(); + let mut units_iter = debug_info.units(); + let mut last_offset = 0; + loop { + let u = match units_iter.next() { + Err(err) => { + let _ = writeln!( + w, + "Can't read unit header at offset {:#x}, stopping reading units: {}", + last_offset, err + ); + break; + } + Ok(None) => break, + Ok(Some(u)) => u, + }; + last_offset = u.offset().0 + u.length_including_self(); + units.push(u); + } + let process_unit = |unit: gimli::UnitHeader| -> UnitSummary { + let unit_offset = unit.debug_info_offset().unwrap(); + let mut ret = UnitSummary { + internally_valid: false, + offset: unit_offset, + die_offsets: Vec::new(), + global_die_references: Vec::new(), + }; + let abbrevs = match unit.abbreviations(debug_abbrev) { + Ok(abbrevs) => abbrevs, + Err(err) => { + let _ = writeln!(w, "Invalid abbrevs for unit {:#x}: {err}", unit_offset.0); + return ret; + } + }; + let mut entries = unit.entries_raw(&abbrevs, None).unwrap(); + let mut unit_refs = Vec::new(); + while !entries.is_empty() { + let entry_offset = entries.next_offset(); + let abbrev = match entries.read_abbreviation() { + Err(err) => { + let _ = writeln!( + w, + "Invalid DIE for unit {:#x} at DIE {:#x}: {err}", + unit_offset.0, entry_offset.0, + ); + return ret; + } + Ok(None) => continue, + Ok(Some(abbrev)) => abbrev, + }; + ret.die_offsets.push(entry_offset); + + for spec in abbrev.attributes() { + let attr = match entries.read_attribute(*spec) { + Err(err) => { + let _ = writeln!( + w, + "Invalid attribute for unit {:#x} at DIE {:#x}: {err}", + unit_offset.0, entry_offset.0, + ); + return ret; + } + Ok(attr) => attr, + }; + match attr.value() { + gimli::AttributeValue::UnitRef(offset) => { + unit_refs.push((entry_offset, offset)); + } + gimli::AttributeValue::DebugInfoRef(offset) => { + ret.global_die_references.push((entry_offset, offset)); + } + _ => (), + } + } + } + ret.internally_valid = true; + ret.die_offsets.shrink_to_fit(); + ret.global_die_references.shrink_to_fit(); + + // Check intra-unit references + for (from, to) in unit_refs { + if ret.die_offsets.binary_search(&to).is_err() { + let _ = writeln!( + w, + "Invalid intra-unit reference in unit {:#x} from DIE {:#x} to {:#x}", + unit_offset.0, from.0, to.0 + ); + } + } + + ret + }; + let processed_units = units.into_iter().map(process_unit).collect::>(); + + let check_unit = |summary: &UnitSummary| { + if !summary.internally_valid { + return; + } + for &(from, to) in summary.global_die_references.iter() { + let u = match processed_units.binary_search_by_key(&to, |v| v.offset) { + Ok(i) => &processed_units[i], + Err(i) => { + if i > 0 { + &processed_units[i - 1] + } else { + let _ = writeln!(w, "Invalid cross-unit reference in unit {:#x} from DIE {:#x} to global DIE {:#x}: no unit found", + summary.offset.0, from.0, to.0); + continue; + } + } + }; + if !u.internally_valid { + continue; + } + let to_offset = gimli::UnitOffset(to.0 - u.offset.0); + if u.die_offsets.binary_search(&to_offset).is_err() { + let _ = writeln!(w, "Invalid cross-unit reference in unit {:#x} from DIE {:#x} to global DIE {:#x}: unit at {:#x} contains no DIE {:#x}", + summary.offset.0, from.0, to.0, u.offset.0, to_offset.0); + } + } + }; + processed_units.iter().for_each(check_unit); +} diff --git a/crates/wasm_split_cli/src/graph_utils/tarjan_scc.rs b/crates/wasm_split_cli/src/graph_utils/tarjan_scc.rs index 2733e88..44257c5 100644 --- a/crates/wasm_split_cli/src/graph_utils/tarjan_scc.rs +++ b/crates/wasm_split_cli/src/graph_utils/tarjan_scc.rs @@ -67,78 +67,6 @@ impl TarjanSccResult { } } -#[cfg(test)] -mod tests { - use std::collections::{HashMap, HashSet}; - - use crate::graph_utils::tarjan_scc::SccEvent; - - use super::TarjanSccResult; - - #[test] - fn test_small_graph() { - let mut searcher = TarjanSccResult::new(); - let graph = HashMap::from([ - (0, HashSet::from([1, 2])), - (1, HashSet::from([2])), - (2, HashSet::from([1, 5])), - (3, HashSet::from([5])), - (5, HashSet::from([6])), - (6, HashSet::from([7])), - (7, HashSet::from([5])), - (99, HashSet::from([100])), - (100, HashSet::from([99])), - ]); - let roots = searcher.explore([0, 3], &graph, &HashSet::new()); - - assert_eq!( - HashSet::from([0, 1, 2, 3, 5, 6, 7]), - searcher.fully_explored.keys().cloned().collect(), - ); - let mut scc_graph = HashMap::new(); - let it = searcher.into_topsort(); - let mut it = it.iter(); - 'graph: loop { - let mut component = vec![]; - loop { - match it.next() { - None => break 'graph, - Some(&SccEvent::Member { node: _node }) => { - component.push(_node); - } - Some(&SccEvent::Next { - this, - ref out_edges, - .. - }) => { - component.sort(); - scc_graph.insert(component, (this, out_edges.clone())); - break; - } - } - } - } - let root_scc = scc_graph - .get(&[0][..]) - .expect("to find a component for [0]"); - let inner_12 = scc_graph - .get(&[1, 2][..]) - .expect("to find a component for [1, 2]"); - let inner_567 = scc_graph - .get(&[5, 6, 7][..]) - .expect("to find a component for [5, 6, 7]"); - let inner_3 = scc_graph - .get(&[3][..]) - .expect("to find a component for [3]"); - assert!(root_scc.1.contains(&inner_12.0)); - assert!(!root_scc.1.contains(&inner_567.0)); - assert!(inner_12.1.contains(&inner_567.0)); - assert!(inner_3.1.contains(&inner_567.0)); - - assert_eq!(HashSet::from([root_scc.0, inner_3.0]), roots); - } -} - enum WorkItem { Explore { vertex: Node, @@ -316,3 +244,75 @@ impl<'r, Node: Copy + Eq + Hash> TarjanState<'r, Node> { } } } + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + + use crate::graph_utils::tarjan_scc::SccEvent; + + use super::TarjanSccResult; + + #[test] + fn test_small_graph() { + let mut searcher = TarjanSccResult::new(); + let graph = HashMap::from([ + (0, HashSet::from([1, 2])), + (1, HashSet::from([2])), + (2, HashSet::from([1, 5])), + (3, HashSet::from([5])), + (5, HashSet::from([6])), + (6, HashSet::from([7])), + (7, HashSet::from([5])), + (99, HashSet::from([100])), + (100, HashSet::from([99])), + ]); + let roots = searcher.explore([0, 3], &graph, &HashSet::new()); + + assert_eq!( + HashSet::from([0, 1, 2, 3, 5, 6, 7]), + searcher.fully_explored.keys().cloned().collect(), + ); + let mut scc_graph = HashMap::new(); + let it = searcher.into_topsort(); + let mut it = it.iter(); + 'graph: loop { + let mut component = vec![]; + loop { + match it.next() { + None => break 'graph, + Some(&SccEvent::Member { node: _node }) => { + component.push(_node); + } + Some(&SccEvent::Next { + this, + ref out_edges, + .. + }) => { + component.sort(); + scc_graph.insert(component, (this, out_edges.clone())); + break; + } + } + } + } + let root_scc = scc_graph + .get(&[0][..]) + .expect("to find a component for [0]"); + let inner_12 = scc_graph + .get(&[1, 2][..]) + .expect("to find a component for [1, 2]"); + let inner_567 = scc_graph + .get(&[5, 6, 7][..]) + .expect("to find a component for [5, 6, 7]"); + let inner_3 = scc_graph + .get(&[3][..]) + .expect("to find a component for [3]"); + assert!(root_scc.1.contains(&inner_12.0)); + assert!(!root_scc.1.contains(&inner_567.0)); + assert!(inner_12.1.contains(&inner_567.0)); + assert!(inner_3.1.contains(&inner_567.0)); + + assert_eq!(HashSet::from([root_scc.0, inner_3.0]), roots); + } +} diff --git a/crates/wasm_split_cli/src/lib.rs b/crates/wasm_split_cli/src/lib.rs index 1e9af5c..18d8d7e 100644 --- a/crates/wasm_split_cli/src/lib.rs +++ b/crates/wasm_split_cli/src/lib.rs @@ -42,6 +42,11 @@ pub struct Options<'a> { /// /// Default: false pub verbose: bool, + /// Switch to transform and emit `.debug_` sections. + /// + /// This option is experimental. + /// Default: `true` if the `WASM_SPLIT_CLI_ENABLE_DWARF` environment variable is non-empty. + pub emit_dwarf: bool, /// Enables explicit tests for assumptions we make about the input wasm file during integration testing. #[doc(hidden)] pub strict_tests: bool, @@ -56,6 +61,8 @@ impl<'wasm> Options<'wasm> { link_name: "./__wasm_split.js", main_module: "./main.js", verbose: false, + emit_dwarf: std::env::var_os("WASM_SPLIT_CLI_ENABLE_DWARF") + .is_some_and(|v| !v.is_empty()), strict_tests: false, } } diff --git a/crates/wasm_split_cli/src/read.rs b/crates/wasm_split_cli/src/read.rs index ccf40c5..a9c05fe 100644 --- a/crates/wasm_split_cli/src/read.rs +++ b/crates/wasm_split_cli/src/read.rs @@ -1,7 +1,13 @@ -use eyre::{bail, Result}; -use std::collections::HashMap; +use eyre::{bail, ensure, Result}; +use gimli::DwarfSections; +use std::{ + collections::HashMap, + fmt::Debug, + ops::{Deref, Range}, +}; use wasmparser::{ - BinaryReader, Imports, NameSectionReader, Payload, Subsection, Subsections, TypeRef, + BinaryReader, CustomSectionReader, Imports, KnownCustom, NameSectionReader, Payload, + ProducersSectionReader, Subsection, Subsections, TypeRef, }; pub use wasmparser::{ Data, Element, Export, FuncType, FunctionBody, Global, Import, MemoryType, Table, TagType, @@ -15,12 +21,6 @@ use crate::{ reloc::{RelocInfo, RelocInfoParser}, }; -pub struct CustomSection<'a> { - pub name: &'a str, - pub data_offset: usize, - pub data: &'a [u8], -} - pub type FuncTypeId = usize; pub type InputFuncId = usize; pub type TableId = usize; @@ -31,6 +31,7 @@ pub type GlobalId = usize; pub type ElementId = usize; pub type DataSegmentId = usize; pub type TagId = usize; +pub type SectionId = usize; #[derive(Debug)] pub struct ImportedFunc { @@ -80,9 +81,9 @@ fn convert_indirect_name_map<'a>( } impl<'a> Names<'a> { - fn new(data: &'a [u8], original_offset: usize) -> Result { + fn from_reader(rdr: NameSectionReader<'a>) -> Result { let mut names: Self = Default::default(); - for part in NameSectionReader::new(BinaryReader::new(data, original_offset)) { + for part in rdr { use wasmparser::Name; match part? { Name::Module { name, .. } => { @@ -133,6 +134,217 @@ impl<'a> Names<'a> { pub type InputOffset = usize; pub use crate::reloc::SymbolIndex; +// We use our own struct here instead of a simple slice to track input positions and ranges +#[derive(Clone, Copy, Default)] +pub struct DwarfReader<'a> { + data: &'a [u8], + input_position: usize, +} +impl Deref for DwarfReader<'_> { + type Target = [u8]; + fn deref(&self) -> &Self::Target { + self.data + } +} +// TODO(MSRV): use std::fmt::from_fn +struct DebugByte(u8); +impl Debug for DebugByte { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:02x}", self.0) + } +} +struct DebugLen(usize); +impl Debug for DebugLen { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "...; {}", self.0) + } +} +struct DebugBytes<'a>(&'a [u8]); +impl Debug for DebugBytes<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut list = f.debug_list(); + list.entries(self.0.iter().take(8).copied().map(DebugByte)); + if self.0.len() > 8 { + list.entry(&DebugLen(self.0.len())); + } + list.finish() + } +} +impl Debug for DwarfReader<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DwarfReader") + .field("data", &DebugBytes(self.data)) + .field("pos", &self.input_position) + .finish() + } +} +impl DwarfReader<'_> { + fn offset_id_enc(&self, offset: usize) -> u64 { + self.data.as_ptr().wrapping_byte_add(offset) as u64 + } + fn offset_id_dec(id: u64) -> *const u8 { + std::ptr::null::().wrapping_byte_offset(id as isize) + } + fn offset_of_addr(&self, ptr: *const u8) -> Option { + let offset = ptr as isize - self.data.as_ptr() as isize; + // TODO(MSRV): diff.cast_unsigned/diff.strict_cast_unsigned + if offset >= 0 && offset as usize <= gimli::Reader::len(self) { + Some(offset as usize) + } else { + None + } + } + /// range in the input of the byte range + pub fn range(&self) -> Range { + self.input_position..self.input_position + self.data.len() + } +} +impl<'a> From> for DwarfReader<'a> { + fn from(custom: CustomSectionReader<'a>) -> Self { + DwarfReader { + data: custom.data(), + input_position: custom.data_offset(), + } + } +} +impl<'a> gimli::Reader for DwarfReader<'a> { + type Endian = gimli::LittleEndian; + type Offset = usize; + + fn endian(&self) -> Self::Endian { + gimli::LittleEndian + } + + fn len(&self) -> Self::Offset { + self.data.len() + } + + fn empty(&mut self) { + let _ = self.truncate(0); + } + + fn truncate(&mut self, len: Self::Offset) -> gimli::Result<()> { + let prefix = self.split(len)?; + *self = prefix; + Ok(()) + } + + fn offset_from(&self, base: &Self) -> Self::Offset { + let offset = base.offset_of_addr(self.data.as_ptr()).unwrap(); + assert!(offset + self.len() <= base.len()); + offset + } + + fn offset_id(&self) -> gimli::ReaderOffsetId { + gimli::ReaderOffsetId(self.offset_id_enc(0)) + } + + fn lookup_offset_id(&self, id: gimli::ReaderOffsetId) -> Option { + let ptr = Self::offset_id_dec(id.0); + let offset = ptr as isize - self.data.as_ptr() as isize; + if offset >= 0 && offset as usize <= self.len() { + Some(offset as usize) + } else { + None + } + } + + fn find(&self, byte: u8) -> gimli::Result { + self.data + .iter() + .position(|&c| c == byte) + .ok_or_else(|| gimli::Error::UnexpectedEof(self.offset_id())) + } + + fn skip(&mut self, len: Self::Offset) -> gimli::Result<()> { + let _ = self.split(len)?; + Ok(()) + } + + fn split(&mut self, len: Self::Offset) -> gimli::Result { + if len > self.data.len() { + return Err(gimli::Error::UnexpectedEof(self.offset_id())); + } + let (prefix, more) = self.data.split_at(len); + *self = Self { + data: more, + input_position: self.input_position + len, + }; + Ok(Self { + data: prefix, + input_position: self.input_position, + }) + } + + fn to_slice(&self) -> gimli::Result> { + Ok(self.data.into()) + } + + fn to_string(&self) -> gimli::Result> { + match str::from_utf8(self.data) { + Ok(s) => Ok(s.into()), + _ => Err(gimli::Error::BadUtf8), + } + } + + fn to_string_lossy(&self) -> gimli::Result> { + Ok(String::from_utf8_lossy(self.data)) + } + + fn read_slice(&mut self, buf: &mut [u8]) -> gimli::Result<()> { + let prefix = self.split(buf.len())?; + buf.copy_from_slice(prefix.data); + Ok(()) + } +} + +#[derive(Default)] +pub enum DwarfState<'a> { + #[default] + None, + Inline(Box>>), + External, +} + +impl<'a> From> for DwarfState<'a> { + fn from(value: DwarfParseState<'a>) -> Self { + match value { + DwarfParseState::Undecided => Self::None, + DwarfParseState::Inline(sections) => Self::Inline(sections), + DwarfParseState::External => Self::External, + } + } +} + +#[derive(Default)] +enum DwarfParseState<'a> { + #[default] + Undecided, + Inline(Box>>), + External, +} + +impl<'a> DwarfParseState<'a> { + fn as_internal(&mut self) -> Option<&mut DwarfSections>> { + if let Self::Undecided = self { + *self = Self::Inline(Default::default()); + } + match self { + Self::Inline(sections) => Some(sections), + _ => None, + } + } + fn as_external(&mut self) -> Option<()> { + if let Self::Undecided = self { + *self = Self::External; + } + match self { + Self::External => Some(()), + _ => None, + } + } +} + #[derive(Default)] pub struct InputModule<'a> { pub raw: &'a [u8], @@ -163,7 +375,11 @@ pub struct InputModule<'a> { pub exports: Vec>, - pub custom_sections: Vec>, + // interesting custom sections + pub producers: Option>, + pub target_features: Option>, + pub wasm_bindgen_unstable: Vec>, + pub dwarf: DwarfState<'a>, pub names: Names<'a>, pub imported_func_map: HashMap, @@ -186,9 +402,11 @@ impl<'a> InputModule<'a> { let mut reloc_info = RelocInfoParser::default(); let mut function_types: Vec = Vec::new(); let parser = wasmparser::Parser::new(0); + let mut dwarf_state = DwarfParseState::default(); + let mut num_split_sections_found = 0; for payload in parser.parse_all(wasm) { let payload = payload?; - reloc_info.visit_payload(&payload)?; + let was_reloc_section = reloc_info.visit_payload(&payload)?; match payload { Payload::Version { .. } => {} Payload::TypeSection(reader) => { @@ -256,30 +474,77 @@ impl<'a> InputModule<'a> { }); } Payload::CustomSection(reader) => { - module.custom_sections.push(CustomSection { - name: reader.name(), - data: reader.data(), - data_offset: reader.data_offset(), - }); + macro_rules! dwarf_match { + ($name:ident) => {{ + let Some(dwarf) = dwarf_state.as_internal() else { + bail!("can't have dwarf sections with external dwarf data"); + }; + dwarf.$name = DwarfReader::from(reader).into(); + }}; + } + match (reader.as_known(), reader.name()) { + (KnownCustom::Name(names), _) => module.names = Names::from_reader(names)?, + (KnownCustom::Producers(producers), _) => { + module.producers = Some(producers); + } + (_, "target_features") => { + module.target_features = Some(reader); + } + (_, "__wasm_bindgen_unstable") => { + module.wasm_bindgen_unstable.push(reader); + } + (_, crate::magic_constants::LINK_SECTION) => { + let rdr = BinaryReader::new(reader.data(), reader.data_offset()); + let () = read_wasm_split_section(rdr, &mut module.options)?; + num_split_sections_found += 1; + } + (_, ".debug_abbrev") => dwarf_match!(debug_abbrev), + (_, ".debug_addr") => dwarf_match!(debug_addr), + (_, ".debug_aranges") => dwarf_match!(debug_aranges), + (_, ".debug_info") => dwarf_match!(debug_info), + (_, ".debug_line") => dwarf_match!(debug_line), + (_, ".debug_line_str") => dwarf_match!(debug_line_str), + (_, ".debug_macinfo") => dwarf_match!(debug_macinfo), + (_, ".debug_macro") => dwarf_match!(debug_macro), + (_, ".debug_names") => dwarf_match!(debug_names), + (_, ".debug_str") => dwarf_match!(debug_str), + (_, ".debug_str_offsets") => dwarf_match!(debug_str_offsets), + (_, ".debug_types") => dwarf_match!(debug_types), + (_, ".debug_loc") => dwarf_match!(debug_loc), + (_, ".debug_loclists") => dwarf_match!(debug_loclists), + (_, ".debug_ranges") => dwarf_match!(debug_ranges), + (_, ".debug_rnglists") => dwarf_match!(debug_rnglists), + // https://github.com/WebAssembly/tool-conventions/blob/main/Dwarf.md + (_, "external_debug_info") => { + ensure!( + dwarf_state.as_external().is_some(), + "can't have both external and internal dwarf data" + ); + // TODO: the file location "should" be relative to the input file + // currently, we don't do any file loading in this tool though, + // there is no protocol either to ask the caller about this either. + tracing::warn!( + "external DWARF data detected that will not be relocated" + ); + } + _ => { + if !was_reloc_section { + tracing::warn!( + "Ignoring custom section {} at [{:x}]", + reader.name(), + reader.data_offset() + ); + } + } + }; } Payload::End(_) => {} section => { - bail!("Unknown section: {:?}", section); + tracing::warn!("Ignoring unknown section {section:?}"); } } } - let mut num_split_sections_found = 0; - for section in module.custom_sections.iter() { - if section.name == "name" { - module.names = Names::new(section.data, section.data_offset)?; - } - if section.name == crate::magic_constants::LINK_SECTION { - let rdr = BinaryReader::new(section.data, section.data_offset); - let () = read_wasm_split_section(rdr, &mut module.options)?; - num_split_sections_found += 1; - } - } if matches!(strict, Strictness::IntegrationTesting) && num_split_sections_found != 1 { bail!( "Wrong number of custom sections for wasm-split found, expected 1 got {}", @@ -287,6 +552,7 @@ impl<'a> InputModule<'a> { ); } module.reloc_info = reloc_info.finish(&module)?; + module.dwarf = dwarf_state.into(); for (import_id, import) in module.imports.iter().enumerate() { let import_id = import_id as ImportId; diff --git a/crates/wasm_split_cli/src/reloc.rs b/crates/wasm_split_cli/src/reloc.rs index 4a7190c..a71c8b9 100644 --- a/crates/wasm_split_cli/src/reloc.rs +++ b/crates/wasm_split_cli/src/reloc.rs @@ -1,24 +1,25 @@ use std::{ + cell::OnceCell, collections::{HashMap, HashSet}, ops::Range, }; -use eyre::{anyhow, bail, Result}; +use eyre::{anyhow, bail, ensure, Result}; use tracing::trace; use wasmparser::{ - BinaryReader, CustomSectionReader, Data, DefinedDataSymbol, ElementItems, ElementKind, Export, - ExternalKind, Linking, LinkingSectionReader, Payload, RelocAddendKind, RelocSectionReader, - RelocationEntry, RelocationType, Segment, SymbolFlags, SymbolInfo, + CustomSectionReader, Data, DefinedDataSymbol, ElementItems, ElementKind, Export, ExternalKind, + KnownCustom, Linking, Payload, RelocAddendKind, RelocationEntry, RelocationType, Segment, + SymbolFlags, SymbolInfo, }; use crate::{ magic_constants, - read::{GlobalId, InputFuncId, InputModule, InputOffset, TableId, TagId}, + read::{GlobalId, InputFuncId, InputModule, InputOffset, SectionId, TableId, TagId}, util::{find_subrange, shift_range}, }; // An offset (index) into the bytes of the input module -pub type SectionIndex = usize; +pub type SectionIndex = SectionId; pub type SymbolIndex = usize; #[derive(Default)] @@ -30,70 +31,88 @@ pub struct RelocInfoParser<'a> { } impl<'a> RelocInfoParser<'a> { - fn visit_custom(&mut self, custom: &CustomSectionReader<'a>) -> Result<()> { - if custom.name() == "linking" { - self.has_linking_section = true; - let reader = - LinkingSectionReader::new(BinaryReader::new(custom.data(), custom.data_offset()))?; - let reader = reader.subsections(); - for subsection in reader { - let subsection = subsection?; - if let Linking::SegmentInfo(segments) = subsection { - assert!(self.info.segments.is_empty(), "duplicate segments info"); - self.info.segments = segments.into_iter().collect::>()?; - continue; - } - if let Linking::SymbolTable(map) = subsection { - assert!(self.info.symbols.is_empty(), "duplicate symbol table"); - self.info.symbols = map.into_iter().collect::, _>>()?; - for sym in &self.info.symbols { - #[allow(clippy::single_match)] // other special cases might follow - match *sym { - SymbolInfo::Table { - name: Some("__indirect_function_table"), - index, - .. - } => { - self.indirect_function_table = Some(index as TableId); - } - _ => {} + fn visit_linking(&mut self, subsection: Linking<'a>) -> Result<()> { + match subsection { + Linking::SegmentInfo(segments) => { + ensure!(self.info.segments.is_empty(), "duplicate segments info"); + self.info.segments = segments.into_iter().collect::>()?; + return Ok(()); + } + Linking::SymbolTable(map) => { + ensure!(self.info.symbols.is_empty(), "duplicate symbol table"); + self.info.symbols = map.into_iter().collect::, _>>()?; + for sym in &self.info.symbols { + #[allow(clippy::single_match)] // other special cases might follow + match *sym { + SymbolInfo::Table { + name: Some("__indirect_function_table"), + index, + .. + } => { + self.indirect_function_table = Some(index as TableId); } + _ => {} } } } - } else if custom.name().starts_with("reloc.") { - let reader = - RelocSectionReader::new(BinaryReader::new(custom.data(), custom.data_offset()))?; - let mut reloc_entries = reader - .entries() - .into_iter() - .collect::, _>>()?; - // We need to slices of entries when we search for a specific offset - // We *might* be fine with assuming that the entries are already sorted, but a single pass to correct this - // doesn't cost a lot of performance. - reloc_entries.sort_unstable_by_key(|entry| entry.offset); - self.info - .relocs - .insert(reader.section_index() as SectionIndex, reloc_entries); + _ => {} } Ok(()) } - pub fn visit_payload(&mut self, payload: &Payload<'a>) -> Result<()> { - let section_index = self.info.section_ranges.len(); - if let Some((_, section_range)) = payload.as_section() { - self.info.section_ranges.push(section_range); + fn visit_custom(&mut self, custom: &CustomSectionReader<'a>) -> Result { + match custom.as_known() { + KnownCustom::Linking(reader) => { + self.has_linking_section = true; + for subsection in reader.subsections() { + let () = self.visit_linking(subsection?)?; + } + Ok(true) + } + KnownCustom::Reloc(reader) => { + let mut reloc_entries = reader + .entries() + .into_iter() + .collect::, _>>()?; + // We need to slices of entries when we search for a specific offset + // We *might* be fine with assuming that the entries are already sorted, but a single pass to correct this + // doesn't cost a lot of performance. + reloc_entries.sort_unstable_by_key(|entry| entry.offset); + self.info + .relocs + .insert(reader.section_index() as SectionIndex, reloc_entries); + Ok(true) + } + _ => Ok(false), + } + } + pub fn visit_payload(&mut self, payload: &Payload<'a>) -> Result { + let section_index = self.info.relocatable_ranges.len(); + if let Some((_, mut section_range)) = payload.as_section() { + if let Payload::CustomSection(custom) = payload { + // Not sure where this is specified, and it might even be wrong. + // https://github.com/WebAssembly/tool-conventions/blob/main/Linking.md#relocation-sections says: + // > offset: relative to the relevant section's contents: offset zero is immediately after the id and size of the section + // well, the data offset comes after the name of the custom section, not "immediately after the id and size". + // alas, llvm seems to generate relocations relative to data offset though. + section_range.start = custom.data_offset(); + } + self.info.relocatable_ranges.push(section_range); } match payload { Payload::DataSection(_) => { self.info.data_section_index = section_index; + Ok(true) } Payload::CodeSectionStart { .. } => { self.info.code_section_index = section_index; + Ok(true) } - Payload::CustomSection(reader) => self.visit_custom(reader)?, - _ => {} - }; - Ok(()) + Payload::CustomSection(reader) => { + self.info.custom_sections.insert(section_index); + self.visit_custom(reader) + } + _ => Ok(false), + } } pub fn finish(self, module: &InputModule<'a>) -> Result> { let mut info = self.info; @@ -309,8 +328,12 @@ fn reconstruct_global_symbols(reloc_info: &mut RelocInfo<'_>, module: &InputModu #[derive(Default)] pub struct RelocInfo<'a> { - pub section_ranges: Vec>, + // The relocatable range within each section. The start offset is the base from which + // the relocation entry is offset from. + pub relocatable_ranges: Vec>, pub segments: Vec>, + pub custom_sections: HashSet, + invalid_reloc_warn: OnceCell<()>, pub code_section_index: SectionIndex, pub data_section_index: SectionIndex, @@ -365,14 +388,14 @@ impl RelocInfo<'_> { trace!(%section, "Reloc section <<<<<<<<<<<<<<<<<<<<<<<<"); } } - pub fn section_offset(&self, section: SectionIndex) -> InputOffset { - self.section_ranges[section].start + pub fn reloc_base(&self, section: SectionIndex) -> InputOffset { + self.relocatable_ranges[section].start } - pub fn data_section_offset(&self) -> InputOffset { - self.section_offset(self.data_section_index) + pub fn data_section_reloc_base(&self) -> InputOffset { + self.reloc_base(self.data_section_index) } - pub fn code_section_offset(&self) -> InputOffset { - self.section_offset(self.code_section_index) + pub fn code_section_reloc_base(&self) -> InputOffset { + self.reloc_base(self.code_section_index) } pub fn iter_section_relocs(&self, section: SectionIndex) -> &'_ [RelocationEntry] { self.relocs @@ -394,18 +417,18 @@ impl RelocInfo<'_> { impl Iterator + use<'_>, ) { let target_sections = find_subrange( - &self.section_ranges, + &self.relocatable_ranges, |section_range| section_range.end >= range.end, |section_range| section_range.start < range.end, ); let section = target_sections.start; - let section_range = &self.section_ranges[section]; + let section_range = &self.relocatable_ranges[section]; + let reloc_base = section_range.start; assert!( section_range.start <= range.start && range.end <= section_range.end, "range to rellocate should be fully contained in one section" ); - let section_subrange = - (range.start - section_range.start)..(range.end - section_range.start); + let section_subrange = (range.start - reloc_base)..(range.end - reloc_base); let section_relocs = self.iter_section_relocs(section); let reloc_range = find_subrange( @@ -414,7 +437,7 @@ impl RelocInfo<'_> { |reloc| (reloc.offset as usize) < section_subrange.end, ); - (section_range.start, section_relocs[reloc_range].iter()) + (reloc_base, section_relocs[reloc_range].iter()) } pub fn get_relocated_data( @@ -425,8 +448,9 @@ impl RelocInfo<'_> { let this = &module.reloc_info; let mut data = Vec::from(&module.raw[range.clone()]); let (reloc_base, relocs) = this.get_relocations_for_range(&range); + let reloc_base_to_data_off = range.start - reloc_base; for relocation in relocs { - this.apply_relocation(target, &mut data, range.start, reloc_base, relocation)?; + this.apply_relocation(target, &mut data, reloc_base_to_data_off, relocation)?; } Ok(data) } @@ -434,6 +458,9 @@ impl RelocInfo<'_> { pub fn expand_relocation(&self, relocation: &RelocationEntry) -> Result> { use wasmparser::RelocationType::*; let symbol_index = relocation.index as usize; + if symbol_index > self.symbols.len() { + bail!("Found {relocation:?} with invalid symbol index?"); + } let symbol = &self.symbols[symbol_index]; let ty = relocation.ty; match ty { @@ -527,13 +554,38 @@ impl RelocInfo<'_> { _symbol_idx: symbol_index, _symbol: *symbol, }), - wasmparser::RelocationType::SectionOffsetI32 - | wasmparser::RelocationType::FunctionOffsetI32 + wasmparser::RelocationType::FunctionOffsetI32 | wasmparser::RelocationType::FunctionOffsetI64 => { - bail!( - "unhandled relocation ty {:?} in module relocation", - relocation.ty + let wasmparser::SymbolInfo::Func { flags, index, name } = *symbol else { + bail!("Expected a func symbol as target of a FUNCTION_OFFSET relocation, got {symbol:?}"); + }; + Ok(RelocDetails::FunctionOffset(SymbolDetails { + _symbol_index: symbol_index, + _flags: flags, + index: index as InputFuncId, + _name: name, + })) + } + wasmparser::RelocationType::SectionOffsetI32 => { + let wasmparser::SymbolInfo::Section { flags, section } = *symbol else { + bail!("Expected a section symbol as target of a SECTION_OFFSET relocation, got {symbol:?}"); + }; + let section = section as SectionId; + // If the reloc points to a section with semantic context (such as data/code), we expect to relocate against a symbol, + // not by offset. It is used e.g. to point to DIE in debug_info sections. Why this happens via offset and not by some + // kind of "symbol" "inside" that custom section with an associated range similar to data symbols is a question I can + // not answer. + // NOTE: it seems newer compiler versions do not bother to emit this in any case. + ensure!( + self.custom_sections.contains(§ion), + "Expect a SECTION_OFFSET relocation to point into a custom section" ); + Ok(RelocDetails::SectionOffset(SymbolDetails { + _symbol_index: symbol_index, + index: section, + _flags: flags, + _name: None, + })) } // [relocate data segments] // TODO: there is no relocation for data segment indices. As such, we'd have to parse the opcodes to find // references to passive and declarative data segments. The solution: only handling active segments @@ -541,35 +593,56 @@ impl RelocInfo<'_> { } } - fn apply_relocation( + fn apply_relocation( &self, - reloc_target: &impl RelocTarget, + reloc_target: &T, data: &mut [u8], - data_offset: InputOffset, - reloc_base: InputOffset, + reloc_base_to_data_off: usize, relocation: &RelocationEntry, ) -> Result<()> { + // TODO(MSRV): -1i32.cast_unsigned() since rust 1.87 + let relocated = if relocation.index == (-1i32 as u32) { + // We have found a relocation against a symbol that isn't in the symbol table. + // This most likely means that we will miss out on correct relocations. + // We must handle this though, as some compilers will emit relocations in the debug section + // against non-public symbols and use index -1 for those. + let fixup = reloc_target.fixup_reloc_entry(relocation)?; + if fixup.is_none_or(|addr| addr == SENTINEL_UNDEF) { + self.invalid_reloc_warn.get_or_init(|| { + tracing::warn!( + "Skipping relocation {relocation:?} with tombstone symbol (others omitted)" + ); + }); + } + fixup + } else { + let details = self.expand_relocation(relocation)?; + reloc_target.reloc_value(details)? + }; let relocation_range = relocation.relocation_range()?; - let target = &mut data[(reloc_base + relocation_range.start - data_offset) - ..(reloc_base + relocation_range.end - data_offset)]; + let target = &mut data[(relocation_range.start - reloc_base_to_data_off) + ..(relocation_range.end - reloc_base_to_data_off)]; let ty = relocation.ty; - let relocated = reloc_target.reloc_value(self.expand_relocation(relocation)?)?; let Some(value) = relocated else { return Ok(()); }; - // handle overflow maybe? Not sure if wrapping would be correct - let value: i64 = value.try_into().expect("relocated value too big"); debug_assert!( relocation.addend == 0 || ty.addend_kind() != RelocAddendKind::None, "relocation {relocation:?} without addend should have addend == 0, not {}", relocation.addend, ); - let value = value + relocation.addend; - encode_for_ty(ty)(value, target); + let () = encode_for_ty( + ty, + value, + relocation.addend as isize, + target, + T::SENTINEL_UNDEF, + )?; Ok(()) } } +#[derive(Debug)] pub struct SymbolDetails<'a, Idx> { pub _symbol_index: usize, pub index: Idx, @@ -577,6 +650,7 @@ pub struct SymbolDetails<'a, Idx> { pub _name: Option<&'a str>, } +#[derive(Debug)] pub struct DataDetails<'a> { pub symbol_index: usize, pub _flags: SymbolFlags, @@ -584,6 +658,7 @@ pub struct DataDetails<'a> { pub definition: Option<&'a DefinedDataSymbol>, } +#[derive(Debug)] pub enum RelocDetails<'a> { TypeIndex { _symbol_idx: usize, @@ -596,9 +671,24 @@ pub enum RelocDetails<'a> { TableNumber(SymbolDetails<'a, TableId>), GlobalIndex(SymbolDetails<'a, GlobalId>), TagIndex(SymbolDetails<'a, TagId>), + FunctionOffset(SymbolDetails<'a, InputFuncId>), + SectionOffset(SymbolDetails<'a, SectionId>), } +pub const SENTINEL_UNDEF: usize = usize::MAX; pub trait RelocTarget { + const SENTINEL_UNDEF: bool = false; + /// Fixup a relocation entry with an invalid symbol. If this fixup fails, + /// we warn and relocate to a tombstone address or error if tombstones + /// are not enabled for this relocation context. + fn fixup_reloc_entry(&self, entry: &RelocationEntry) -> Result> { + // assert: entry.index == (-1i32 as u32) + ensure!( + Self::SENTINEL_UNDEF, + "Invalid relocation {entry:?} with tombstone symbol couldn't be fixed" + ); + Ok(Some(SENTINEL_UNDEF)) + } fn reloc_value(&self, reloc: RelocDetails<'_>) -> Result>; } @@ -650,47 +740,65 @@ fn encode_u64(value: u64, buf: &mut [u8; 8]) { *buf = value.to_le_bytes(); } -fn encode_for_ty(ty: RelocationType) -> fn(i64, &mut [u8]) { +fn encode_for_ty( + ty: RelocationType, + value: usize, + addend: isize, + target: &mut [u8], + allow_undef: bool, +) -> Result<()> { use RelocationType::*; + let resolved = if allow_undef && value == SENTINEL_UNDEF { + Some(SENTINEL_UNDEF) + } else { + value.checked_add_signed(addend) + }; + let Some(resolved) = resolved else { + bail!("reloc {ty:?} <{value:x}{addend:+}> overflows") + }; + macro_rules! try_into_value { + ($resolved:ident as $t:ty, $msg:literal) => { + match $resolved { + SENTINEL_UNDEF if allow_undef => -1isize as $t, + resolved if let Ok(resolved) = resolved.try_into() => resolved, + resolved => bail!("{}: {resolved:x}", $msg), + } + }; + } match ty { TableIndexI32 | MemoryAddrI32 | FunctionOffsetI32 | SectionOffsetI32 | GlobalIndexI32 - | FunctionIndexI32 | MemoryAddrLocrelI32 => |value, target| { - encode_u32( - value.try_into().expect("invalid value for I32 relocation"), - target.try_into().unwrap(), - ) - }, + | FunctionIndexI32 | MemoryAddrLocrelI32 => { + let resolved = try_into_value!(resolved as u32, "invalid value for I32 relocation"); + encode_u32(resolved, target.try_into().unwrap()); + Ok(()) + } FunctionIndexLeb | MemoryAddrLeb | TypeIndexLeb | GlobalIndexLeb | EventIndexLeb - | TableNumberLeb => |value, target| { - encode_leb128_u32_5byte( - value.try_into().expect("invalid value for leb relocation"), - target.try_into().unwrap(), - ); - }, + | TableNumberLeb => { + let resolved = try_into_value!(resolved as u32, "invalid value for leb relocation"); + encode_leb128_u32_5byte(resolved, target.try_into().unwrap()); + Ok(()) + } TableIndexSleb | MemoryAddrSleb | MemoryAddrRelSleb | TableIndexRelSleb - | MemoryAddrTlsSleb => |value, target| { - encode_leb128_i32_5byte( - value.try_into().expect("invalid value for sleb relocation"), - target.try_into().unwrap(), - ); - }, - FunctionOffsetI64 | MemoryAddrI64 | TableIndexI64 => |value, target| { - encode_u64( - value.try_into().expect("invalid value for I64 relocation"), - target.try_into().unwrap(), - ); - }, - MemoryAddrLeb64 => |value, target| { - encode_leb128_u64_10byte( - value - .try_into() - .expect("invalid value for leb64 relocation"), - target.try_into().unwrap(), - ); - }, + | MemoryAddrTlsSleb => { + let resolved = try_into_value!(resolved as i32, "invalid value for sleb relocation"); + encode_leb128_i32_5byte(resolved, target.try_into().unwrap()); + Ok(()) + } + FunctionOffsetI64 | MemoryAddrI64 | TableIndexI64 => { + let resolved = try_into_value!(resolved as u64, "invalid value for I64 relocation"); + encode_u64(resolved, target.try_into().unwrap()); + Ok(()) + } + MemoryAddrLeb64 => { + let resolved = try_into_value!(resolved as u64, "invalid value for leb64 relocation"); + encode_leb128_u64_10byte(resolved, target.try_into().unwrap()); + Ok(()) + } MemoryAddrRelSleb64 | TableIndexSleb64 | TableIndexRelSleb64 | MemoryAddrTlsSleb64 - | MemoryAddrSleb64 => |value, target| { - encode_leb128_i64_10byte(value, target.try_into().unwrap()); - }, + | MemoryAddrSleb64 => { + let resolved = try_into_value!(resolved as i64, "invalid value for sleb64 relocation"); + encode_leb128_i64_10byte(resolved, target.try_into().unwrap()); + Ok(()) + } } } diff --git a/integration/simple/src/lib.rs b/integration/simple/src/lib.rs index 61c85c3..fe7f868 100644 --- a/integration/simple/src/lib.rs +++ b/integration/simple/src/lib.rs @@ -10,9 +10,17 @@ fn lazy() -> u32 { 42 } +fn run_computation(a: u32, b: u32) -> u32 { + // this function is longer than necessary to allow more debugger targets :) + dbg!(a, b); + let c = a + b; + dbg!(c); + c +} + #[wasm_split(split)] -fn args_test((a, b): (u32, u32), _: &str) -> u32 { - a + b +pub fn args_test((a, b): (u32, u32), _: &str) -> u32 { + run_computation(a, b) } #[wasm_split( diff --git a/test-runner/Cargo.lock b/test-runner/Cargo.lock index 0103694..779d946 100644 --- a/test-runner/Cargo.lock +++ b/test-runner/Cargo.lock @@ -511,6 +511,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.2.0" @@ -584,6 +590,18 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "gimli" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033caf0b349c518623b5396bfb2cf0bddf44f0306d543a250e5743297aafd10" +dependencies = [ + "fnv", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "stable_deref_trait", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1778,7 +1796,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e151599d689dac80e85c66a7cfa6ffd1b2ab79220517f9161040a87a5041aee3" dependencies = [ "anyhow", - "gimli", + "gimli 0.32.3", "id-arena", "leb128", "log", @@ -1956,6 +1974,7 @@ name = "wasm_split_cli_support" version = "0.2.2" dependencies = [ "eyre", + "gimli 0.34.0", "lazy_static", "regex", "tracing", diff --git a/test-runner/src/main.rs b/test-runner/src/main.rs index 0bff036..fc1876a 100644 --- a/test-runner/src/main.rs +++ b/test-runner/src/main.rs @@ -122,6 +122,7 @@ fn wasm_split_cli(target: &Path, dir: &Path) -> Result<(PathBuf, Report)> { split_opts.main_module = "./wasm-bindgen-test"; split_opts.verbose = true; split_opts.strict_tests = true; + split_opts.emit_dwarf = true; split_opts })?; let time_taken = Instant::now().duration_since(start_time);