From 71fb77148d336f7efe575c23c27b3af385a33279 Mon Sep 17 00:00:00 2001 From: Martin Molzer Date: Fri, 3 Jul 2026 17:48:59 +0200 Subject: [PATCH 1/2] add reproducibility check to testing --- test-runner/Cargo.lock | 71 +++++++++++++++++++++++++++++++++++++++++ test-runner/Cargo.toml | 1 + test-runner/src/main.rs | 61 +++++++++++++++++++++++++++++------ 3 files changed, 124 insertions(+), 9 deletions(-) diff --git a/test-runner/Cargo.lock b/test-runner/Cargo.lock index 3730fc1..0103694 100644 --- a/test-runner/Cargo.lock +++ b/test-runner/Cargo.lock @@ -151,6 +151,15 @@ version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brotli-decompressor" version = "5.0.0" @@ -272,12 +281,27 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -312,6 +336,15 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "darling" version = "0.23.0" @@ -355,6 +388,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -608,6 +652,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1390,6 +1443,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1617,6 +1681,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicase" version = "2.8.1" @@ -1874,6 +1944,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "sha2", "tempfile", "tracing-subscriber", "wasm-bindgen-cli", diff --git a/test-runner/Cargo.toml b/test-runner/Cargo.toml index ca3f275..bae8fdf 100644 --- a/test-runner/Cargo.toml +++ b/test-runner/Cargo.toml @@ -18,3 +18,4 @@ tracing-subscriber = { version = "0.3.23", features = ["fmt"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" serde_with = "3.21.0" +sha2 = "0.11.0" diff --git a/test-runner/src/main.rs b/test-runner/src/main.rs index 4075586..744dadd 100644 --- a/test-runner/src/main.rs +++ b/test-runner/src/main.rs @@ -1,4 +1,5 @@ -use eyre::{Result, bail}; +use eyre::{Result, ensure}; +use sha2::Digest; use std::{ collections::HashMap, env::args_os, @@ -12,10 +13,12 @@ use std::{ struct Report { #[serde_as(as = "Vec<(_, _)>")] file_sizes: HashMap, + #[serde_as(as = "Vec<(_, _)>")] + file_hashes: HashMap, cli_runtime: Duration, } -fn print_report(report: Report, report_dir: &Path) -> Result<()> { +fn print_report(report: &Report, report_dir: &Path) -> Result<()> { let branded_report_name = format!( "report-{}{}.json", std::env::var("XRUSTUP_TOOLCHAIN").unwrap(), @@ -43,6 +46,24 @@ fn wasm_bindgen_test_runner() -> Command { ) } +type ContentHash = [u8; 32]; +fn hash_file(path: &Path) -> Result { + struct Sink(sha2::Sha256); + impl std::io::Write for Sink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut sink = Sink(sha2::Sha256::new()); + let _written = std::io::copy(&mut std::fs::File::open(path)?, &mut sink)?; + Ok(sink.0.finalize().0) +} + fn collect_file_sizes( report: &mut Report, main_file: &Path, @@ -54,12 +75,25 @@ fn collect_file_sizes( .map(|p| p.as_ref()) .chain([main_file]) { - let file_size = std::fs::File::open(module)?.metadata()?.len(); + let meta = std::fs::File::open(module)?.metadata()?; + let file_hash = hash_file(module)?; + let file_size = meta.len(); report.file_sizes.insert(module.to_path_buf(), file_size); + report.file_hashes.insert(module.to_path_buf(), file_hash); } Ok(()) } +fn check_reproducible(first_report: &Report, second_report: &Report) -> Result<()> { + ensure!( + first_report.file_hashes == second_report.file_hashes, + "mismatching file sizes. Expected `left` but got `right`\n left = {:#?}\n right = {:#?}", + first_report.file_sizes, + second_report.file_sizes + ); + Ok(()) +} + fn wasm_split_cli(target: &Path, dir: &Path) -> Result<(PathBuf, Report)> { let main_file = dir.join("main.wasm"); let input = std::fs::read(target)?; @@ -102,7 +136,7 @@ pub fn main() -> Result<()> { let mut args = args_os(); let _ = args.next().expect("args[0] to be the name of this runner"); let target = args.next().expect("args[1] to be a wasm program to test"); - let target_manifest_dir = std::env::var("XCARGO_MANIFEST_DIR") + let target_manifest_dir = std::env::var_os("XCARGO_MANIFEST_DIR") .expect("env variable to manifest should be set by runner script"); let target = Path::new(&target); @@ -110,12 +144,20 @@ pub fn main() -> Result<()> { let mut tempdir = tempfile::Builder::new().tempdir_in(&target_report_dir)?; tempdir.disable_cleanup(true); // keep the dir for debugging eprintln!( - "Splitting wasm from {target_manifest_dir} in {}", + "Splitting wasm from {} in {}", + target_manifest_dir.display(), tempdir.path().display() ); let (split_main, report) = wasm_split_cli(target, tempdir.path())?; - print_report(report, &target_report_dir)?; + print_report(&report, &target_report_dir)?; + if !std::env::var_os("XTEST_SKIP_REPRODUCTION").is_some_and(|skip| !skip.is_empty()) { + // check that the result is reproducible. + // we could do this in its own directory. However, when the result is reproducible, + // the second output should not have overwritten anything from the first either way. + let (_, second_report) = wasm_split_cli(target, tempdir.path())?; + check_reproducible(&report, &second_report)?; + } let mut wbg = wasm_bindgen_test_runner(); // Currently, testing is ONLY supported in browser mode. For node and others, the wrapper script needs to be @@ -128,9 +170,10 @@ pub fn main() -> Result<()> { wbg.arg(&split_main).args(args); let wbg_exit = wbg.status()?; - if !wbg_exit.success() { - bail!("Failed to execute wasm-bindgen-test-runner"); - } + ensure!( + wbg_exit.success(), + "Failed to execute wasm-bindgen-test-runner" + ); tempdir.disable_cleanup(false); Ok(()) From e978b5bdd6291735d7c5722d55516cb7ba8bec27 Mon Sep 17 00:00:00 2001 From: Martin Molzer Date: Fri, 3 Jul 2026 18:55:37 +0200 Subject: [PATCH 2/2] sort in appropriate places to ensure a stable binary result, the outputs are processed in a defined order. We run a test by simply compiling twice and checking for matching file hashes to check that we indeed preserve the output. --- crates/wasm_split_cli/src/emit.rs | 51 +++++++++++++++++++----- crates/wasm_split_cli/src/reloc.rs | 8 +++- crates/wasm_split_cli/src/split_point.rs | 8 +++- test-runner/src/main.rs | 23 +++++++++-- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/crates/wasm_split_cli/src/emit.rs b/crates/wasm_split_cli/src/emit.rs index cc9eb63..def690f 100644 --- a/crates/wasm_split_cli/src/emit.rs +++ b/crates/wasm_split_cli/src/emit.rs @@ -45,11 +45,13 @@ impl<'a> EmitState<'a> { let mut shared_names = HashMap::new(); // We potentially overwrite the mapping later on again, but that's okay. // We could also check the reloc section for better names. - let shared_by_import = program_info + let mut shared_by_import = program_info .shared_deps .iter() - .filter(|dep| !matches!(dep, DepNode::Function(_) | DepNode::DataSymbol(_))); - for (unique_id, dep) in shared_by_import.enumerate() { + .filter(|dep| !matches!(dep, DepNode::Function(_) | DepNode::DataSymbol(_))) + .collect::>(); + shared_by_import.sort(); + for (unique_id, dep) in shared_by_import.into_iter().enumerate() { let chosen_name = Cow::Owned(format!("__wasm_split_shared{unique_id}")); shared_names.insert(*dep, chosen_name); } @@ -177,13 +179,28 @@ impl<'a> EmitState<'a> { } } -#[derive(Debug, Default)] +#[derive(Default)] struct IndirectFunctionEmitInfo { table_entries: Vec, function_table_index: HashMap, table_range_for_output_module: Vec>, } +impl std::fmt::Debug for IndirectFunctionEmitInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let _ = self.function_table_index; + f.debug_struct("IndirectFunctionEmitInfo") + .field("table_entries", &self.table_entries) + // Omit this field since it's derived from table_entries + // .field("function_table_index", &self.function_table_index) + .field( + "table_range_for_output_module", + &self.table_range_for_output_module, + ) + .finish() + } +} + impl IndirectFunctionEmitInfo { fn new(module: &InputModule, program_info: &SplitProgramInfo) -> Result { let mut indirect_functions = module @@ -217,7 +234,8 @@ impl IndirectFunctionEmitInfo { }; let mut table_entries: Vec<_> = indirect_functions.into_iter().collect(); - table_entries.sort_unstable_by_key(|&func_id| module_for_func(func_id)); + // all keys are unique, hence this sort is actually stable + table_entries.sort_unstable_by_key(|&func_id| (module_for_func(func_id), func_id)); let function_table_index: HashMap<_, _> = table_entries .iter() .enumerate() @@ -282,6 +300,7 @@ enum DataSegmentEmitInfo { }, } +#[derive(Debug)] struct DataEmitInfo { per_segment: Vec, } @@ -364,7 +383,10 @@ impl DataEmitInfo { Some(Ok((symbol_index, def_data))) }) .collect::>>()?; - included_symbols.sort_by_key(|(_, def_data)| (def_data.index, def_data.offset)); + // all keys are unique by the inclusion of the symbol index + included_symbols.sort_unstable_by_key(|&(sym_index, ref def_data)| { + (def_data.index, def_data.offset, sym_index) + }); for (symbol_index, def_data) in included_symbols { let segment_index = def_data.index as usize; let DataSegmentAnalysis::Ranges { @@ -701,7 +723,13 @@ impl<'a> ModuleEmitState<'a> { }); } - for dep in &output_module_info.included_symbols { + let mut included_symbols = output_module_info + .included_symbols + .iter() + .cloned() + .collect::>(); + included_symbols.sort(); + for dep in &included_symbols { if let &dep @ DepNode::Function(input_func) = dep { let is_import = input_func < emit_state.input_module.imported_funcs.len(); if !is_import { @@ -730,7 +758,12 @@ impl<'a> ModuleEmitState<'a> { let mut also_needs_indirect_table = !emit_state.indirect_functions.table_range_for_output_module[output_module_index] .is_empty(); - for used_shared in &output_module_info.used_shared_deps { + let mut used_shared_deps = output_module_info + .used_shared_deps + .iter() + .collect::>(); + used_shared_deps.sort(); + for used_shared in used_shared_deps { if output_module_info.included_symbols.contains(used_shared) { continue; } @@ -809,7 +842,7 @@ impl<'a> ModuleEmitState<'a> { }); exported_dont_share.insert(dep); } - for dep in &output_module_info.included_symbols { + for dep in &included_symbols { if !program_info.shared_deps.contains(dep) || exported_dont_share.contains(dep) { continue; } diff --git a/crates/wasm_split_cli/src/reloc.rs b/crates/wasm_split_cli/src/reloc.rs index b769a39..4a7190c 100644 --- a/crates/wasm_split_cli/src/reloc.rs +++ b/crates/wasm_split_cli/src/reloc.rs @@ -68,7 +68,10 @@ impl<'a> RelocInfoParser<'a> { .entries() .into_iter() .collect::, _>>()?; - reloc_entries.sort_by_key(|entry| entry.offset); + // We need to slices of entries when we search for a specific offset + // We *might* be fine with assuming that the entries are already sorted, but a single pass to correct this + // doesn't cost a lot of performance. + reloc_entries.sort_unstable_by_key(|entry| entry.offset); self.info .relocs .insert(reader.section_index() as SectionIndex, reloc_entries); @@ -211,7 +214,8 @@ fn get_data_symbols(data_segments: &[Data], symbols: &[SymbolInfo]) -> Result Result> { ); } + // This sort here should not influence downstream results, + // but we do it anyway for good measure since its cheap + let mut split_points = split_points; + split_points.sort_unstable_by_key(|split| split.export); Ok(split_points) } @@ -572,9 +576,11 @@ pub fn compute_split_modules( program_info.split_points = split_points; program_info.output_modules = split_module_contents.into_iter().collect(); + // We sort by split module here to get a stable assignment from split->output index + // An unstable sort is sufficient, no two keys compare equal. program_info .output_modules - .sort_by_key(|(identifier, _)| (*identifier).clone()); + .sort_unstable_by_key(|(identifier, _)| (*identifier).clone()); for (output_index, (_, info)) in program_info.output_modules.iter().enumerate() { for &symbol in info.included_symbols.iter() { diff --git a/test-runner/src/main.rs b/test-runner/src/main.rs index 744dadd..0bff036 100644 --- a/test-runner/src/main.rs +++ b/test-runner/src/main.rs @@ -1,4 +1,5 @@ use eyre::{Result, ensure}; +use serde::Serialize; use sha2::Digest; use std::{ collections::HashMap, @@ -46,7 +47,19 @@ fn wasm_bindgen_test_runner() -> Command { ) } -type ContentHash = [u8; 32]; +#[derive(PartialEq, Eq, Serialize)] +#[repr(transparent)] +#[serde(transparent)] +struct ContentHash([u8; 32]); +impl std::fmt::Debug for ContentHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for d in &self.0 { + write!(f, "{:02x}", d)?; + } + Ok(()) + } +} + fn hash_file(path: &Path) -> Result { struct Sink(sha2::Sha256); impl std::io::Write for Sink { @@ -61,7 +74,7 @@ fn hash_file(path: &Path) -> Result { } let mut sink = Sink(sha2::Sha256::new()); let _written = std::io::copy(&mut std::fs::File::open(path)?, &mut sink)?; - Ok(sink.0.finalize().0) + Ok(ContentHash(sink.0.finalize().0)) } fn collect_file_sizes( @@ -87,9 +100,11 @@ fn collect_file_sizes( fn check_reproducible(first_report: &Report, second_report: &Report) -> Result<()> { ensure!( first_report.file_hashes == second_report.file_hashes, - "mismatching file sizes. Expected `left` but got `right`\n left = {:#?}\n right = {:#?}", + "mismatching file hashes/sizes. Expected `left` but got `right`\n left-sizes = {:#?}\n left-hashes = {:#?}\n right-sizes = {:#?}\n right-hashes = {:#?}", first_report.file_sizes, - second_report.file_sizes + first_report.file_hashes, + second_report.file_sizes, + second_report.file_hashes, ); Ok(()) }