From 8f43b61cf617688123aaf92b07d6c90ab89362fb Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 23 Dec 2025 08:58:04 -0500 Subject: [PATCH 01/24] Proper handling for windows mingw flags --- rust/platform/triple_mappings.bzl | 14 +- rust/private/repository_utils.bzl | 2 +- rust/private/rustc.bzl | 51 +++-- test/unit/windows_lib_name/BUILD.bazel | 3 + .../windows_lib_name_test.bzl | 185 ++++++++++++++++++ test/unit/windows_stdlib/BUILD.bazel | 3 + .../windows_stdlib/windows_stdlib_test.bzl | 122 ++++++++++++ 7 files changed, 350 insertions(+), 30 deletions(-) create mode 100644 test/unit/windows_lib_name/BUILD.bazel create mode 100644 test/unit/windows_lib_name/windows_lib_name_test.bzl create mode 100644 test/unit/windows_stdlib/BUILD.bazel create mode 100644 test/unit/windows_stdlib/windows_stdlib_test.bzl diff --git a/rust/platform/triple_mappings.bzl b/rust/platform/triple_mappings.bzl index e213cb1f82..46c928a429 100644 --- a/rust/platform/triple_mappings.bzl +++ b/rust/platform/triple_mappings.bzl @@ -284,7 +284,12 @@ _SYSTEM_TO_STDLIB_LINKFLAGS = { "wasi": [], "wasip1": [], "wasip2": [], - "windows": ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"], + "windows": { + # see https://github.com/rust-lang/rust/blob/c4aa646f15e40bd3e64ddb5017b7b89b3646ac99/src/tools/run-make-support/src/external_deps/c_cxx_compiler/extras.rs#L14-L23 + "gnu": ["-lws2_32", "-luserenv", "-lbcrypt", "-lntdll", "-lsynchronization"], + "gnullvm": ["-lws2_32", "-luserenv", "-lbcrypt", "-lntdll", "-lsynchronization"], + "msvc": ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"], + }, } def cpu_arch_to_constraints(cpu_arch, *, system = None, abi = None): @@ -410,8 +415,11 @@ def system_to_staticlib_ext(system): def system_to_binary_ext(system): return _SYSTEM_TO_BINARY_EXT[system] -def system_to_stdlib_linkflags(system): - return _SYSTEM_TO_STDLIB_LINKFLAGS[system] +def system_to_stdlib_linkflags(target_triple): + val = _SYSTEM_TO_STDLIB_LINKFLAGS[target_triple.system] + if type(val) == "list": + return val + return val[target_triple.abi] def triple_to_constraint_set(target_triple): """Returns a set of constraints for a given platform triple diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index 67ce739e6a..4f3b89377a 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -417,7 +417,7 @@ def BUILD_for_rust_toolchain( str: A rendered template of a `rust_toolchain` declaration """ if stdlib_linkflags == None: - stdlib_linkflags = ", ".join(['"%s"' % x for x in system_to_stdlib_linkflags(target_triple.system)]) + stdlib_linkflags = ", ".join(['"%s"' % x for x in system_to_stdlib_linkflags(target_triple)]) rustfmt_label = None if include_rustfmt: diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index a51c21e4b2..43695b4ef6 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -560,9 +560,9 @@ def _symlink_for_ambiguous_lib(actions, toolchain, crate_info, lib): # Take the absolute value of hash() since it could be negative. path_hash = abs(hash(lib.path)) - lib_name = get_lib_name_for_windows(lib) if toolchain.target_os.startswith("windows") else get_lib_name_default(lib) + lib_name = get_lib_name_for_windows(lib) if toolchain.target_abi == "msvc" else get_lib_name_default(lib) - if toolchain.target_os.startswith("windows"): + if toolchain.target_abi == "msvc": prefix = "" extension = ".lib" elif lib_name.endswith(".pic"): @@ -1500,7 +1500,7 @@ def rustc_compile_action( pdb_file = None dsym_folder = None if crate_info.type in ("cdylib", "bin") and not experimental_use_cc_common_link: - if toolchain.target_os == "windows" and compilation_mode.strip_level == "none": + if toolchain.target_abi == "msvc" and compilation_mode.strip_level == "none": pdb_file = ctx.actions.declare_file(crate_info.output.basename[:-len(crate_info.output.extension)] + "pdb", sibling = crate_info.output) action_outputs.append(pdb_file) elif toolchain.target_os in ["macos", "darwin"]: @@ -2204,7 +2204,7 @@ def _get_crate_dirname(crate): """ return crate.output.dirname -def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = False, for_darwin = False, flavor_msvc = False): +def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_darwin = False, flavor_msvc = False): artifact = get_preferred_artifact(lib, use_pic) if ambiguous_libs and artifact.path in ambiguous_libs: artifact = ambiguous_libs[artifact.path] @@ -2244,17 +2244,11 @@ def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows ): return [] if for_darwin else ["-lstatic=%s" % get_lib_name(artifact)] - if for_windows: - if flavor_msvc: - return [ - "-lstatic=%s" % get_lib_name(artifact), - "-Clink-arg={}".format(artifact.basename), - ] - else: - return [ - "-lstatic=%s" % get_lib_name(artifact), - "-Clink-arg=-l{}".format(artifact.basename), - ] + if flavor_msvc: + return [ + "-lstatic=%s" % get_lib_name(artifact), + "-Clink-arg={}".format(artifact.basename), + ] else: return [ "-lstatic=%s" % get_lib_name(artifact), @@ -2285,7 +2279,8 @@ def _make_link_flags_windows(make_link_flags_args, flavor_msvc, use_direct_drive ("-Clink-arg=%s--no-whole-archive" % prefix), ]) elif include_link_flags: - ret.extend(_portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name_for_windows, for_windows = True, flavor_msvc = flavor_msvc)) + get_lib_name = get_lib_name_for_windows if flavor_msvc else get_lib_name_default + ret.extend(_portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, flavor_msvc = flavor_msvc)) _add_user_link_flags(ret, linker_input) return ret @@ -2365,19 +2360,19 @@ def _get_make_link_flag_funcs(target_os, target_abi, use_direct_link_driver): - callable: The function for producing link args. - callable: The function for formatting link library names. """ + + get_lib_name = get_lib_name_default + if target_os == "windows": - make_link_flags_windows_msvc = _make_link_flags_windows_msvc_direct if use_direct_link_driver else _make_link_flags_windows_msvc_indirect - make_link_flags_windows_gnu = _make_link_flags_windows_gnu_direct if use_direct_link_driver else _make_link_flags_windows_gnu_indirect - make_link_flags = make_link_flags_windows_msvc if target_abi == "msvc" else make_link_flags_windows_gnu - get_lib_name = get_lib_name_for_windows + if target_abi == "msvc": + make_link_flags = _make_link_flags_windows_msvc_direct if use_direct_link_driver else _make_link_flags_windows_msvc_indirect + get_lib_name = get_lib_name_for_windows + else: + make_link_flags = _make_link_flags_windows_gnu_direct if use_direct_link_driver else _make_link_flags_windows_gnu_indirect elif target_os.startswith(("mac", "darwin", "ios")): - make_link_flags_darwin = _make_link_flags_darwin_direct if use_direct_link_driver else _make_link_flags_darwin_indirect - make_link_flags = make_link_flags_darwin - get_lib_name = get_lib_name_default + make_link_flags = _make_link_flags_darwin_direct if use_direct_link_driver else _make_link_flags_darwin_indirect else: - make_link_flags_default = _make_link_flags_default_direct if use_direct_link_driver else _make_link_flags_default_indirect - make_link_flags = make_link_flags_default - get_lib_name = get_lib_name_default + make_link_flags = _make_link_flags_default_direct if use_direct_link_driver else _make_link_flags_default_indirect return (make_link_flags, get_lib_name) @@ -2710,3 +2705,7 @@ no_std = rule( }, implementation = _no_std_impl, ) + +# Test-only exports for private helpers. +portable_link_flags_for_testing = _portable_link_flags +symlink_for_ambiguous_lib_for_testing = _symlink_for_ambiguous_lib diff --git a/test/unit/windows_lib_name/BUILD.bazel b/test/unit/windows_lib_name/BUILD.bazel new file mode 100644 index 0000000000..e2a5113ec1 --- /dev/null +++ b/test/unit/windows_lib_name/BUILD.bazel @@ -0,0 +1,3 @@ +load(":windows_lib_name_test.bzl", "windows_lib_name_test_suite") + +windows_lib_name_test_suite(name = "windows_lib_name_test_suite") diff --git a/test/unit/windows_lib_name/windows_lib_name_test.bzl b/test/unit/windows_lib_name/windows_lib_name_test.bzl new file mode 100644 index 0000000000..1749e16700 --- /dev/null +++ b/test/unit/windows_lib_name/windows_lib_name_test.bzl @@ -0,0 +1,185 @@ +"""Analysistests for Windows-specific library naming and link flags.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") + +# buildifier: disable=bzl-visibility +load("//rust/private:rustc.bzl", "portable_link_flags_for_testing", "symlink_for_ambiguous_lib_for_testing") + +# buildifier: disable=bzl-visibility +load("//rust/private:utils.bzl", "get_lib_name_default", "get_lib_name_for_windows") + +# buildifier: disable=provider-params +LinkFlagsInfo = provider(fields = {"flags": "List[str]"}) + +# buildifier: disable=provider-params +SymlinkInfo = provider(fields = {"symlink": "File"}) + +def _portable_link_flags_probe_impl(ctx): + lib_artifact = ctx.actions.declare_file(ctx.attr.lib_basename) + ctx.actions.write(lib_artifact, "", is_executable = False) + library_to_link = struct( + static_library = lib_artifact, + pic_static_library = None, + dynamic_library = None, + interface_library = None, + alwayslink = False, + ) + + get_lib_name = get_lib_name_for_windows if ctx.attr.flavor_msvc else get_lib_name_default + flags = portable_link_flags_for_testing( + lib = library_to_link, + use_pic = False, + ambiguous_libs = {}, + get_lib_name = get_lib_name, + for_windows = True, + flavor_msvc = ctx.attr.flavor_msvc, + ) + + return [ + DefaultInfo(files = depset([])), + LinkFlagsInfo(flags = flags), + ] + +portable_link_flags_probe = rule( + implementation = _portable_link_flags_probe_impl, + attrs = { + "flavor_msvc": attr.bool(default = False), + "lib_basename": attr.string(mandatory = True), + }, +) + +def _symlink_probe_impl(ctx): + lib_artifact = ctx.actions.declare_file(ctx.attr.lib_basename) + ctx.actions.write(lib_artifact, "", is_executable = False) + crate_output = ctx.actions.declare_file("crate.rlib") + ctx.actions.write(crate_output, "", is_executable = False) + symlink = symlink_for_ambiguous_lib_for_testing( + ctx.actions, + toolchain = struct(target_abi = ctx.attr.target_abi), + crate_info = struct(output = crate_output), + lib = lib_artifact, + ) + + return [ + SymlinkInfo(symlink = symlink), + DefaultInfo(files = depset([symlink])), + ] + +symlink_probe = rule( + implementation = _symlink_probe_impl, + attrs = { + "lib_basename": attr.string(mandatory = True), + "target_abi": attr.string(mandatory = True), + }, +) + +def _portable_link_flags_windows_gnu_test_impl(ctx): + env = analysistest.begin(ctx) + flags = analysistest.target_under_test(env)[LinkFlagsInfo].flags + + asserts.equals( + env, + ["-lstatic=foo.dll", "-Clink-arg=-lfoo.dll"], + flags, + ) + return analysistest.end(env) + +portable_link_flags_windows_gnu_test = analysistest.make( + _portable_link_flags_windows_gnu_test_impl, +) + +def _portable_link_flags_windows_msvc_test_impl(ctx): + env = analysistest.begin(ctx) + flags = analysistest.target_under_test(env)[LinkFlagsInfo].flags + + asserts.equals( + env, + ["-lstatic=libfoo.dll", "-Clink-arg=libfoo.dll.lib"], + flags, + ) + return analysistest.end(env) + +portable_link_flags_windows_msvc_test = analysistest.make( + _portable_link_flags_windows_msvc_test_impl, +) + +def _symlink_name_windows_gnu_test_impl(ctx): + env = analysistest.begin(ctx) + symlink = analysistest.target_under_test(env)[SymlinkInfo].symlink + + asserts.true(env, symlink.basename.startswith("libfoo.dll-")) + asserts.true(env, symlink.basename.endswith(".a")) + asserts.false(env, symlink.basename.startswith("liblib")) + + return analysistest.end(env) + +symlink_name_windows_gnu_test = analysistest.make(_symlink_name_windows_gnu_test_impl) + +def _symlink_name_windows_msvc_test_impl(ctx): + env = analysistest.begin(ctx) + symlink = analysistest.target_under_test(env)[SymlinkInfo].symlink + + asserts.true(env, symlink.basename.startswith("native_dep-")) + asserts.true(env, symlink.basename.endswith(".lib")) + + return analysistest.end(env) + +symlink_name_windows_msvc_test = analysistest.make(_symlink_name_windows_msvc_test_impl) + +def _define_targets(): + portable_link_flags_probe( + name = "portable_link_flags_windows_gnu_probe", + flavor_msvc = False, + lib_basename = "libfoo.dll.a", + ) + portable_link_flags_probe( + name = "portable_link_flags_windows_msvc_probe", + flavor_msvc = True, + lib_basename = "libfoo.dll.lib", + ) + + symlink_probe( + name = "symlink_windows_gnu_probe", + lib_basename = "libfoo.dll.a", + target_abi = "gnu", + ) + symlink_probe( + name = "symlink_windows_msvc_probe", + lib_basename = "native_dep.lib", + target_abi = "msvc", + ) + +def windows_lib_name_test_suite(name): + """Entry-point macro for Windows library naming tests. + + Args: + name: test suite name + """ + _define_targets() + + portable_link_flags_windows_gnu_test( + name = "portable_link_flags_windows_gnu_test", + target_under_test = ":portable_link_flags_windows_gnu_probe", + ) + portable_link_flags_windows_msvc_test( + name = "portable_link_flags_windows_msvc_test", + target_under_test = ":portable_link_flags_windows_msvc_probe", + ) + symlink_name_windows_gnu_test( + name = "symlink_name_windows_gnu_test", + target_under_test = ":symlink_windows_gnu_probe", + ) + symlink_name_windows_msvc_test( + name = "symlink_name_windows_msvc_test", + target_under_test = ":symlink_windows_msvc_probe", + ) + + native.test_suite( + name = name, + tests = [ + ":portable_link_flags_windows_gnu_test", + ":portable_link_flags_windows_msvc_test", + ":symlink_name_windows_gnu_test", + ":symlink_name_windows_msvc_test", + ], + ) diff --git a/test/unit/windows_stdlib/BUILD.bazel b/test/unit/windows_stdlib/BUILD.bazel new file mode 100644 index 0000000000..91b803e6a7 --- /dev/null +++ b/test/unit/windows_stdlib/BUILD.bazel @@ -0,0 +1,3 @@ +load(":windows_stdlib_test.bzl", "windows_stdlib_test_suite") + +windows_stdlib_test_suite(name = "windows_stdlib_test_suite") diff --git a/test/unit/windows_stdlib/windows_stdlib_test.bzl b/test/unit/windows_stdlib/windows_stdlib_test.bzl new file mode 100644 index 0000000000..7008a86aa6 --- /dev/null +++ b/test/unit/windows_stdlib/windows_stdlib_test.bzl @@ -0,0 +1,122 @@ +"""Analysistests covering Windows-specific stdlib link flags.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("//rust/platform:triple.bzl", "triple") +load("//rust/platform:triple_mappings.bzl", "system_to_stdlib_linkflags") + +# buildifier: disable=bzl-visibility +load("//rust/private:repository_utils.bzl", "BUILD_for_rust_toolchain") + +def _stdlib_linkflags_windows_test_impl(ctx): + env = analysistest.begin(ctx) + analysistest.target_under_test(env) # Ensure target is configured. + + msvc_flags = system_to_stdlib_linkflags(triple("x86_64-pc-windows-msvc")) + gnu_flags = system_to_stdlib_linkflags(triple("x86_64-pc-windows-gnu")) + gnullvm_flags = system_to_stdlib_linkflags(triple("aarch64-pc-windows-gnullvm")) + + asserts.equals( + env, + ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"], + msvc_flags, + ) + asserts.equals( + env, + ["-ladvapi32", "-lws2_32", "-luserenv"], + gnu_flags, + ) + asserts.equals(env, gnu_flags, gnullvm_flags) + + return analysistest.end(env) + +stdlib_linkflags_windows_test = analysistest.make(_stdlib_linkflags_windows_test_impl) + +def _build_for_rust_toolchain_windows_flags_test_impl(ctx): + env = analysistest.begin(ctx) + analysistest.target_under_test(env) + + msvc_triple = triple("x86_64-pc-windows-msvc") + gnu_triple = triple("x86_64-pc-windows-gnu") + + rendered_msvc = BUILD_for_rust_toolchain( + name = "tc_msvc", + exec_triple = msvc_triple, + target_triple = msvc_triple, + version = "1.75.0", + allocator_library = None, + global_allocator_library = None, + default_edition = "2021", + include_rustfmt = False, + include_llvm_tools = False, + include_linker = False, + stdlib_linkflags = None, + extra_rustc_flags = None, + extra_exec_rustc_flags = None, + opt_level = None, + strip_level = None, + ) + rendered_gnu = BUILD_for_rust_toolchain( + name = "tc_gnu", + exec_triple = gnu_triple, + target_triple = gnu_triple, + version = "1.75.0", + allocator_library = None, + global_allocator_library = None, + default_edition = "2021", + include_rustfmt = False, + include_llvm_tools = False, + include_linker = False, + stdlib_linkflags = None, + extra_rustc_flags = None, + extra_exec_rustc_flags = None, + opt_level = None, + strip_level = None, + ) + + asserts.true( + env, + 'stdlib_linkflags = ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"],' in rendered_msvc, + "MSVC toolchain should render .lib stdlib linkflags:\n%s" % rendered_msvc, + ) + asserts.true( + env, + 'stdlib_linkflags = ["-ladvapi32", "-lws2_32", "-luserenv"],' in rendered_gnu, + "GNU toolchain should render -l stdlib linkflags:\n%s" % rendered_gnu, + ) + + return analysistest.end(env) + +build_for_rust_toolchain_windows_flags_test = analysistest.make( + _build_for_rust_toolchain_windows_flags_test_impl, +) + +def _define_targets(): + # Target under test is unused beyond satisfying analysistest requirements. + native.filegroup( + name = "dummy_target", + srcs = [], + ) + +def windows_stdlib_test_suite(name): + """Entry-point macro for Windows stdlib linkflag tests. + + Args: + name: test suite name""" + _define_targets() + + stdlib_linkflags_windows_test( + name = "stdlib_linkflags_windows_test", + target_under_test = ":dummy_target", + ) + build_for_rust_toolchain_windows_flags_test( + name = "build_for_rust_toolchain_windows_flags_test", + target_under_test = ":dummy_target", + ) + + native.test_suite( + name = name, + tests = [ + ":build_for_rust_toolchain_windows_flags_test", + ":stdlib_linkflags_windows_test", + ], + ) From a9e3b96af78c98d5dd74759ee1df57517ed6b816 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 18 Dec 2025 14:04:34 -0500 Subject: [PATCH 02/24] Properly handle artifact_name_patterns --- rust/private/rustc.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 43695b4ef6..c7100dae9e 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1643,6 +1643,7 @@ def rustc_compile_action( compilation_outputs = compilation_outputs, name = output_relative_to_package, stamp = ctx.attr.stamp, + main_output = crate_info.output, output_type = "executable" if crate_info.type == "bin" else "dynamic_library", additional_outputs = additional_linker_outputs, ) From e061256c79ca0718dec23d355ddaac322743bdd7 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 20 Jan 2026 10:31:15 -0500 Subject: [PATCH 03/24] Rearrange link paths on Windows to reduce size overruns and fix errors --- util/process_wrapper/main.rs | 186 ++++++++++++++++++++++++++++++++++- 1 file changed, 184 insertions(+), 2 deletions(-) diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 39a6d6db16..7ffd66a8c1 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -19,10 +19,15 @@ mod rustc; mod util; use std::collections::HashMap; +#[cfg(windows)] +use std::collections::HashSet; use std::fmt; -use std::fs::{copy, OpenOptions}; +use std::fs::{self, copy, OpenOptions}; use std::io; +use std::path::PathBuf; use std::process::{exit, Command, ExitStatus, Stdio}; +#[cfg(windows)] +use std::time::{SystemTime, UNIX_EPOCH}; use tinyjson::JsonValue; @@ -73,6 +78,175 @@ macro_rules! debug_log { }; } +#[cfg(windows)] +struct TemporaryDirectoryGuard { + path: Option, +} + +#[cfg(windows)] +impl TemporaryDirectoryGuard { + fn new(path: Option) -> Self { + Self { path } + } + + fn take(&mut self) -> Option { + self.path.take() + } +} + +#[cfg(windows)] +impl Drop for TemporaryDirectoryGuard { + fn drop(&mut self) { + if let Some(path) = self.path.take() { + let _ = fs::remove_dir_all(path); + } + } +} + +#[cfg(not(windows))] +struct TemporaryDirectoryGuard; + +#[cfg(not(windows))] +impl TemporaryDirectoryGuard { + fn new(_: Option) -> Self { + TemporaryDirectoryGuard + } + + fn take(&mut self) -> Option { + None + } +} + +#[cfg(windows)] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + let mut dependency_paths = Vec::new(); + let mut filtered_args = Vec::with_capacity(args.len()); + + let mut i = 0; + while i < args.len() { + let arg = &args[i]; + if arg == "-L" { + if let Some(next) = args.get(i + 1) { + if let Some(path) = next.strip_prefix("dependency=") { + dependency_paths.push(PathBuf::from(path)); + i += 2; + continue; + } + } + } + + if let Some(path) = arg.strip_prefix("-Ldependency=") { + dependency_paths.push(PathBuf::from(path)); + i += 1; + continue; + } + + filtered_args.push(arg.clone()); + i += 1; + } + + if dependency_paths.is_empty() { + return Ok((filtered_args, None)); + } + + let unique_suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let dir_name = format!( + "rules_rust_process_wrapper_deps_{}_{}", + std::process::id(), + unique_suffix + ); + + let base_dir = std::env::current_dir().map_err(|e| { + ProcessWrapperError(format!("unable to read current working directory: {}", e)) + })?; + let unified_dir = base_dir.join(&dir_name); + fs::create_dir_all(&unified_dir).map_err(|e| { + ProcessWrapperError(format!( + "unable to create unified dependency directory {}: {}", + unified_dir.display(), + e + )) + })?; + + let mut seen = HashSet::new(); + for path in dependency_paths { + let entries = fs::read_dir(&path).map_err(|e| { + ProcessWrapperError(format!( + "unable to read dependency search path {}: {}", + path.display(), + e + )) + })?; + + for entry in entries { + let entry = entry.map_err(|e| { + ProcessWrapperError(format!( + "unable to iterate dependency search path {}: {}", + path.display(), + e + )) + })?; + let file_type = entry.file_type().map_err(|e| { + ProcessWrapperError(format!( + "unable to inspect dependency search path {}: {}", + path.display(), + e + )) + })?; + if !(file_type.is_file() || file_type.is_symlink()) { + continue; + } + + let file_name = entry.file_name(); + let file_name_lower = file_name + .to_string_lossy() + .to_ascii_lowercase(); + if !seen.insert(file_name_lower) { + continue; + } + + let dest = unified_dir.join(&file_name); + let src = entry.path(); + match fs::hard_link(&src, &dest) { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(err) => { + debug_log!( + "failed to hardlink {} to {} ({}), falling back to copy", + src.display(), + dest.display(), + err + ); + fs::copy(&src, &dest).map_err(|copy_err| { + ProcessWrapperError(format!( + "unable to copy {} into unified dependency dir {}: {}", + src.display(), + dest.display(), + copy_err + )) + })?; + } + } + } + } + + filtered_args.push(format!("-Ldependency={}", unified_dir.display())); + + Ok((filtered_args, Some(unified_dir))) +} + +#[cfg(not(windows))] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + Ok((args.to_vec(), None)) +} + fn json_warning(line: &str) -> JsonValue { JsonValue::Object(HashMap::from([ ( @@ -120,9 +294,13 @@ fn process_line( fn main() -> Result<(), ProcessWrapperError> { let opts = options().map_err(|e| ProcessWrapperError(e.to_string()))?; + let (child_arguments, dep_dir_cleanup) = + consolidate_dependency_search_paths(&opts.child_arguments)?; + let mut temp_dir_guard = TemporaryDirectoryGuard::new(dep_dir_cleanup); + let mut command = Command::new(opts.executable); command - .args(opts.child_arguments) + .args(child_arguments) .env_clear() .envs(opts.child_environment) .stdout(if let Some(stdout_file) = opts.stdout_file { @@ -228,6 +406,10 @@ fn main() -> Result<(), ProcessWrapperError> { } } + if let Some(path) = temp_dir_guard.take() { + let _ = fs::remove_dir_all(path); + } + exit(code) } From cbcb6f0620b7842b8552aecd7b51ac10dc5ea1c2 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 31 Jan 2026 18:34:15 -0500 Subject: [PATCH 04/24] Fix some triple-mapping errors --- rust/platform/triple_mappings.bzl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rust/platform/triple_mappings.bzl b/rust/platform/triple_mappings.bzl index 46c928a429..d6265f0a51 100644 --- a/rust/platform/triple_mappings.bzl +++ b/rust/platform/triple_mappings.bzl @@ -121,8 +121,8 @@ _CPU_ARCH_TO_BUILTIN_PLAT_SUFFIX = { "le32": None, "mips": None, "mipsel": None, - "powerpc": "ppc", - "powerpc64": None, + "powerpc": "ppc32", + "powerpc64": "ppc", "powerpc64le": "ppc64le", "riscv32": "riscv32", "riscv32imc": "riscv32", @@ -154,7 +154,7 @@ _SYSTEM_TO_BUILTIN_SYS_SUFFIX = { "linux": "linux", "macos": "osx", "nacl": None, - "netbsd": None, + "netbsd": "netbsd", "nixos": "nixos", "none": "none", "nto": "qnx", @@ -162,9 +162,9 @@ _SYSTEM_TO_BUILTIN_SYS_SUFFIX = { "solaris": None, "uefi": "uefi", "unknown": None, - "wasi": None, - "wasip1": None, - "wasip2": None, + "wasi": "wasi", + "wasip1": "wasi", + "wasip2": "wasi", "windows": "windows", } From 7d8d8505281f60df129f68b3b8d5b890de95d2e6 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 6 Feb 2026 19:30:18 -0500 Subject: [PATCH 05/24] Strip -pthread for Windows link args --- rust/private/rustc.bzl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index c7100dae9e..4210832c62 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -2282,7 +2282,13 @@ def _make_link_flags_windows(make_link_flags_args, flavor_msvc, use_direct_drive elif include_link_flags: get_lib_name = get_lib_name_for_windows if flavor_msvc else get_lib_name_default ret.extend(_portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, flavor_msvc = flavor_msvc)) - _add_user_link_flags(ret, linker_input) + + # Windows toolchains can inherit POSIX defaults like -pthread from C deps, + # which fails to link with the MinGW/LLD toolchain. Drop them here. + for flag in linker_input.user_link_flags: + if flag in ("-pthread", "-lpthread"): + continue + ret.append("--codegen=link-arg={}".format(flag)) return ret def _make_link_flags_windows_msvc(make_link_flags_args, use_direct_driver): From 98d0a304b37e171920393834ca2d1ecfcbf66471 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 6 Feb 2026 20:00:57 -0500 Subject: [PATCH 06/24] Revert "Fix stamping for rules that don't have a stamp attribute (#3829)" This reverts commit f198ddee7f49ac351d27204b1488df0af2512fac. --- rust/private/stamp.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/private/stamp.bzl b/rust/private/stamp.bzl index bff7cbadf3..a05c255c9a 100644 --- a/rust/private/stamp.bzl +++ b/rust/private/stamp.bzl @@ -10,7 +10,7 @@ def is_stamping_enabled(ctx, attr): Returns: bool: The stamp value """ - stamp_num = getattr(attr, "stamp", 0) + stamp_num = getattr(attr, "stamp", -1) if stamp_num == 1: return True elif stamp_num == 0: From ca6b834f6f311886c77930c3d04edd0dc05fd920 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 6 Feb 2026 20:01:07 -0500 Subject: [PATCH 07/24] Revert "Switch stamping detection to ctx.configuration.stamp_binaries() (#3816)" This reverts commit 9586468d1eaa8d22f527966cb5f43a4870463649. --- rust/private/BUILD.bazel | 3 ++ rust/private/rust.bzl | 4 +++ rust/private/rustc.bzl | 2 +- rust/private/stamp.bzl | 62 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index 89444bb9bc..d18e895493 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//rust/private:rust_analyzer.bzl", "rust_analyzer_detect_sysroot") load("//rust/private:rustc.bzl", "is_proc_macro_dep", "is_proc_macro_dep_enabled") +load("//rust/private:stamp.bzl", "stamp_build_setting") # Exported for docs exports_files(["providers.bzl"]) @@ -32,6 +33,8 @@ bzl_library( ], ) +stamp_build_setting(name = "stamp") + # This setting may be used to identify dependencies of proc-macro-s. # This feature is only enabled if `is_proc_macro_dep_enabled` is true. # Its value controls the BAZEL_RULES_RUST_IS_PROC_MACRO_DEP environment variable diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index d50abbf180..cb34e0ed01 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -808,6 +808,10 @@ _COMMON_ATTRS = { doc = "Enable collection of cfg flags with results stored in CrateInfo.cfgs.", default = Label("//rust/settings:collect_cfgs"), ), + "_stamp_flag": attr.label( + doc = "A setting used to determine whether or not the `--stamp` flag is enabled", + default = Label("//rust/private:stamp"), + ), } | RUSTC_ATTRS | RUSTC_ALLOCATOR_LIBRARIES_ATTRS _PLATFORM_ATTRS = { diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 4210832c62..1443d3455f 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1350,7 +1350,7 @@ def rustc_compile_action( linkstamps = depset([]) # Determine if the build is currently running with --stamp - stamp = is_stamping_enabled(ctx, attr) + stamp = is_stamping_enabled(attr) # Add flags for any 'rustc' lints that are specified. # diff --git a/rust/private/stamp.bzl b/rust/private/stamp.bzl index a05c255c9a..1a7cab65cc 100644 --- a/rust/private/stamp.bzl +++ b/rust/private/stamp.bzl @@ -1,10 +1,63 @@ -"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled""" +"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled -def is_stamping_enabled(ctx, attr): +This module can be removed likely after the following PRs ar addressed: +- https://github.com/bazelbuild/bazel/issues/11164 +""" + +load("//rust/private:utils.bzl", "dedent") + +StampSettingInfo = provider( + doc = "Information about the `--stamp` command line flag", + fields = { + "value": "bool: Whether or not the `--stamp` flag was enabled", + }, +) + +def _stamp_build_setting_impl(ctx): + return StampSettingInfo(value = ctx.attr.value) + +_stamp_build_setting = rule( + doc = dedent("""\ + Whether to encode build information into the binary. Possible values: + + - stamp = 1: Always stamp the build information into the binary, even in [--nostamp][stamp] builds. \ + This setting should be avoided, since it potentially kills remote caching for the binary and \ + any downstream actions that depend on it. + - stamp = 0: Always replace build information by constant values. This gives good build result caching. + - stamp = -1: Embedding of build information is controlled by the [--[no]stamp][stamp] flag. + + Stamped binaries are not rebuilt unless their dependencies change. + [stamp]: https://docs.bazel.build/versions/main/user-manual.html#flag--stamp + """), + implementation = _stamp_build_setting_impl, + attrs = { + "value": attr.bool( + doc = "The default value of the stamp build flag", + mandatory = True, + ), + }, +) + +def stamp_build_setting(name, visibility = ["//visibility:public"]): + native.config_setting( + name = "stamp_detect", + values = {"stamp": "1"}, + visibility = visibility, + ) + + _stamp_build_setting( + name = name, + value = select({ + ":stamp_detect": True, + "//conditions:default": False, + }), + visibility = visibility, + ) + +def is_stamping_enabled(attr): """Determine whether or not build stamping is enabled Args: - ctx (ctx): The rule's context object attr (struct): A rule's struct of attributes (`ctx.attr`) Returns: @@ -16,6 +69,7 @@ def is_stamping_enabled(ctx, attr): elif stamp_num == 0: return False elif stamp_num == -1: - return ctx.configuration.stamp_binaries() + stamp_flag = getattr(attr, "_stamp_flag", None) + return stamp_flag[StampSettingInfo].value if stamp_flag else False else: fail("Unexpected `stamp` value: {}".format(stamp_num)) From 8865aa365f449689a1f74a738b8e73057402dd71 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Mon, 16 Feb 2026 15:50:51 -0500 Subject: [PATCH 08/24] Fix process-wrapper link lib handling when using argfiles --- util/process_wrapper/main.rs | 73 +++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 7ffd66a8c1..5d057b08cf 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -20,7 +20,7 @@ mod util; use std::collections::HashMap; #[cfg(windows)] -use std::collections::HashSet; +use std::collections::{HashSet, VecDeque}; use std::fmt; use std::fs::{self, copy, OpenOptions}; use std::io; @@ -34,6 +34,8 @@ use tinyjson::JsonValue; use crate::options::options; use crate::output::{process_output, LineOutput}; use crate::rustc::ErrorFormat; +#[cfg(windows)] +use crate::util::read_file_to_array; #[cfg(windows)] fn status_code(status: ExitStatus, was_killed: bool) -> i32 { @@ -118,35 +120,64 @@ impl TemporaryDirectoryGuard { } #[cfg(windows)] -fn consolidate_dependency_search_paths( - args: &[String], -) -> Result<(Vec, Option), ProcessWrapperError> { +fn get_dependency_search_paths_from_args( + initial_args: &[String], +) -> Result<(Vec, Vec), ProcessWrapperError> { let mut dependency_paths = Vec::new(); - let mut filtered_args = Vec::with_capacity(args.len()); + let mut filtered_args = Vec::new(); + let mut argfile_contents: HashMap> = HashMap::new(); + + let mut queue: VecDeque<(String, Option)> = initial_args + .iter() + .map(|arg| (arg.clone(), None)) + .collect(); + + while let Some((arg, parent_argfile)) = queue.pop_front() { + let target = match &parent_argfile { + Some(p) => argfile_contents.entry(format!("{}.filtered", p)).or_default(), + None => &mut filtered_args, + }; - let mut i = 0; - while i < args.len() { - let arg = &args[i]; if arg == "-L" { - if let Some(next) = args.get(i + 1) { - if let Some(path) = next.strip_prefix("dependency=") { - dependency_paths.push(PathBuf::from(path)); - i += 2; - continue; - } + let next_arg = queue.front().map(|(a, _)| a.as_str()); + if let Some(path) = next_arg.and_then(|n| n.strip_prefix("dependency=")) { + dependency_paths.push(PathBuf::from(path)); + queue.pop_front(); + } else { + target.push(arg); } - } - - if let Some(path) = arg.strip_prefix("-Ldependency=") { + } else if let Some(path) = arg.strip_prefix("-Ldependency=") { dependency_paths.push(PathBuf::from(path)); - i += 1; - continue; + } else if let Some(argfile_path) = arg.strip_prefix('@') { + let lines = read_file_to_array(argfile_path).map_err(|e| { + ProcessWrapperError(format!("unable to read argfile {}: {}", argfile_path, e)) + })?; + + for line in lines { + queue.push_back((line, Some(argfile_path.to_string()))); + } + + target.push(format!("@{}.filtered", argfile_path)); + } else { + target.push(arg); } + } - filtered_args.push(arg.clone()); - i += 1; + for (path, content) in argfile_contents { + fs::write(&path, content.join("\n")).map_err(|e| { + ProcessWrapperError(format!("unable to write filtered argfile {}: {}", path, e)) + })?; } + Ok((dependency_paths, filtered_args)) +} + +#[cfg(windows)] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + let (dependency_paths, mut filtered_args) = get_dependency_search_paths_from_args(args)?; + if dependency_paths.is_empty() { return Ok((filtered_args, None)); } From 59d689cfe8042efcd77e55cf86cbac2465eebddf Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 18 Feb 2026 05:12:26 -0500 Subject: [PATCH 09/24] Rewrite process_wrapper_bootstrap to cc --- rust/settings/BUILD.bazel | 3 - rust/settings/settings.bzl | 9 -- test/process_wrapper_bootstrap/BUILD.bazel | 20 +++-- .../bootstrap_process_wrapper_probe.rs | 10 +++ .../bootstrap_process_wrapper_test.rs | 60 +++++++++---- .../process_wrapper_bootstrap_test.bzl | 78 ---------------- util/process_wrapper/BUILD.bazel | 8 +- util/process_wrapper/private/BUILD.bazel | 11 ++- .../private/bootstrap_process_wrapper.bzl | 73 --------------- .../private/bootstrap_process_wrapper.cc | 90 +++++++++++++++++++ .../private/process_wrapper.bat | 31 ------- .../private/process_wrapper.sh | 21 ----- 12 files changed, 169 insertions(+), 245 deletions(-) create mode 100644 test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs delete mode 100644 test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl delete mode 100644 util/process_wrapper/private/bootstrap_process_wrapper.bzl create mode 100644 util/process_wrapper/private/bootstrap_process_wrapper.cc delete mode 100755 util/process_wrapper/private/process_wrapper.bat delete mode 100755 util/process_wrapper/private/process_wrapper.sh diff --git a/rust/settings/BUILD.bazel b/rust/settings/BUILD.bazel index d2b84debcc..6f0cd9dbca 100644 --- a/rust/settings/BUILD.bazel +++ b/rust/settings/BUILD.bazel @@ -18,7 +18,6 @@ load( "experimental_use_cc_common_link", "experimental_use_coverage_metadata_files", "experimental_use_global_allocator", - "experimental_use_sh_toolchain_for_bootstrap_process_wrapper", "extra_exec_rustc_env", "extra_exec_rustc_flag", "extra_exec_rustc_flags", @@ -94,8 +93,6 @@ experimental_use_global_allocator() experimental_use_allocator_libraries_with_mangled_symbols() -experimental_use_sh_toolchain_for_bootstrap_process_wrapper() - extra_exec_rustc_env() extra_exec_rustc_flag() diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index 2769272dc5..b6d1c0aea4 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -265,15 +265,6 @@ def experimental_link_std_dylib(): build_setting_default = False, ) -def experimental_use_sh_toolchain_for_bootstrap_process_wrapper(): - """A flag to control whether the shell path from a shell toolchain (`@bazel_tools//tools/sh:toolchain_type`) \ - is embedded into the bootstrap process wrapper for the `.sh` file. - """ - bool_flag( - name = "experimental_use_sh_toolchain_for_bootstrap_process_wrapper", - build_setting_default = False, - ) - def toolchain_linker_preference(): """A flag to control which linker is preferred for linking Rust binaries. diff --git a/test/process_wrapper_bootstrap/BUILD.bazel b/test/process_wrapper_bootstrap/BUILD.bazel index d2c8419b81..6a0c167adc 100644 --- a/test/process_wrapper_bootstrap/BUILD.bazel +++ b/test/process_wrapper_bootstrap/BUILD.bazel @@ -1,12 +1,22 @@ -load("//rust:defs.bzl", "rust_test") -load(":process_wrapper_bootstrap_test.bzl", "process_wrapper_bootstrap_test_suite") +load("//rust:defs.bzl", "rust_binary", "rust_test") + +rust_binary( + name = "bootstrap_process_wrapper_probe", + srcs = ["bootstrap_process_wrapper_probe.rs"], + edition = "2021", +) rust_test( name = "bootstrap_process_wrapper_test", srcs = ["bootstrap_process_wrapper_test.rs"], - data = ["//util/process_wrapper/private:process_wrapper.sh"], + data = [ + ":bootstrap_process_wrapper_probe", + "//util/process_wrapper:bootstrap_process_wrapper", + ], edition = "2021", + env = { + "BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH": "$(rlocationpath :bootstrap_process_wrapper_probe)", + "BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH": "$(rlocationpath //util/process_wrapper:bootstrap_process_wrapper)", + }, deps = ["//rust/runfiles"], ) - -process_wrapper_bootstrap_test_suite(name = "process_wrapper_bootstrap_test_suite") diff --git a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs new file mode 100644 index 0000000000..35ecc496cf --- /dev/null +++ b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs @@ -0,0 +1,10 @@ +fn main() { + let arg = std::env::args().nth(1).unwrap_or_default(); + println!("{arg}"); + + let exit_code = std::env::var("BOOTSTRAP_PROCESS_WRAPPER_PROBE_EXIT_CODE") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); + std::process::exit(exit_code); +} diff --git a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs index 4352d8d5de..43935b1b6e 100644 --- a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs +++ b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs @@ -1,24 +1,54 @@ -//! Tests for the bootstrap process wrapper +//! Tests for the bootstrap process wrapper. -use std::fs::read_to_string; +use std::env; +use std::process::Command; use runfiles::Runfiles; -/// Test that the shell process wrapper starts with the expected shebang to -/// avoid breaking the contract with the `bootstrap_process_wrapper` rule. -#[test] -fn test_shebang() { +fn resolve_runfile(env_var: &str) -> String { let rfiles = Runfiles::create().unwrap(); + let rlocationpath = env::var(env_var).unwrap(); + runfiles::rlocation!(rfiles, rlocationpath.as_str()) + .unwrap() + .display() + .to_string() +} - let script = runfiles::rlocation!( - rfiles, - "rules_rust/util/process_wrapper/private/process_wrapper.sh" - ) - .unwrap(); +#[test] +fn test_substitutes_pwd() { + let wrapper = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH"); + let probe = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH"); + let pwd = env::current_dir().unwrap().display().to_string(); + + let output = Command::new(wrapper) + .arg("--") + .arg(probe) + .arg("${pwd}/suffix") + .output() + .unwrap(); - let content = read_to_string(script).unwrap(); assert!( - content.starts_with("#!/bin/sh"), - "The shell script does not start with the expected shebang." - ) + output.status.success(), + "wrapper failed: status={:?}, stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr), + ); + + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.trim_end(), format!("{}/suffix", pwd)); +} + +#[test] +fn test_propagates_exit_code() { + let wrapper = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH"); + let probe = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH"); + + let status = Command::new(wrapper) + .arg("--") + .arg(probe) + .env("BOOTSTRAP_PROCESS_WRAPPER_PROBE_EXIT_CODE", "23") + .status() + .unwrap(); + + assert_eq!(status.code(), Some(23)); } diff --git a/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl b/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl deleted file mode 100644 index 7e0e4ea571..0000000000 --- a/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl +++ /dev/null @@ -1,78 +0,0 @@ -"""Starlark unit tests for the bootstrap process wrapper""" - -load("@bazel_skylib//lib:unittest.bzl", "analysistest") -load("//test/unit:common.bzl", "assert_action_mnemonic") - -def _enable_sh_toolchain_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - - if ctx.attr.expected_ext == ".bat": - assert_action_mnemonic(env, target.actions[0], "ExecutableSymlink") - else: - assert_action_mnemonic(env, target.actions[0], "TemplateExpand") - - return analysistest.end(env) - -_enable_sh_toolchain_test = analysistest.make( - _enable_sh_toolchain_test_impl, - config_settings = { - str(Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper")): True, - }, - attrs = { - "expected_ext": attr.string( - doc = "The expected extension for the bootstrap script.", - mandatory = True, - values = [ - ".bat", - ".sh", - ], - ), - }, -) - -def _disable_sh_toolchain_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - - assert_action_mnemonic(env, target.actions[0], "ExecutableSymlink") - - return analysistest.end(env) - -_disable_sh_toolchain_test = analysistest.make( - _disable_sh_toolchain_test_impl, - config_settings = { - str(Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper")): False, - }, -) - -def process_wrapper_bootstrap_test_suite(name, **kwargs): - """Entry-point macro called from the BUILD file. - - Args: - name (str): Name of the macro. - **kwargs (dict): Additional keyword arguments. - """ - - _enable_sh_toolchain_test( - name = "enable_sh_toolchain_test", - target_under_test = Label("//util/process_wrapper:bootstrap_process_wrapper"), - expected_ext = select({ - "@platforms//os:windows": ".bat", - "//conditions:default": ".sh", - }), - ) - - _disable_sh_toolchain_test( - name = "disable_sh_toolchain_test", - target_under_test = Label("//util/process_wrapper:bootstrap_process_wrapper"), - ) - - native.test_suite( - name = name, - tests = [ - ":disable_sh_toolchain_test", - ":enable_sh_toolchain_test", - ], - **kwargs - ) diff --git a/util/process_wrapper/BUILD.bazel b/util/process_wrapper/BUILD.bazel index af56264e4c..08dd7ecadf 100644 --- a/util/process_wrapper/BUILD.bazel +++ b/util/process_wrapper/BUILD.bazel @@ -2,7 +2,6 @@ load("@bazel_skylib//lib:selects.bzl", "selects") # buildifier: disable=bzl-visibility load("//rust/private:rust.bzl", "rust_binary_without_process_wrapper", "rust_test_without_process_wrapper_test") -load("//util/process_wrapper/private:bootstrap_process_wrapper.bzl", "bootstrap_process_wrapper") config_setting( name = "compilation_mode_opt", @@ -55,11 +54,8 @@ rust_test_without_process_wrapper_test( edition = "2018", ) -bootstrap_process_wrapper( +alias( name = "bootstrap_process_wrapper", - is_windows = select({ - "@platforms//os:windows": True, - "//conditions:default": False, - }), + actual = "//util/process_wrapper/private:bootstrap_process_wrapper", visibility = ["//visibility:public"], ) diff --git a/util/process_wrapper/private/BUILD.bazel b/util/process_wrapper/private/BUILD.bazel index badd4a695d..6cbbbc07da 100644 --- a/util/process_wrapper/private/BUILD.bazel +++ b/util/process_wrapper/private/BUILD.bazel @@ -1,4 +1,7 @@ -exports_files([ - "process_wrapper.sh", - "process_wrapper.bat", -]) +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + +cc_binary( + name = "bootstrap_process_wrapper", + srcs = ["bootstrap_process_wrapper.cc"], + visibility = ["//util/process_wrapper:__pkg__"], +) diff --git a/util/process_wrapper/private/bootstrap_process_wrapper.bzl b/util/process_wrapper/private/bootstrap_process_wrapper.bzl deleted file mode 100644 index ca5047079c..0000000000 --- a/util/process_wrapper/private/bootstrap_process_wrapper.bzl +++ /dev/null @@ -1,73 +0,0 @@ -"""Bootstrap rustc process wrapper""" - -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") - -def _bootstrap_process_wrapper_impl_unix(ctx): - output = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - setting = ctx.attr._use_sh_toolchain_for_bootstrap_process_wrapper[BuildSettingInfo].value - sh_toolchain = ctx.toolchains["@bazel_tools//tools/sh:toolchain_type"] - if setting and sh_toolchain: - shebang = "#!{}".format(sh_toolchain.path) - ctx.actions.expand_template( - output = output, - template = ctx.file._bash, - substitutions = { - # Replace the shebang with one constructed from the configured - # shell toolchain. - "#!/bin/sh": shebang, - }, - ) - else: - ctx.actions.symlink( - output = output, - target_file = ctx.file._bash, - is_executable = True, - ) - - return [DefaultInfo( - files = depset([output]), - executable = output, - )] - -def _bootstrap_process_wrapper_impl_windows(ctx): - output = ctx.actions.declare_file("{}.bat".format(ctx.label.name)) - ctx.actions.symlink( - output = output, - target_file = ctx.file._batch, - is_executable = True, - ) - - return [DefaultInfo( - files = depset([output]), - executable = output, - )] - -def _bootstrap_process_wrapper_impl(ctx): - if ctx.attr.is_windows: - return _bootstrap_process_wrapper_impl_windows(ctx) - return _bootstrap_process_wrapper_impl_unix(ctx) - -bootstrap_process_wrapper = rule( - doc = "A rule which produces a bootstrapping script for the rustc process wrapper.", - implementation = _bootstrap_process_wrapper_impl, - attrs = { - "is_windows": attr.bool( - doc = "Indicate whether or not the target platform is windows.", - mandatory = True, - ), - "_bash": attr.label( - allow_single_file = True, - default = Label("//util/process_wrapper/private:process_wrapper.sh"), - ), - "_batch": attr.label( - allow_single_file = True, - default = Label("//util/process_wrapper/private:process_wrapper.bat"), - ), - "_use_sh_toolchain_for_bootstrap_process_wrapper": attr.label( - default = Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper"), - ), - }, - toolchains = [config_common.toolchain_type("@bazel_tools//tools/sh:toolchain_type", mandatory = False)], - executable = True, -) diff --git a/util/process_wrapper/private/bootstrap_process_wrapper.cc b/util/process_wrapper/private/bootstrap_process_wrapper.cc new file mode 100644 index 0000000000..e6e58513e3 --- /dev/null +++ b/util/process_wrapper/private/bootstrap_process_wrapper.cc @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#define getcwd _getcwd +#else +#include +#endif + +namespace { + +constexpr const char* kPwdPlaceholder = "${pwd}"; + +std::string replace_pwd_placeholder(const std::string& arg, + const std::string& pwd) { + std::string out = arg; + std::string::size_type pos = 0; + while ((pos = out.find(kPwdPlaceholder, pos)) != std::string::npos) { + out.replace(pos, std::strlen(kPwdPlaceholder), pwd); + pos += pwd.size(); + } + return out; +} + +std::vector build_exec_argv(const std::vector& args) { + std::vector exec_argv; + exec_argv.reserve(args.size() + 1); + for (const std::string& arg : args) { + exec_argv.push_back(const_cast(arg.c_str())); + } + exec_argv.push_back(nullptr); + return exec_argv; +} + +} // namespace + +int main(int argc, char** argv) { + int first_arg_index = 1; + if (argc > 1 && std::strcmp(argv[1], "--") == 0) { + first_arg_index = 2; + } + + if (first_arg_index >= argc) { + std::fprintf(stderr, "bootstrap_process_wrapper: missing command\n"); + return 1; + } + + char* pwd_raw = getcwd(nullptr, 0); + if (pwd_raw == nullptr) { + std::perror("bootstrap_process_wrapper: getcwd"); + return 1; + } + std::string pwd = pwd_raw; + std::free(pwd_raw); + + std::vector command_args; + command_args.reserve(static_cast(argc - first_arg_index)); + for (int i = first_arg_index; i < argc; ++i) { + command_args.push_back(replace_pwd_placeholder(argv[i], pwd)); + } + +#if defined(_WIN32) + for (char& c : command_args[0]) { + if (c == '/') { + c = '\\'; + } + } +#endif + + std::vector exec_argv = build_exec_argv(command_args); + +#if defined(_WIN32) + int exit_code = _spawnvp(_P_WAIT, exec_argv[0], exec_argv.data()); + if (exit_code == -1) { + std::perror("bootstrap_process_wrapper: _spawnvp"); + return 1; + } + return exit_code; +#else + execvp(exec_argv[0], exec_argv.data()); + std::perror("bootstrap_process_wrapper: execvp"); + return 1; +#endif +} diff --git a/util/process_wrapper/private/process_wrapper.bat b/util/process_wrapper/private/process_wrapper.bat deleted file mode 100755 index 36fff8699a..0000000000 --- a/util/process_wrapper/private/process_wrapper.bat +++ /dev/null @@ -1,31 +0,0 @@ -@ECHO OFF -SETLOCAL enabledelayedexpansion - -SET command=%* - -:: Resolve the `${pwd}` placeholders -SET command=!command:${pwd}=%CD%! - -:: Strip out the leading `--` argument. -SET command=!command:~3! - -:: Find the rustc.exe argument and sanitize it's path -for %%A in (%*) do ( - SET arg=%%~A - if "!arg:~-9!"=="rustc.exe" ( - SET sanitized=!arg:/=\! - - SET command=!sanitized! !command:%%~A=! - goto :break - ) -) - -:break - -%command% - -:: Capture the exit code of rustc.exe -SET exit_code=!errorlevel! - -:: Exit with the same exit code -EXIT /b %exit_code% diff --git a/util/process_wrapper/private/process_wrapper.sh b/util/process_wrapper/private/process_wrapper.sh deleted file mode 100755 index 27fb44bbd8..0000000000 --- a/util/process_wrapper/private/process_wrapper.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -set -eu - -# Skip the first argument which is expected to be `--` -shift - -for arg in "$@"; do - case "$arg" in - *'${pwd}'*) - # Split on '${pwd}' and rejoin with the actual PWD value - prefix="${arg%%\$\{pwd\}*}" - suffix="${arg#*\$\{pwd\}}" - arg="${prefix}${PWD}${suffix}" - ;; - esac - set -- "$@" "$arg" - shift -done - -exec "$@" From 77190f06afaf32d7e687d8c42bf3fa33c8b3f4d2 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 18 Feb 2026 09:09:52 -0500 Subject: [PATCH 10/24] Attempt to fix CopyFile for windows --- MODULE.bazel | 1 + cargo/private/BUILD.bazel | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 35ed301ed8..87372c618e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,7 @@ module( ## Core ############################################################################### +bazel_dep(name = "bazel_lib", version = "3.0.0") bazel_dep(name = "bazel_features", version = "1.32.0") bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "platforms", version = "1.0.0") diff --git a/cargo/private/BUILD.bazel b/cargo/private/BUILD.bazel index fd60c4ff62..536490441d 100644 --- a/cargo/private/BUILD.bazel +++ b/cargo/private/BUILD.bazel @@ -1,5 +1,5 @@ +load("@bazel_lib//lib:copy_file.bzl", "copy_file") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("//rust:defs.bzl", "rust_binary") rust_binary( From ad29c7ba4847b060ffce7a14dd3671d3a584925a Mon Sep 17 00:00:00 2001 From: isaacparker0 <128327439+isaacparker0@users.noreply.github.com> Date: Fri, 20 Feb 2026 09:19:32 -0500 Subject: [PATCH 11/24] Apply lint config in exec configuration (#2) --- rust/private/rustc.bzl | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 1443d3455f..35bbe73b66 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1353,11 +1353,8 @@ def rustc_compile_action( stamp = is_stamping_enabled(attr) # Add flags for any 'rustc' lints that are specified. - # - # Exclude lints if we're building in the exec configuration to prevent crates - # used in build scripts from generating warnings. lint_files = [] - if hasattr(ctx.attr, "lint_config") and ctx.attr.lint_config and not is_exec_configuration(ctx): + if hasattr(ctx.attr, "lint_config") and ctx.attr.lint_config: rust_flags = rust_flags + ctx.attr.lint_config[LintsInfo].rustc_lint_flags lint_files = lint_files + ctx.attr.lint_config[LintsInfo].rustc_lint_files From a76f5793dfdaf86b756e77fda15785f272cd3da3 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 20 Feb 2026 09:43:40 -0500 Subject: [PATCH 12/24] Fix up rules_rust bzl_library targets --- cargo/private/BUILD.bazel | 5 ++++- rust/platform/BUILD.bazel | 1 - rust/private/BUILD.bazel | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cargo/private/BUILD.bazel b/cargo/private/BUILD.bazel index 536490441d..db04afa7b1 100644 --- a/cargo/private/BUILD.bazel +++ b/cargo/private/BUILD.bazel @@ -39,6 +39,9 @@ copy_file( bzl_library( name = "bzl_lib", + deps = [ + "//rust:bzl_lib", + ], srcs = glob(["**/*.bzl"]), - visibility = ["//:__subpackages__"], + visibility = ["//visibility:public"], ) diff --git a/rust/platform/BUILD.bazel b/rust/platform/BUILD.bazel index d5513b1e79..91b095ebed 100644 --- a/rust/platform/BUILD.bazel +++ b/rust/platform/BUILD.bazel @@ -31,5 +31,4 @@ package_group( bzl_library( name = "bzl_lib", srcs = glob(["**/*.bzl"]), - visibility = ["//rust:__subpackages__"], ) diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index d18e895493..d3cea4650d 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -22,7 +22,7 @@ bzl_library( bzl_library( name = "bzl_lib", srcs = glob(["**/*.bzl"]), - visibility = ["//rust:__subpackages__"], + visibility = ["//visibility:public"], deps = [ ":bazel_tools_bzl_lib", ":rules_cc_bzl_lib", From 51e04ab4b2c3f8cfcb1fc909e3a7a92633c4ee13 Mon Sep 17 00:00:00 2001 From: isaacparker0 <128327439+isaacparker0@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:32:15 -0500 Subject: [PATCH 13/24] rust-analyzer: include Bazel package dir in crate source include_dirs (#3) * 0 * Add rust analyzer test coverage --- rust/private/rust_analyzer.bzl | 9 +++ .../rust_project_json_test.rs | 32 ++++---- .../BUILD.bazel | 29 +++++++ .../subdir_test_crates_same_package/lib.rs | 1 + .../rust_project_json_test.rs | 76 +++++++++++++++++++ .../subdir/subdir_test.rs | 5 ++ 6 files changed, 138 insertions(+), 14 deletions(-) create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/lib.rs create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs create mode 100644 test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs diff --git a/rust/private/rust_analyzer.bzl b/rust/private/rust_analyzer.bzl index da41867985..1dd8244c98 100644 --- a/rust/private/rust_analyzer.bzl +++ b/rust/private/rust_analyzer.bzl @@ -257,6 +257,15 @@ def _create_single_crate(ctx, attrs, info): if info.crate.root.short_path in src_map: crate["root_module"] = _WORKSPACE_TEMPLATE + src_map[info.crate.root.short_path].path crate["source"]["include_dirs"].append(path_prefix + info.crate.root.dirname) + + # Ensure workspace crates in the same Bazel package share one source root. + # + # rust-analyzer picks candidate crates by source root (`relevant_crates`). + # Widening include_dirs at the package level keeps related crates in a + # shared candidate set; final membership is still resolved by each crate's + # module tree. + if not is_external: + crate["source"]["include_dirs"].append(_WORKSPACE_TEMPLATE + ctx.label.package) if info.build_info != None and info.build_info.out_dir != None: out_dir_path = info.build_info.out_dir.path diff --git a/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs b/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs index d9ce7579c5..2138c62e19 100644 --- a/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs +++ b/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs @@ -2,11 +2,12 @@ mod tests { use serde::Deserialize; use std::env; + use std::fs; + use std::path::Path; use std::path::PathBuf; #[derive(Deserialize)] struct Project { - sysroot_src: String, crates: Vec, } @@ -25,22 +26,12 @@ mod tests { #[test] fn test_generated_srcs() { let rust_project_path = PathBuf::from(env::var("RUST_PROJECT_JSON").unwrap()); + let rust_project_path = fs::canonicalize(&rust_project_path).unwrap(); let content = std::fs::read_to_string(&rust_project_path) .unwrap_or_else(|_| panic!("couldn't open {:?}", &rust_project_path)); let project: Project = serde_json::from_str(&content).expect("Failed to deserialize project JSON"); - // /tmp/_bazel/12345678/external/tools/rustlib/library => /tmp/_bazel - let output_base = project - .sysroot_src - .rsplitn(2, "/external/") - .last() - .unwrap() - .rsplitn(2, '/') - .last() - .unwrap(); - println!("output_base: {output_base}"); - let with_gen = project .crates .iter() @@ -50,7 +41,20 @@ mod tests { assert!(with_gen.root_module.ends_with("/lib.rs")); let include_dirs = &with_gen.source.as_ref().unwrap().include_dirs; - assert!(include_dirs.len() == 1); - assert!(include_dirs[0].starts_with(output_base)); + assert_eq!(include_dirs.len(), 2); + + let root_module_parent = Path::new(&with_gen.root_module).parent().unwrap(); + let workspace_dir = rust_project_path.parent().unwrap(); + + assert!( + include_dirs.iter().any(|p| Path::new(p) == root_module_parent), + "expected include_dirs to contain root_module parent, got include_dirs={include_dirs:?}, root_module={}", + with_gen.root_module, + ); + assert!( + include_dirs.iter().any(|p| Path::new(p) == workspace_dir), + "expected include_dirs to contain workspace dir, got include_dirs={include_dirs:?}, workspace_dir={}", + workspace_dir.display(), + ); } } diff --git a/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel b/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel new file mode 100644 index 0000000000..eac2fe6e4e --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "mylib", + srcs = ["lib.rs"], + edition = "2018", +) + +rust_test( + name = "mylib_test", + srcs = ["subdir/subdir_test.rs"], + edition = "2018", +) + +rust_test( + name = "rust_project_json_test", + srcs = ["rust_project_json_test.rs"], + data = [":rust-project.json"], + edition = "2018", + env = {"RUST_PROJECT_JSON": "$(rootpath :rust-project.json)"}, + # This target is tagged as manual since it's not expected to pass in + # contexts outside of `//test/rust_analyzer:rust_analyzer_test`. Run + # that target to execute this test. + tags = ["manual"], + deps = [ + "//test/rust_analyzer/3rdparty/crates:serde", + "//test/rust_analyzer/3rdparty/crates:serde_json", + ], +) diff --git a/test/rust_analyzer/subdir_test_crates_same_package/lib.rs b/test/rust_analyzer/subdir_test_crates_same_package/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/lib.rs @@ -0,0 +1 @@ + diff --git a/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs b/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs new file mode 100644 index 0000000000..77a99d75ed --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs @@ -0,0 +1,76 @@ +#[cfg(test)] +mod tests { + use serde::Deserialize; + use std::collections::BTreeSet; + use std::env; + use std::path::PathBuf; + + #[derive(Deserialize)] + struct Project { + crates: Vec, + } + + #[derive(Deserialize)] + struct Crate { + root_module: String, + is_workspace_member: Option, + source: Option, + } + + #[derive(Deserialize)] + struct Source { + include_dirs: Vec, + } + + fn normalize(path: &str) -> String { + path.trim_end_matches('/').to_owned() + } + + #[test] + fn test_same_package_crates_share_include_dir() { + let rust_project_path = PathBuf::from(env::var("RUST_PROJECT_JSON").unwrap()); + let content = std::fs::read_to_string(&rust_project_path) + .unwrap_or_else(|_| panic!("couldn't open {:?}", &rust_project_path)); + let project: Project = + serde_json::from_str(&content).expect("Failed to deserialize project JSON"); + + let lib = project + .crates + .iter() + .find(|c| c.is_workspace_member == Some(true) && c.root_module.ends_with("/lib.rs")) + .expect("missing library crate"); + let test = project + .crates + .iter() + .find(|c| { + c.is_workspace_member == Some(true) + && c.root_module.ends_with("/subdir/subdir_test.rs") + }) + .expect("missing subdir test crate"); + + let lib_include_dirs: BTreeSet<_> = lib + .source + .as_ref() + .expect("lib crate missing source field") + .include_dirs + .iter() + .map(|p| normalize(p)) + .collect(); + let test_include_dirs: BTreeSet<_> = test + .source + .as_ref() + .expect("test crate missing source field") + .include_dirs + .iter() + .map(|p| normalize(p)) + .collect(); + + let shared_dir = lib_include_dirs + .intersection(&test_include_dirs) + .next() + .expect("expected crates in same package to share an include_dir"); + + assert!(lib.root_module.starts_with(&format!("{}/", shared_dir))); + assert!(test.root_module.starts_with(&format!("{}/", shared_dir))); + } +} diff --git a/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs b/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs new file mode 100644 index 0000000000..915d7c6130 --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs @@ -0,0 +1,5 @@ +#[test] +fn test_subdir_fixture() { + let marker = String::from("ok"); + assert_eq!(marker.len(), 2); +} From 173d384f112d9842d54ff071c9040536ecb45a19 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Tue, 9 Sep 2025 09:19:46 -0400 Subject: [PATCH 14/24] Improve proc_macro_deps ergonomics --- .../private/wasm_bindgen_test.bzl | 7 +- rust/defs.bzl | 26 ++++-- rust/private/rust.bzl | 81 +++++++------------ rust/private/rustc.bzl | 2 +- rust/private/rustdoc/BUILD.bazel | 2 +- rust/private/rustdoc_test.bzl | 8 +- rust/private/utils.bzl | 28 ++++++- 7 files changed, 85 insertions(+), 69 deletions(-) diff --git a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl index 925a11a2f4..6c26f6c3cf 100644 --- a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl +++ b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl @@ -13,6 +13,7 @@ load( "@rules_rust//rust/private:utils.bzl", "determine_output_hash", "expand_dict_value_locations", + "filter_deps", "find_toolchain", "generate_output_diagnostics", "get_import_macro_deps", @@ -61,8 +62,10 @@ def _rust_wasm_bindgen_test_binary_impl(ctx): toolchain = find_toolchain(ctx) crate_type = "bin" - deps = transform_deps(ctx.attr.deps + [wb_toolchain.wasm_bindgen_test]) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + + deps, proc_macro_deps = filter_deps(ctx) + deps = transform_deps(deps + [wb_toolchain.wasm_bindgen_test]) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) # Target is building the crate in `test` config if WasmBindgenTestCrateInfo in ctx.attr.wasm: diff --git a/rust/defs.bzl b/rust/defs.bzl index 4f2ef72582..09be66d253 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -78,25 +78,37 @@ load( _rust_unpretty_aspect = "rust_unpretty_aspect", ) -rust_library = _rust_library +def _rule_wrapper(rule): + def _wrapped(name, deps = [], proc_macro_deps = [], **kwargs): + rule( + name = name, + deps = deps + proc_macro_deps, + # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. + proc_macro_deps = deps + proc_macro_deps, + **kwargs + ) + + return _wrapped + +rust_library = _rule_wrapper(_rust_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_static_library = _rust_static_library +rust_static_library = _rule_wrapper(_rust_static_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_shared_library = _rust_shared_library +rust_shared_library = _rule_wrapper(_rust_shared_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_proc_macro = _rust_proc_macro +rust_proc_macro = _rule_wrapper(_rust_proc_macro) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_binary = _rust_binary +rust_binary = _rule_wrapper(_rust_binary) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_library_group = _rust_library_group # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_test = _rust_test +rust_test = _rule_wrapper(_rust_test) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_test_suite = _rust_test_suite @@ -105,7 +117,7 @@ rust_test_suite = _rust_test_suite rust_doc = _rust_doc # See @rules_rust//rust/private:rustdoc.bzl for a complete description. -rust_doc_test = _rust_doc_test +rust_doc_test = _rule_wrapper(_rust_doc_test) # See @rules_rust//rust/private:rustdoc_test.bzl for a complete description. clippy_flag = _clippy_flag diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index cb34e0ed01..22cdddc09e 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -23,6 +23,7 @@ load( "BuildInfo", "CrateGroupInfo", "CrateInfo", + "DepInfo", "LintsInfo", ) load( @@ -46,6 +47,7 @@ load( "determine_lib_name", "determine_output_hash", "expand_dict_value_locations", + "filter_deps", "find_toolchain", "generate_output_diagnostics", "get_edition", @@ -64,41 +66,6 @@ def _assert_no_deprecated_attributes(_ctx): """ pass -def _assert_correct_dep_mapping(ctx): - """Forces a failure if proc_macro_deps and deps are mixed inappropriately - - Args: - ctx (ctx): The current rule's context object - """ - for dep in ctx.attr.deps: - if rust_common.crate_info in dep: - if dep[rust_common.crate_info].type == "proc-macro": - fail( - "{} listed {} in its deps, but it is a proc-macro. It should instead be in the bazel property proc_macro_deps.".format( - ctx.label, - dep.label, - ), - ) - for dep in ctx.attr.proc_macro_deps: - if CrateInfo in dep: - types = [dep[CrateInfo].type] - else: - types = [ - dep_variant_info.crate_info.type - for dep_variant_info in dep[CrateGroupInfo].dep_variant_infos.to_list() - if dep_variant_info.crate_info - ] - - for type in types: - if type != "proc-macro": - fail( - "{} listed {} in its proc_macro_deps, but it is not proc-macro, it is a {}. It should probably instead be listed in deps.".format( - ctx.label, - dep.label, - type, - ), - ) - def _rust_library_impl(ctx): """The implementation of the `rust_library` rule. @@ -168,7 +135,7 @@ def _rust_library_common(ctx, crate_type): list: A list of providers. See `rustc_compile_action` """ _assert_no_deprecated_attributes(ctx) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) toolchain = find_toolchain(ctx) @@ -215,8 +182,8 @@ def _rust_library_common(ctx, crate_type): not ctx.attr.disable_pipelining ) - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) return rustc_compile_action( ctx = ctx, @@ -259,7 +226,7 @@ def _rust_binary_impl(ctx): """ toolchain = find_toolchain(ctx) crate_name = compute_crate_name(ctx.workspace_name, ctx.label, toolchain, ctx.attr.crate_name) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) if ctx.attr.binary_name: output_filename = ctx.attr.binary_name @@ -267,8 +234,8 @@ def _rust_binary_impl(ctx): output_filename = ctx.label.name output = ctx.actions.declare_file(output_filename + toolchain.binary_ext) - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) crate_root = getattr(ctx.file, "crate_root", None) if not crate_root: @@ -349,13 +316,13 @@ def _rust_test_impl(ctx): list: The list of providers. See `rustc_compile_action` """ _assert_no_deprecated_attributes(ctx) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) toolchain = find_toolchain(ctx) crate_type = "bin" - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) if ctx.attr.crate and ctx.attr.srcs: fail("rust_test.crate and rust_test.srcs are mutually exclusive. Update {} to use only one of these attributes".format( @@ -539,16 +506,16 @@ def _rust_library_group_impl(ctx): runfiles = [] for dep in ctx.attr.deps: - if rust_common.crate_info in dep: + if CrateInfo in dep: dep_variant_infos.append(rust_common.dep_variant_info( - crate_info = dep[rust_common.crate_info] if rust_common.crate_info in dep else None, - dep_info = dep[rust_common.dep_info] if rust_common.crate_info in dep else None, + crate_info = dep[CrateInfo] if CrateInfo in dep else None, + dep_info = dep[DepInfo] if DepInfo in dep else None, build_info = dep[BuildInfo] if BuildInfo in dep else None, cc_info = dep[CcInfo] if CcInfo in dep else None, crate_group_info = None, )) - elif rust_common.crate_group_info in dep: - dep_variant_transitive_infos.append(dep[rust_common.crate_group_info].dep_variant_infos) + elif CrateGroupInfo in dep: + dep_variant_transitive_infos.append(dep[CrateGroupInfo].dep_variant_infos) else: fail("crate_group_info targets can only depend on rust_library or rust_library_group targets.") @@ -724,10 +691,12 @@ _COMMON_ATTRS = { # `@local_config_platform//:exec` exposed. "proc_macro_deps": attr.label_list( doc = dedent("""\ - List of `rust_proc_macro` targets used to help build this library target. + Copy of deps in exec configuration. This should really be called `exec_configured_deps`. + + Rule implementations use this to select exec-configured `rust_proc_macro` targets. + User code should pass all deps to `deps` for the macros loaded from `defs.bzl`. """), cfg = "exec", - providers = [[CrateInfo], [CrateGroupInfo]], ), "require_explicit_unstable_features": attr.int( doc = ( @@ -1322,7 +1291,9 @@ rust_binary_without_process_wrapper = rule( implementation = _rust_binary_without_process_wrapper_impl, doc = "A variant of `rust_binary` that uses a minimal process wrapper for `Rustc` actions.", provides = COMMON_PROVIDERS + [_RustBuiltWithoutProcessWrapperInfo], - attrs = _common_attrs_for_binary_without_process_wrapper(_COMMON_ATTRS | _RUST_BINARY_ATTRS), + attrs = _common_attrs_for_binary_without_process_wrapper(_COMMON_ATTRS | _RUST_BINARY_ATTRS | { + "_skip_deps_verification": attr.bool(default = True), + }), executable = True, fragments = ["cpp"], toolchains = [ @@ -1558,7 +1529,7 @@ rust_test = rule( """), ) -def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): +def rust_test_suite(name, srcs, shared_srcs = [], deps = [], proc_macro_deps = [], **kwargs): """A rule for creating a test suite for a set of `rust_test` targets. This rule can be used for setting up typical rust [integration tests][it]. Given the following @@ -1611,6 +1582,8 @@ def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): name (str): The name of the `test_suite`. srcs (list): All test sources, typically `glob(["tests/**/*.rs"])`. shared_srcs (list): Optional argument for sources shared among tests, typically helper functions. + deps (list): Deps and proc_macro_deps for underlying test. + proc_macro_deps (list): Deprecated; do not use. **kwargs (dict): Additional keyword arguments for the underlying [rust_test](#rust_test) targets. The `tags` argument is also passed to the generated `test_suite` target. """ @@ -1641,6 +1614,8 @@ def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): srcs = [src] + shared_srcs, tags = tags, crate_name = crate_name, + deps = deps + proc_macro_deps, + proc_macro_deps = deps + proc_macro_deps, **kwargs ) tests.append(test_name) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 35bbe73b66..24fc36f2b1 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -226,7 +226,7 @@ def collect_deps( Args: deps (list): The deps from ctx.attr.deps. - proc_macro_deps (list): The proc_macro deps from ctx.attr.proc_macro_deps. + proc_macro_deps (list): The proc_macro deps from `filter_deps(ctx)`. aliases (dict): A dict mapping aliased targets to their actual Crate information. Returns: diff --git a/rust/private/rustdoc/BUILD.bazel b/rust/private/rustdoc/BUILD.bazel index ee2067a87d..85cda1cd7e 100644 --- a/rust/private/rustdoc/BUILD.bazel +++ b/rust/private/rustdoc/BUILD.bazel @@ -1,4 +1,4 @@ -load("//rust/private:rust.bzl", "rust_binary") +load("//rust:defs.bzl", "rust_binary") package(default_visibility = ["//visibility:public"]) diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index 522c048cd1..dcf847f10a 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -18,7 +18,7 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//rust/private:common.bzl", "rust_common") load("//rust/private:providers.bzl", "CrateInfo") load("//rust/private:rustdoc.bzl", "rustdoc_compile_action") -load("//rust/private:utils.bzl", "dedent", "find_toolchain", "transform_deps") +load("//rust/private:utils.bzl", "dedent", "filter_deps", "find_toolchain", "transform_deps") def _construct_writer_arguments(ctx, test_runner, opt_test_params, action, crate_info): """Construct arguments and environment variables specific to `rustdoc_test_writer`. @@ -110,8 +110,10 @@ def _rust_doc_test_impl(ctx): toolchain = find_toolchain(ctx) crate = ctx.attr.crate[rust_common.crate_info] - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps) + + deps, proc_macro_deps = filter_deps(ctx) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps) crate_info = rust_common.create_crate_info( name = crate.name, diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl index b3f5b8437c..4d710b7fa9 100644 --- a/rust/private/utils.bzl +++ b/rust/private/utils.bzl @@ -511,14 +511,38 @@ def is_exec_configuration(ctx): # TODO(djmarcin): Is there any better way to determine cfg=exec? return ctx.genfiles_dir.path.find("-exec") != -1 +def filter_deps(ctx): + """Filters the provided (combined) deps into normal deps and proc_macro deps. + + Args: + ctx (ctx): The current rule's context object + + Returns: + deps and proc_macro_deps + """ + if len(ctx.attr.deps) != len(ctx.attr.proc_macro_deps) and not getattr(ctx.attr, "_skip_deps_verification", False): + fail("All deps should be passed to both `deps` and `proc_macro_deps`; please use the macros in //rust:defs.bzl") + + deps = [] + for dep in ctx.attr.deps: + if CrateInfo not in dep or dep[CrateInfo].type != "proc-macro": + deps.append(dep) + + proc_macro_deps = [] + for dep in ctx.attr.proc_macro_deps: + if CrateInfo in dep and dep[CrateInfo].type == "proc-macro": + proc_macro_deps.append(dep) + + return deps, proc_macro_deps + def transform_deps(deps): """Transforms a [Target] into [DepVariantInfo]. - This helper function is used to transform ctx.attr.deps and ctx.attr.proc_macro_deps into + This helper function is used to transform deps and .proc_macro_deps coming from `filter_deps` into [DepVariantInfo]. Args: - deps (list of Targets): Dependencies coming from ctx.attr.deps or ctx.attr.proc_macro_deps + deps (list of Targets): Dependencies coming from `filter_deps` Returns: list of DepVariantInfos. From 9c6dcd016b749c44833974353245e30e9177e917 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 25 Feb 2026 04:15:34 -0500 Subject: [PATCH 15/24] Always use param file for process wrapper --- rust/private/rustc.bzl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 24fc36f2b1..7b4957288d 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -912,6 +912,7 @@ def construct_arguments( force_depend_on_objects = False, skip_expanding_rustc_env = False, require_explicit_unstable_features = False, + always_use_param_file = False, error_format = None): """Builds an Args object containing common rustc flags @@ -1017,7 +1018,7 @@ def construct_arguments( # Rustc arguments rustc_flags = ctx.actions.args() rustc_flags.set_param_file_format("multiline") - rustc_flags.use_param_file("@%s", use_always = False) + rustc_flags.use_param_file("@%s", use_always = always_use_param_file) rustc_flags.add(crate_info.root) rustc_flags.add(crate_info.name, format = "--crate-name=%s") rustc_flags.add(crate_info.type, format = "--crate-type=%s") @@ -1421,6 +1422,7 @@ def rustc_compile_action( use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, + always_use_param_file = not ctx.executable._process_wrapper, ) args_metadata = None From aa60b14fb6b6cd5e84b7b131970cf53626ed8ae2 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 26 Feb 2026 10:49:57 -0500 Subject: [PATCH 16/24] Avoid hashing RustAnalyzerInfo in rust_analyzer alias mapping --- rust/private/rust_analyzer.bzl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rust/private/rust_analyzer.bzl b/rust/private/rust_analyzer.bzl index 1dd8244c98..12bc6fa5ab 100644 --- a/rust/private/rust_analyzer.bzl +++ b/rust/private/rust_analyzer.bzl @@ -132,10 +132,13 @@ def _rust_analyzer_aspect_impl(target, ctx): else: fail("Unexpected target type: {}".format(target)) - aliases = {} + # Keep aliases as a list of (RustAnalyzerInfo, alias_name) tuples. + # Using RustAnalyzerInfo as a dict key can trigger expensive recursive hashing. + aliases = [] for aliased_target, aliased_name in getattr(ctx.rule.attr, "aliases", {}).items(): - if aliased_target.label in labels_to_rais: - aliases[labels_to_rais[aliased_target.label]] = aliased_name + dep_info = labels_to_rais.get(aliased_target.label) + if dep_info: + aliases.append((dep_info, aliased_name)) proc_macro_dylib = find_proc_macro_dylib(toolchain, target) proc_macro_dylibs = [proc_macro_dylib] if proc_macro_dylib else None @@ -294,7 +297,7 @@ def _create_single_crate(ctx, attrs, info): # the crate being processed, we don't add it as a dependency to itself. This is # common and expected - `rust_test.crate` pointing to the `rust_library`. crate["deps"] = [_crate_id(dep.crate) for dep in info.deps if _crate_id(dep.crate) != crate_id] - crate["aliases"] = {_crate_id(alias_target.crate): alias_name for alias_target, alias_name in info.aliases.items()} + crate["aliases"] = {_crate_id(alias_target.crate): alias_name for alias_target, alias_name in info.aliases} crate["cfg"] = info.cfgs toolchain = find_toolchain(ctx) crate["target"] = (_EXEC_ROOT_TEMPLATE + toolchain.target_json.path) if toolchain.target_json else toolchain.target_flag_value From 8616e334e790d804a0f216b54eb8fdbef3ed7e7a Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Thu, 26 Feb 2026 11:33:43 -0500 Subject: [PATCH 17/24] Convert wrappers to symbolic macros --- rust/defs.bzl | 27 ++++++++++++++++++++++++--- rust/rust_binary.bzl | 2 +- rust/rust_library.bzl | 2 +- rust/rust_test.bzl | 2 +- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/rust/defs.bzl b/rust/defs.bzl index 09be66d253..8c6c14813d 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -14,6 +14,7 @@ """Public entry point to all Rust rules and supported APIs.""" +load("@bazel_features//:features.bzl", "bazel_features") load( "//rust:toolchain.bzl", _rust_stdlib_filegroup = "rust_stdlib_filegroup", @@ -90,7 +91,27 @@ def _rule_wrapper(rule): return _wrapped -rust_library = _rule_wrapper(_rust_library) +def _symbolic_rule_wrapper(rule, macro_fn): + def _wrapped(name, visibility, deps, proc_macro_deps, **kwargs): + rule( + name = name, + visibility = visibility, + deps = deps + proc_macro_deps, + # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. + proc_macro_deps = deps + proc_macro_deps, + **kwargs + ) + + return macro_fn( + implementation = _wrapped, + inherit_attrs = rule, + attrs = { + "deps": attr.label_list(default = []), + "proc_macro_deps": attr.label_list(default = []), + }, + ) + +rust_library = _symbolic_rule_wrapper(_rust_library, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_library) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_static_library = _rule_wrapper(_rust_static_library) @@ -102,13 +123,13 @@ rust_shared_library = _rule_wrapper(_rust_shared_library) rust_proc_macro = _rule_wrapper(_rust_proc_macro) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_binary = _rule_wrapper(_rust_binary) +rust_binary = _symbolic_rule_wrapper(_rust_binary, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_binary) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_library_group = _rust_library_group # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_test = _rule_wrapper(_rust_test) +rust_test = _symbolic_rule_wrapper(_rust_test, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_test) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_test_suite = _rust_test_suite diff --git a/rust/rust_binary.bzl b/rust/rust_binary.bzl index dbeba03911..518c7c3f01 100644 --- a/rust/rust_binary.bzl +++ b/rust/rust_binary.bzl @@ -1,7 +1,7 @@ """rust_binary""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_binary = "rust_binary", ) diff --git a/rust/rust_library.bzl b/rust/rust_library.bzl index b1e63494a6..a694ed970c 100644 --- a/rust/rust_library.bzl +++ b/rust/rust_library.bzl @@ -1,7 +1,7 @@ """rust_library""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_library = "rust_library", ) diff --git a/rust/rust_test.bzl b/rust/rust_test.bzl index 001963fccc..abd61ba8c5 100644 --- a/rust/rust_test.bzl +++ b/rust/rust_test.bzl @@ -1,7 +1,7 @@ """rust_test""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_test = "rust_test", ) From d9dfb1e1fc3f0e8eb03b728bafffa7f6df41dcf9 Mon Sep 17 00:00:00 2001 From: Titouan BION Date: Sat, 28 Feb 2026 08:47:56 +0100 Subject: [PATCH 18/24] Add missing system keys to `triple_mappings.bzl` (#5) --- rust/platform/triple_mappings.bzl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/platform/triple_mappings.bzl b/rust/platform/triple_mappings.bzl index d6265f0a51..72dd64225c 100644 --- a/rust/platform/triple_mappings.bzl +++ b/rust/platform/triple_mappings.bzl @@ -179,6 +179,7 @@ _SYSTEM_TO_BINARY_EXT = { "ios": "", "linux": "", "macos": "", + "netbsd": "", "nixos": "", "none": "", "nto": "", @@ -187,6 +188,7 @@ _SYSTEM_TO_BINARY_EXT = { # generated extension for the wasm target, similarly to the # windows target "unknown": ".wasm", + "threads": ".wasm", "wasi": ".wasm", "wasip1": ".wasm", "wasip2": ".wasm", @@ -204,11 +206,13 @@ _SYSTEM_TO_STATICLIB_EXT = { "ios": ".a", "linux": ".a", "macos": ".a", + "netbsd": ".a", "nixos": ".a", "none": ".a", "nto": ".a", "uefi": ".lib", "unknown": "", + "threads": "", "wasi": "", "wasip1": "", "wasip2": "", @@ -226,11 +230,13 @@ _SYSTEM_TO_DYLIB_EXT = { "ios": ".dylib", "linux": ".so", "macos": ".dylib", + "netbsd": ".so", "nixos": ".so", "none": ".so", "nto": ".a", "uefi": "", # UEFI doesn't have dynamic linking "unknown": ".wasm", + "threads": ".wasm", "wasi": ".wasm", "wasip1": ".wasm", "wasip2": ".wasm", From 63593213eec731ef0a75e53da6cf9d5568d420de Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Sat, 28 Feb 2026 12:52:38 -0500 Subject: [PATCH 19/24] Handle toolchain registration when not registered as a bazel_dep --- rust/extensions.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/extensions.bzl b/rust/extensions.bzl index 749740032e..b059d6e02e 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -26,8 +26,6 @@ def _find_modules(module_ctx): our_module = mod if root == None: root = our_module - if our_module == None: - fail("Unable to find rules_rust module") return root, our_module @@ -93,7 +91,9 @@ def _rust_impl(module_ctx): if toolchain_triples.get(repository_set["exec_triple"]) == repository_set["name"]: toolchain_triples.pop(repository_set["exec_triple"], None) - toolchains = root.tags.toolchain or rules_rust.tags.toolchain + toolchains = root.tags.toolchain + if not toolchains and rules_rust: + toolchains = rules_rust.tags.toolchain for toolchain in toolchains: if toolchain.extra_rustc_flags and toolchain.extra_rustc_flags_triples: From 40c17fcb5ad2ccaaa464d95f469ad7e77f1f4241 Mon Sep 17 00:00:00 2001 From: Walter Gray Date: Wed, 25 Feb 2026 18:32:17 -0800 Subject: [PATCH 20/24] Switch pipelining metadata action to hollow rlib (-Zno-codegen) Replace the --rustc-quit-on-rmeta / .rmeta approach with Buck2-style hollow rlibs: the RustcMetadata action runs rustc to completion with -Zno-codegen, emitting a .rlib archive (named -hollow.rlib) that contains only metadata and optimized MIR. Key fixes: - Main Rustc action uses full rlib --extern deps so the SVH embedded in the full rlib matches what downstream binaries expect (avoiding E0460 with non-deterministic proc macros). - RUSTC_BOOTSTRAP=1 is set on both actions: it changes the crate hash, so inconsistent use would cause SVH mismatch even for deterministic crates. - -Ldependency= points to the _hollow/ subdirectory so the full rlib and hollow rlib never appear in the same search path (avoids E0463). - transitive_metadata_outputs are always included in the sandbox so rustc can resolve transitive -Ldependency= references. Adds an SVH mismatch test that demonstrates the problem with non-deterministic proc macros (uses a HashMap-based derive macro whose output varies by OS-seeded random seed). --- rust/private/rust.bzl | 8 +- rust/private/rustc.bzl | 151 ++++++++++++-- rust/settings/settings.bzl | 21 +- .../pipelined_compilation_test.bzl | 194 ++++++++++++++---- .../svh_mismatch/svh_mismatch_consumer.rs | 6 + .../svh_mismatch/svh_mismatch_lib.rs | 8 + .../svh_mismatch_nondeterministic_macro.rs | 37 ++++ .../svh_mismatch/svh_mismatch_test.rs | 28 +++ test/unit/pipelined_compilation/wrap.bzl | 26 ++- 9 files changed, 410 insertions(+), 69 deletions(-) create mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs create mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs create mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs create mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 22cdddc09e..3afba08713 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -172,11 +172,15 @@ def _rust_library_common(ctx, crate_type): crate_type, disable_pipelining = getattr(ctx.attr, "disable_pipelining", False), ): + # The hollow rlib uses .rlib extension (not .rmeta) so rustc reads it as an + # rlib archive containing lib.rmeta with optimized MIR. It is placed in a + # "_hollow/" subdirectory so the full rlib and hollow rlib never appear in the + # same -Ldependency= search directory (which would cause E0463). rust_metadata = ctx.actions.declare_file( - paths.replace_extension(rust_lib_name, ".rmeta"), - sibling = rust_lib, + "_hollow/" + rust_lib_name[:-len(".rlib")] + "-hollow.rlib", ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + metadata_supports_pipelining = ( can_use_metadata_for_pipelining(toolchain, crate_type) and not ctx.attr.disable_pipelining diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 7b4957288d..9e7adba219 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -663,7 +663,7 @@ def _disambiguate_libs(actions, toolchain, crate_info, dep_info, use_pic): visited_libs[name] = artifact return ambiguous_libs -def _depend_on_metadata(crate_info, force_depend_on_objects): +def _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link = False): """Determines if we can depend on metadata for this crate. By default (when pipelining is disabled or when the crate type needs to link against @@ -673,9 +673,22 @@ def _depend_on_metadata(crate_info, force_depend_on_objects): In some rare cases, even if both of those conditions are true, we still want to depend on objects. This is what force_depend_on_objects is. + When experimental_use_cc_common_link is True, bin/cdylib crates also use hollow + rlib deps. The rustc step only emits .o files (no rustc linking), so SVH chain + consistency is sufficient; the actual linking is done by cc_common.link, which + does not check SVH. + + Callers are responsible for zeroing out experimental_use_cc_common_link for + exec-platform builds before calling this function (see rustc_compile_action). + Exec-platform binaries (build scripts) must use full rlib deps because their + CcInfo linking contexts may lack a CC toolchain. + Args: crate_info (CrateInfo): The Crate to determine this for. force_depend_on_objects (bool): if set we will not depend on metadata. + experimental_use_cc_common_link (bool): if set, bin/cdylib crates also use + hollow rlib deps for SVH consistency. Must already be False for + exec-platform builds when this function is called. Returns: Whether we can depend on metadata for this crate. @@ -683,6 +696,11 @@ def _depend_on_metadata(crate_info, force_depend_on_objects): if force_depend_on_objects: return False + if experimental_use_cc_common_link and crate_info.type in ("bin", "cdylib"): + # cc_common.link: rustc only emits .o files, so hollow rlib deps are safe and + # keep the SVH chain consistent (avoiding E0460 from nondeterministic proc macros). + return True + return crate_info.type in ("rlib", "lib") def collect_inputs( @@ -770,7 +788,7 @@ def collect_inputs( linkstamp_outs = [] transitive_crate_outputs = dep_info.transitive_crate_outputs - if _depend_on_metadata(crate_info, force_depend_on_objects): + if _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link): transitive_crate_outputs = dep_info.transitive_metadata_outputs nolinkstamp_compile_direct_inputs = [] @@ -806,6 +824,12 @@ def collect_inputs( transitive = [ crate_info.srcs, transitive_crate_outputs, + # Always include hollow rlibs so they are present in the sandbox for + # -Ldependency= resolution. Binaries and proc-macros compile against full + # rlib --extern deps but need hollow rlibs available for transitive + # dependency resolution when those rlibs were themselves compiled against + # hollow deps. For rlib/lib crates this is a no-op (already included above). + dep_info.transitive_metadata_outputs, crate_info.compile_data, dep_info.transitive_proc_macro_data, toolchain.all_files, @@ -910,6 +934,7 @@ def construct_arguments( use_json_output = False, build_metadata = False, force_depend_on_objects = False, + experimental_use_cc_common_link = False, skip_expanding_rustc_env = False, require_explicit_unstable_features = False, always_use_param_file = False, @@ -1046,8 +1071,14 @@ def construct_arguments( error_format = "json" if build_metadata: - # Configure process_wrapper to terminate rustc when metadata are emitted - process_wrapper_flags.add("--rustc-quit-on-rmeta", "true") + if crate_info.type in ("rlib", "lib"): + # Hollow rlib approach (Buck2-style): rustc runs to completion with -Zno-codegen, + # producing a hollow .rlib (metadata only, no object code) via --emit=link=. + # No need to kill rustc — -Zno-codegen skips codegen entirely and exits quickly. + rustc_flags.add("-Zno-codegen") + + # else: IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits + # naturally after writing .rmeta via --emit=dep-info,metadata (no kill needed). if crate_info.rustc_rmeta_output: process_wrapper_flags.add("--output-file", crate_info.rustc_rmeta_output.path) elif crate_info.rustc_output: @@ -1080,7 +1111,14 @@ def construct_arguments( emit_without_paths = [] for kind in emit: - if kind == "link" and crate_info.type == "bin" and crate_info.output != None: + if kind == "link" and build_metadata and crate_info.type in ("rlib", "lib") and crate_info.metadata: + # Hollow rlib: direct rustc's link output to the -hollow.rlib path. + # The file has .rlib extension so rustc reads it as an rlib archive + # (with optimized MIR in lib.rmeta). Using a .rmeta path would cause + # E0786 "found invalid metadata files" because rustc parses .rmeta files + # as raw metadata blobs, not rlib archives. + rustc_flags.add(crate_info.metadata, format = "--emit=link=%s") + elif kind == "link" and crate_info.type == "bin" and crate_info.output != None: rustc_flags.add(crate_info.output, format = "--emit=link=%s") else: emit_without_paths.append(kind) @@ -1155,7 +1193,7 @@ def construct_arguments( include_link_flags = include_link_flags, ) - use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects) + use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link) # These always need to be added, even if not linking this crate. add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct, use_metadata) @@ -1324,6 +1362,13 @@ def rustc_compile_action( rustc_output = crate_info.rustc_output rustc_rmeta_output = crate_info.rustc_rmeta_output + # Use the hollow rlib approach (Buck2-style) for rlib/lib crate types when a metadata + # action is being created. This always applies for rlib/lib regardless of whether + # pipelining is globally enabled — the hollow rlib is simpler than killing rustc. + # Non-rlib types (bin, proc-macro, etc.) use --emit=dep-info,metadata instead + # (rustc exits naturally after writing .rmeta, no process-wrapper kill needed). + use_hollow_rlib = bool(build_metadata) and crate_info.type in ("rlib", "lib") + # Determine whether to use cc_common.link: # * either if experimental_use_cc_common_link is 1, # * or if experimental_use_cc_common_link is -1 and @@ -1337,6 +1382,12 @@ def rustc_compile_action( elif ctx.attr.experimental_use_cc_common_link == -1: experimental_use_cc_common_link = toolchain._experimental_use_cc_common_link + # Exec-platform binaries (build scripts) skip cc_common.link: exec-configuration + # rlib deps may lack a CC toolchain, causing empty CcInfo linking contexts. They + # use standard rustc linking with full rlib deps instead. + if experimental_use_cc_common_link and is_exec_configuration(ctx): + experimental_use_cc_common_link = False + dep_info, build_info, linkstamps = collect_deps( deps = deps, proc_macro_deps = proc_macro_deps, @@ -1375,17 +1426,34 @@ def rustc_compile_action( experimental_use_cc_common_link = experimental_use_cc_common_link, ) - # The types of rustc outputs to emit. - # If we build metadata, we need to keep the command line of the two invocations - # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as - # a candidate. - # Because of that we need to add emit=metadata to both the rlib and rmeta invocation. - # - # When cc_common linking is enabled, emit a `.o` file, which is later - # passed to the cc_common.link action. + # The main Rustc action uses FULL rlib deps so the full rlib it produces records + # full-rlib SVHs. A downstream binary links against full rlibs; if the Rustc action + # had used hollow rlib deps instead, nondeterministic proc macros could produce + # different SVHs for the hollow vs full rlib, causing E0460 in the binary build. + # The RustcMetadata action still uses hollow rlibs (compile_inputs_for_metadata) + # so it can start before full codegen of its deps completes. + compile_inputs_for_metadata = compile_inputs + if use_hollow_rlib: + compile_inputs, _, _, _, _, _ = collect_inputs( + ctx = ctx, + file = ctx.file, + files = ctx.files, + linkstamps = linkstamps, + toolchain = toolchain, + cc_toolchain = cc_toolchain, + feature_configuration = feature_configuration, + crate_info = crate_info, + dep_info = dep_info, + build_info = build_info, + lint_files = lint_files, + stamp = stamp, + force_depend_on_objects = True, + experimental_use_cc_common_link = experimental_use_cc_common_link, + ) + + # The main Rustc action emits dep-info and link (the full rlib/binary/cdylib). + # When cc_common linking is enabled, emit a `.o` file instead. emit = ["dep-info", "link"] - if build_metadata: - emit.append("metadata") if experimental_use_cc_common_link: emit = ["obj"] @@ -1420,6 +1488,9 @@ def rustc_compile_action( force_all_deps_direct = force_all_deps_direct, stamp = stamp, use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), + # Force full rlib --extern deps so the full rlib records full-rlib SVHs. + force_depend_on_objects = use_hollow_rlib, + experimental_use_cc_common_link = experimental_use_cc_common_link, skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, always_use_param_file = not ctx.executable._process_wrapper, @@ -1427,6 +1498,15 @@ def rustc_compile_action( args_metadata = None if build_metadata: + if use_hollow_rlib: + # Hollow rlib: emit dep-info and link (directed to the -hollow.rlib path via + # -Zno-codegen). dep-info must be included: it affects the SVH stored in the + # rlib, so both actions must include it to keep SVHs consistent. + metadata_emit = ["dep-info", "link"] + else: + # IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits + # naturally after writing .rmeta with --emit=dep-info,metadata. + metadata_emit = ["dep-info", "metadata"] args_metadata, _ = construct_arguments( ctx = ctx, attr = attr, @@ -1434,7 +1514,7 @@ def rustc_compile_action( toolchain = toolchain, tool_path = toolchain.rustc.path, cc_toolchain = cc_toolchain, - emit = emit, + emit = metadata_emit, feature_configuration = feature_configuration, crate_info = crate_info, dep_info = dep_info, @@ -1449,6 +1529,7 @@ def rustc_compile_action( stamp = stamp, use_json_output = True, build_metadata = True, + experimental_use_cc_common_link = experimental_use_cc_common_link, require_explicit_unstable_features = require_explicit_unstable_features, ) @@ -1457,6 +1538,13 @@ def rustc_compile_action( # this is the final list of env vars env.update(env_from_args) + if use_hollow_rlib: + # Both the metadata action and the full Rustc action must have RUSTC_BOOTSTRAP=1 + # for SVH compatibility. RUSTC_BOOTSTRAP=1 changes the crate hash — setting it + # on only one action would cause SVH mismatch even for deterministic crates. + # This enables -Zno-codegen on stable Rust compilers for the metadata action. + env["RUSTC_BOOTSTRAP"] = "1" + if hasattr(attr, "version") and attr.version != "0.0.0": formatted_version = " v{}".format(attr.version) else: @@ -1528,7 +1616,7 @@ def rustc_compile_action( if args_metadata: ctx.actions.run( executable = ctx.executable._process_wrapper, - inputs = compile_inputs, + inputs = compile_inputs_for_metadata, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, arguments = args_metadata.all, @@ -2144,9 +2232,14 @@ def add_crate_link_flags(args, dep_info, force_all_deps_direct = False, use_meta crate_to_link_flags = _crate_to_link_flag_metadata if use_metadata else _crate_to_link_flag args.add_all(direct_crates, uniquify = True, map_each = crate_to_link_flags) + # Use hollow rlib directories for -Ldependency= when use_metadata=True (rlib/lib) + # so that both --extern= and -Ldependency= point to the same hollow rlib file. + # When use_metadata=False (bins, proc-macros), use full rlib directories; pointing + # to hollow dirs alongside full --extern= args would cause E0463 (ambiguous crate). + get_dirname = _get_crate_dirname_pipelined if use_metadata else _get_crate_dirname args.add_all( dep_info.transitive_crates, - map_each = _get_crate_dirname, + map_each = get_dirname, uniquify = True, format_each = "-Ldependency=%s", ) @@ -2204,7 +2297,25 @@ def _get_crate_dirname(crate): """ return crate.output.dirname -def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_darwin = False, flavor_msvc = False): +def _get_crate_dirname_pipelined(crate): + """For pipelined compilation: returns the _hollow/ directory for pipelined crates + + When a crate supports pipelining and has a hollow rlib in its _hollow/ subdirectory, + pointing -Ldependency= to that subdirectory lets rustc find the hollow rlib (which has + the correct SVH matching downstream metadata). Pointing to the parent directory instead + would expose the full rlib (compiled separately, with a different SVH), causing E0460. + + Args: + crate (CrateInfo): A CrateInfo provider from the current rule + + Returns: + str: The directory to use for -Ldependency= search. + """ + if crate.metadata and crate.metadata_supports_pipelining: + return crate.metadata.dirname + return crate.output.dirname + +def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = False, for_darwin = False, flavor_msvc = False): artifact = get_preferred_artifact(lib, use_pic) if ambiguous_libs and artifact.path in ambiguous_libs: artifact = ambiguous_libs[artifact.path] diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index b6d1c0aea4..1c9bcb1d06 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -112,10 +112,18 @@ def use_real_import_macro(): ) def pipelined_compilation(): - """When set, this flag causes rustc to emit `*.rmeta` files and use them for `rlib -> rlib` dependencies. - - While this involves one extra (short) rustc invocation to build the rmeta file, - it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation. + """When set, this flag enables pipelined compilation for rlib/lib crates. + + For each rlib/lib, a separate RustcMetadata action produces a hollow rlib + (via `-Zno-codegen`) containing only metadata. Downstream rlib/lib crates + can begin compiling against the hollow rlib before the upstream full codegen + action completes, increasing build parallelism. + + Pipelining applies to rlib→rlib dependencies by default. To also pipeline + bin/cdylib crates (starting their compile step before upstream full codegen + finishes), enable `experimental_use_cc_common_link` alongside this flag. + With cc_common.link, rustc only emits `.o` files for binaries (linking is + handled separately), so hollow rlib deps are safe for bins too. """ bool_flag( name = "pipelined_compilation", @@ -126,6 +134,11 @@ def pipelined_compilation(): def experimental_use_cc_common_link(): """A flag to control whether to link rust_binary and rust_test targets using \ cc_common.link instead of rustc. + + When combined with `pipelined_compilation`, bin/cdylib crates also participate + in the hollow-rlib dependency chain: rustc only emits `.o` files (linking is + done by cc_common.link and does not check SVH), so bin compile steps can start + as soon as upstream hollow rlibs are ready rather than waiting for full codegen. """ bool_flag( name = "experimental_use_cc_common_link", diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl index 36a3de891b..0f638c3ee6 100644 --- a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl +++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl @@ -1,8 +1,8 @@ """Unittests for rust rules.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") -load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro") -load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not") +load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") +load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements_not") load(":wrap.bzl", "wrap") ENABLE_PIPELINING = { @@ -22,49 +22,77 @@ def _second_lib_test_impl(ctx): rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - # Both actions should use the same --emit= - assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata") - assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata") + # Hollow rlib approach: Rustc action uses --emit=dep-info,link (no metadata). + assert_argv_contains(env, rlib_action, "--emit=dep-info,link") - # The metadata action should have a .rmeta as output and the rlib action a .rlib + # Metadata action uses --emit=link=-hollow.rlib (hollow rlib, .rlib extension). + # The .rlib extension is required so rustc reads it as an rlib archive (extracting + # lib.rmeta with optimized MIR). Using .rmeta extension causes E0786, and using + # --emit=metadata produces raw .rmeta without optimized MIR (causes "missing + # optimized MIR" errors on Rust 1.85+). + metadata_emit_link = [arg for arg in metadata_action.argv if arg.startswith("--emit=link=") and arg.endswith("-hollow.rlib")] + asserts.true( + env, + len(metadata_emit_link) == 1, + "expected --emit=link=*-hollow.rlib for hollow rlib, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--emit=")]), + ) + + # The rlib action produces a .rlib; the metadata action produces a -hollow.rlib. path = rlib_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rlib"), - "expected Rustc to output .rlib, got " + path, + path.endswith(".rlib") and not path.endswith("-hollow.rlib"), + "expected Rustc to output .rlib (not hollow), got " + path, ) path = metadata_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rmeta"), - "expected RustcMetadata to output .rmeta, got " + path, + path.endswith("-hollow.rlib"), + "expected RustcMetadata to output -hollow.rlib, got " + path, ) - # Only the action building metadata should contain --rustc-quit-on-rmeta + # Neither action should use --rustc-quit-on-rmeta (hollow rlib exits naturally). assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"]) - assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) - - # Check that both actions refer to the metadata of :first, not the rlib - extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + assert_list_contains_adjacent_elements_not(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) + + # The metadata action should use -Zno-codegen for the hollow rlib approach. + assert_argv_contains(env, metadata_action, "-Zno-codegen") + + # The Rustc action should NOT use -Zno-codegen. + no_codegen_in_rlib = [arg for arg in rlib_action.argv if arg == "-Zno-codegen"] + asserts.true(env, len(no_codegen_in_rlib) == 0, "Rustc action should not have -Zno-codegen") + + # The metadata action references first's hollow rlib for --extern (pipelining: starts + # before first's full codegen finishes). The Rustc action uses the full rlib for + # --extern so the full rlib's embedded SVH matches the full rlib that downstream + # binaries (without cc_common.link) see in their -Ldependency path. If both actions + # used the hollow rlib, nondeterministic proc macros could produce different SVHs + # for the hollow vs full rlib, causing E0460 in downstream binary builds. + extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("-hollow.rlib")] asserts.true( env, len(extern_metadata) == 1, - "did not find a --extern=first=*.rmeta but expected one", + "did not find --extern=first=*-hollow.rlib for metadata action, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--extern=first=")]), ) - extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + extern_rlib_full = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and not arg.endswith("-hollow.rlib")] asserts.true( env, - len(extern_rlib) == 1, - "did not find a --extern=first=*.rlib but expected one", + len(extern_rlib_full) == 1, + "expected --extern=first=libfirst*.rlib (full rlib) for rlib action, got: " + str([arg for arg in rlib_action.argv if arg.startswith("--extern=first=")]), ) - # Check that the input to both actions is the metadata of :first + # The metadata action's input is first's hollow rlib only (no full rlib needed). input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata])) - asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path) - input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib])) - asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path) + asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input for metadata, found " + str([i.path for i in input_metadata])) + asserts.true(env, input_metadata[0].basename.endswith("-hollow.rlib"), "expected hollow rlib for metadata action, found " + input_metadata[0].path) + + # The Rustc action's inputs contain the full rlib (referenced by --extern) and the + # hollow rlib (present in the sandbox for -Ldependency=<_hollow_dir> resolution of + # transitive deps that were compiled against hollow rlibs). + input_rlib_full = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and not i.basename.endswith("-hollow.rlib")] + input_rlib_hollow = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("-hollow.rlib")] + asserts.true(env, len(input_rlib_full) == 1, "expected full rlib in rlib action inputs, found " + str([i.path for i in input_rlib_full])) + asserts.true(env, len(input_rlib_hollow) == 1, "expected hollow rlib in rlib action inputs (for sandbox), found " + str([i.path for i in input_rlib_hollow])) return analysistest.end(env) @@ -124,10 +152,16 @@ def _pipelined_compilation_test(): target_under_test = ":bin", target_compatible_with = _NO_WINDOWS, ) + hollow_rlib_env_test( + name = "hollow_rlib_env_test", + target_under_test = ":second", + target_compatible_with = _NO_WINDOWS, + ) return [ ":second_lib_test", ":bin_test", + ":hollow_rlib_env_test", ] def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): @@ -138,8 +172,8 @@ def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): # also depend on metadata for 'wrapper'. rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")] - rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")] + metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith("-hollow.rlib")] + rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib") and not i.path.endswith("-hollow.rlib")] seen_wrapper_metadata = False seen_to_wrap_metadata = False @@ -176,22 +210,30 @@ def _rmeta_is_used_when_building_custom_rule_test_impl(ctx): # This is the custom rule invocation of rustc. rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata seen_to_wrap_rlib = False - seen_to_wrap_rmeta = False + seen_to_wrap_hollow = False for act in rust_action.inputs.to_list(): - if "libto_wrap" in act.path and act.path.endswith(".rlib"): + if "libto_wrap" in act.path and act.path.endswith("-hollow.rlib"): + seen_to_wrap_hollow = True + elif "libto_wrap" in act.path and act.path.endswith(".rlib") and not act.path.endswith("-hollow.rlib"): seen_to_wrap_rlib = True - elif "libto_wrap" in act.path and act.path.endswith(".rmeta"): - seen_to_wrap_rmeta = True - asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found") - asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") + if ctx.attr.generate_metadata: + # When wrapper generates its own hollow rlib, the Rustc action uses the full + # rlib of to_wrap for --extern (SVH consistency) and also has the hollow rlib + # in the sandbox for -Ldependency= resolution. + asserts.true(env, seen_to_wrap_hollow, "expected hollow rlib in inputs (for sandbox) when generate_metadata=True") + asserts.true(env, seen_to_wrap_rlib, "expected full rlib in inputs for --extern when generate_metadata=True") + else: + # When wrapper does not generate its own hollow rlib, the Rustc action uses + # hollow rlib deps via normal _depend_on_metadata logic (pipelined rlib deps). + asserts.true(env, seen_to_wrap_hollow, "expected dependency on metadata for 'to_wrap' but not found") + asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") return analysistest.end(env) rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) -rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING) +rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): env = analysistest.begin(ctx) @@ -204,6 +246,33 @@ def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): rmeta_not_produced_if_pipelining_disabled_test = analysistest.make(_rmeta_not_produced_if_pipelining_disabled_test_impl, config_settings = ENABLE_PIPELINING) +def _hollow_rlib_env_test_impl(ctx): + """Verify RUSTC_BOOTSTRAP=1 is set consistently on both Rustc and RustcMetadata actions. + + RUSTC_BOOTSTRAP=1 changes the crate hash (SVH), so it must be set on both actions + to keep the hollow rlib and full rlib SVHs consistent.""" + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] + rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] + + asserts.equals( + env, + "1", + metadata_action.env.get("RUSTC_BOOTSTRAP", ""), + "Metadata action should have RUSTC_BOOTSTRAP=1 for hollow rlib approach", + ) + asserts.equals( + env, + "1", + rlib_action.env.get("RUSTC_BOOTSTRAP", ""), + "Rustc action should have RUSTC_BOOTSTRAP=1 for SVH compatibility with hollow rlib", + ) + + return analysistest.end(env) + +hollow_rlib_env_test = analysistest.make(_hollow_rlib_env_test_impl, config_settings = ENABLE_PIPELINING) + def _disable_pipelining_test(): rust_library( name = "lib", @@ -249,6 +318,7 @@ def _custom_rule_test(generate_metadata, suffix): rmeta_is_used_when_building_custom_rule_test( name = "rmeta_is_used_when_building_custom_rule_test" + suffix, + generate_metadata = generate_metadata, target_under_test = ":wrapper" + suffix, target_compatible_with = _NO_WINDOWS, ) @@ -258,6 +328,59 @@ def _custom_rule_test(generate_metadata, suffix): ":rmeta_is_used_when_building_custom_rule_test" + suffix, ] +def _svh_mismatch_test(): + """Creates a rust_test demonstrating SVH mismatch with non-deterministic proc macros. + + Without pipelining (default): each library is compiled exactly once, SVH + is consistent across the dependency graph, and the test builds and passes. + + With pipelining (//rust/settings:pipelined_compilation=true): rules_rust + compiles svh_lib twice in separate rustc invocations — once for the hollow + metadata (.rmeta), once for the full .rlib. Because the proc macro uses + HashMap with OS-seeded randomness, these two invocations typically produce + different token streams and therefore different SVH values. The consumer is + compiled against the hollow .rmeta (recording SVH_1); when rustc links the + test binary against the full .rlib (SVH_2), it detects SVH_1 ≠ SVH_2 and + fails with E0460. The test is therefore expected to FAIL TO BUILD most of + the time (~99.2% with 5 HashMap entries) when pipelining is enabled. + + The test is marked flaky because the SVH mismatch is non-deterministic: + on rare occasions (~0.8%) both rustc invocations produce the same HashMap + iteration order and the build succeeds even with pipelining enabled. + """ + + rust_proc_macro( + name = "svh_nondeterministic_macro", + srcs = ["svh_mismatch/svh_mismatch_nondeterministic_macro.rs"], + crate_name = "nondeterministic_macro", + edition = "2021", + ) + + rust_library( + name = "svh_lib", + srcs = ["svh_mismatch/svh_mismatch_lib.rs"], + edition = "2021", + proc_macro_deps = [":svh_nondeterministic_macro"], + ) + + rust_library( + name = "svh_consumer", + srcs = ["svh_mismatch/svh_mismatch_consumer.rs"], + edition = "2021", + deps = [":svh_lib"], + ) + + rust_test( + name = "svh_mismatch_test", + srcs = ["svh_mismatch/svh_mismatch_test.rs"], + edition = "2021", + deps = [":svh_consumer"], + flaky = True, + target_compatible_with = _NO_WINDOWS, + ) + + return [":svh_mismatch_test"] + def pipelined_compilation_test_suite(name): """Entry-point macro called from the BUILD file. @@ -269,6 +392,7 @@ def pipelined_compilation_test_suite(name): tests.extend(_disable_pipelining_test()) tests.extend(_custom_rule_test(generate_metadata = True, suffix = "_with_metadata")) tests.extend(_custom_rule_test(generate_metadata = False, suffix = "_without_metadata")) + tests.extend(_svh_mismatch_test()) native.test_suite( name = name, diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs new file mode 100644 index 0000000000..99b0ea9bf4 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs @@ -0,0 +1,6 @@ +/// A library that depends on svh_lib. When compiled against a hollow `.rmeta` +/// of svh_lib, this crate's metadata records svh_lib's SVH at that point in +/// time. If the full `.rlib` of svh_lib was produced by a separate rustc +/// invocation (with a different HashMap seed), it may have a different SVH, +/// causing a mismatch when a downstream binary tries to link against both. +pub use svh_lib::Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs new file mode 100644 index 0000000000..e2f3985399 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs @@ -0,0 +1,8 @@ +use nondeterministic_macro::NondeterministicHash; + +/// A struct whose derivation runs the non-deterministic proc macro. +/// The macro generates a public constant whose value depends on HashMap +/// iteration order, so this crate's SVH varies between separate rustc +/// invocations. +#[derive(NondeterministicHash)] +pub struct Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs new file mode 100644 index 0000000000..7ba44425b7 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs @@ -0,0 +1,37 @@ +extern crate proc_macro; +use proc_macro::TokenStream; +use std::collections::HashMap; + +/// A derive macro that produces non-deterministic output due to HashMap's +/// random iteration order. Each separate process invocation initializes +/// `HashMap` with a different OS-seeded `RandomState`, so iteration order +/// varies between invocations. This makes the generated constant—and thus +/// the crate's SVH—differ when the macro is run twice (e.g., once for a +/// hollow `.rmeta` and once for a full `.rlib` in pipelined compilation). +#[proc_macro_derive(NondeterministicHash)] +pub fn nondeterministic_hash_derive(_input: TokenStream) -> TokenStream { + // HashMap::new() uses RandomState, which seeds from OS entropy. + // Each separate process invocation gets a different seed, so iteration + // order over the map is non-deterministic across invocations. + let mut map = HashMap::new(); + map.insert("alpha", 1u64); + map.insert("beta", 2u64); + map.insert("gamma", 4u64); + map.insert("delta", 8u64); + map.insert("epsilon", 16u64); + + // Position-weighted sum: not commutative, so different iteration orders + // produce different values. With 5 entries (5! = 120 orderings), the + // probability of identical output in two separate invocations is ~0.8%. + let fingerprint: u64 = map + .iter() + .enumerate() + .map(|(pos, (_, &val))| val.wrapping_mul(pos as u64 + 1)) + .fold(0u64, u64::wrapping_add); + + // Exposing this as a public constant makes it part of the crate's + // exported API, which is included in the SVH computation. + format!("pub const NONDETERMINISTIC_HASH_FINGERPRINT: u64 = {};", fingerprint) + .parse() + .unwrap() +} diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs new file mode 100644 index 0000000000..6ecfe83553 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs @@ -0,0 +1,28 @@ +/// Demonstrates SVH (Strict Version Hash) mismatch with pipelined compilation. +/// +/// Without pipelining this test always builds and passes: each library is +/// compiled exactly once, so the SVH embedded in every `.rmeta` and `.rlib` +/// is identical. +/// +/// With `//rust/settings:pipelined_compilation=true` rules_rust compiles +/// `svh_lib` **twice** in separate rustc processes — once to emit the hollow +/// `.rmeta` (metadata only), once to emit the full `.rlib`. Because +/// `nondeterministic_macro` uses `HashMap` with OS-seeded randomness, the two +/// rustc invocations typically produce different token streams and therefore +/// different SVH values. `svh_consumer` is compiled against the hollow `.rmeta` +/// and records SVH_1 in its own metadata; when rustc later tries to link the +/// test binary against the full `.rlib` (which carries SVH_2), it detects the +/// mismatch and fails with E0460. The test therefore **fails to build** most of +/// the time (~99.2% probability) when pipelining is enabled. +/// +/// The `flaky = True` attribute on this target acknowledges that the mismatch +/// is non-deterministic: on rare occasions (~0.8%) both rustc invocations +/// happen to produce the same HashMap iteration order, the SVHs agree, and the +/// build succeeds. +use svh_consumer::Widget; + +#[test] +fn svh_consistent() { + // If we reach here the SVH was consistent (no pipelining, or a lucky run). + let _: Widget = Widget; +} diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index f24a0e421a..e3f4ac5482 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -40,12 +40,23 @@ def _wrap_impl(ctx): lib_hash = output_hash, extension = ".rlib", ) - rust_metadata_name = "{prefix}{name}-{lib_hash}{extension}".format( - prefix = "lib", - name = crate_name, - lib_hash = output_hash, - extension = ".rmeta", - ) + + # Use -hollow.rlib extension (not .rmeta) so rustc reads it as an rlib archive + # containing optimized MIR. See rust/private/rust.bzl for the same logic. + # The hollow rlib is placed in a "_hollow/" subdirectory to avoid the full rlib + # and hollow rlib appearing in the same -Ldependency= search directory, which + # would cause E0463 "can't find crate" errors due to ambiguous crate candidates. + metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and ctx.attr.generate_metadata + if metadata_supports_pipelining: + rust_metadata_name = "_hollow/lib{name}-{lib_hash}-hollow.rlib".format( + name = crate_name, + lib_hash = output_hash, + ) + else: + rust_metadata_name = "lib{name}-{lib_hash}.rmeta".format( + name = crate_name, + lib_hash = output_hash, + ) tgt = ctx.attr.target deps = [DepVariantInfo( @@ -73,8 +84,7 @@ def _wrap_impl(ctx): aliases = {}, output = rust_lib, metadata = rust_metadata, - metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and - ctx.attr.generate_metadata, + metadata_supports_pipelining = metadata_supports_pipelining, owner = ctx.label, edition = "2018", compile_data = depset([]), From 46cf9603d76a032d4b0002a1e1a11e4373adc6ca Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Mon, 2 Mar 2026 11:32:58 -0500 Subject: [PATCH 21/24] Cleanup some process_wrapper code --- rust/private/rustc.bzl | 3 +- util/process_wrapper/main.rs | 78 ++++----------------------------- util/process_wrapper/options.rs | 16 +------ util/process_wrapper/output.rs | 7 +-- util/process_wrapper/rustc.rs | 60 +++---------------------- 5 files changed, 17 insertions(+), 147 deletions(-) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 9e7adba219..d593325e8b 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -1057,8 +1057,7 @@ def construct_arguments( process_wrapper_flags.add("--rustc-output-format", "json" if error_format == "json" else "rendered") # Configure rustc json output by adding artifact notifications. - # These will always be filtered out by process_wrapper and will be use to terminate - # rustc when appropriate. + # These are filtered out by process_wrapper. json = ["artifacts"] if error_format == "short": json.append("diagnostic-short") diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 5d057b08cf..2a7cbd8565 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -25,7 +25,7 @@ use std::fmt; use std::fs::{self, copy, OpenOptions}; use std::io; use std::path::PathBuf; -use std::process::{exit, Command, ExitStatus, Stdio}; +use std::process::{exit, Command, Stdio}; #[cfg(windows)] use std::time::{SystemTime, UNIX_EPOCH}; @@ -37,30 +37,6 @@ use crate::rustc::ErrorFormat; #[cfg(windows)] use crate::util::read_file_to_array; -#[cfg(windows)] -fn status_code(status: ExitStatus, was_killed: bool) -> i32 { - // On windows, there's no good way to know if the process was killed by a signal. - // If we killed the process, we override the code to signal success. - if was_killed { - 0 - } else { - status.code().unwrap_or(1) - } -} - -#[cfg(not(windows))] -fn status_code(status: ExitStatus, was_killed: bool) -> i32 { - // On unix, if code is None it means that the process was killed by a signal. - // https://doc.rust-lang.org/std/process/struct.ExitStatus.html#method.success - match status.code() { - Some(code) => code, - // If we killed the process, we expect None here - None if was_killed => 0, - // Otherwise it's some unexpected signal - None => 1, - } -} - #[derive(Debug)] struct ProcessWrapperError(String); @@ -298,9 +274,7 @@ fn json_warning(line: &str) -> JsonValue { fn process_line( mut line: String, - quit_on_rmeta: bool, format: ErrorFormat, - metadata_emitted: &mut bool, ) -> Result { // LLVM can emit lines that look like the following, and these will be interspersed // with the regular JSON output. Arguably, rustc should be fixed not to emit lines @@ -315,11 +289,7 @@ fn process_line( return Ok(LineOutput::Skip); } } - if quit_on_rmeta { - rustc::stop_on_rmeta_completion(line, format, metadata_emitted) - } else { - rustc::process_json(line, format) - } + rustc::process_json(line, format) } fn main() -> Result<(), ProcessWrapperError> { @@ -381,26 +351,13 @@ fn main() -> Result<(), ProcessWrapperError> { None }; - let mut was_killed = false; let result = if let Some(format) = opts.rustc_output_format { - let quit_on_rmeta = opts.rustc_quit_on_rmeta; - // Process json rustc output and kill the subprocess when we get a signal - // that we emitted a metadata file. - let mut me = false; - let metadata_emitted = &mut me; - let result = process_output( + process_output( &mut child_stderr, stderr.as_mut(), output_file.as_mut(), - move |line| process_line(line, quit_on_rmeta, format, metadata_emitted), - ); - if me { - // If recv returns Ok(), a signal was sent in this channel so we should terminate the child process. - // We can safely ignore the Result from kill() as we don't care if the process already terminated. - let _ = child.kill(); - was_killed = true; - } - result + move |line| process_line(line, format), + ) } else { // Process output normally by forwarding stderr process_output( @@ -415,10 +372,8 @@ fn main() -> Result<(), ProcessWrapperError> { let status = child .wait() .map_err(|e| ProcessWrapperError(format!("failed to wait for child process: {}", e)))?; - // If the child process is rustc and is killed after metadata generation, that's also a success. - let code = status_code(status, was_killed); - let success = code == 0; - if success { + let code = status.code().unwrap_or(1); + if code == 0 { if let Some(tf) = opts.touch_file { OpenOptions::new() .create(true) @@ -454,7 +409,6 @@ mod test { #[test] fn test_process_line_diagnostic_json() -> Result<(), String> { - let mut metadata_emitted = false; let LineOutput::Message(msg) = process_line( r#" { @@ -463,9 +417,7 @@ mod test { } "# .to_string(), - false, ErrorFormat::Json, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -486,7 +438,6 @@ mod test { #[test] fn test_process_line_diagnostic_rendered() -> Result<(), String> { - let mut metadata_emitted = false; let LineOutput::Message(msg) = process_line( r#" { @@ -495,9 +446,7 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ false, ErrorFormat::Rendered, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -508,16 +457,13 @@ mod test { #[test] fn test_process_line_noise() -> Result<(), String> { - let mut metadata_emitted = false; for text in [ "'+zaamo' is not a recognized feature for this target (ignoring feature)", " WARN rustc_errors::emitter Invalid span...", ] { let LineOutput::Message(msg) = process_line( text.to_string(), - /*quit_on_rmeta=*/ false, ErrorFormat::Json, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -543,7 +489,6 @@ mod test { #[test] fn test_process_line_emit_link() -> Result<(), String> { - let mut metadata_emitted = false; assert!(matches!( process_line( r#" @@ -553,19 +498,15 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ true, ErrorFormat::Rendered, - &mut metadata_emitted, )?, LineOutput::Skip )); - assert!(!metadata_emitted); Ok(()) } #[test] fn test_process_line_emit_metadata() -> Result<(), String> { - let mut metadata_emitted = false; assert!(matches!( process_line( r#" @@ -575,13 +516,10 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ true, ErrorFormat::Rendered, - &mut metadata_emitted, )?, - LineOutput::Terminate + LineOutput::Skip )); - assert!(metadata_emitted); Ok(()) } } diff --git a/util/process_wrapper/options.rs b/util/process_wrapper/options.rs index 2f252cadc7..6dbc898a11 100644 --- a/util/process_wrapper/options.rs +++ b/util/process_wrapper/options.rs @@ -44,9 +44,6 @@ pub(crate) struct Options { // If set, also logs all unprocessed output from the rustc output to this file. // Meant to be used to get json output out of rustc for tooling usage. pub(crate) output_file: Option, - // If set, it configures rustc to emit an rmeta file and then - // quit. - pub(crate) rustc_quit_on_rmeta: bool, // This controls the output format of rustc messages. pub(crate) rustc_output_format: Option, } @@ -64,7 +61,6 @@ pub(crate) fn options() -> Result { let mut stdout_file = None; let mut stderr_file = None; let mut output_file = None; - let mut rustc_quit_on_rmeta_raw = None; let mut rustc_output_format_raw = None; let mut flags = Flags::new(); let mut require_explicit_unstable_features = None; @@ -102,17 +98,9 @@ pub(crate) fn options() -> Result { "Log all unprocessed subprocess stderr in this file.", &mut output_file, ); - flags.define_flag( - "--rustc-quit-on-rmeta", - "If enabled, this wrapper will terminate rustc after rmeta has been emitted.", - &mut rustc_quit_on_rmeta_raw, - ); flags.define_flag( "--rustc-output-format", - "Controls the rustc output format if --rustc-quit-on-rmeta is set.\n\ - 'json' will cause the json output to be output, \ - 'rendered' will extract the rendered message and print that.\n\ - Default: `rendered`", + "The expected rustc output format. Valid values: json, rendered.", &mut rustc_output_format_raw, ); flags.define_flag( @@ -179,7 +167,6 @@ pub(crate) fn options() -> Result { }) .transpose()?; - let rustc_quit_on_rmeta = rustc_quit_on_rmeta_raw.is_some_and(|s| s == "true"); let rustc_output_format = rustc_output_format_raw .map(|v| match v.as_str() { "json" => Ok(rustc::ErrorFormat::Json), @@ -227,7 +214,6 @@ pub(crate) fn options() -> Result { stdout_file, stderr_file, output_file, - rustc_quit_on_rmeta, rustc_output_format, }) } diff --git a/util/process_wrapper/output.rs b/util/process_wrapper/output.rs index 4b3604b18d..5dabad8179 100644 --- a/util/process_wrapper/output.rs +++ b/util/process_wrapper/output.rs @@ -18,15 +18,11 @@ use std::io::{self, prelude::*}; /// LineOutput tells process_output what to do when a line is processed. /// If a Message is returned, it will be written to write_end, if -/// Skip is returned nothing will be printed and execution continues, -/// if Terminate is returned, process_output returns immediately. -/// Terminate is used to stop processing when we see an emit metadata -/// message. +/// Skip is returned nothing will be printed and execution continues. #[derive(Debug)] pub(crate) enum LineOutput { Message(String), Skip, - Terminate, } #[derive(Debug)] @@ -95,7 +91,6 @@ where match process_line(line.clone()) { Ok(LineOutput::Message(to_write)) => output_writer.write_all(to_write.as_bytes())?, Ok(LineOutput::Skip) => {} - Ok(LineOutput::Terminate) => return Ok(()), Err(msg) => { failed_on = Some((line, msg)); break; diff --git a/util/process_wrapper/rustc.rs b/util/process_wrapper/rustc.rs index 97ee466337..3bb4a8c2d9 100644 --- a/util/process_wrapper/rustc.rs +++ b/util/process_wrapper/rustc.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::convert::{TryFrom, TryInto}; - use tinyjson::JsonValue; use crate::output::{LineOutput, LineResult}; @@ -37,66 +35,20 @@ fn get_key(value: &JsonValue, key: &str) -> Option { } } -#[derive(Debug)] -enum RustcMessage { - Emit(String), - Message(String), -} - -impl TryFrom for RustcMessage { - type Error = (); - fn try_from(val: JsonValue) -> Result { - if let Some(emit) = get_key(&val, "emit") { - return Ok(Self::Emit(emit)); - } - if let Some(rendered) = get_key(&val, "rendered") { - return Ok(Self::Message(rendered)); - } - Err(()) - } -} - /// process_rustc_json takes an output line from rustc configured with /// --error-format=json, parses the json and returns the appropriate output /// according to the original --error-format supplied. -/// Only messages are returned, emits are ignored. +/// Only diagnostics with a rendered message are returned. /// Returns an errors if parsing json fails. pub(crate) fn process_json(line: String, error_format: ErrorFormat) -> LineResult { let parsed: JsonValue = line .parse() .map_err(|_| "error parsing rustc output as json".to_owned())?; - Ok(match parsed.try_into() { - Ok(RustcMessage::Message(rendered)) => { - output_based_on_error_format(line, rendered, error_format) - } - _ => LineOutput::Skip, - }) -} - -/// stop_on_rmeta_completion parses the json output of rustc in the same way -/// process_rustc_json does. In addition, it will signal to stop when metadata -/// is emitted so the compiler can be terminated. -/// This is used to implement pipelining in rules_rust, please see -/// https://internals.rust-lang.org/t/evaluating-pipelined-rustc-compilation/10199 -/// Returns an error if parsing json fails. -/// TODO: pass a function to handle the emit event and merge with process_json -pub(crate) fn stop_on_rmeta_completion( - line: String, - error_format: ErrorFormat, - kill: &mut bool, -) -> LineResult { - let parsed: JsonValue = line - .parse() - .map_err(|_| "error parsing rustc output as json".to_owned())?; - Ok(match parsed.try_into() { - Ok(RustcMessage::Emit(emit)) if emit == "metadata" => { - *kill = true; - LineOutput::Terminate - } - Ok(RustcMessage::Message(rendered)) => { - output_based_on_error_format(line, rendered, error_format) - } - _ => LineOutput::Skip, + Ok(if let Some(rendered) = get_key(&parsed, "rendered") { + output_based_on_error_format(line, rendered, error_format) + } else { + // Ignore non-diagnostic messages such as artifact notifications. + LineOutput::Skip }) } From 5935451ac8681d045c4665a4d90451f5d5a77217 Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Fri, 13 Mar 2026 14:19:28 -0400 Subject: [PATCH 22/24] Fix prost to be compatible with multiplatform --- extensions/prost/private/BUILD.bazel | 2 ++ extensions/prost/private/prost.bzl | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/extensions/prost/private/BUILD.bazel b/extensions/prost/private/BUILD.bazel index ed8e508537..aaf6c22d8d 100644 --- a/extensions/prost/private/BUILD.bazel +++ b/extensions/prost/private/BUILD.bazel @@ -5,6 +5,8 @@ load("//:defs.bzl", "rust_prost_toolchain") load(":legacy_proto_toolchain.bzl", "legacy_proto_toolchain") load(":prost.bzl", "RUST_EDITION", "current_prost_runtime") +exports_files(["protoc_wrapper.rs"]) + current_prost_runtime( name = "current_prost_runtime", ) diff --git a/extensions/prost/private/prost.bzl b/extensions/prost/private/prost.bzl index 4eecf85a06..386f0d7d72 100644 --- a/extensions/prost/private/prost.bzl +++ b/extensions/prost/private/prost.bzl @@ -31,14 +31,14 @@ RUST_EDITION = "2021" TOOLCHAIN_TYPE = "@rules_rust_prost//:toolchain_type" -def _create_proto_lang_toolchain(ctx, prost_toolchain): +def _create_proto_lang_toolchain(prost_toolchain): proto_lang_toolchain = proto_common.ProtoLangToolchainInfo( out_replacement_format_flag = "--prost_out=%s", plugin_format_flag = prost_toolchain.prost_plugin_flag, plugin = prost_toolchain.prost_plugin[DefaultInfo].files_to_run, runtime = prost_toolchain.prost_runtime, provided_proto_sources = depset(), - proto_compiler = ctx.attr._prost_process_wrapper[DefaultInfo].files_to_run, + proto_compiler = prost_toolchain.prost_process_wrapper[DefaultInfo].files_to_run, protoc_opts = prost_toolchain.protoc_opts, progress_message = "ProstGenProto %{label}", mnemonic = "ProstGenProto", @@ -118,7 +118,7 @@ def _compile_proto( additional_inputs = additional_inputs, additional_args = additional_args, generated_files = [lib_rs, package_info_file], - proto_lang_toolchain_info = _create_proto_lang_toolchain(ctx, prost_toolchain), + proto_lang_toolchain_info = _create_proto_lang_toolchain(prost_toolchain), plugin_output = ctx.bin_dir.path, ) @@ -377,12 +377,6 @@ rust_prost_aspect = aspect( default = Label("@bazel_tools//tools/cpp:grep-includes"), cfg = "exec", ), - "_prost_process_wrapper": attr.label( - doc = "The wrapper script for the Prost protoc plugin.", - cfg = "exec", - executable = True, - default = Label("//private:protoc_wrapper"), - ), } | RUSTC_ATTRS | { # Need to override this attribute to explicitly set the workspace. "_always_enable_metadata_output_groups": attr.label( @@ -473,6 +467,7 @@ def _rust_prost_toolchain_impl(ctx): prost_plugin = ctx.attr.prost_plugin, prost_plugin_flag = ctx.attr.prost_plugin_flag, prost_runtime = ctx.attr.prost_runtime, + prost_process_wrapper = ctx.attr._prost_process_wrapper, prost_types = ctx.attr.prost_types, proto_compiler = proto_compiler, protoc_opts = ctx.fragments.proto.experimental_protoc_opts, @@ -516,6 +511,12 @@ rust_prost_toolchain = rule( mandatory = True, aspects = [rust_analyzer_aspect], ), + "_prost_process_wrapper": attr.label( + doc = "The wrapper script for the Prost protoc plugin.", + cfg = "exec", + executable = True, + default = Label("@rules_rust_prost//private:protoc_wrapper"), + ), "prost_types": attr.label( doc = "The Prost types crates to use.", providers = [[rust_common.crate_info], [rust_common.crate_group_info]], From e1ed0e436cb319700887823c7df421c5684bc6da Mon Sep 17 00:00:00 2001 From: David Zbarsky Date: Wed, 1 Apr 2026 15:10:09 -0400 Subject: [PATCH 23/24] Avoid vendored crates for rust-analyzer binary --- tools/rust_analyzer/BUILD.bazel | 120 +++++--------------------------- tools/rust_analyzer/aquery.rs | 2 +- tools/rust_analyzer/lib.rs | 2 +- 3 files changed, 18 insertions(+), 106 deletions(-) diff --git a/tools/rust_analyzer/BUILD.bazel b/tools/rust_analyzer/BUILD.bazel index 8b54c09c2f..6d56ffca26 100644 --- a/tools/rust_analyzer/BUILD.bazel +++ b/tools/rust_analyzer/BUILD.bazel @@ -1,106 +1,18 @@ -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//rust:defs.bzl", "rust_binary", "rust_clippy", "rust_library", "rust_test") -load("//tools/private:tool_utils.bzl", "aspect_repository") +package(default_visibility = ["//visibility:public"]) -rust_binary( - name = "discover_bazel_rust_project", - srcs = ["bin/discover_rust_project.rs"], - edition = "2018", - rustc_env = { - "ASPECT_REPOSITORY": aspect_repository(), - }, - visibility = ["//visibility:public"], - deps = [ - ":gen_rust_project_lib", - "//tools/rust_analyzer/3rdparty/crates:anyhow", - "//tools/rust_analyzer/3rdparty/crates:camino", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:env_logger", - "//tools/rust_analyzer/3rdparty/crates:log", - "//tools/rust_analyzer/3rdparty/crates:serde_json", - ], -) +exports_files([ + "aquery.rs", + "lib.rs", + "rust_project.rs", + "bin/discover_rust_project.rs", + "bin/gen_rust_project.rs", + "bin/validate.rs", +]) -rust_binary( - name = "gen_rust_project", - srcs = ["bin/gen_rust_project.rs"], - edition = "2018", - rustc_env = { - "ASPECT_REPOSITORY": aspect_repository(), - }, - visibility = ["//visibility:public"], - deps = [ - ":gen_rust_project_lib", - "//tools/rust_analyzer/3rdparty/crates:anyhow", - "//tools/rust_analyzer/3rdparty/crates:camino", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:env_logger", - "//tools/rust_analyzer/3rdparty/crates:log", - "//tools/rust_analyzer/3rdparty/crates:serde_json", - ], -) - -rust_binary( - name = "validate", - srcs = ["bin/validate.rs"], - data = [ - "//rust/toolchain:current_rust_analyzer_toolchain", - ], - edition = "2021", - rustc_env = { - "RUST_ANALYZER_RLOCATIONPATH": "$(RUST_ANALYZER_RLOCATIONPATH)", - }, - toolchains = ["//rust/toolchain:current_rust_analyzer_toolchain"], - visibility = ["//visibility:public"], - deps = [ - "//rust/runfiles", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:env_logger", - "//tools/rust_analyzer/3rdparty/crates:log", - ], -) - -rust_library( - name = "gen_rust_project_lib", - srcs = glob( - ["**/*.rs"], - exclude = ["bin"], - ), - data = [ - "//rust/private:rust_analyzer_detect_sysroot", - ], - edition = "2018", - deps = [ - "//rust/runfiles", - "//tools/rust_analyzer/3rdparty/crates:anyhow", - "//tools/rust_analyzer/3rdparty/crates:camino", - "//tools/rust_analyzer/3rdparty/crates:clap", - "//tools/rust_analyzer/3rdparty/crates:log", - "//tools/rust_analyzer/3rdparty/crates:serde", - "//tools/rust_analyzer/3rdparty/crates:serde_json", - ], -) - -rust_test( - name = "gen_rust_project_lib_test", - crate = ":gen_rust_project_lib", - deps = [ - "//tools/rust_analyzer/3rdparty/crates:itertools", - ], -) - -rust_clippy( - name = "gen_rust_project_clippy", - testonly = True, - visibility = ["//visibility:private"], - deps = [ - ":gen_rust_project", - ], -) - -bzl_library( - name = "bzl_lib", - srcs = glob(["**/*.bzl"]), - visibility = ["//visibility:public"], - deps = ["//tools/rust_analyzer/3rdparty:bzl_lib"], -) +alias(name = "discover_bazel_rust_project", actual = "@rules_rs//tools/rust_analyzer:discover_bazel_rust_project") +alias(name = "gen_rust_project", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project") +alias(name = "validate", actual = "@rules_rs//tools/rust_analyzer:validate") +alias(name = "gen_rust_project_lib", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project_lib") +alias(name = "gen_rust_project_lib_test", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project_lib_test") +alias(name = "gen_rust_project_clippy", actual = "@rules_rs//tools/rust_analyzer:gen_rust_project_clippy") +alias(name = "bzl_lib", actual = "@rules_rs//tools/rust_analyzer:bzl_lib") diff --git a/tools/rust_analyzer/aquery.rs b/tools/rust_analyzer/aquery.rs index f306aa5b75..c1ebeddbd1 100644 --- a/tools/rust_analyzer/aquery.rs +++ b/tools/rust_analyzer/aquery.rs @@ -104,7 +104,7 @@ pub fn get_crate_specs( .arg("--include_aspects") .arg("--include_artifacts") .arg(format!( - "--aspects={rules_rust_name}//rust:defs.bzl%rust_analyzer_aspect" + "--aspects={rules_rust_name}//tools/rust_analyzer:defs.bzl%rust_analyzer_aspect" )) .arg("--output_groups=rust_analyzer_crate_spec") .arg(format!( diff --git a/tools/rust_analyzer/lib.rs b/tools/rust_analyzer/lib.rs index 4ea641e6a0..93ca7cf3da 100644 --- a/tools/rust_analyzer/lib.rs +++ b/tools/rust_analyzer/lib.rs @@ -109,7 +109,7 @@ fn generate_crate_info( .arg("--norun_validations") .arg("--remote_download_all") .arg(format!( - "--aspects={rules_rust}//rust:defs.bzl%rust_analyzer_aspect" + "--aspects={rules_rust}//tools/rust_analyzer:defs.bzl%rust_analyzer_aspect" )) .arg("--output_groups=rust_analyzer_crate_spec,rust_generated_srcs,rust_analyzer_proc_macro_dylib,rust_analyzer_src") .args(targets) From 04e12028cbd3403055fde1713c32b9f83d8d1cbc Mon Sep 17 00:00:00 2001 From: Walter Gray Date: Fri, 3 Apr 2026 13:15:46 -0700 Subject: [PATCH 24/24] cargo: improve Windows symlink handling Rewrite remove_symlink on Windows to properly classify file symlinks, directory symlinks, and junctions using FileTypeExt. Merge the separate drain_runfiles_dir_unix/drain_runfiles_dir_windows into a single drain_runfiles_dir_impl(symlinks_used) that tolerates missing entries in both modes. Make symlink_if_not_exists return bool to avoid adding pre-existing paths to the cleanup list, and tolerate PermissionDenied errors during exec_root link cleanup on Windows. --- .../private/cargo_build_script_runner/bin.rs | 49 +-- .../cargo_manifest_dir.rs | 323 ++++++++++++++---- 2 files changed, 283 insertions(+), 89 deletions(-) diff --git a/cargo/private/cargo_build_script_runner/bin.rs b/cargo/private/cargo_build_script_runner/bin.rs index 27f5848edc..d546c29ccc 100644 --- a/cargo/private/cargo_build_script_runner/bin.rs +++ b/cargo/private/cargo_build_script_runner/bin.rs @@ -77,10 +77,12 @@ fn run_buildrs() -> Result<(), String> { .ok_or_else(|| "Failed while getting file name".to_string())?; let link = manifest_dir.join(file_name); - symlink_if_not_exists(&path, &link) + let created = symlink_if_not_exists(&path, &link) .map_err(|err| format!("Failed to symlink {path:?} to {link:?}: {err}"))?; - exec_root_links.push(link) + if created { + exec_root_links.push(link); + } } } @@ -219,15 +221,25 @@ fn run_buildrs() -> Result<(), String> { ) }); - if !exec_root_links.is_empty() { - for link in exec_root_links { - remove_symlink(&link).map_err(|e| { - format!( - "Failed to remove exec_root link '{}' with {:?}", + for link in exec_root_links { + if let Err(e) = remove_symlink(&link) { + if cfg!(target_family = "windows") { + // On Windows, symlink removal can fail with PermissionDenied if + // another process still holds a handle to the target directory. + // These are temporary symlinks in the build sandbox that Bazel + // will clean up, so we log and continue rather than failing. + eprintln!( + "Warning: could not remove exec_root link '{}': {:?}", link.display(), e - ) - })?; + ); + } else { + return Err(format!( + "Failed to remove exec_root link '{}': {:?}", + link.display(), + e + )); + } } } @@ -247,10 +259,13 @@ fn should_symlink_exec_root() -> bool { } /// Create a symlink from `link` to `original` if `link` doesn't already exist. -fn symlink_if_not_exists(original: &Path, link: &Path) -> Result<(), String> { - symlink(original, link) - .or_else(swallow_already_exists) - .map_err(|err| format!("Failed to create symlink: {err}")) +/// Returns `true` if a new symlink was created, `false` if the path already existed. +fn symlink_if_not_exists(original: &Path, link: &Path) -> Result { + match symlink(original, link) { + Ok(()) => Ok(true), + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => Ok(false), + Err(err) => Err(format!("Failed to create symlink: {err}")), + } } fn resolve_rundir(rundir: &str, exec_root: &Path, manifest_dir: &Path) -> Result { @@ -270,14 +285,6 @@ fn resolve_rundir(rundir: &str, exec_root: &Path, manifest_dir: &Path) -> Result Ok(exec_root.join(rundir_path)) } -fn swallow_already_exists(err: std::io::Error) -> std::io::Result<()> { - if err.kind() == std::io::ErrorKind::AlreadyExists { - Ok(()) - } else { - Err(err) - } -} - /// A representation of expected command line arguments. struct Args { progname: String, diff --git a/cargo/private/cargo_build_script_runner/cargo_manifest_dir.rs b/cargo/private/cargo_build_script_runner/cargo_manifest_dir.rs index def343a34e..895ef6155d 100644 --- a/cargo/private/cargo_build_script_runner/cargo_manifest_dir.rs +++ b/cargo/private/cargo_build_script_runner/cargo_manifest_dir.rs @@ -27,14 +27,41 @@ pub fn remove_symlink(path: &Path) -> Result<(), std::io::Error> { std::fs::remove_file(path) } -/// Create a symlink file on windows systems +/// Remove a symlink or junction on Windows. +/// +/// Windows has three kinds of reparse points we may encounter: +/// 1. File symlinks — `remove_file` works. +/// 2. Directory symlinks — `remove_dir` removes the link itself (not the +/// target contents), but `remove_file` also works on some Windows versions. +/// 3. Junctions — similar to directory symlinks; `remove_dir` removes the +/// junction entry. +/// +/// We use `symlink_metadata` + `FileTypeExt` to classify the entry and try +/// the most appropriate removal call first, with a fallback for edge cases. #[cfg(target_family = "windows")] pub fn remove_symlink(path: &Path) -> Result<(), std::io::Error> { - if path.is_dir() { - std::fs::remove_dir(path) - } else { - std::fs::remove_file(path) + use std::os::windows::fs::FileTypeExt; + + let metadata = std::fs::symlink_metadata(path)?; + let ft = metadata.file_type(); + + if ft.is_symlink_file() { + return std::fs::remove_file(path); } + + if ft.is_symlink_dir() { + // remove_dir removes the symlink entry itself, not the target contents. + // Fall back to remove_file if remove_dir fails (some Windows versions). + return std::fs::remove_dir(path).or_else(|_| std::fs::remove_file(path)); + } + + // Junctions appear as directories but are not symlinks per FileTypeExt. + // remove_dir removes the junction entry itself. + if ft.is_dir() { + return std::fs::remove_dir(path).or_else(|_| std::fs::remove_file(path)); + } + + std::fs::remove_file(path) } /// Check if the system supports symlinks by attempting to create one. @@ -227,73 +254,84 @@ impl RunfilesMaker { Ok(()) } - /// Delete runfiles from the runfiles directory that do not match user defined suffixes + /// Strip runfiles that do not match a retained suffix. /// - /// The Unix implementation assumes symlinks are supported and that the runfiles directory - /// was created using symlinks. - fn drain_runfiles_dir_unix(&self) -> Result<(), String> { + /// When `symlinks_used` is true the runfiles directory was populated with + /// symlinks: every entry is removed and only retained entries are copied + /// back as real files. When false, real file copies were used (Windows + /// without symlink support) and only retained entries are deleted so that + /// downstream steps can recreate them. + /// + /// Missing entries are tolerated in either mode — on Windows the runfiles + /// directory may be incomplete (e.g. a Cargo.lock that was never created). + fn drain_runfiles_dir_impl(&self, symlinks_used: bool) -> Result<(), String> { for (src, dest) in &self.runfiles { let abs_dest = self.output_dir.join(dest); - - remove_symlink(&abs_dest).map_err(|e| { - format!( - "Failed to delete symlink '{}' with {:?}", - abs_dest.display(), - e - ) - })?; - - if !self + let should_retain = self .filename_suffixes_to_retain .iter() - .any(|suffix| dest.ends_with(suffix)) - { - if let Some(parent) = abs_dest.parent() { - if is_dir_empty(parent).map_err(|e| { - format!("Failed to determine if directory was empty with: {:?}", e) - })? { - std::fs::remove_dir(parent).map_err(|e| { - format!( - "Failed to delete directory {} with {:?}", - parent.display(), - e - ) - })?; + .any(|suffix| dest.ends_with(suffix)); + + if symlinks_used { + match remove_symlink(&abs_dest) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + if !should_retain { + continue; + } + } + Err(e) => { + return Err(format!( + "Failed to delete symlink '{}' with {:?}", + abs_dest.display(), + e + )); } } - continue; - } - std::fs::copy(src, &abs_dest).map_err(|e| { - format!( - "Failed to copy `{} -> {}` with {:?}", - src.display(), - abs_dest.display(), - e - ) - })?; - } - Ok(()) - } + if !should_retain { + if let Some(parent) = abs_dest.parent() { + if is_dir_empty(parent).map_err(|e| { + format!("Failed to determine if directory was empty with: {:?}", e) + })? { + std::fs::remove_dir(parent).map_err(|e| { + format!( + "Failed to delete directory {} with {:?}", + parent.display(), + e + ) + })?; + } + } + continue; + } - /// Delete runfiles from the runfiles directory that do not match user defined suffixes - /// - /// The Windows implementation assumes symlinks are not supported and real files will have - /// been copied into the runfiles directory. - fn drain_runfiles_dir_windows(&self) -> Result<(), String> { - for dest in self.runfiles.values() { - if !self - .filename_suffixes_to_retain - .iter() - .any(|suffix| dest.ends_with(suffix)) - { + std::fs::copy(src, &abs_dest).map_err(|e| { + format!( + "Failed to copy `{} -> {}` with {:?}", + src.display(), + abs_dest.display(), + e + ) + })?; + } else if !should_retain { + // Non-symlink mode: non-retained files are left as-is (no + // empty-directory cleanup needed since the files were never + // removed in the first place). continue; + } else { + match std::fs::remove_file(&abs_dest) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(format!( + "Failed to remove file {} with {:?}", + abs_dest.display(), + e + )); + } + } } - - let abs_dest = self.output_dir.join(dest); - std::fs::remove_file(&abs_dest).map_err(|e| { - format!("Failed to remove file {} with {:?}", abs_dest.display(), e) - })?; } Ok(()) } @@ -301,15 +339,10 @@ impl RunfilesMaker { /// Delete runfiles from the runfiles directory that do not match user defined suffixes pub fn drain_runfiles_dir(&self, out_dir: &Path) -> Result<(), String> { if cfg!(target_family = "windows") { - // If symlinks are supported then symlinks will have been used. let supports_symlinks = system_supports_symlinks(&self.output_dir)?; - if supports_symlinks { - self.drain_runfiles_dir_unix()?; - } else { - self.drain_runfiles_dir_windows()?; - } + self.drain_runfiles_dir_impl(supports_symlinks)?; } else { - self.drain_runfiles_dir_unix()?; + self.drain_runfiles_dir_impl(true)?; } // Due to the symlinks in `CARGO_MANIFEST_DIR`, some build scripts @@ -409,6 +442,160 @@ mod tests { out_dir } + /// Create a `RunfilesMaker` for testing without needing a param file. + fn make_runfiles_maker( + output_dir: PathBuf, + suffixes: &[&str], + runfiles: Vec<(PathBuf, RlocationPath)>, + ) -> RunfilesMaker { + RunfilesMaker { + output_dir, + filename_suffixes_to_retain: suffixes.iter().map(|s| s.to_string()).collect(), + runfiles: runfiles.into_iter().collect(), + } + } + + /// Helper to create a unique test directory under TEST_TMPDIR. + fn test_dir(name: &str) -> PathBuf { + let test_tmp = PathBuf::from(std::env::var("TEST_TMPDIR").unwrap()); + let dir = test_tmp.join(name); + if dir.exists() { + fs::remove_dir_all(&dir).unwrap(); + } + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[cfg(any(target_family = "windows", target_family = "unix"))] + #[test] + fn drain_symlinks_tolerates_missing_symlinks() { + let base = test_dir("drain_sym_missing"); + let output_dir = base.join("runfiles"); + fs::create_dir_all(&output_dir).unwrap(); + + // Two distinct source files so BTreeMap keeps both entries. + let src_real = base.join("real.txt"); + fs::write(&src_real, "content").unwrap(); + let src_lock = base.join("Cargo.lock"); + fs::write(&src_lock, "lock data").unwrap(); + + // Two runfile entries: one exists as a symlink, one does not. + let existing_dest = "pkg/real.txt"; + let missing_dest = "pkg/Cargo.lock"; + let abs_existing = output_dir.join(existing_dest); + fs::create_dir_all(abs_existing.parent().unwrap()).unwrap(); + symlink(&src_real, &abs_existing).unwrap(); + // Intentionally do NOT create a symlink for missing_dest. + + let maker = make_runfiles_maker( + output_dir.clone(), + &[], // retain nothing + vec![ + (src_real.clone(), existing_dest.to_string()), + (src_lock.clone(), missing_dest.to_string()), + ], + ); + + // Should succeed despite the missing symlink. + maker.drain_runfiles_dir_impl(true).unwrap(); + + // The existing symlink should have been removed. + assert!(!abs_existing.exists()); + } + + #[cfg(any(target_family = "windows", target_family = "unix"))] + #[test] + fn drain_symlinks_retains_matching_suffixes() { + let base = test_dir("drain_sym_retain"); + let output_dir = base.join("runfiles"); + fs::create_dir_all(&output_dir).unwrap(); + + let src_file = base.join("lib.rs"); + fs::write(&src_file, "fn main() {}").unwrap(); + + let src_lock = base.join("Cargo.lock"); + fs::write(&src_lock, "lock contents").unwrap(); + + let rs_dest = "pkg/lib.rs"; + let lock_dest = "pkg/Cargo.lock"; + + // Create symlinks for both entries. + let abs_rs = output_dir.join(rs_dest); + let abs_lock = output_dir.join(lock_dest); + fs::create_dir_all(abs_rs.parent().unwrap()).unwrap(); + symlink(&src_file, &abs_rs).unwrap(); + symlink(&src_lock, &abs_lock).unwrap(); + + let maker = make_runfiles_maker( + output_dir.clone(), + &[".rs"], // only retain .rs files + vec![ + (src_file.clone(), rs_dest.to_string()), + (src_lock.clone(), lock_dest.to_string()), + ], + ); + + maker.drain_runfiles_dir_impl(true).unwrap(); + + // .rs file should be retained (copied back as a real file, not a symlink). + assert!(abs_rs.exists()); + assert!(!abs_rs.is_symlink()); + assert_eq!(fs::read_to_string(&abs_rs).unwrap(), "fn main() {}"); + + // .lock file should have been removed. + assert!(!abs_lock.exists()); + } + + #[cfg(any(target_family = "windows", target_family = "unix"))] + #[test] + fn drain_symlinks_missing_with_retained_suffix_still_copies() { + let base = test_dir("drain_sym_missing_retain"); + let output_dir = base.join("runfiles"); + fs::create_dir_all(&output_dir).unwrap(); + + let src_file = base.join("lib.rs"); + fs::write(&src_file, "fn main() {}").unwrap(); + + let dest = "pkg/lib.rs"; + // Create the parent dir but NOT the symlink. + fs::create_dir_all(output_dir.join("pkg")).unwrap(); + + let maker = make_runfiles_maker( + output_dir.clone(), + &[".rs"], // retain .rs files + vec![(src_file.clone(), dest.to_string())], + ); + + // Should succeed — missing symlink is tolerated, file is still copied. + maker.drain_runfiles_dir_impl(true).unwrap(); + + let abs_dest = output_dir.join(dest); + assert!(abs_dest.exists()); + assert!(!abs_dest.is_symlink()); + assert_eq!(fs::read_to_string(&abs_dest).unwrap(), "fn main() {}"); + } + + #[cfg(any(target_family = "windows", target_family = "unix"))] + #[test] + fn drain_no_symlinks_tolerates_missing_files() { + let base = test_dir("drain_nosym_missing"); + let output_dir = base.join("runfiles"); + fs::create_dir_all(&output_dir).unwrap(); + + let src_file = base.join("real.txt"); + fs::write(&src_file, "content").unwrap(); + + // Retain .txt but the file doesn't exist in the runfiles dir. + let maker = make_runfiles_maker( + output_dir.clone(), + &[".txt"], + vec![(src_file.clone(), "pkg/real.txt".to_string())], + ); + + // Should succeed despite the missing file. + maker.drain_runfiles_dir_impl(false).unwrap(); + } + #[cfg(any(target_family = "windows", target_family = "unix"))] #[test] fn replace_symlinks_in_out_dir() {