From 43ccab5eebe6f95fbac5ea29541bd700824d5e12 Mon Sep 17 00:00:00 2001 From: Walter Gray Date: Fri, 10 Apr 2026 11:16:31 -0700 Subject: [PATCH 1/2] Revert "Switch pipelining metadata action to hollow rlib (-Zno-codegen)" This reverts commit 40c17fcb5ad2ccaaa464d95f469ad7e77f1f4241. --- rust/private/rust.bzl | 8 +- rust/private/rustc.bzl | 151 ++------------ rust/settings/settings.bzl | 21 +- .../pipelined_compilation_test.bzl | 194 ++++-------------- .../svh_mismatch/svh_mismatch_consumer.rs | 6 - .../svh_mismatch/svh_mismatch_lib.rs | 8 - .../svh_mismatch_nondeterministic_macro.rs | 37 ---- .../svh_mismatch/svh_mismatch_test.rs | 28 --- test/unit/pipelined_compilation/wrap.bzl | 26 +-- 9 files changed, 69 insertions(+), 410 deletions(-) delete mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs delete mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs delete mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs delete mode 100644 test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 2878b241f4..3b15974624 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -180,15 +180,11 @@ def _rust_library_common(ctx, crate_type): crate_type, disable_pipelining = getattr(ctx.attr, "disable_pipelining", False), ): - # The hollow rlib uses .rlib extension (not .rmeta) so rustc reads it as an - # rlib archive containing lib.rmeta with optimized MIR. It is placed in a - # "_hollow/" subdirectory so the full rlib and hollow rlib never appear in the - # same -Ldependency= search directory (which would cause E0463). rust_metadata = ctx.actions.declare_file( - "_hollow/" + rust_lib_name[:-len(".rlib")] + "-hollow.rlib", + paths.replace_extension(rust_lib_name, ".rmeta"), + sibling = rust_lib, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) - metadata_supports_pipelining = ( can_use_metadata_for_pipelining(toolchain, crate_type) and not ctx.attr.disable_pipelining diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index de99243e34..962d4cd2ba 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -663,7 +663,7 @@ def _disambiguate_libs(actions, toolchain, crate_info, dep_info, use_pic): visited_libs[name] = artifact return ambiguous_libs -def _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link = False): +def _depend_on_metadata(crate_info, force_depend_on_objects): """Determines if we can depend on metadata for this crate. By default (when pipelining is disabled or when the crate type needs to link against @@ -673,22 +673,9 @@ def _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc In some rare cases, even if both of those conditions are true, we still want to depend on objects. This is what force_depend_on_objects is. - When experimental_use_cc_common_link is True, bin/cdylib crates also use hollow - rlib deps. The rustc step only emits .o files (no rustc linking), so SVH chain - consistency is sufficient; the actual linking is done by cc_common.link, which - does not check SVH. - - Callers are responsible for zeroing out experimental_use_cc_common_link for - exec-platform builds before calling this function (see rustc_compile_action). - Exec-platform binaries (build scripts) must use full rlib deps because their - CcInfo linking contexts may lack a CC toolchain. - Args: crate_info (CrateInfo): The Crate to determine this for. force_depend_on_objects (bool): if set we will not depend on metadata. - experimental_use_cc_common_link (bool): if set, bin/cdylib crates also use - hollow rlib deps for SVH consistency. Must already be False for - exec-platform builds when this function is called. Returns: Whether we can depend on metadata for this crate. @@ -696,11 +683,6 @@ def _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc if force_depend_on_objects: return False - if experimental_use_cc_common_link and crate_info.type in ("bin", "cdylib"): - # cc_common.link: rustc only emits .o files, so hollow rlib deps are safe and - # keep the SVH chain consistent (avoiding E0460 from nondeterministic proc macros). - return True - return crate_info.type in ("rlib", "lib") def collect_inputs( @@ -788,7 +770,7 @@ def collect_inputs( linkstamp_outs = [] transitive_crate_outputs = dep_info.transitive_crate_outputs - if _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link): + if _depend_on_metadata(crate_info, force_depend_on_objects): transitive_crate_outputs = dep_info.transitive_metadata_outputs nolinkstamp_compile_direct_inputs = [] @@ -824,12 +806,6 @@ def collect_inputs( transitive = [ crate_info.srcs, transitive_crate_outputs, - # Always include hollow rlibs so they are present in the sandbox for - # -Ldependency= resolution. Binaries and proc-macros compile against full - # rlib --extern deps but need hollow rlibs available for transitive - # dependency resolution when those rlibs were themselves compiled against - # hollow deps. For rlib/lib crates this is a no-op (already included above). - dep_info.transitive_metadata_outputs, crate_info.compile_data, dep_info.transitive_proc_macro_data, toolchain.all_files, @@ -934,7 +910,6 @@ def construct_arguments( use_json_output = False, build_metadata = False, force_depend_on_objects = False, - experimental_use_cc_common_link = False, skip_expanding_rustc_env = False, require_explicit_unstable_features = False, always_use_param_file = False, @@ -1070,14 +1045,8 @@ def construct_arguments( error_format = "json" if build_metadata: - if crate_info.type in ("rlib", "lib"): - # Hollow rlib approach (Buck2-style): rustc runs to completion with -Zno-codegen, - # producing a hollow .rlib (metadata only, no object code) via --emit=link=. - # No need to kill rustc — -Zno-codegen skips codegen entirely and exits quickly. - rustc_flags.add("-Zno-codegen") - - # else: IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits - # naturally after writing .rmeta via --emit=dep-info,metadata (no kill needed). + # Configure process_wrapper to terminate rustc when metadata are emitted + process_wrapper_flags.add("--rustc-quit-on-rmeta", "true") if crate_info.rustc_rmeta_output: process_wrapper_flags.add("--output-file", crate_info.rustc_rmeta_output.path) elif crate_info.rustc_output: @@ -1110,14 +1079,7 @@ def construct_arguments( emit_without_paths = [] for kind in emit: - if kind == "link" and build_metadata and crate_info.type in ("rlib", "lib") and crate_info.metadata: - # Hollow rlib: direct rustc's link output to the -hollow.rlib path. - # The file has .rlib extension so rustc reads it as an rlib archive - # (with optimized MIR in lib.rmeta). Using a .rmeta path would cause - # E0786 "found invalid metadata files" because rustc parses .rmeta files - # as raw metadata blobs, not rlib archives. - rustc_flags.add(crate_info.metadata, format = "--emit=link=%s") - elif kind == "link" and crate_info.type == "bin" and crate_info.output != None: + if kind == "link" and crate_info.type == "bin" and crate_info.output != None: rustc_flags.add(crate_info.output, format = "--emit=link=%s") else: emit_without_paths.append(kind) @@ -1192,7 +1154,7 @@ def construct_arguments( include_link_flags = include_link_flags, ) - use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link) + use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects) # These always need to be added, even if not linking this crate. add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct, use_metadata) @@ -1361,13 +1323,6 @@ def rustc_compile_action( rustc_output = crate_info.rustc_output rustc_rmeta_output = crate_info.rustc_rmeta_output - # Use the hollow rlib approach (Buck2-style) for rlib/lib crate types when a metadata - # action is being created. This always applies for rlib/lib regardless of whether - # pipelining is globally enabled — the hollow rlib is simpler than killing rustc. - # Non-rlib types (bin, proc-macro, etc.) use --emit=dep-info,metadata instead - # (rustc exits naturally after writing .rmeta, no process-wrapper kill needed). - use_hollow_rlib = bool(build_metadata) and crate_info.type in ("rlib", "lib") - # Determine whether to use cc_common.link: # * either if experimental_use_cc_common_link is 1, # * or if experimental_use_cc_common_link is -1 and @@ -1381,12 +1336,6 @@ def rustc_compile_action( elif ctx.attr.experimental_use_cc_common_link == -1: experimental_use_cc_common_link = toolchain._experimental_use_cc_common_link - # Exec-platform binaries (build scripts) skip cc_common.link: exec-configuration - # rlib deps may lack a CC toolchain, causing empty CcInfo linking contexts. They - # use standard rustc linking with full rlib deps instead. - if experimental_use_cc_common_link and is_exec_configuration(ctx): - experimental_use_cc_common_link = False - dep_info, build_info, linkstamps = collect_deps( deps = deps, proc_macro_deps = proc_macro_deps, @@ -1425,34 +1374,17 @@ def rustc_compile_action( experimental_use_cc_common_link = experimental_use_cc_common_link, ) - # The main Rustc action uses FULL rlib deps so the full rlib it produces records - # full-rlib SVHs. A downstream binary links against full rlibs; if the Rustc action - # had used hollow rlib deps instead, nondeterministic proc macros could produce - # different SVHs for the hollow vs full rlib, causing E0460 in the binary build. - # The RustcMetadata action still uses hollow rlibs (compile_inputs_for_metadata) - # so it can start before full codegen of its deps completes. - compile_inputs_for_metadata = compile_inputs - if use_hollow_rlib: - compile_inputs, _, _, _, _, _ = collect_inputs( - ctx = ctx, - file = ctx.file, - files = ctx.files, - linkstamps = linkstamps, - toolchain = toolchain, - cc_toolchain = cc_toolchain, - feature_configuration = feature_configuration, - crate_info = crate_info, - dep_info = dep_info, - build_info = build_info, - lint_files = lint_files, - stamp = stamp, - force_depend_on_objects = True, - experimental_use_cc_common_link = experimental_use_cc_common_link, - ) - - # The main Rustc action emits dep-info and link (the full rlib/binary/cdylib). - # When cc_common linking is enabled, emit a `.o` file instead. + # The types of rustc outputs to emit. + # If we build metadata, we need to keep the command line of the two invocations + # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as + # a candidate. + # Because of that we need to add emit=metadata to both the rlib and rmeta invocation. + # + # When cc_common linking is enabled, emit a `.o` file, which is later + # passed to the cc_common.link action. emit = ["dep-info", "link"] + if build_metadata: + emit.append("metadata") if experimental_use_cc_common_link: emit = ["obj"] @@ -1487,9 +1419,6 @@ def rustc_compile_action( force_all_deps_direct = force_all_deps_direct, stamp = stamp, use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), - # Force full rlib --extern deps so the full rlib records full-rlib SVHs. - force_depend_on_objects = use_hollow_rlib, - experimental_use_cc_common_link = experimental_use_cc_common_link, skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, always_use_param_file = not ctx.executable._process_wrapper, @@ -1497,15 +1426,6 @@ def rustc_compile_action( args_metadata = None if build_metadata: - if use_hollow_rlib: - # Hollow rlib: emit dep-info and link (directed to the -hollow.rlib path via - # -Zno-codegen). dep-info must be included: it affects the SVH stored in the - # rlib, so both actions must include it to keep SVHs consistent. - metadata_emit = ["dep-info", "link"] - else: - # IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits - # naturally after writing .rmeta with --emit=dep-info,metadata. - metadata_emit = ["dep-info", "metadata"] args_metadata, _ = construct_arguments( ctx = ctx, attr = attr, @@ -1513,7 +1433,7 @@ def rustc_compile_action( toolchain = toolchain, tool_path = toolchain.rustc.path, cc_toolchain = cc_toolchain, - emit = metadata_emit, + emit = emit, feature_configuration = feature_configuration, crate_info = crate_info, dep_info = dep_info, @@ -1528,7 +1448,6 @@ def rustc_compile_action( stamp = stamp, use_json_output = True, build_metadata = True, - experimental_use_cc_common_link = experimental_use_cc_common_link, require_explicit_unstable_features = require_explicit_unstable_features, ) @@ -1537,13 +1456,6 @@ def rustc_compile_action( # this is the final list of env vars env.update(env_from_args) - if use_hollow_rlib: - # Both the metadata action and the full Rustc action must have RUSTC_BOOTSTRAP=1 - # for SVH compatibility. RUSTC_BOOTSTRAP=1 changes the crate hash — setting it - # on only one action would cause SVH mismatch even for deterministic crates. - # This enables -Zno-codegen on stable Rust compilers for the metadata action. - env["RUSTC_BOOTSTRAP"] = "1" - if hasattr(attr, "version") and attr.version != "0.0.0": formatted_version = " v{}".format(attr.version) else: @@ -1614,7 +1526,7 @@ def rustc_compile_action( if args_metadata: ctx.actions.run( executable = ctx.executable._process_wrapper, - inputs = compile_inputs_for_metadata, + inputs = compile_inputs, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, arguments = args_metadata.all, @@ -2228,14 +2140,9 @@ def add_crate_link_flags(args, dep_info, force_all_deps_direct = False, use_meta crate_to_link_flags = _crate_to_link_flag_metadata if use_metadata else _crate_to_link_flag args.add_all(direct_crates, uniquify = True, map_each = crate_to_link_flags) - # Use hollow rlib directories for -Ldependency= when use_metadata=True (rlib/lib) - # so that both --extern= and -Ldependency= point to the same hollow rlib file. - # When use_metadata=False (bins, proc-macros), use full rlib directories; pointing - # to hollow dirs alongside full --extern= args would cause E0463 (ambiguous crate). - get_dirname = _get_crate_dirname_pipelined if use_metadata else _get_crate_dirname args.add_all( dep_info.transitive_crates, - map_each = get_dirname, + map_each = _get_crate_dirname, uniquify = True, format_each = "-Ldependency=%s", ) @@ -2293,25 +2200,7 @@ def _get_crate_dirname(crate): """ return crate.output.dirname -def _get_crate_dirname_pipelined(crate): - """For pipelined compilation: returns the _hollow/ directory for pipelined crates - - When a crate supports pipelining and has a hollow rlib in its _hollow/ subdirectory, - pointing -Ldependency= to that subdirectory lets rustc find the hollow rlib (which has - the correct SVH matching downstream metadata). Pointing to the parent directory instead - would expose the full rlib (compiled separately, with a different SVH), causing E0460. - - Args: - crate (CrateInfo): A CrateInfo provider from the current rule - - Returns: - str: The directory to use for -Ldependency= search. - """ - if crate.metadata and crate.metadata_supports_pipelining: - return crate.metadata.dirname - return crate.output.dirname - -def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = False, for_darwin = False, flavor_msvc = False): +def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_darwin = False, flavor_msvc = False): artifact = get_preferred_artifact(lib, use_pic) if ambiguous_libs and artifact.path in ambiguous_libs: artifact = ambiguous_libs[artifact.path] diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index 1c9bcb1d06..b6d1c0aea4 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -112,18 +112,10 @@ def use_real_import_macro(): ) def pipelined_compilation(): - """When set, this flag enables pipelined compilation for rlib/lib crates. - - For each rlib/lib, a separate RustcMetadata action produces a hollow rlib - (via `-Zno-codegen`) containing only metadata. Downstream rlib/lib crates - can begin compiling against the hollow rlib before the upstream full codegen - action completes, increasing build parallelism. - - Pipelining applies to rlib→rlib dependencies by default. To also pipeline - bin/cdylib crates (starting their compile step before upstream full codegen - finishes), enable `experimental_use_cc_common_link` alongside this flag. - With cc_common.link, rustc only emits `.o` files for binaries (linking is - handled separately), so hollow rlib deps are safe for bins too. + """When set, this flag causes rustc to emit `*.rmeta` files and use them for `rlib -> rlib` dependencies. + + While this involves one extra (short) rustc invocation to build the rmeta file, + it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation. """ bool_flag( name = "pipelined_compilation", @@ -134,11 +126,6 @@ def pipelined_compilation(): def experimental_use_cc_common_link(): """A flag to control whether to link rust_binary and rust_test targets using \ cc_common.link instead of rustc. - - When combined with `pipelined_compilation`, bin/cdylib crates also participate - in the hollow-rlib dependency chain: rustc only emits `.o` files (linking is - done by cc_common.link and does not check SVH), so bin compile steps can start - as soon as upstream hollow rlibs are ready rather than waiting for full codegen. """ bool_flag( name = "experimental_use_cc_common_link", diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl index 0f638c3ee6..36a3de891b 100644 --- a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl +++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl @@ -1,8 +1,8 @@ """Unittests for rust rules.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") -load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") -load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements_not") +load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro") +load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not") load(":wrap.bzl", "wrap") ENABLE_PIPELINING = { @@ -22,77 +22,49 @@ def _second_lib_test_impl(ctx): rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - # Hollow rlib approach: Rustc action uses --emit=dep-info,link (no metadata). - assert_argv_contains(env, rlib_action, "--emit=dep-info,link") + # Both actions should use the same --emit= + assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata") + assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata") - # Metadata action uses --emit=link=-hollow.rlib (hollow rlib, .rlib extension). - # The .rlib extension is required so rustc reads it as an rlib archive (extracting - # lib.rmeta with optimized MIR). Using .rmeta extension causes E0786, and using - # --emit=metadata produces raw .rmeta without optimized MIR (causes "missing - # optimized MIR" errors on Rust 1.85+). - metadata_emit_link = [arg for arg in metadata_action.argv if arg.startswith("--emit=link=") and arg.endswith("-hollow.rlib")] - asserts.true( - env, - len(metadata_emit_link) == 1, - "expected --emit=link=*-hollow.rlib for hollow rlib, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--emit=")]), - ) - - # The rlib action produces a .rlib; the metadata action produces a -hollow.rlib. + # The metadata action should have a .rmeta as output and the rlib action a .rlib path = rlib_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rlib") and not path.endswith("-hollow.rlib"), - "expected Rustc to output .rlib (not hollow), got " + path, + path.endswith(".rlib"), + "expected Rustc to output .rlib, got " + path, ) path = metadata_action.outputs.to_list()[0].path asserts.true( env, - path.endswith("-hollow.rlib"), - "expected RustcMetadata to output -hollow.rlib, got " + path, + path.endswith(".rmeta"), + "expected RustcMetadata to output .rmeta, got " + path, ) - # Neither action should use --rustc-quit-on-rmeta (hollow rlib exits naturally). + # Only the action building metadata should contain --rustc-quit-on-rmeta assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"]) - assert_list_contains_adjacent_elements_not(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) - - # The metadata action should use -Zno-codegen for the hollow rlib approach. - assert_argv_contains(env, metadata_action, "-Zno-codegen") - - # The Rustc action should NOT use -Zno-codegen. - no_codegen_in_rlib = [arg for arg in rlib_action.argv if arg == "-Zno-codegen"] - asserts.true(env, len(no_codegen_in_rlib) == 0, "Rustc action should not have -Zno-codegen") - - # The metadata action references first's hollow rlib for --extern (pipelining: starts - # before first's full codegen finishes). The Rustc action uses the full rlib for - # --extern so the full rlib's embedded SVH matches the full rlib that downstream - # binaries (without cc_common.link) see in their -Ldependency path. If both actions - # used the hollow rlib, nondeterministic proc macros could produce different SVHs - # for the hollow vs full rlib, causing E0460 in downstream binary builds. - extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("-hollow.rlib")] + assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) + + # Check that both actions refer to the metadata of :first, not the rlib + extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] asserts.true( env, len(extern_metadata) == 1, - "did not find --extern=first=*-hollow.rlib for metadata action, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--extern=first=")]), + "did not find a --extern=first=*.rmeta but expected one", ) - extern_rlib_full = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and not arg.endswith("-hollow.rlib")] + extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] asserts.true( env, - len(extern_rlib_full) == 1, - "expected --extern=first=libfirst*.rlib (full rlib) for rlib action, got: " + str([arg for arg in rlib_action.argv if arg.startswith("--extern=first=")]), + len(extern_rlib) == 1, + "did not find a --extern=first=*.rlib but expected one", ) - # The metadata action's input is first's hollow rlib only (no full rlib needed). + # Check that the input to both actions is the metadata of :first input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input for metadata, found " + str([i.path for i in input_metadata])) - asserts.true(env, input_metadata[0].basename.endswith("-hollow.rlib"), "expected hollow rlib for metadata action, found " + input_metadata[0].path) - - # The Rustc action's inputs contain the full rlib (referenced by --extern) and the - # hollow rlib (present in the sandbox for -Ldependency=<_hollow_dir> resolution of - # transitive deps that were compiled against hollow rlibs). - input_rlib_full = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and not i.basename.endswith("-hollow.rlib")] - input_rlib_hollow = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("-hollow.rlib")] - asserts.true(env, len(input_rlib_full) == 1, "expected full rlib in rlib action inputs, found " + str([i.path for i in input_rlib_full])) - asserts.true(env, len(input_rlib_hollow) == 1, "expected hollow rlib in rlib action inputs (for sandbox), found " + str([i.path for i in input_rlib_hollow])) + asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata])) + asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path) + input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")] + asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib])) + asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path) return analysistest.end(env) @@ -152,16 +124,10 @@ def _pipelined_compilation_test(): target_under_test = ":bin", target_compatible_with = _NO_WINDOWS, ) - hollow_rlib_env_test( - name = "hollow_rlib_env_test", - target_under_test = ":second", - target_compatible_with = _NO_WINDOWS, - ) return [ ":second_lib_test", ":bin_test", - ":hollow_rlib_env_test", ] def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): @@ -172,8 +138,8 @@ def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): # also depend on metadata for 'wrapper'. rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith("-hollow.rlib")] - rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib") and not i.path.endswith("-hollow.rlib")] + metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")] + rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")] seen_wrapper_metadata = False seen_to_wrap_metadata = False @@ -210,30 +176,22 @@ def _rmeta_is_used_when_building_custom_rule_test_impl(ctx): # This is the custom rule invocation of rustc. rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] + # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata seen_to_wrap_rlib = False - seen_to_wrap_hollow = False + seen_to_wrap_rmeta = False for act in rust_action.inputs.to_list(): - if "libto_wrap" in act.path and act.path.endswith("-hollow.rlib"): - seen_to_wrap_hollow = True - elif "libto_wrap" in act.path and act.path.endswith(".rlib") and not act.path.endswith("-hollow.rlib"): + if "libto_wrap" in act.path and act.path.endswith(".rlib"): seen_to_wrap_rlib = True + elif "libto_wrap" in act.path and act.path.endswith(".rmeta"): + seen_to_wrap_rmeta = True - if ctx.attr.generate_metadata: - # When wrapper generates its own hollow rlib, the Rustc action uses the full - # rlib of to_wrap for --extern (SVH consistency) and also has the hollow rlib - # in the sandbox for -Ldependency= resolution. - asserts.true(env, seen_to_wrap_hollow, "expected hollow rlib in inputs (for sandbox) when generate_metadata=True") - asserts.true(env, seen_to_wrap_rlib, "expected full rlib in inputs for --extern when generate_metadata=True") - else: - # When wrapper does not generate its own hollow rlib, the Rustc action uses - # hollow rlib deps via normal _depend_on_metadata logic (pipelined rlib deps). - asserts.true(env, seen_to_wrap_hollow, "expected dependency on metadata for 'to_wrap' but not found") - asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") + asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found") + asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") return analysistest.end(env) rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) -rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) +rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING) def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): env = analysistest.begin(ctx) @@ -246,33 +204,6 @@ def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): rmeta_not_produced_if_pipelining_disabled_test = analysistest.make(_rmeta_not_produced_if_pipelining_disabled_test_impl, config_settings = ENABLE_PIPELINING) -def _hollow_rlib_env_test_impl(ctx): - """Verify RUSTC_BOOTSTRAP=1 is set consistently on both Rustc and RustcMetadata actions. - - RUSTC_BOOTSTRAP=1 changes the crate hash (SVH), so it must be set on both actions - to keep the hollow rlib and full rlib SVHs consistent.""" - env = analysistest.begin(ctx) - tut = analysistest.target_under_test(env) - metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - - asserts.equals( - env, - "1", - metadata_action.env.get("RUSTC_BOOTSTRAP", ""), - "Metadata action should have RUSTC_BOOTSTRAP=1 for hollow rlib approach", - ) - asserts.equals( - env, - "1", - rlib_action.env.get("RUSTC_BOOTSTRAP", ""), - "Rustc action should have RUSTC_BOOTSTRAP=1 for SVH compatibility with hollow rlib", - ) - - return analysistest.end(env) - -hollow_rlib_env_test = analysistest.make(_hollow_rlib_env_test_impl, config_settings = ENABLE_PIPELINING) - def _disable_pipelining_test(): rust_library( name = "lib", @@ -318,7 +249,6 @@ def _custom_rule_test(generate_metadata, suffix): rmeta_is_used_when_building_custom_rule_test( name = "rmeta_is_used_when_building_custom_rule_test" + suffix, - generate_metadata = generate_metadata, target_under_test = ":wrapper" + suffix, target_compatible_with = _NO_WINDOWS, ) @@ -328,59 +258,6 @@ def _custom_rule_test(generate_metadata, suffix): ":rmeta_is_used_when_building_custom_rule_test" + suffix, ] -def _svh_mismatch_test(): - """Creates a rust_test demonstrating SVH mismatch with non-deterministic proc macros. - - Without pipelining (default): each library is compiled exactly once, SVH - is consistent across the dependency graph, and the test builds and passes. - - With pipelining (//rust/settings:pipelined_compilation=true): rules_rust - compiles svh_lib twice in separate rustc invocations — once for the hollow - metadata (.rmeta), once for the full .rlib. Because the proc macro uses - HashMap with OS-seeded randomness, these two invocations typically produce - different token streams and therefore different SVH values. The consumer is - compiled against the hollow .rmeta (recording SVH_1); when rustc links the - test binary against the full .rlib (SVH_2), it detects SVH_1 ≠ SVH_2 and - fails with E0460. The test is therefore expected to FAIL TO BUILD most of - the time (~99.2% with 5 HashMap entries) when pipelining is enabled. - - The test is marked flaky because the SVH mismatch is non-deterministic: - on rare occasions (~0.8%) both rustc invocations produce the same HashMap - iteration order and the build succeeds even with pipelining enabled. - """ - - rust_proc_macro( - name = "svh_nondeterministic_macro", - srcs = ["svh_mismatch/svh_mismatch_nondeterministic_macro.rs"], - crate_name = "nondeterministic_macro", - edition = "2021", - ) - - rust_library( - name = "svh_lib", - srcs = ["svh_mismatch/svh_mismatch_lib.rs"], - edition = "2021", - proc_macro_deps = [":svh_nondeterministic_macro"], - ) - - rust_library( - name = "svh_consumer", - srcs = ["svh_mismatch/svh_mismatch_consumer.rs"], - edition = "2021", - deps = [":svh_lib"], - ) - - rust_test( - name = "svh_mismatch_test", - srcs = ["svh_mismatch/svh_mismatch_test.rs"], - edition = "2021", - deps = [":svh_consumer"], - flaky = True, - target_compatible_with = _NO_WINDOWS, - ) - - return [":svh_mismatch_test"] - def pipelined_compilation_test_suite(name): """Entry-point macro called from the BUILD file. @@ -392,7 +269,6 @@ def pipelined_compilation_test_suite(name): tests.extend(_disable_pipelining_test()) tests.extend(_custom_rule_test(generate_metadata = True, suffix = "_with_metadata")) tests.extend(_custom_rule_test(generate_metadata = False, suffix = "_without_metadata")) - tests.extend(_svh_mismatch_test()) native.test_suite( name = name, diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs deleted file mode 100644 index 99b0ea9bf4..0000000000 --- a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs +++ /dev/null @@ -1,6 +0,0 @@ -/// A library that depends on svh_lib. When compiled against a hollow `.rmeta` -/// of svh_lib, this crate's metadata records svh_lib's SVH at that point in -/// time. If the full `.rlib` of svh_lib was produced by a separate rustc -/// invocation (with a different HashMap seed), it may have a different SVH, -/// causing a mismatch when a downstream binary tries to link against both. -pub use svh_lib::Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs deleted file mode 100644 index e2f3985399..0000000000 --- a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -use nondeterministic_macro::NondeterministicHash; - -/// A struct whose derivation runs the non-deterministic proc macro. -/// The macro generates a public constant whose value depends on HashMap -/// iteration order, so this crate's SVH varies between separate rustc -/// invocations. -#[derive(NondeterministicHash)] -pub struct Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs deleted file mode 100644 index 7ba44425b7..0000000000 --- a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs +++ /dev/null @@ -1,37 +0,0 @@ -extern crate proc_macro; -use proc_macro::TokenStream; -use std::collections::HashMap; - -/// A derive macro that produces non-deterministic output due to HashMap's -/// random iteration order. Each separate process invocation initializes -/// `HashMap` with a different OS-seeded `RandomState`, so iteration order -/// varies between invocations. This makes the generated constant—and thus -/// the crate's SVH—differ when the macro is run twice (e.g., once for a -/// hollow `.rmeta` and once for a full `.rlib` in pipelined compilation). -#[proc_macro_derive(NondeterministicHash)] -pub fn nondeterministic_hash_derive(_input: TokenStream) -> TokenStream { - // HashMap::new() uses RandomState, which seeds from OS entropy. - // Each separate process invocation gets a different seed, so iteration - // order over the map is non-deterministic across invocations. - let mut map = HashMap::new(); - map.insert("alpha", 1u64); - map.insert("beta", 2u64); - map.insert("gamma", 4u64); - map.insert("delta", 8u64); - map.insert("epsilon", 16u64); - - // Position-weighted sum: not commutative, so different iteration orders - // produce different values. With 5 entries (5! = 120 orderings), the - // probability of identical output in two separate invocations is ~0.8%. - let fingerprint: u64 = map - .iter() - .enumerate() - .map(|(pos, (_, &val))| val.wrapping_mul(pos as u64 + 1)) - .fold(0u64, u64::wrapping_add); - - // Exposing this as a public constant makes it part of the crate's - // exported API, which is included in the SVH computation. - format!("pub const NONDETERMINISTIC_HASH_FINGERPRINT: u64 = {};", fingerprint) - .parse() - .unwrap() -} diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs deleted file mode 100644 index 6ecfe83553..0000000000 --- a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs +++ /dev/null @@ -1,28 +0,0 @@ -/// Demonstrates SVH (Strict Version Hash) mismatch with pipelined compilation. -/// -/// Without pipelining this test always builds and passes: each library is -/// compiled exactly once, so the SVH embedded in every `.rmeta` and `.rlib` -/// is identical. -/// -/// With `//rust/settings:pipelined_compilation=true` rules_rust compiles -/// `svh_lib` **twice** in separate rustc processes — once to emit the hollow -/// `.rmeta` (metadata only), once to emit the full `.rlib`. Because -/// `nondeterministic_macro` uses `HashMap` with OS-seeded randomness, the two -/// rustc invocations typically produce different token streams and therefore -/// different SVH values. `svh_consumer` is compiled against the hollow `.rmeta` -/// and records SVH_1 in its own metadata; when rustc later tries to link the -/// test binary against the full `.rlib` (which carries SVH_2), it detects the -/// mismatch and fails with E0460. The test therefore **fails to build** most of -/// the time (~99.2% probability) when pipelining is enabled. -/// -/// The `flaky = True` attribute on this target acknowledges that the mismatch -/// is non-deterministic: on rare occasions (~0.8%) both rustc invocations -/// happen to produce the same HashMap iteration order, the SVHs agree, and the -/// build succeeds. -use svh_consumer::Widget; - -#[test] -fn svh_consistent() { - // If we reach here the SVH was consistent (no pipelining, or a lucky run). - let _: Widget = Widget; -} diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index e3f4ac5482..f24a0e421a 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -40,23 +40,12 @@ def _wrap_impl(ctx): lib_hash = output_hash, extension = ".rlib", ) - - # Use -hollow.rlib extension (not .rmeta) so rustc reads it as an rlib archive - # containing optimized MIR. See rust/private/rust.bzl for the same logic. - # The hollow rlib is placed in a "_hollow/" subdirectory to avoid the full rlib - # and hollow rlib appearing in the same -Ldependency= search directory, which - # would cause E0463 "can't find crate" errors due to ambiguous crate candidates. - metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and ctx.attr.generate_metadata - if metadata_supports_pipelining: - rust_metadata_name = "_hollow/lib{name}-{lib_hash}-hollow.rlib".format( - name = crate_name, - lib_hash = output_hash, - ) - else: - rust_metadata_name = "lib{name}-{lib_hash}.rmeta".format( - name = crate_name, - lib_hash = output_hash, - ) + rust_metadata_name = "{prefix}{name}-{lib_hash}{extension}".format( + prefix = "lib", + name = crate_name, + lib_hash = output_hash, + extension = ".rmeta", + ) tgt = ctx.attr.target deps = [DepVariantInfo( @@ -84,7 +73,8 @@ def _wrap_impl(ctx): aliases = {}, output = rust_lib, metadata = rust_metadata, - metadata_supports_pipelining = metadata_supports_pipelining, + metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and + ctx.attr.generate_metadata, owner = ctx.label, edition = "2018", compile_data = depset([]), From 6b800e4d44c4cf69ffbb48581db5a8cd8aa50ddf Mon Sep 17 00:00:00 2001 From: Walter Gray Date: Fri, 10 Apr 2026 11:31:45 -0700 Subject: [PATCH 2/2] Switch pipelined compilation from --rustc-quit-on-rmeta to -Zno-codegen Replace the process wrapper's rmeta-interception approach (kill rustc after metadata emission) with rustc's -Zno-codegen flag, which produces a hollow rlib containing metadata and MIR but no object code. This is the same approach used by Buck2. Key changes: - Metadata action uses to produce a hollow rlib (_meta.rlib) instead of raw .rmeta files - Remove --rustc-quit-on-rmeta flag, LineOutput::Terminate, and all associated kill logic from the process wrapper - Full action emits only --emit=link (no longer includes metadata) - Set RUSTC_BOOTSTRAP=1 on both metadata and full actions for SVH compatibility (required for the unstable -Zno-codegen flag) --- extensions/prost/private/prost.bzl | 4 +- rust/private/clippy.bzl | 2 +- rust/private/rust.bzl | 8 +- rust/private/rustc.bzl | 39 +++--- rust/private/unpretty.bzl | 2 +- rust/settings/settings.bzl | 11 +- test/process_wrapper/BUILD.bazel | 4 +- ...uit_on_rmeta.rs => rustc_output_format.rs} | 54 +++------ .../metadata_output_groups.bzl | 4 +- .../pipelined_compilation_test.bzl | 112 ++++++++++-------- test/unit/pipelined_compilation/wrap.bzl | 2 +- .../codegen_disambiguation_test.bzl | 2 +- util/process_wrapper/main.rs | 35 ------ 13 files changed, 128 insertions(+), 151 deletions(-) rename test/process_wrapper/{rustc_quit_on_rmeta.rs => rustc_output_format.rs} (63%) diff --git a/extensions/prost/private/prost.bzl b/extensions/prost/private/prost.bzl index 386f0d7d72..e67ff89fcf 100644 --- a/extensions/prost/private/prost.bzl +++ b/extensions/prost/private/prost.bzl @@ -180,7 +180,7 @@ def _compile_rust( prefix = "lib", name = crate_name, lib_hash = output_hash, - extension = ".rmeta", + extension = "_meta.rlib", ) lib = ctx.actions.declare_file(lib_name) @@ -193,7 +193,7 @@ def _compile_rust( prefix = "lib", name = crate_name, lib_hash = output_hash, - extension = ".rmeta", + extension = "_meta.rlib", ) rmeta = ctx.actions.declare_file(rmeta_name) rustc_rmeta_output = generate_output_diagnostics(ctx, rmeta) diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl index 318c05af8d..16285e7987 100644 --- a/rust/private/clippy.bzl +++ b/rust/private/clippy.bzl @@ -177,7 +177,7 @@ def rust_clippy_action(ctx, clippy_executable, process_wrapper, crate_info, conf out_dir = out_dir, build_env_files = build_env_files, build_flags_files = build_flags_files, - emit = ["dep-info", "metadata"], + emit = ["metadata"], skip_expanding_rustc_env = True, use_json_output = bool(clippy_diagnostics_file), error_format = error_format, diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index 3b15974624..6e21b40281 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -181,7 +181,7 @@ def _rust_library_common(ctx, crate_type): disable_pipelining = getattr(ctx.attr, "disable_pipelining", False), ): rust_metadata = ctx.actions.declare_file( - paths.replace_extension(rust_lib_name, ".rmeta"), + paths.replace_extension(rust_lib_name, "_meta.rlib"), sibling = rust_lib, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) @@ -254,7 +254,7 @@ def _rust_binary_impl(ctx): rustc_rmeta_output = None if can_build_metadata(toolchain, ctx, ctx.attr.crate_type): rust_metadata = ctx.actions.declare_file( - paths.replace_extension("lib" + crate_name, ".rmeta"), + paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) @@ -356,7 +356,7 @@ def _rust_test_impl(ctx): rustc_rmeta_output = None if can_build_metadata(toolchain, ctx, crate_type): rust_metadata = ctx.actions.declare_file( - paths.replace_extension("lib" + crate_name, ".rmeta"), + paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) @@ -426,7 +426,7 @@ def _rust_test_impl(ctx): rustc_rmeta_output = None if can_build_metadata(toolchain, ctx, crate_type): rust_metadata = ctx.actions.declare_file( - paths.replace_extension("lib" + crate_name, ".rmeta"), + paths.replace_extension("lib" + crate_name, "_meta.rlib"), sibling = output, ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 962d4cd2ba..2c0098a652 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -901,7 +901,7 @@ def construct_arguments( out_dir, build_env_files, build_flags_files, - emit = ["dep-info", "link"], + emit = ["link"], force_all_deps_direct = False, add_flags_for_binary = False, include_link_flags = True, @@ -1045,8 +1045,11 @@ def construct_arguments( error_format = "json" if build_metadata: - # Configure process_wrapper to terminate rustc when metadata are emitted - process_wrapper_flags.add("--rustc-quit-on-rmeta", "true") + # Build a hollow rlib (metadata-full, Buck2 equivalent) using -Zno-codegen. + # This produces an rlib with metadata but no object code, allowing downstream + # crates to start compiling without waiting for codegen. + # RUSTC_BOOTSTRAP=1 must be set in the action env for this unstable flag. + rustc_flags.add("-Zno-codegen") if crate_info.rustc_rmeta_output: process_wrapper_flags.add("--output-file", crate_info.rustc_rmeta_output.path) elif crate_info.rustc_output: @@ -1079,7 +1082,12 @@ def construct_arguments( emit_without_paths = [] for kind in emit: - if kind == "link" and crate_info.type == "bin" and crate_info.output != None: + if kind == "link" and build_metadata and crate_info.metadata != None: + # Redirect hollow rlib output to the declared metadata file path, + # since -Zno-codegen --emit=link would otherwise write lib.rlib + # which collides with the full action's output. + rustc_flags.add(crate_info.metadata, format = "--emit=link=%s") + elif kind == "link" and crate_info.type == "bin" and crate_info.output != None: rustc_flags.add(crate_info.output, format = "--emit=link=%s") else: emit_without_paths.append(kind) @@ -1374,17 +1382,12 @@ def rustc_compile_action( experimental_use_cc_common_link = experimental_use_cc_common_link, ) + compile_inputs_metadata = compile_inputs + # The types of rustc outputs to emit. - # If we build metadata, we need to keep the command line of the two invocations - # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as - # a candidate. - # Because of that we need to add emit=metadata to both the rlib and rmeta invocation. - # # When cc_common linking is enabled, emit a `.o` file, which is later # passed to the cc_common.link action. - emit = ["dep-info", "link"] - if build_metadata: - emit.append("metadata") + emit = ["link"] if experimental_use_cc_common_link: emit = ["obj"] @@ -1433,7 +1436,7 @@ def rustc_compile_action( toolchain = toolchain, tool_path = toolchain.rustc.path, cc_toolchain = cc_toolchain, - emit = emit, + emit = ["link"], feature_configuration = feature_configuration, crate_info = crate_info, dep_info = dep_info, @@ -1456,6 +1459,12 @@ def rustc_compile_action( # this is the final list of env vars env.update(env_from_args) + if build_metadata: + # RUSTC_BOOTSTRAP=1 is required for -Zno-codegen on stable rustc, and must + # be set on both the metadata and full actions for SVH compatibility (since + # RUSTC_BOOTSTRAP affects the crate hash). + env["RUSTC_BOOTSTRAP"] = "1" + if hasattr(attr, "version") and attr.version != "0.0.0": formatted_version = " v{}".format(attr.version) else: @@ -1526,7 +1535,7 @@ def rustc_compile_action( if args_metadata: ctx.actions.run( executable = ctx.executable._process_wrapper, - inputs = compile_inputs, + inputs = compile_inputs_metadata, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, arguments = args_metadata.all, @@ -2200,7 +2209,7 @@ def _get_crate_dirname(crate): """ return crate.output.dirname -def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_darwin = False, flavor_msvc = False): +def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = False, for_darwin = False, flavor_msvc = False): artifact = get_preferred_artifact(lib, use_pic) if ambiguous_libs and artifact.path in ambiguous_libs: artifact = ambiguous_libs[artifact.path] diff --git a/rust/private/unpretty.bzl b/rust/private/unpretty.bzl index be111c83a3..dbb2d28a85 100644 --- a/rust/private/unpretty.bzl +++ b/rust/private/unpretty.bzl @@ -202,7 +202,7 @@ def _rust_unpretty_aspect_impl(target, ctx): out_dir = out_dir, build_env_files = build_env_files, build_flags_files = build_flags_files, - emit = ["dep-info", "metadata"], + emit = ["metadata"], skip_expanding_rustc_env = True, ) diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index b6d1c0aea4..bb232bebe2 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -112,10 +112,15 @@ def use_real_import_macro(): ) def pipelined_compilation(): - """When set, this flag causes rustc to emit `*.rmeta` files and use them for `rlib -> rlib` dependencies. + """When set, this flag enables pipelined compilation for `rlib -> rlib` dependencies. - While this involves one extra (short) rustc invocation to build the rmeta file, - it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation. + For each library target, a second RustcMetadata action is created that runs rustc with + `-Zno-codegen --emit=link` to produce a hollow rlib (metadata & MIR, no object code). + Downstream library compilations can start as soon as this hollow rlib is available, + increasing parallelism during compilation. + + Requires RUSTC_BOOTSTRAP=1, which is set automatically on both the metadata and full + actions for pipelined targets. """ bool_flag( name = "pipelined_compilation", diff --git a/test/process_wrapper/BUILD.bazel b/test/process_wrapper/BUILD.bazel index 43e2420062..1ab6510841 100644 --- a/test/process_wrapper/BUILD.bazel +++ b/test/process_wrapper/BUILD.bazel @@ -157,8 +157,8 @@ rust_binary( ) rust_test( - name = "rustc_quit_on_rmeta", - srcs = ["rustc_quit_on_rmeta.rs"], + name = "rustc_output_format", + srcs = ["rustc_output_format.rs"], data = [ ":fake_rustc", "//util/process_wrapper", diff --git a/test/process_wrapper/rustc_quit_on_rmeta.rs b/test/process_wrapper/rustc_output_format.rs similarity index 63% rename from test/process_wrapper/rustc_quit_on_rmeta.rs rename to test/process_wrapper/rustc_output_format.rs index b804ba1ebb..a3b175a48a 100644 --- a/test/process_wrapper/rustc_quit_on_rmeta.rs +++ b/test/process_wrapper/rustc_output_format.rs @@ -40,54 +40,38 @@ mod test { } #[test] - fn test_rustc_quit_on_rmeta_quits() { - let out_content = fake_rustc( - &[ - "--rustc-quit-on-rmeta", - "true", - "--rustc-output-format", - "rendered", - ], - &[], - true, + fn test_rustc_output_format_rendered() { + let out_content = fake_rustc(&["--rustc-output-format", "rendered"], &[], true); + assert!( + out_content.contains("should be\nin output"), + "output should contain the first rendered message", ); assert!( - !out_content.contains("should not be in output"), - "output should not contain 'should not be in output' but did", + out_content.contains("should not be in output"), + "output should contain the second rendered message", + ); + assert!( + !out_content.contains(r#""rendered""#), + "rendered mode should not print raw json", ); } #[test] - fn test_rustc_quit_on_rmeta_output_json() { + fn test_rustc_output_format_json() { let json_content = fake_rustc( - &[ - "--rustc-quit-on-rmeta", - "true", - "--rustc-output-format", - "json", - ], + &["--rustc-output-format", "json"], &[], true, ); assert_eq!( json_content, - concat!(r#"{"rendered": "should be\nin output"}"#, "\n") - ); - } - - #[test] - fn test_rustc_quit_on_rmeta_output_rendered() { - let rendered_content = fake_rustc( - &[ - "--rustc-quit-on-rmeta", - "true", - "--rustc-output-format", - "rendered", - ], - &[], - true, + concat!( + r#"{"rendered": "should be\nin output"}"#, + "\n", + r#"{"rendered": "should not be in output"}"#, + "\n" + ) ); - assert_eq!(rendered_content, "should be\nin output"); } #[test] diff --git a/test/unit/metadata_output_groups/metadata_output_groups.bzl b/test/unit/metadata_output_groups/metadata_output_groups.bzl index ec89e0f132..a20c9ac5b2 100644 --- a/test/unit/metadata_output_groups/metadata_output_groups.bzl +++ b/test/unit/metadata_output_groups/metadata_output_groups.bzl @@ -13,8 +13,8 @@ def _metadata_output_groups_present_test_impl(ctx): asserts.equals(env, 1, len(build_metadata), "Expected 1 build_metadata file") asserts.true( - build_metadata[0].basename.endswith(".rmeta"), - "Expected %s to end with .rmeta" % build_metadata[0], + build_metadata[0].basename.endswith("_meta.rlib"), + "Expected %s to end with _meta.rlib" % build_metadata[0], ) asserts.equals(env, 1, len(rustc_rmeta_output), "Expected 1 rustc_rmeta_output file") diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl index 36a3de891b..a10dad44e2 100644 --- a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl +++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl @@ -2,7 +2,7 @@ load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro") -load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not") +load("//test/unit:common.bzl", "assert_argv_contains") load(":wrap.bzl", "wrap") ENABLE_PIPELINING = { @@ -22,49 +22,56 @@ def _second_lib_test_impl(ctx): rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - # Both actions should use the same --emit= - assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata") - assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata") + # The full action emits link; the metadata action emits only + # link with an explicit path (--emit=link=) and uses -Zno-codegen to + # produce a hollow rlib (metadata-full). + assert_argv_contains(env, rlib_action, "--emit=link") - # The metadata action should have a .rmeta as output and the rlib action a .rlib + # RustcMetadata uses --emit=link= to redirect the hollow rlib to the + # declared .rmeta output. Check that it contains the flag with a path. + metadata_emit = [arg for arg in metadata_action.argv if arg.startswith("--emit=link=")] + asserts.true( + env, + len(metadata_emit) == 1, + "expected RustcMetadata to have --emit=link=, got " + str(metadata_emit), + ) + + # RustcMetadata must use -Zno-codegen to produce a hollow rlib + assert_argv_contains(env, metadata_action, "-Zno-codegen") + + # The metadata action outputs a hollow .rlib (_meta.rlib), the full action a normal .rlib path = rlib_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rlib"), - "expected Rustc to output .rlib, got " + path, + path.endswith(".rlib") and not path.endswith("_meta.rlib"), + "expected Rustc to output .rlib (not _meta.rlib), got " + path, ) path = metadata_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rmeta"), - "expected RustcMetadata to output .rmeta, got " + path, + path.endswith("_meta.rlib"), + "expected RustcMetadata to output _meta.rlib, got " + path, ) - # Only the action building metadata should contain --rustc-quit-on-rmeta - assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"]) - assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) - - # Check that both actions refer to the metadata of :first, not the rlib - extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + # Both actions should refer to the metadata artifact of :first. + extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("_meta.rlib")] asserts.true( env, len(extern_metadata) == 1, - "did not find a --extern=first=*.rmeta but expected one", + "expected RustcMetadata --extern=first=*_meta.rlib, got " + str([a for a in metadata_action.argv if "--extern=first=" in a]), ) - extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("_meta.rlib")] asserts.true( env, len(extern_rlib) == 1, - "did not find a --extern=first=*.rlib but expected one", + "expected Rustc --extern=first=*_meta.rlib, got " + str([a for a in rlib_action.argv if "--extern=first=" in a]), ) - # Check that the input to both actions is the metadata of :first - input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata])) - asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path) - input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib])) - asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path) + # Both actions should take the metadata artifact as input. + input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("_meta.rlib")] + asserts.true(env, len(input_metadata) == 1, "expected one libfirst _meta.rlib input to RustcMetadata, found " + str([i.path for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")])) + input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("_meta.rlib")] + asserts.true(env, len(input_rlib) == 1, "expected one libfirst _meta.rlib input to Rustc, found " + str([i.path for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")])) return analysistest.end(env) @@ -73,10 +80,10 @@ def _bin_test_impl(ctx): tut = analysistest.target_under_test(env) bin_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # Check that no inputs to this binary are .rmeta files. - metadata_inputs = [i.path for i in bin_action.inputs.to_list() if i.path.endswith(".rmeta")] + # Check that no inputs to this binary are hollow rlib (_meta.rlib) files. + metadata_inputs = [i.path for i in bin_action.inputs.to_list() if i.path.endswith("_meta.rlib")] - # Filter out toolchain targets. This test intends to only check for rmeta files of `deps`. + # Filter out toolchain targets. This test intends to only check for metadata files of `deps`. metadata_inputs = [i for i in metadata_inputs if "/lib/rustlib" not in i] asserts.false(env, metadata_inputs, "expected no metadata inputs, found " + json.encode_indent(metadata_inputs, indent = " " * 4)) @@ -130,6 +137,14 @@ def _pipelined_compilation_test(): ":bin_test", ] +def _is_metadata_file(path): + """Returns True if the path is a hollow rlib (metadata-full) file.""" + return path.endswith("_meta.rlib") + +def _is_full_rlib(path): + """Returns True if the path is a full rlib (not a hollow rlib).""" + return path.endswith(".rlib") and not path.endswith("_meta.rlib") + def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): env = analysistest.begin(ctx) tut = analysistest.target_under_test(env) @@ -138,24 +153,21 @@ def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): # also depend on metadata for 'wrapper'. rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")] - rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")] - seen_wrapper_metadata = False seen_to_wrap_metadata = False - for mi in metadata_inputs: - if "libwrapper" in mi.path: - seen_wrapper_metadata = True - if "libto_wrap" in mi.path: - seen_to_wrap_metadata = True - seen_wrapper_rlib = False seen_to_wrap_rlib = False - for ri in rlib_inputs: - if "libwrapper" in ri.path: - seen_wrapper_rlib = True - if "libto_wrap" in ri.path: - seen_to_wrap_rlib = True + for i in rust_action.inputs.to_list(): + if "libwrapper" in i.path: + if _is_metadata_file(i.path): + seen_wrapper_metadata = True + elif _is_full_rlib(i.path): + seen_wrapper_rlib = True + if "libto_wrap" in i.path: + if _is_metadata_file(i.path): + seen_to_wrap_metadata = True + elif _is_full_rlib(i.path): + seen_to_wrap_rlib = True if ctx.attr.generate_metadata: asserts.true(env, seen_wrapper_metadata, "expected dependency on metadata for 'wrapper' but not found") @@ -176,22 +188,23 @@ def _rmeta_is_used_when_building_custom_rule_test_impl(ctx): # This is the custom rule invocation of rustc. rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata + # The custom rule invocation should depend on metadata, regardless of whether + # the wrapper itself generates metadata. seen_to_wrap_rlib = False - seen_to_wrap_rmeta = False + seen_to_wrap_metadata = False for act in rust_action.inputs.to_list(): - if "libto_wrap" in act.path and act.path.endswith(".rlib"): + if "libto_wrap" in act.path and _is_full_rlib(act.path): seen_to_wrap_rlib = True - elif "libto_wrap" in act.path and act.path.endswith(".rmeta"): - seen_to_wrap_rmeta = True + elif "libto_wrap" in act.path and _is_metadata_file(act.path): + seen_to_wrap_metadata = True - asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found") + asserts.true(env, seen_to_wrap_metadata, "expected dependency on metadata for 'to_wrap' but not found") asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") return analysistest.end(env) rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) -rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING) +rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): env = analysistest.begin(ctx) @@ -249,6 +262,7 @@ def _custom_rule_test(generate_metadata, suffix): rmeta_is_used_when_building_custom_rule_test( name = "rmeta_is_used_when_building_custom_rule_test" + suffix, + generate_metadata = generate_metadata, target_under_test = ":wrapper" + suffix, target_compatible_with = _NO_WINDOWS, ) diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index f24a0e421a..89b75b1850 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -44,7 +44,7 @@ def _wrap_impl(ctx): prefix = "lib", name = crate_name, lib_hash = output_hash, - extension = ".rmeta", + extension = "_meta.rlib", ) tgt = ctx.attr.target diff --git a/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl b/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl index 1b0c353f61..f263a4538c 100644 --- a/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl +++ b/test/unit/rust_test_codegen_disambiguation/codegen_disambiguation_test.bzl @@ -1,7 +1,7 @@ """Tests that rust_test targets receive codegen disambiguation flags. rust_test targets pass --codegen=metadata and --codegen=extra-filename to rustc -so that intermediate compilation artifacts (.o, .d files) get unique names. This +so that intermediate compilation artifacts (such as .o files) get unique names. This prevents collisions with rust_binary or rust_library targets that share the same crate name, which would otherwise cause link failures on non-sandboxed builds (e.g. Windows or --spawn_strategy=standalone). See diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 2a7cbd8565..8b10955464 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -487,39 +487,4 @@ mod test { Ok(()) } - #[test] - fn test_process_line_emit_link() -> Result<(), String> { - assert!(matches!( - process_line( - r#" - { - "$message_type": "artifact", - "emit": "link" - } - "# - .to_string(), - ErrorFormat::Rendered, - )?, - LineOutput::Skip - )); - Ok(()) - } - - #[test] - fn test_process_line_emit_metadata() -> Result<(), String> { - assert!(matches!( - process_line( - r#" - { - "$message_type": "artifact", - "emit": "metadata" - } - "# - .to_string(), - ErrorFormat::Rendered, - )?, - LineOutput::Skip - )); - Ok(()) - } }