diff --git a/MODULE.bazel b/MODULE.bazel index 9876238c48..5fce02f767 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,6 +9,7 @@ module( ## Core ############################################################################### +bazel_dep(name = "bazel_lib", version = "3.0.0") bazel_dep(name = "bazel_features", version = "1.32.0") bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "platforms", version = "1.0.0") diff --git a/cargo/private/BUILD.bazel b/cargo/private/BUILD.bazel index fd60c4ff62..db04afa7b1 100644 --- a/cargo/private/BUILD.bazel +++ b/cargo/private/BUILD.bazel @@ -1,5 +1,5 @@ +load("@bazel_lib//lib:copy_file.bzl", "copy_file") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("//rust:defs.bzl", "rust_binary") rust_binary( @@ -39,6 +39,9 @@ copy_file( bzl_library( name = "bzl_lib", + deps = [ + "//rust:bzl_lib", + ], srcs = glob(["**/*.bzl"]), - visibility = ["//:__subpackages__"], + visibility = ["//visibility:public"], ) diff --git a/extensions/prost/private/BUILD.bazel b/extensions/prost/private/BUILD.bazel index ed8e508537..aaf6c22d8d 100644 --- a/extensions/prost/private/BUILD.bazel +++ b/extensions/prost/private/BUILD.bazel @@ -5,6 +5,8 @@ load("//:defs.bzl", "rust_prost_toolchain") load(":legacy_proto_toolchain.bzl", "legacy_proto_toolchain") load(":prost.bzl", "RUST_EDITION", "current_prost_runtime") +exports_files(["protoc_wrapper.rs"]) + current_prost_runtime( name = "current_prost_runtime", ) diff --git a/extensions/prost/private/prost.bzl b/extensions/prost/private/prost.bzl index 4eecf85a06..386f0d7d72 100644 --- a/extensions/prost/private/prost.bzl +++ b/extensions/prost/private/prost.bzl @@ -31,14 +31,14 @@ RUST_EDITION = "2021" TOOLCHAIN_TYPE = "@rules_rust_prost//:toolchain_type" -def _create_proto_lang_toolchain(ctx, prost_toolchain): +def _create_proto_lang_toolchain(prost_toolchain): proto_lang_toolchain = proto_common.ProtoLangToolchainInfo( out_replacement_format_flag = "--prost_out=%s", plugin_format_flag = prost_toolchain.prost_plugin_flag, plugin = prost_toolchain.prost_plugin[DefaultInfo].files_to_run, runtime = prost_toolchain.prost_runtime, provided_proto_sources = depset(), - proto_compiler = ctx.attr._prost_process_wrapper[DefaultInfo].files_to_run, + proto_compiler = prost_toolchain.prost_process_wrapper[DefaultInfo].files_to_run, protoc_opts = prost_toolchain.protoc_opts, progress_message = "ProstGenProto %{label}", mnemonic = "ProstGenProto", @@ -118,7 +118,7 @@ def _compile_proto( additional_inputs = additional_inputs, additional_args = additional_args, generated_files = [lib_rs, package_info_file], - proto_lang_toolchain_info = _create_proto_lang_toolchain(ctx, prost_toolchain), + proto_lang_toolchain_info = _create_proto_lang_toolchain(prost_toolchain), plugin_output = ctx.bin_dir.path, ) @@ -377,12 +377,6 @@ rust_prost_aspect = aspect( default = Label("@bazel_tools//tools/cpp:grep-includes"), cfg = "exec", ), - "_prost_process_wrapper": attr.label( - doc = "The wrapper script for the Prost protoc plugin.", - cfg = "exec", - executable = True, - default = Label("//private:protoc_wrapper"), - ), } | RUSTC_ATTRS | { # Need to override this attribute to explicitly set the workspace. "_always_enable_metadata_output_groups": attr.label( @@ -473,6 +467,7 @@ def _rust_prost_toolchain_impl(ctx): prost_plugin = ctx.attr.prost_plugin, prost_plugin_flag = ctx.attr.prost_plugin_flag, prost_runtime = ctx.attr.prost_runtime, + prost_process_wrapper = ctx.attr._prost_process_wrapper, prost_types = ctx.attr.prost_types, proto_compiler = proto_compiler, protoc_opts = ctx.fragments.proto.experimental_protoc_opts, @@ -516,6 +511,12 @@ rust_prost_toolchain = rule( mandatory = True, aspects = [rust_analyzer_aspect], ), + "_prost_process_wrapper": attr.label( + doc = "The wrapper script for the Prost protoc plugin.", + cfg = "exec", + executable = True, + default = Label("@rules_rust_prost//private:protoc_wrapper"), + ), "prost_types": attr.label( doc = "The Prost types crates to use.", providers = [[rust_common.crate_info], [rust_common.crate_group_info]], diff --git a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl index 8a2dcb5738..ee8591d71d 100644 --- a/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl +++ b/extensions/wasm_bindgen/private/wasm_bindgen_test.bzl @@ -13,6 +13,7 @@ load( "@rules_rust//rust/private:utils.bzl", "determine_output_hash", "expand_dict_value_locations", + "filter_deps", "find_toolchain", "generate_output_diagnostics", "get_import_macro_deps", @@ -64,8 +65,10 @@ def _rust_wasm_bindgen_test_impl(ctx): toolchain = find_toolchain(ctx) crate_type = "bin" - deps = transform_deps(ctx.attr.deps + [wb_toolchain.wasm_bindgen_test]) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + + deps, proc_macro_deps = filter_deps(ctx) + deps = transform_deps(deps + [wb_toolchain.wasm_bindgen_test]) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) # Target is building the crate in `test` config if WasmBindgenTestCrateInfo in ctx.attr.wasm: diff --git a/rust/defs.bzl b/rust/defs.bzl index 4f2ef72582..8c6c14813d 100644 --- a/rust/defs.bzl +++ b/rust/defs.bzl @@ -14,6 +14,7 @@ """Public entry point to all Rust rules and supported APIs.""" +load("@bazel_features//:features.bzl", "bazel_features") load( "//rust:toolchain.bzl", _rust_stdlib_filegroup = "rust_stdlib_filegroup", @@ -78,25 +79,57 @@ load( _rust_unpretty_aspect = "rust_unpretty_aspect", ) -rust_library = _rust_library +def _rule_wrapper(rule): + def _wrapped(name, deps = [], proc_macro_deps = [], **kwargs): + rule( + name = name, + deps = deps + proc_macro_deps, + # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. + proc_macro_deps = deps + proc_macro_deps, + **kwargs + ) + + return _wrapped + +def _symbolic_rule_wrapper(rule, macro_fn): + def _wrapped(name, visibility, deps, proc_macro_deps, **kwargs): + rule( + name = name, + visibility = visibility, + deps = deps + proc_macro_deps, + # TODO(zbarsky): This attribute would ideally be called `exec_configured_deps` or similar. + proc_macro_deps = deps + proc_macro_deps, + **kwargs + ) + + return macro_fn( + implementation = _wrapped, + inherit_attrs = rule, + attrs = { + "deps": attr.label_list(default = []), + "proc_macro_deps": attr.label_list(default = []), + }, + ) + +rust_library = _symbolic_rule_wrapper(_rust_library, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_static_library = _rust_static_library +rust_static_library = _rule_wrapper(_rust_static_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_shared_library = _rust_shared_library +rust_shared_library = _rule_wrapper(_rust_shared_library) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_proc_macro = _rust_proc_macro +rust_proc_macro = _rule_wrapper(_rust_proc_macro) # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_binary = _rust_binary +rust_binary = _symbolic_rule_wrapper(_rust_binary, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_binary) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_library_group = _rust_library_group # See @rules_rust//rust/private:rust.bzl for a complete description. -rust_test = _rust_test +rust_test = _symbolic_rule_wrapper(_rust_test, bazel_features.globals.macro) if bazel_features.globals.macro else _rule_wrapper(_rust_test) # See @rules_rust//rust/private:rust.bzl for a complete description. rust_test_suite = _rust_test_suite @@ -105,7 +138,7 @@ rust_test_suite = _rust_test_suite rust_doc = _rust_doc # See @rules_rust//rust/private:rustdoc.bzl for a complete description. -rust_doc_test = _rust_doc_test +rust_doc_test = _rule_wrapper(_rust_doc_test) # See @rules_rust//rust/private:rustdoc_test.bzl for a complete description. clippy_flag = _clippy_flag diff --git a/rust/extensions.bzl b/rust/extensions.bzl index 749740032e..b059d6e02e 100644 --- a/rust/extensions.bzl +++ b/rust/extensions.bzl @@ -26,8 +26,6 @@ def _find_modules(module_ctx): our_module = mod if root == None: root = our_module - if our_module == None: - fail("Unable to find rules_rust module") return root, our_module @@ -93,7 +91,9 @@ def _rust_impl(module_ctx): if toolchain_triples.get(repository_set["exec_triple"]) == repository_set["name"]: toolchain_triples.pop(repository_set["exec_triple"], None) - toolchains = root.tags.toolchain or rules_rust.tags.toolchain + toolchains = root.tags.toolchain + if not toolchains and rules_rust: + toolchains = rules_rust.tags.toolchain for toolchain in toolchains: if toolchain.extra_rustc_flags and toolchain.extra_rustc_flags_triples: diff --git a/rust/platform/BUILD.bazel b/rust/platform/BUILD.bazel index d5513b1e79..91b095ebed 100644 --- a/rust/platform/BUILD.bazel +++ b/rust/platform/BUILD.bazel @@ -31,5 +31,4 @@ package_group( bzl_library( name = "bzl_lib", srcs = glob(["**/*.bzl"]), - visibility = ["//rust:__subpackages__"], ) diff --git a/rust/platform/triple_mappings.bzl b/rust/platform/triple_mappings.bzl index e213cb1f82..72dd64225c 100644 --- a/rust/platform/triple_mappings.bzl +++ b/rust/platform/triple_mappings.bzl @@ -121,8 +121,8 @@ _CPU_ARCH_TO_BUILTIN_PLAT_SUFFIX = { "le32": None, "mips": None, "mipsel": None, - "powerpc": "ppc", - "powerpc64": None, + "powerpc": "ppc32", + "powerpc64": "ppc", "powerpc64le": "ppc64le", "riscv32": "riscv32", "riscv32imc": "riscv32", @@ -154,7 +154,7 @@ _SYSTEM_TO_BUILTIN_SYS_SUFFIX = { "linux": "linux", "macos": "osx", "nacl": None, - "netbsd": None, + "netbsd": "netbsd", "nixos": "nixos", "none": "none", "nto": "qnx", @@ -162,9 +162,9 @@ _SYSTEM_TO_BUILTIN_SYS_SUFFIX = { "solaris": None, "uefi": "uefi", "unknown": None, - "wasi": None, - "wasip1": None, - "wasip2": None, + "wasi": "wasi", + "wasip1": "wasi", + "wasip2": "wasi", "windows": "windows", } @@ -179,6 +179,7 @@ _SYSTEM_TO_BINARY_EXT = { "ios": "", "linux": "", "macos": "", + "netbsd": "", "nixos": "", "none": "", "nto": "", @@ -187,6 +188,7 @@ _SYSTEM_TO_BINARY_EXT = { # generated extension for the wasm target, similarly to the # windows target "unknown": ".wasm", + "threads": ".wasm", "wasi": ".wasm", "wasip1": ".wasm", "wasip2": ".wasm", @@ -204,11 +206,13 @@ _SYSTEM_TO_STATICLIB_EXT = { "ios": ".a", "linux": ".a", "macos": ".a", + "netbsd": ".a", "nixos": ".a", "none": ".a", "nto": ".a", "uefi": ".lib", "unknown": "", + "threads": "", "wasi": "", "wasip1": "", "wasip2": "", @@ -226,11 +230,13 @@ _SYSTEM_TO_DYLIB_EXT = { "ios": ".dylib", "linux": ".so", "macos": ".dylib", + "netbsd": ".so", "nixos": ".so", "none": ".so", "nto": ".a", "uefi": "", # UEFI doesn't have dynamic linking "unknown": ".wasm", + "threads": ".wasm", "wasi": ".wasm", "wasip1": ".wasm", "wasip2": ".wasm", @@ -284,7 +290,12 @@ _SYSTEM_TO_STDLIB_LINKFLAGS = { "wasi": [], "wasip1": [], "wasip2": [], - "windows": ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"], + "windows": { + # see https://github.com/rust-lang/rust/blob/c4aa646f15e40bd3e64ddb5017b7b89b3646ac99/src/tools/run-make-support/src/external_deps/c_cxx_compiler/extras.rs#L14-L23 + "gnu": ["-lws2_32", "-luserenv", "-lbcrypt", "-lntdll", "-lsynchronization"], + "gnullvm": ["-lws2_32", "-luserenv", "-lbcrypt", "-lntdll", "-lsynchronization"], + "msvc": ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"], + }, } def cpu_arch_to_constraints(cpu_arch, *, system = None, abi = None): @@ -410,8 +421,11 @@ def system_to_staticlib_ext(system): def system_to_binary_ext(system): return _SYSTEM_TO_BINARY_EXT[system] -def system_to_stdlib_linkflags(system): - return _SYSTEM_TO_STDLIB_LINKFLAGS[system] +def system_to_stdlib_linkflags(target_triple): + val = _SYSTEM_TO_STDLIB_LINKFLAGS[target_triple.system] + if type(val) == "list": + return val + return val[target_triple.abi] def triple_to_constraint_set(target_triple): """Returns a set of constraints for a given platform triple diff --git a/rust/private/BUILD.bazel b/rust/private/BUILD.bazel index 89444bb9bc..d3cea4650d 100644 --- a/rust/private/BUILD.bazel +++ b/rust/private/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//rust/private:rust_analyzer.bzl", "rust_analyzer_detect_sysroot") load("//rust/private:rustc.bzl", "is_proc_macro_dep", "is_proc_macro_dep_enabled") +load("//rust/private:stamp.bzl", "stamp_build_setting") # Exported for docs exports_files(["providers.bzl"]) @@ -21,7 +22,7 @@ bzl_library( bzl_library( name = "bzl_lib", srcs = glob(["**/*.bzl"]), - visibility = ["//rust:__subpackages__"], + visibility = ["//visibility:public"], deps = [ ":bazel_tools_bzl_lib", ":rules_cc_bzl_lib", @@ -32,6 +33,8 @@ bzl_library( ], ) +stamp_build_setting(name = "stamp") + # This setting may be used to identify dependencies of proc-macro-s. # This feature is only enabled if `is_proc_macro_dep_enabled` is true. # Its value controls the BAZEL_RULES_RUST_IS_PROC_MACRO_DEP environment variable diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl index 318c05af8d..58a82b600f 100644 --- a/rust/private/clippy.bzl +++ b/rust/private/clippy.bzl @@ -165,7 +165,7 @@ def rust_clippy_action(ctx, clippy_executable, process_wrapper, crate_info, conf attr = ctx.rule.attr, file = ctx.file, toolchain = toolchain, - tool_path = clippy_executable.path, + tool_path = clippy_executable, cc_toolchain = cc_toolchain, feature_configuration = feature_configuration, crate_info = crate_info, diff --git a/rust/private/repository_utils.bzl b/rust/private/repository_utils.bzl index 67ce739e6a..4f3b89377a 100644 --- a/rust/private/repository_utils.bzl +++ b/rust/private/repository_utils.bzl @@ -417,7 +417,7 @@ def BUILD_for_rust_toolchain( str: A rendered template of a `rust_toolchain` declaration """ if stdlib_linkflags == None: - stdlib_linkflags = ", ".join(['"%s"' % x for x in system_to_stdlib_linkflags(target_triple.system)]) + stdlib_linkflags = ", ".join(['"%s"' % x for x in system_to_stdlib_linkflags(target_triple)]) rustfmt_label = None if include_rustfmt: diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl index d06f5fa71f..c7a04a2515 100644 --- a/rust/private/rust.bzl +++ b/rust/private/rust.bzl @@ -23,6 +23,7 @@ load( "BuildInfo", "CrateGroupInfo", "CrateInfo", + "DepInfo", "LintsInfo", ) load( @@ -46,6 +47,7 @@ load( "determine_lib_name", "determine_output_hash", "expand_dict_value_locations", + "filter_deps", "find_toolchain", "generate_output_diagnostics", "get_edition", @@ -64,41 +66,6 @@ def _assert_no_deprecated_attributes(_ctx): """ pass -def _assert_correct_dep_mapping(ctx): - """Forces a failure if proc_macro_deps and deps are mixed inappropriately - - Args: - ctx (ctx): The current rule's context object - """ - for dep in ctx.attr.deps: - if rust_common.crate_info in dep: - if dep[rust_common.crate_info].type == "proc-macro": - fail( - "{} listed {} in its deps, but it is a proc-macro. It should instead be in the bazel property proc_macro_deps.".format( - ctx.label, - dep.label, - ), - ) - for dep in ctx.attr.proc_macro_deps: - if CrateInfo in dep: - types = [dep[CrateInfo].type] - else: - types = [ - dep_variant_info.crate_info.type - for dep_variant_info in dep[CrateGroupInfo].dep_variant_infos.to_list() - if dep_variant_info.crate_info - ] - - for type in types: - if type != "proc-macro": - fail( - "{} listed {} in its proc_macro_deps, but it is not proc-macro, it is a {}. It should probably instead be listed in deps.".format( - ctx.label, - dep.label, - type, - ), - ) - def _rust_library_impl(ctx): """The implementation of the `rust_library` rule. @@ -168,7 +135,7 @@ def _rust_library_common(ctx, crate_type): list: A list of providers. See `rustc_compile_action` """ _assert_no_deprecated_attributes(ctx) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) toolchain = find_toolchain(ctx) @@ -205,18 +172,22 @@ def _rust_library_common(ctx, crate_type): crate_type, disable_pipelining = getattr(ctx.attr, "disable_pipelining", False), ): + # The hollow rlib uses .rlib extension (not .rmeta) so rustc reads it as an + # rlib archive containing lib.rmeta with optimized MIR. It is placed in a + # "_hollow/" subdirectory so the full rlib and hollow rlib never appear in the + # same -Ldependency= search directory (which would cause E0463). rust_metadata = ctx.actions.declare_file( - paths.replace_extension(rust_lib_name, ".rmeta"), - sibling = rust_lib, + "_hollow/" + rust_lib_name[:-len(".rlib")] + "-hollow.rlib", ) rustc_rmeta_output = generate_output_diagnostics(ctx, rust_metadata) + metadata_supports_pipelining = ( can_use_metadata_for_pipelining(toolchain, crate_type) and not ctx.attr.disable_pipelining ) - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) return rustc_compile_action( ctx = ctx, @@ -259,7 +230,7 @@ def _rust_binary_impl(ctx): """ toolchain = find_toolchain(ctx) crate_name = compute_crate_name(ctx.workspace_name, ctx.label, toolchain, ctx.attr.crate_name) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) if ctx.attr.binary_name: output_filename = ctx.attr.binary_name @@ -267,8 +238,8 @@ def _rust_binary_impl(ctx): output_filename = ctx.label.name output = ctx.actions.declare_file(output_filename + toolchain.binary_ext) - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) crate_root = getattr(ctx.file, "crate_root", None) if not crate_root: @@ -349,13 +320,13 @@ def _rust_test_impl(ctx): list: The list of providers. See `rustc_compile_action` """ _assert_no_deprecated_attributes(ctx) - _assert_correct_dep_mapping(ctx) + deps, proc_macro_deps = filter_deps(ctx) toolchain = find_toolchain(ctx) crate_type = "bin" - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps + get_import_macro_deps(ctx)) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps + get_import_macro_deps(ctx)) if ctx.attr.crate and ctx.attr.srcs: fail("rust_test.crate and rust_test.srcs are mutually exclusive. Update {} to use only one of these attributes".format( @@ -557,16 +528,16 @@ def _rust_library_group_impl(ctx): runfiles = [] for dep in ctx.attr.deps: - if rust_common.crate_info in dep: + if CrateInfo in dep: dep_variant_infos.append(rust_common.dep_variant_info( - crate_info = dep[rust_common.crate_info] if rust_common.crate_info in dep else None, - dep_info = dep[rust_common.dep_info] if rust_common.crate_info in dep else None, + crate_info = dep[CrateInfo] if CrateInfo in dep else None, + dep_info = dep[DepInfo] if DepInfo in dep else None, build_info = dep[BuildInfo] if BuildInfo in dep else None, cc_info = dep[CcInfo] if CcInfo in dep else None, crate_group_info = None, )) - elif rust_common.crate_group_info in dep: - dep_variant_transitive_infos.append(dep[rust_common.crate_group_info].dep_variant_infos) + elif CrateGroupInfo in dep: + dep_variant_transitive_infos.append(dep[CrateGroupInfo].dep_variant_infos) else: fail("crate_group_info targets can only depend on rust_library or rust_library_group targets.") @@ -742,10 +713,12 @@ _common_attrs = { # `@local_config_platform//:exec` exposed. "proc_macro_deps": attr.label_list( doc = dedent("""\ - List of `rust_proc_macro` targets used to help build this library target. + Copy of deps in exec configuration. This should really be called `exec_configured_deps`. + + Rule implementations use this to select exec-configured `rust_proc_macro` targets. + User code should pass all deps to `deps` for the macros loaded from `defs.bzl`. """), cfg = "exec", - providers = [[CrateInfo], [CrateGroupInfo]], ), "require_explicit_unstable_features": attr.int( doc = ( @@ -826,6 +799,10 @@ _common_attrs = { doc = "Enable collection of cfg flags with results stored in CrateInfo.cfgs.", default = Label("//rust/settings:collect_cfgs"), ), + "_stamp_flag": attr.label( + doc = "A setting used to determine whether or not the `--stamp` flag is enabled", + default = Label("//rust/private:stamp"), + ), } | RUSTC_ATTRS | RUSTC_ALLOCATOR_LIBRARIES_ATTRS _coverage_attrs = { @@ -1349,7 +1326,9 @@ rust_binary_without_process_wrapper = rule( implementation = _rust_binary_without_process_wrapper_impl, doc = "A variant of `rust_binary` that uses a minimal process wrapper for `Rustc` actions.", provides = COMMON_PROVIDERS + [_RustBuiltWithoutProcessWrapperInfo], - attrs = _common_attrs_for_binary_without_process_wrapper(_common_attrs | _rust_binary_attrs), + attrs = _common_attrs_for_binary_without_process_wrapper(_common_attrs | _rust_binary_attrs | { + "_skip_deps_verification": attr.bool(default = True), + }), executable = True, fragments = ["cpp"], toolchains = [ @@ -1593,7 +1572,7 @@ rust_test = rule( """), ) -def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): +def rust_test_suite(name, srcs, shared_srcs = [], deps = [], proc_macro_deps = [], **kwargs): """A rule for creating a test suite for a set of `rust_test` targets. This rule can be used for setting up typical rust [integration tests][it]. Given the following @@ -1646,6 +1625,8 @@ def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): name (str): The name of the `test_suite`. srcs (list): All test sources, typically `glob(["tests/**/*.rs"])`. shared_srcs (list): Optional argument for sources shared among tests, typically helper functions. + deps (list): Deps and proc_macro_deps for underlying test. + proc_macro_deps (list): Deprecated; do not use. **kwargs (dict): Additional keyword arguments for the underlying [rust_test](#rust_test) targets. The `tags` argument is also passed to the generated `test_suite` target. """ @@ -1676,6 +1657,8 @@ def rust_test_suite(name, srcs, shared_srcs = [], **kwargs): srcs = [src] + shared_srcs, tags = tags, crate_name = crate_name, + deps = deps + proc_macro_deps, + proc_macro_deps = deps + proc_macro_deps, **kwargs ) tests.append(test_name) diff --git a/rust/private/rust_analyzer.bzl b/rust/private/rust_analyzer.bzl index da41867985..39d65dae75 100644 --- a/rust/private/rust_analyzer.bzl +++ b/rust/private/rust_analyzer.bzl @@ -132,10 +132,13 @@ def _rust_analyzer_aspect_impl(target, ctx): else: fail("Unexpected target type: {}".format(target)) - aliases = {} + # Keep aliases as a list of (RustAnalyzerInfo, alias_name) tuples. + # Using RustAnalyzerInfo as a dict key can trigger expensive recursive hashing. + aliases = [] for aliased_target, aliased_name in getattr(ctx.rule.attr, "aliases", {}).items(): - if aliased_target.label in labels_to_rais: - aliases[labels_to_rais[aliased_target.label]] = aliased_name + dep_info = labels_to_rais.get(aliased_target.label) + if dep_info: + aliases.append((dep_info, aliased_name)) proc_macro_dylib = find_proc_macro_dylib(toolchain, target) proc_macro_dylibs = [proc_macro_dylib] if proc_macro_dylib else None @@ -258,6 +261,15 @@ def _create_single_crate(ctx, attrs, info): crate["root_module"] = _WORKSPACE_TEMPLATE + src_map[info.crate.root.short_path].path crate["source"]["include_dirs"].append(path_prefix + info.crate.root.dirname) + # Ensure workspace crates in the same Bazel package share one source root. + # + # rust-analyzer picks candidate crates by source root (`relevant_crates`). + # Widening include_dirs at the package level keeps related crates in a + # shared candidate set; final membership is still resolved by each crate's + # module tree. + if not is_external: + crate["source"]["include_dirs"].append(_WORKSPACE_TEMPLATE + ctx.label.package) + if info.build_info != None and info.build_info.out_dir != None: out_dir_path = info.build_info.out_dir.path crate["env"].update({"OUT_DIR": _EXEC_ROOT_TEMPLATE + out_dir_path}) @@ -285,7 +297,7 @@ def _create_single_crate(ctx, attrs, info): # the crate being processed, we don't add it as a dependency to itself. This is # common and expected - `rust_test.crate` pointing to the `rust_library`. crate["deps"] = [_crate_id(dep.crate) for dep in info.deps if _crate_id(dep.crate) != crate_id] - crate["aliases"] = {_crate_id(alias_target.crate): alias_name for alias_target, alias_name in info.aliases.items()} + crate["aliases"] = {_crate_id(alias_target.crate): alias_name for alias_target, alias_name in info.aliases} crate["cfg"] = info.cfgs toolchain = find_toolchain(ctx) crate["target"] = (_EXEC_ROOT_TEMPLATE + toolchain.target_json.path) if toolchain.target_json else toolchain.target_flag_value diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl index 776dce73b4..08c632e663 100644 --- a/rust/private/rustc.bzl +++ b/rust/private/rustc.bzl @@ -226,7 +226,7 @@ def collect_deps( Args: deps (list): The deps from ctx.attr.deps. - proc_macro_deps (list): The proc_macro deps from ctx.attr.proc_macro_deps. + proc_macro_deps (list): The proc_macro deps from `filter_deps(ctx)`. aliases (dict): A dict mapping aliased targets to their actual Crate information. Returns: @@ -560,9 +560,9 @@ def _symlink_for_ambiguous_lib(actions, toolchain, crate_info, lib): # Take the absolute value of hash() since it could be negative. path_hash = abs(hash(lib.path)) - lib_name = get_lib_name_for_windows(lib) if toolchain.target_os.startswith("windows") else get_lib_name_default(lib) + lib_name = get_lib_name_for_windows(lib) if toolchain.target_abi == "msvc" else get_lib_name_default(lib) - if toolchain.target_os.startswith("windows"): + if toolchain.target_abi == "msvc": prefix = "" extension = ".lib" elif lib_name.endswith(".pic"): @@ -663,7 +663,7 @@ def _disambiguate_libs(actions, toolchain, crate_info, dep_info, use_pic): visited_libs[name] = artifact return ambiguous_libs -def _depend_on_metadata(crate_info, force_depend_on_objects): +def _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link = False): """Determines if we can depend on metadata for this crate. By default (when pipelining is disabled or when the crate type needs to link against @@ -673,9 +673,22 @@ def _depend_on_metadata(crate_info, force_depend_on_objects): In some rare cases, even if both of those conditions are true, we still want to depend on objects. This is what force_depend_on_objects is. + When experimental_use_cc_common_link is True, bin/cdylib crates also use hollow + rlib deps. The rustc step only emits .o files (no rustc linking), so SVH chain + consistency is sufficient; the actual linking is done by cc_common.link, which + does not check SVH. + + Callers are responsible for zeroing out experimental_use_cc_common_link for + exec-platform builds before calling this function (see rustc_compile_action). + Exec-platform binaries (build scripts) must use full rlib deps because their + CcInfo linking contexts may lack a CC toolchain. + Args: crate_info (CrateInfo): The Crate to determine this for. force_depend_on_objects (bool): if set we will not depend on metadata. + experimental_use_cc_common_link (bool): if set, bin/cdylib crates also use + hollow rlib deps for SVH consistency. Must already be False for + exec-platform builds when this function is called. Returns: Whether we can depend on metadata for this crate. @@ -683,6 +696,11 @@ def _depend_on_metadata(crate_info, force_depend_on_objects): if force_depend_on_objects: return False + if experimental_use_cc_common_link and crate_info.type in ("bin", "cdylib"): + # cc_common.link: rustc only emits .o files, so hollow rlib deps are safe and + # keep the SVH chain consistent (avoiding E0460 from nondeterministic proc macros). + return True + return crate_info.type in ("rlib", "lib") def collect_inputs( @@ -770,7 +788,7 @@ def collect_inputs( linkstamp_outs = [] transitive_crate_outputs = dep_info.transitive_crate_outputs - if _depend_on_metadata(crate_info, force_depend_on_objects): + if _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link): transitive_crate_outputs = dep_info.transitive_metadata_outputs nolinkstamp_compile_direct_inputs = [] @@ -806,6 +824,12 @@ def collect_inputs( transitive = [ crate_info.srcs, transitive_crate_outputs, + # Always include hollow rlibs so they are present in the sandbox for + # -Ldependency= resolution. Binaries and proc-macros compile against full + # rlib --extern deps but need hollow rlibs available for transitive + # dependency resolution when those rlibs were themselves compiled against + # hollow deps. For rlib/lib crates this is a no-op (already included above). + dep_info.transitive_metadata_outputs, crate_info.compile_data, dep_info.transitive_proc_macro_data, toolchain.all_files, @@ -909,8 +933,10 @@ def construct_arguments( use_json_output = False, build_metadata = False, force_depend_on_objects = False, + experimental_use_cc_common_link = False, skip_expanding_rustc_env = False, require_explicit_unstable_features = False, + always_use_param_file = False, error_format = None): """Builds an Args object containing common rustc flags @@ -1016,7 +1042,7 @@ def construct_arguments( # Rustc arguments rustc_flags = ctx.actions.args() rustc_flags.set_param_file_format("multiline") - rustc_flags.use_param_file("@%s", use_always = False) + rustc_flags.use_param_file("@%s", use_always = always_use_param_file) rustc_flags.add(crate_info.root) rustc_flags.add(crate_info.name, format = "--crate-name=%s") rustc_flags.add(crate_info.type, format = "--crate-type=%s") @@ -1030,8 +1056,7 @@ def construct_arguments( process_wrapper_flags.add("--rustc-output-format", "json" if error_format == "json" else "rendered") # Configure rustc json output by adding artifact notifications. - # These will always be filtered out by process_wrapper and will be use to terminate - # rustc when appropriate. + # These are filtered out by process_wrapper. json = ["artifacts"] if error_format == "short": json.append("diagnostic-short") @@ -1044,8 +1069,14 @@ def construct_arguments( error_format = "json" if build_metadata: - # Configure process_wrapper to terminate rustc when metadata are emitted - process_wrapper_flags.add("--rustc-quit-on-rmeta", "true") + if crate_info.type in ("rlib", "lib"): + # Hollow rlib approach (Buck2-style): rustc runs to completion with -Zno-codegen, + # producing a hollow .rlib (metadata only, no object code) via --emit=link=. + # No need to kill rustc — -Zno-codegen skips codegen entirely and exits quickly. + rustc_flags.add("-Zno-codegen") + + # else: IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits + # naturally after writing .rmeta via --emit=dep-info,metadata (no kill needed). if crate_info.rustc_rmeta_output: process_wrapper_flags.add("--output-file", crate_info.rustc_rmeta_output.path) elif crate_info.rustc_output: @@ -1078,7 +1109,14 @@ def construct_arguments( emit_without_paths = [] for kind in emit: - if kind == "link" and crate_info.type == "bin" and crate_info.output != None: + if kind == "link" and build_metadata and crate_info.type in ("rlib", "lib") and crate_info.metadata: + # Hollow rlib: direct rustc's link output to the -hollow.rlib path. + # The file has .rlib extension so rustc reads it as an rlib archive + # (with optimized MIR in lib.rmeta). Using a .rmeta path would cause + # E0786 "found invalid metadata files" because rustc parses .rmeta files + # as raw metadata blobs, not rlib archives. + rustc_flags.add(crate_info.metadata, format = "--emit=link=%s") + elif kind == "link" and crate_info.type == "bin" and crate_info.output != None: rustc_flags.add(crate_info.output, format = "--emit=link=%s") else: emit_without_paths.append(kind) @@ -1153,7 +1191,7 @@ def construct_arguments( include_link_flags = include_link_flags, ) - use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects) + use_metadata = _depend_on_metadata(crate_info, force_depend_on_objects, experimental_use_cc_common_link) # These always need to be added, even if not linking this crate. add_crate_link_flags(rustc_flags, dep_info, force_all_deps_direct, use_metadata) @@ -1322,6 +1360,13 @@ def rustc_compile_action( rustc_output = crate_info.rustc_output rustc_rmeta_output = crate_info.rustc_rmeta_output + # Use the hollow rlib approach (Buck2-style) for rlib/lib crate types when a metadata + # action is being created. This always applies for rlib/lib regardless of whether + # pipelining is globally enabled — the hollow rlib is simpler than killing rustc. + # Non-rlib types (bin, proc-macro, etc.) use --emit=dep-info,metadata instead + # (rustc exits naturally after writing .rmeta, no process-wrapper kill needed). + use_hollow_rlib = bool(build_metadata) and crate_info.type in ("rlib", "lib") + # Determine whether to use cc_common.link: # * either if experimental_use_cc_common_link is 1, # * or if experimental_use_cc_common_link is -1 and @@ -1335,6 +1380,12 @@ def rustc_compile_action( elif ctx.attr.experimental_use_cc_common_link == -1: experimental_use_cc_common_link = toolchain._experimental_use_cc_common_link + # Exec-platform binaries (build scripts) skip cc_common.link: exec-configuration + # rlib deps may lack a CC toolchain, causing empty CcInfo linking contexts. They + # use standard rustc linking with full rlib deps instead. + if experimental_use_cc_common_link and is_exec_configuration(ctx): + experimental_use_cc_common_link = False + dep_info, build_info, linkstamps = collect_deps( deps = deps, proc_macro_deps = proc_macro_deps, @@ -1349,14 +1400,11 @@ def rustc_compile_action( linkstamps = depset([]) # Determine if the build is currently running with --stamp - stamp = is_stamping_enabled(ctx, attr) + stamp = is_stamping_enabled(attr) # Add flags for any 'rustc' lints that are specified. - # - # Exclude lints if we're building in the exec configuration to prevent crates - # used in build scripts from generating warnings. lint_files = [] - if hasattr(ctx.attr, "lint_config") and ctx.attr.lint_config and not is_exec_configuration(ctx): + if hasattr(ctx.attr, "lint_config") and ctx.attr.lint_config: rust_flags = rust_flags + ctx.attr.lint_config[LintsInfo].rustc_lint_flags lint_files = lint_files + ctx.attr.lint_config[LintsInfo].rustc_lint_files @@ -1376,17 +1424,34 @@ def rustc_compile_action( experimental_use_cc_common_link = experimental_use_cc_common_link, ) - # The types of rustc outputs to emit. - # If we build metadata, we need to keep the command line of the two invocations - # (rlib and rmeta) as similar as possible, otherwise rustc rejects the rmeta as - # a candidate. - # Because of that we need to add emit=metadata to both the rlib and rmeta invocation. - # - # When cc_common linking is enabled, emit a `.o` file, which is later - # passed to the cc_common.link action. + # The main Rustc action uses FULL rlib deps so the full rlib it produces records + # full-rlib SVHs. A downstream binary links against full rlibs; if the Rustc action + # had used hollow rlib deps instead, nondeterministic proc macros could produce + # different SVHs for the hollow vs full rlib, causing E0460 in the binary build. + # The RustcMetadata action still uses hollow rlibs (compile_inputs_for_metadata) + # so it can start before full codegen of its deps completes. + compile_inputs_for_metadata = compile_inputs + if use_hollow_rlib: + compile_inputs, _, _, _, _, _ = collect_inputs( + ctx = ctx, + file = ctx.file, + files = ctx.files, + linkstamps = linkstamps, + toolchain = toolchain, + cc_toolchain = cc_toolchain, + feature_configuration = feature_configuration, + crate_info = crate_info, + dep_info = dep_info, + build_info = build_info, + lint_files = lint_files, + stamp = stamp, + force_depend_on_objects = True, + experimental_use_cc_common_link = experimental_use_cc_common_link, + ) + + # The main Rustc action emits dep-info and link (the full rlib/binary/cdylib). + # When cc_common linking is enabled, emit a `.o` file instead. emit = ["dep-info", "link"] - if build_metadata: - emit.append("metadata") if experimental_use_cc_common_link: emit = ["obj"] @@ -1421,12 +1486,25 @@ def rustc_compile_action( force_all_deps_direct = force_all_deps_direct, stamp = stamp, use_json_output = bool(build_metadata) or bool(rustc_output) or bool(rustc_rmeta_output), + # Force full rlib --extern deps so the full rlib records full-rlib SVHs. + force_depend_on_objects = use_hollow_rlib, + experimental_use_cc_common_link = experimental_use_cc_common_link, skip_expanding_rustc_env = skip_expanding_rustc_env, require_explicit_unstable_features = require_explicit_unstable_features, + always_use_param_file = not ctx.executable._process_wrapper, ) args_metadata = None if build_metadata: + if use_hollow_rlib: + # Hollow rlib: emit dep-info and link (directed to the -hollow.rlib path via + # -Zno-codegen). dep-info must be included: it affects the SVH stored in the + # rlib, so both actions must include it to keep SVHs consistent. + metadata_emit = ["dep-info", "link"] + else: + # IDE-only metadata for non-rlib types (bin, proc-macro, etc.): rustc exits + # naturally after writing .rmeta with --emit=dep-info,metadata. + metadata_emit = ["dep-info", "metadata"] args_metadata, _ = construct_arguments( ctx = ctx, attr = attr, @@ -1434,7 +1512,7 @@ def rustc_compile_action( toolchain = toolchain, tool_path = toolchain.rustc.path, cc_toolchain = cc_toolchain, - emit = emit, + emit = metadata_emit, feature_configuration = feature_configuration, crate_info = crate_info, dep_info = dep_info, @@ -1449,6 +1527,7 @@ def rustc_compile_action( stamp = stamp, use_json_output = True, build_metadata = True, + experimental_use_cc_common_link = experimental_use_cc_common_link, require_explicit_unstable_features = require_explicit_unstable_features, ) @@ -1457,6 +1536,13 @@ def rustc_compile_action( # this is the final list of env vars env.update(env_from_args) + if use_hollow_rlib: + # Both the metadata action and the full Rustc action must have RUSTC_BOOTSTRAP=1 + # for SVH compatibility. RUSTC_BOOTSTRAP=1 changes the crate hash — setting it + # on only one action would cause SVH mismatch even for deterministic crates. + # This enables -Zno-codegen on stable Rust compilers for the metadata action. + env["RUSTC_BOOTSTRAP"] = "1" + if hasattr(attr, "version") and attr.version != "0.0.0": formatted_version = " v{}".format(attr.version) else: @@ -1496,7 +1582,7 @@ def rustc_compile_action( pdb_file = None dsym_folder = None if crate_info.type in ("cdylib", "bin") and not experimental_use_cc_common_link: - if toolchain.target_os == "windows" and compilation_mode.strip_level == "none": + if toolchain.target_abi == "msvc" and compilation_mode.strip_level == "none": pdb_file = ctx.actions.declare_file(crate_info.output.basename[:-len(crate_info.output.extension)] + "pdb", sibling = crate_info.output) action_outputs.append(pdb_file) elif toolchain.target_os in ["macos", "darwin"]: @@ -1525,7 +1611,7 @@ def rustc_compile_action( if args_metadata: ctx.actions.run( executable = ctx.executable._process_wrapper, - inputs = compile_inputs, + inputs = compile_inputs_for_metadata, outputs = [build_metadata] + [x for x in [rustc_rmeta_output] if x], env = env, arguments = args_metadata.all, @@ -1639,6 +1725,7 @@ def rustc_compile_action( compilation_outputs = compilation_outputs, name = output_relative_to_package, stamp = ctx.attr.stamp, + main_output = crate_info.output, output_type = "executable" if crate_info.type == "bin" else "dynamic_library", additional_outputs = additional_linker_outputs, ) @@ -2135,9 +2222,14 @@ def add_crate_link_flags(args, dep_info, force_all_deps_direct = False, use_meta crate_to_link_flags = _crate_to_link_flag_metadata if use_metadata else _crate_to_link_flag args.add_all(direct_crates, uniquify = True, map_each = crate_to_link_flags) + # Use hollow rlib directories for -Ldependency= when use_metadata=True (rlib/lib) + # so that both --extern= and -Ldependency= point to the same hollow rlib file. + # When use_metadata=False (bins, proc-macros), use full rlib directories; pointing + # to hollow dirs alongside full --extern= args would cause E0463 (ambiguous crate). + get_dirname = _get_crate_dirname_pipelined if use_metadata else _get_crate_dirname args.add_all( dep_info.transitive_crates, - map_each = _get_crate_dirname, + map_each = get_dirname, uniquify = True, format_each = "-Ldependency=%s", ) @@ -2195,6 +2287,24 @@ def _get_crate_dirname(crate): """ return crate.output.dirname +def _get_crate_dirname_pipelined(crate): + """For pipelined compilation: returns the _hollow/ directory for pipelined crates + + When a crate supports pipelining and has a hollow rlib in its _hollow/ subdirectory, + pointing -Ldependency= to that subdirectory lets rustc find the hollow rlib (which has + the correct SVH matching downstream metadata). Pointing to the parent directory instead + would expose the full rlib (compiled separately, with a different SVH), causing E0460. + + Args: + crate (CrateInfo): A CrateInfo provider from the current rule + + Returns: + str: The directory to use for -Ldependency= search. + """ + if crate.metadata and crate.metadata_supports_pipelining: + return crate.metadata.dirname + return crate.output.dirname + def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows = False, for_darwin = False, flavor_msvc = False): artifact = get_preferred_artifact(lib, use_pic) if ambiguous_libs and artifact.path in ambiguous_libs: @@ -2235,17 +2345,11 @@ def _portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, for_windows ): return [] if for_darwin else ["-lstatic=%s" % get_lib_name(artifact)] - if for_windows: - if flavor_msvc: - return [ - "-lstatic=%s" % get_lib_name(artifact), - "-Clink-arg={}".format(artifact.basename), - ] - else: - return [ - "-lstatic=%s" % get_lib_name(artifact), - "-Clink-arg=-l{}".format(artifact.basename), - ] + if flavor_msvc: + return [ + "-lstatic=%s" % get_lib_name(artifact), + "-Clink-arg={}".format(artifact.basename), + ] else: return [ "-lstatic=%s" % get_lib_name(artifact), @@ -2276,8 +2380,15 @@ def _make_link_flags_windows(make_link_flags_args, flavor_msvc, use_direct_drive ("-Clink-arg=%s--no-whole-archive" % prefix), ]) elif include_link_flags: - ret.extend(_portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name_for_windows, for_windows = True, flavor_msvc = flavor_msvc)) - _add_user_link_flags(ret, linker_input) + get_lib_name = get_lib_name_for_windows if flavor_msvc else get_lib_name_default + ret.extend(_portable_link_flags(lib, use_pic, ambiguous_libs, get_lib_name, flavor_msvc = flavor_msvc)) + + # Windows toolchains can inherit POSIX defaults like -pthread from C deps, + # which fails to link with the MinGW/LLD toolchain. Drop them here. + for flag in linker_input.user_link_flags: + if flag in ("-pthread", "-lpthread"): + continue + ret.append("--codegen=link-arg={}".format(flag)) return ret def _make_link_flags_windows_msvc(make_link_flags_args, use_direct_driver): @@ -2356,19 +2467,19 @@ def _get_make_link_flag_funcs(target_os, target_abi, use_direct_link_driver): - callable: The function for producing link args. - callable: The function for formatting link library names. """ + + get_lib_name = get_lib_name_default + if target_os == "windows": - make_link_flags_windows_msvc = _make_link_flags_windows_msvc_direct if use_direct_link_driver else _make_link_flags_windows_msvc_indirect - make_link_flags_windows_gnu = _make_link_flags_windows_gnu_direct if use_direct_link_driver else _make_link_flags_windows_gnu_indirect - make_link_flags = make_link_flags_windows_msvc if target_abi == "msvc" else make_link_flags_windows_gnu - get_lib_name = get_lib_name_for_windows + if target_abi == "msvc": + make_link_flags = _make_link_flags_windows_msvc_direct if use_direct_link_driver else _make_link_flags_windows_msvc_indirect + get_lib_name = get_lib_name_for_windows + else: + make_link_flags = _make_link_flags_windows_gnu_direct if use_direct_link_driver else _make_link_flags_windows_gnu_indirect elif target_os.startswith(("mac", "darwin", "ios")): - make_link_flags_darwin = _make_link_flags_darwin_direct if use_direct_link_driver else _make_link_flags_darwin_indirect - make_link_flags = make_link_flags_darwin - get_lib_name = get_lib_name_default + make_link_flags = _make_link_flags_darwin_direct if use_direct_link_driver else _make_link_flags_darwin_indirect else: - make_link_flags_default = _make_link_flags_default_direct if use_direct_link_driver else _make_link_flags_default_indirect - make_link_flags = make_link_flags_default - get_lib_name = get_lib_name_default + make_link_flags = _make_link_flags_default_direct if use_direct_link_driver else _make_link_flags_default_indirect return (make_link_flags, get_lib_name) @@ -2701,3 +2812,7 @@ no_std = rule( }, implementation = _no_std_impl, ) + +# Test-only exports for private helpers. +portable_link_flags_for_testing = _portable_link_flags +symlink_for_ambiguous_lib_for_testing = _symlink_for_ambiguous_lib diff --git a/rust/private/rustdoc.bzl b/rust/private/rustdoc.bzl index f302bab743..c530752cd8 100644 --- a/rust/private/rustdoc.bzl +++ b/rust/private/rustdoc.bzl @@ -131,7 +131,7 @@ def rustdoc_compile_action( attr = ctx.attr, file = ctx.file, toolchain = toolchain, - tool_path = toolchain.rust_doc.short_path if is_test else toolchain.rust_doc.path, + tool_path = toolchain.rust_doc.short_path if is_test else toolchain.rust_doc, cc_toolchain = cc_toolchain, feature_configuration = feature_configuration, crate_info = rustdoc_crate_info, diff --git a/rust/private/rustdoc/BUILD.bazel b/rust/private/rustdoc/BUILD.bazel index ee2067a87d..85cda1cd7e 100644 --- a/rust/private/rustdoc/BUILD.bazel +++ b/rust/private/rustdoc/BUILD.bazel @@ -1,4 +1,4 @@ -load("//rust/private:rust.bzl", "rust_binary") +load("//rust:defs.bzl", "rust_binary") package(default_visibility = ["//visibility:public"]) diff --git a/rust/private/rustdoc_test.bzl b/rust/private/rustdoc_test.bzl index 522c048cd1..dcf847f10a 100644 --- a/rust/private/rustdoc_test.bzl +++ b/rust/private/rustdoc_test.bzl @@ -18,7 +18,7 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//rust/private:common.bzl", "rust_common") load("//rust/private:providers.bzl", "CrateInfo") load("//rust/private:rustdoc.bzl", "rustdoc_compile_action") -load("//rust/private:utils.bzl", "dedent", "find_toolchain", "transform_deps") +load("//rust/private:utils.bzl", "dedent", "filter_deps", "find_toolchain", "transform_deps") def _construct_writer_arguments(ctx, test_runner, opt_test_params, action, crate_info): """Construct arguments and environment variables specific to `rustdoc_test_writer`. @@ -110,8 +110,10 @@ def _rust_doc_test_impl(ctx): toolchain = find_toolchain(ctx) crate = ctx.attr.crate[rust_common.crate_info] - deps = transform_deps(ctx.attr.deps) - proc_macro_deps = transform_deps(ctx.attr.proc_macro_deps) + + deps, proc_macro_deps = filter_deps(ctx) + deps = transform_deps(deps) + proc_macro_deps = transform_deps(proc_macro_deps) crate_info = rust_common.create_crate_info( name = crate.name, diff --git a/rust/private/stamp.bzl b/rust/private/stamp.bzl index bff7cbadf3..1a7cab65cc 100644 --- a/rust/private/stamp.bzl +++ b/rust/private/stamp.bzl @@ -1,21 +1,75 @@ -"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled""" +"""A small utility module dedicated to detecting whether or not the `--stamp` flag is enabled -def is_stamping_enabled(ctx, attr): +This module can be removed likely after the following PRs ar addressed: +- https://github.com/bazelbuild/bazel/issues/11164 +""" + +load("//rust/private:utils.bzl", "dedent") + +StampSettingInfo = provider( + doc = "Information about the `--stamp` command line flag", + fields = { + "value": "bool: Whether or not the `--stamp` flag was enabled", + }, +) + +def _stamp_build_setting_impl(ctx): + return StampSettingInfo(value = ctx.attr.value) + +_stamp_build_setting = rule( + doc = dedent("""\ + Whether to encode build information into the binary. Possible values: + + - stamp = 1: Always stamp the build information into the binary, even in [--nostamp][stamp] builds. \ + This setting should be avoided, since it potentially kills remote caching for the binary and \ + any downstream actions that depend on it. + - stamp = 0: Always replace build information by constant values. This gives good build result caching. + - stamp = -1: Embedding of build information is controlled by the [--[no]stamp][stamp] flag. + + Stamped binaries are not rebuilt unless their dependencies change. + [stamp]: https://docs.bazel.build/versions/main/user-manual.html#flag--stamp + """), + implementation = _stamp_build_setting_impl, + attrs = { + "value": attr.bool( + doc = "The default value of the stamp build flag", + mandatory = True, + ), + }, +) + +def stamp_build_setting(name, visibility = ["//visibility:public"]): + native.config_setting( + name = "stamp_detect", + values = {"stamp": "1"}, + visibility = visibility, + ) + + _stamp_build_setting( + name = name, + value = select({ + ":stamp_detect": True, + "//conditions:default": False, + }), + visibility = visibility, + ) + +def is_stamping_enabled(attr): """Determine whether or not build stamping is enabled Args: - ctx (ctx): The rule's context object attr (struct): A rule's struct of attributes (`ctx.attr`) Returns: bool: The stamp value """ - stamp_num = getattr(attr, "stamp", 0) + stamp_num = getattr(attr, "stamp", -1) if stamp_num == 1: return True elif stamp_num == 0: return False elif stamp_num == -1: - return ctx.configuration.stamp_binaries() + stamp_flag = getattr(attr, "_stamp_flag", None) + return stamp_flag[StampSettingInfo].value if stamp_flag else False else: fail("Unexpected `stamp` value: {}".format(stamp_num)) diff --git a/rust/private/unpretty.bzl b/rust/private/unpretty.bzl index be111c83a3..66c7a4b63f 100644 --- a/rust/private/unpretty.bzl +++ b/rust/private/unpretty.bzl @@ -190,7 +190,7 @@ def _rust_unpretty_aspect_impl(target, ctx): attr = ctx.rule.attr, file = ctx.file, toolchain = toolchain, - tool_path = toolchain.rustc.path, + tool_path = toolchain.rustc, cc_toolchain = cc_toolchain, feature_configuration = feature_configuration, crate_info = crate_info, diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl index b3f5b8437c..4d710b7fa9 100644 --- a/rust/private/utils.bzl +++ b/rust/private/utils.bzl @@ -511,14 +511,38 @@ def is_exec_configuration(ctx): # TODO(djmarcin): Is there any better way to determine cfg=exec? return ctx.genfiles_dir.path.find("-exec") != -1 +def filter_deps(ctx): + """Filters the provided (combined) deps into normal deps and proc_macro deps. + + Args: + ctx (ctx): The current rule's context object + + Returns: + deps and proc_macro_deps + """ + if len(ctx.attr.deps) != len(ctx.attr.proc_macro_deps) and not getattr(ctx.attr, "_skip_deps_verification", False): + fail("All deps should be passed to both `deps` and `proc_macro_deps`; please use the macros in //rust:defs.bzl") + + deps = [] + for dep in ctx.attr.deps: + if CrateInfo not in dep or dep[CrateInfo].type != "proc-macro": + deps.append(dep) + + proc_macro_deps = [] + for dep in ctx.attr.proc_macro_deps: + if CrateInfo in dep and dep[CrateInfo].type == "proc-macro": + proc_macro_deps.append(dep) + + return deps, proc_macro_deps + def transform_deps(deps): """Transforms a [Target] into [DepVariantInfo]. - This helper function is used to transform ctx.attr.deps and ctx.attr.proc_macro_deps into + This helper function is used to transform deps and .proc_macro_deps coming from `filter_deps` into [DepVariantInfo]. Args: - deps (list of Targets): Dependencies coming from ctx.attr.deps or ctx.attr.proc_macro_deps + deps (list of Targets): Dependencies coming from `filter_deps` Returns: list of DepVariantInfos. diff --git a/rust/rust_binary.bzl b/rust/rust_binary.bzl index dbeba03911..518c7c3f01 100644 --- a/rust/rust_binary.bzl +++ b/rust/rust_binary.bzl @@ -1,7 +1,7 @@ """rust_binary""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_binary = "rust_binary", ) diff --git a/rust/rust_library.bzl b/rust/rust_library.bzl index b1e63494a6..a694ed970c 100644 --- a/rust/rust_library.bzl +++ b/rust/rust_library.bzl @@ -1,7 +1,7 @@ """rust_library""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_library = "rust_library", ) diff --git a/rust/rust_test.bzl b/rust/rust_test.bzl index 001963fccc..abd61ba8c5 100644 --- a/rust/rust_test.bzl +++ b/rust/rust_test.bzl @@ -1,7 +1,7 @@ """rust_test""" load( - "//rust/private:rust.bzl", + "//rust:defs.bzl", _rust_test = "rust_test", ) diff --git a/rust/settings/BUILD.bazel b/rust/settings/BUILD.bazel index 06cfb39e4c..e57f5ec5d3 100644 --- a/rust/settings/BUILD.bazel +++ b/rust/settings/BUILD.bazel @@ -18,7 +18,6 @@ load( "experimental_use_cc_common_link", "experimental_use_coverage_metadata_files", "experimental_use_global_allocator", - "experimental_use_sh_toolchain_for_bootstrap_process_wrapper", "extra_exec_rustc_env", "extra_exec_rustc_flag", "extra_exec_rustc_flags", @@ -94,8 +93,6 @@ experimental_use_global_allocator() experimental_use_allocator_libraries_with_mangled_symbols() -experimental_use_sh_toolchain_for_bootstrap_process_wrapper() - extra_exec_rustc_env() extra_exec_rustc_flag() diff --git a/rust/settings/settings.bzl b/rust/settings/settings.bzl index aaed525204..b155eb172a 100644 --- a/rust/settings/settings.bzl +++ b/rust/settings/settings.bzl @@ -112,10 +112,18 @@ def use_real_import_macro(): ) def pipelined_compilation(): - """When set, this flag causes rustc to emit `*.rmeta` files and use them for `rlib -> rlib` dependencies. + """When set, this flag enables pipelined compilation for rlib/lib crates. - While this involves one extra (short) rustc invocation to build the rmeta file, - it allows library dependencies to be unlocked much sooner, increasing parallelism during compilation. + For each rlib/lib, a separate RustcMetadata action produces a hollow rlib + (via `-Zno-codegen`) containing only metadata. Downstream rlib/lib crates + can begin compiling against the hollow rlib before the upstream full codegen + action completes, increasing build parallelism. + + Pipelining applies to rlib→rlib dependencies by default. To also pipeline + bin/cdylib crates (starting their compile step before upstream full codegen + finishes), enable `experimental_use_cc_common_link` alongside this flag. + With cc_common.link, rustc only emits `.o` files for binaries (linking is + handled separately), so hollow rlib deps are safe for bins too. """ bool_flag( name = "pipelined_compilation", @@ -126,6 +134,11 @@ def pipelined_compilation(): def experimental_use_cc_common_link(): """A flag to control whether to link rust_binary and rust_test targets using \ cc_common.link instead of rustc. + + When combined with `pipelined_compilation`, bin/cdylib crates also participate + in the hollow-rlib dependency chain: rustc only emits `.o` files (linking is + done by cc_common.link and does not check SVH), so bin compile steps can start + as soon as upstream hollow rlibs are ready rather than waiting for full codegen. """ bool_flag( name = "experimental_use_cc_common_link", @@ -275,15 +288,6 @@ def experimental_link_std_dylib(): build_setting_default = False, ) -def experimental_use_sh_toolchain_for_bootstrap_process_wrapper(): - """A flag to control whether the shell path from a shell toolchain (`@bazel_tools//tools/sh:toolchain_type`) \ - is embedded into the bootstrap process wrapper for the `.sh` file. - """ - bool_flag( - name = "experimental_use_sh_toolchain_for_bootstrap_process_wrapper", - build_setting_default = False, - ) - def toolchain_linker_preference(): """A flag to control which linker is preferred for linking Rust binaries. diff --git a/rust/toolchain.bzl b/rust/toolchain.bzl index c2aa484ac2..4e9377c616 100644 --- a/rust/toolchain.bzl +++ b/rust/toolchain.bzl @@ -604,6 +604,7 @@ def _rust_toolchain_impl(ctx): extra_exec_rustc_flags = expanded_extra_exec_rustc_flags, per_crate_rustc_flags = ctx.attr.per_crate_rustc_flags, sysroot = sysroot_path, + sysroot_anchor = sysroot.sysroot_anchor, sysroot_short_path = sysroot_short_path, target_arch = target_arch, target_flag_value = target_json.path if target_json else target_triple.str, diff --git a/test/process_wrapper_bootstrap/BUILD.bazel b/test/process_wrapper_bootstrap/BUILD.bazel index d2c8419b81..6a0c167adc 100644 --- a/test/process_wrapper_bootstrap/BUILD.bazel +++ b/test/process_wrapper_bootstrap/BUILD.bazel @@ -1,12 +1,22 @@ -load("//rust:defs.bzl", "rust_test") -load(":process_wrapper_bootstrap_test.bzl", "process_wrapper_bootstrap_test_suite") +load("//rust:defs.bzl", "rust_binary", "rust_test") + +rust_binary( + name = "bootstrap_process_wrapper_probe", + srcs = ["bootstrap_process_wrapper_probe.rs"], + edition = "2021", +) rust_test( name = "bootstrap_process_wrapper_test", srcs = ["bootstrap_process_wrapper_test.rs"], - data = ["//util/process_wrapper/private:process_wrapper.sh"], + data = [ + ":bootstrap_process_wrapper_probe", + "//util/process_wrapper:bootstrap_process_wrapper", + ], edition = "2021", + env = { + "BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH": "$(rlocationpath :bootstrap_process_wrapper_probe)", + "BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH": "$(rlocationpath //util/process_wrapper:bootstrap_process_wrapper)", + }, deps = ["//rust/runfiles"], ) - -process_wrapper_bootstrap_test_suite(name = "process_wrapper_bootstrap_test_suite") diff --git a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs new file mode 100644 index 0000000000..35ecc496cf --- /dev/null +++ b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_probe.rs @@ -0,0 +1,10 @@ +fn main() { + let arg = std::env::args().nth(1).unwrap_or_default(); + println!("{arg}"); + + let exit_code = std::env::var("BOOTSTRAP_PROCESS_WRAPPER_PROBE_EXIT_CODE") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(0); + std::process::exit(exit_code); +} diff --git a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs index 3fbf45484c..43935b1b6e 100644 --- a/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs +++ b/test/process_wrapper_bootstrap/bootstrap_process_wrapper_test.rs @@ -1,24 +1,54 @@ -//! Tests for the bootstrap process wrapper +//! Tests for the bootstrap process wrapper. -use std::fs::read_to_string; +use std::env; +use std::process::Command; use runfiles::Runfiles; -/// Test that the shell process wrapper starts with the expected shebang to -/// avoid breaking the contract with the `bootstrap_process_wrapper` rule. -#[test] -fn test_shebang() { +fn resolve_runfile(env_var: &str) -> String { let rfiles = Runfiles::create().unwrap(); + let rlocationpath = env::var(env_var).unwrap(); + runfiles::rlocation!(rfiles, rlocationpath.as_str()) + .unwrap() + .display() + .to_string() +} - let script = runfiles::rlocation!( - rfiles, - "rules_rust/util/process_wrapper/private/process_wrapper.sh" - ) - .unwrap(); +#[test] +fn test_substitutes_pwd() { + let wrapper = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH"); + let probe = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH"); + let pwd = env::current_dir().unwrap().display().to_string(); + + let output = Command::new(wrapper) + .arg("--") + .arg(probe) + .arg("${pwd}/suffix") + .output() + .unwrap(); - let content = read_to_string(script).unwrap(); assert!( - content.starts_with("#!/usr/bin/env bash"), - "The shell script does not start with the expected shebang." - ) + output.status.success(), + "wrapper failed: status={:?}, stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr), + ); + + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.trim_end(), format!("{}/suffix", pwd)); +} + +#[test] +fn test_propagates_exit_code() { + let wrapper = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_RLOCATIONPATH"); + let probe = resolve_runfile("BOOTSTRAP_PROCESS_WRAPPER_PROBE_RLOCATIONPATH"); + + let status = Command::new(wrapper) + .arg("--") + .arg(probe) + .env("BOOTSTRAP_PROCESS_WRAPPER_PROBE_EXIT_CODE", "23") + .status() + .unwrap(); + + assert_eq!(status.code(), Some(23)); } diff --git a/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl b/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl deleted file mode 100644 index 7e0e4ea571..0000000000 --- a/test/process_wrapper_bootstrap/process_wrapper_bootstrap_test.bzl +++ /dev/null @@ -1,78 +0,0 @@ -"""Starlark unit tests for the bootstrap process wrapper""" - -load("@bazel_skylib//lib:unittest.bzl", "analysistest") -load("//test/unit:common.bzl", "assert_action_mnemonic") - -def _enable_sh_toolchain_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - - if ctx.attr.expected_ext == ".bat": - assert_action_mnemonic(env, target.actions[0], "ExecutableSymlink") - else: - assert_action_mnemonic(env, target.actions[0], "TemplateExpand") - - return analysistest.end(env) - -_enable_sh_toolchain_test = analysistest.make( - _enable_sh_toolchain_test_impl, - config_settings = { - str(Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper")): True, - }, - attrs = { - "expected_ext": attr.string( - doc = "The expected extension for the bootstrap script.", - mandatory = True, - values = [ - ".bat", - ".sh", - ], - ), - }, -) - -def _disable_sh_toolchain_test_impl(ctx): - env = analysistest.begin(ctx) - target = analysistest.target_under_test(env) - - assert_action_mnemonic(env, target.actions[0], "ExecutableSymlink") - - return analysistest.end(env) - -_disable_sh_toolchain_test = analysistest.make( - _disable_sh_toolchain_test_impl, - config_settings = { - str(Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper")): False, - }, -) - -def process_wrapper_bootstrap_test_suite(name, **kwargs): - """Entry-point macro called from the BUILD file. - - Args: - name (str): Name of the macro. - **kwargs (dict): Additional keyword arguments. - """ - - _enable_sh_toolchain_test( - name = "enable_sh_toolchain_test", - target_under_test = Label("//util/process_wrapper:bootstrap_process_wrapper"), - expected_ext = select({ - "@platforms//os:windows": ".bat", - "//conditions:default": ".sh", - }), - ) - - _disable_sh_toolchain_test( - name = "disable_sh_toolchain_test", - target_under_test = Label("//util/process_wrapper:bootstrap_process_wrapper"), - ) - - native.test_suite( - name = name, - tests = [ - ":disable_sh_toolchain_test", - ":enable_sh_toolchain_test", - ], - **kwargs - ) diff --git a/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs b/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs index d9ce7579c5..2138c62e19 100644 --- a/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs +++ b/test/rust_analyzer/generated_srcs_test/rust_project_json_test.rs @@ -2,11 +2,12 @@ mod tests { use serde::Deserialize; use std::env; + use std::fs; + use std::path::Path; use std::path::PathBuf; #[derive(Deserialize)] struct Project { - sysroot_src: String, crates: Vec, } @@ -25,22 +26,12 @@ mod tests { #[test] fn test_generated_srcs() { let rust_project_path = PathBuf::from(env::var("RUST_PROJECT_JSON").unwrap()); + let rust_project_path = fs::canonicalize(&rust_project_path).unwrap(); let content = std::fs::read_to_string(&rust_project_path) .unwrap_or_else(|_| panic!("couldn't open {:?}", &rust_project_path)); let project: Project = serde_json::from_str(&content).expect("Failed to deserialize project JSON"); - // /tmp/_bazel/12345678/external/tools/rustlib/library => /tmp/_bazel - let output_base = project - .sysroot_src - .rsplitn(2, "/external/") - .last() - .unwrap() - .rsplitn(2, '/') - .last() - .unwrap(); - println!("output_base: {output_base}"); - let with_gen = project .crates .iter() @@ -50,7 +41,20 @@ mod tests { assert!(with_gen.root_module.ends_with("/lib.rs")); let include_dirs = &with_gen.source.as_ref().unwrap().include_dirs; - assert!(include_dirs.len() == 1); - assert!(include_dirs[0].starts_with(output_base)); + assert_eq!(include_dirs.len(), 2); + + let root_module_parent = Path::new(&with_gen.root_module).parent().unwrap(); + let workspace_dir = rust_project_path.parent().unwrap(); + + assert!( + include_dirs.iter().any(|p| Path::new(p) == root_module_parent), + "expected include_dirs to contain root_module parent, got include_dirs={include_dirs:?}, root_module={}", + with_gen.root_module, + ); + assert!( + include_dirs.iter().any(|p| Path::new(p) == workspace_dir), + "expected include_dirs to contain workspace dir, got include_dirs={include_dirs:?}, workspace_dir={}", + workspace_dir.display(), + ); } } diff --git a/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel b/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel new file mode 100644 index 0000000000..eac2fe6e4e --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_rust//rust:defs.bzl", "rust_library", "rust_test") + +rust_library( + name = "mylib", + srcs = ["lib.rs"], + edition = "2018", +) + +rust_test( + name = "mylib_test", + srcs = ["subdir/subdir_test.rs"], + edition = "2018", +) + +rust_test( + name = "rust_project_json_test", + srcs = ["rust_project_json_test.rs"], + data = [":rust-project.json"], + edition = "2018", + env = {"RUST_PROJECT_JSON": "$(rootpath :rust-project.json)"}, + # This target is tagged as manual since it's not expected to pass in + # contexts outside of `//test/rust_analyzer:rust_analyzer_test`. Run + # that target to execute this test. + tags = ["manual"], + deps = [ + "//test/rust_analyzer/3rdparty/crates:serde", + "//test/rust_analyzer/3rdparty/crates:serde_json", + ], +) diff --git a/test/rust_analyzer/subdir_test_crates_same_package/lib.rs b/test/rust_analyzer/subdir_test_crates_same_package/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/lib.rs @@ -0,0 +1 @@ + diff --git a/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs b/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs new file mode 100644 index 0000000000..77a99d75ed --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/rust_project_json_test.rs @@ -0,0 +1,76 @@ +#[cfg(test)] +mod tests { + use serde::Deserialize; + use std::collections::BTreeSet; + use std::env; + use std::path::PathBuf; + + #[derive(Deserialize)] + struct Project { + crates: Vec, + } + + #[derive(Deserialize)] + struct Crate { + root_module: String, + is_workspace_member: Option, + source: Option, + } + + #[derive(Deserialize)] + struct Source { + include_dirs: Vec, + } + + fn normalize(path: &str) -> String { + path.trim_end_matches('/').to_owned() + } + + #[test] + fn test_same_package_crates_share_include_dir() { + let rust_project_path = PathBuf::from(env::var("RUST_PROJECT_JSON").unwrap()); + let content = std::fs::read_to_string(&rust_project_path) + .unwrap_or_else(|_| panic!("couldn't open {:?}", &rust_project_path)); + let project: Project = + serde_json::from_str(&content).expect("Failed to deserialize project JSON"); + + let lib = project + .crates + .iter() + .find(|c| c.is_workspace_member == Some(true) && c.root_module.ends_with("/lib.rs")) + .expect("missing library crate"); + let test = project + .crates + .iter() + .find(|c| { + c.is_workspace_member == Some(true) + && c.root_module.ends_with("/subdir/subdir_test.rs") + }) + .expect("missing subdir test crate"); + + let lib_include_dirs: BTreeSet<_> = lib + .source + .as_ref() + .expect("lib crate missing source field") + .include_dirs + .iter() + .map(|p| normalize(p)) + .collect(); + let test_include_dirs: BTreeSet<_> = test + .source + .as_ref() + .expect("test crate missing source field") + .include_dirs + .iter() + .map(|p| normalize(p)) + .collect(); + + let shared_dir = lib_include_dirs + .intersection(&test_include_dirs) + .next() + .expect("expected crates in same package to share an include_dir"); + + assert!(lib.root_module.starts_with(&format!("{}/", shared_dir))); + assert!(test.root_module.starts_with(&format!("{}/", shared_dir))); + } +} diff --git a/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs b/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs new file mode 100644 index 0000000000..915d7c6130 --- /dev/null +++ b/test/rust_analyzer/subdir_test_crates_same_package/subdir/subdir_test.rs @@ -0,0 +1,5 @@ +#[test] +fn test_subdir_fixture() { + let marker = String::from("ok"); + assert_eq!(marker.len(), 2); +} diff --git a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl index 36a3de891b..0f638c3ee6 100644 --- a/test/unit/pipelined_compilation/pipelined_compilation_test.bzl +++ b/test/unit/pipelined_compilation/pipelined_compilation_test.bzl @@ -1,8 +1,8 @@ """Unittests for rust rules.""" load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") -load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro") -load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements", "assert_list_contains_adjacent_elements_not") +load("//rust:defs.bzl", "rust_binary", "rust_library", "rust_proc_macro", "rust_test") +load("//test/unit:common.bzl", "assert_argv_contains", "assert_list_contains_adjacent_elements_not") load(":wrap.bzl", "wrap") ENABLE_PIPELINING = { @@ -22,49 +22,77 @@ def _second_lib_test_impl(ctx): rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - # Both actions should use the same --emit= - assert_argv_contains(env, rlib_action, "--emit=dep-info,link,metadata") - assert_argv_contains(env, metadata_action, "--emit=dep-info,link,metadata") + # Hollow rlib approach: Rustc action uses --emit=dep-info,link (no metadata). + assert_argv_contains(env, rlib_action, "--emit=dep-info,link") - # The metadata action should have a .rmeta as output and the rlib action a .rlib + # Metadata action uses --emit=link=-hollow.rlib (hollow rlib, .rlib extension). + # The .rlib extension is required so rustc reads it as an rlib archive (extracting + # lib.rmeta with optimized MIR). Using .rmeta extension causes E0786, and using + # --emit=metadata produces raw .rmeta without optimized MIR (causes "missing + # optimized MIR" errors on Rust 1.85+). + metadata_emit_link = [arg for arg in metadata_action.argv if arg.startswith("--emit=link=") and arg.endswith("-hollow.rlib")] + asserts.true( + env, + len(metadata_emit_link) == 1, + "expected --emit=link=*-hollow.rlib for hollow rlib, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--emit=")]), + ) + + # The rlib action produces a .rlib; the metadata action produces a -hollow.rlib. path = rlib_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rlib"), - "expected Rustc to output .rlib, got " + path, + path.endswith(".rlib") and not path.endswith("-hollow.rlib"), + "expected Rustc to output .rlib (not hollow), got " + path, ) path = metadata_action.outputs.to_list()[0].path asserts.true( env, - path.endswith(".rmeta"), - "expected RustcMetadata to output .rmeta, got " + path, + path.endswith("-hollow.rlib"), + "expected RustcMetadata to output -hollow.rlib, got " + path, ) - # Only the action building metadata should contain --rustc-quit-on-rmeta + # Neither action should use --rustc-quit-on-rmeta (hollow rlib exits naturally). assert_list_contains_adjacent_elements_not(env, rlib_action.argv, ["--rustc-quit-on-rmeta", "true"]) - assert_list_contains_adjacent_elements(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) - - # Check that both actions refer to the metadata of :first, not the rlib - extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + assert_list_contains_adjacent_elements_not(env, metadata_action.argv, ["--rustc-quit-on-rmeta", "true"]) + + # The metadata action should use -Zno-codegen for the hollow rlib approach. + assert_argv_contains(env, metadata_action, "-Zno-codegen") + + # The Rustc action should NOT use -Zno-codegen. + no_codegen_in_rlib = [arg for arg in rlib_action.argv if arg == "-Zno-codegen"] + asserts.true(env, len(no_codegen_in_rlib) == 0, "Rustc action should not have -Zno-codegen") + + # The metadata action references first's hollow rlib for --extern (pipelining: starts + # before first's full codegen finishes). The Rustc action uses the full rlib for + # --extern so the full rlib's embedded SVH matches the full rlib that downstream + # binaries (without cc_common.link) see in their -Ldependency path. If both actions + # used the hollow rlib, nondeterministic proc macros could produce different SVHs + # for the hollow vs full rlib, causing E0460 in downstream binary builds. + extern_metadata = [arg for arg in metadata_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith("-hollow.rlib")] asserts.true( env, len(extern_metadata) == 1, - "did not find a --extern=first=*.rmeta but expected one", + "did not find --extern=first=*-hollow.rlib for metadata action, got: " + str([arg for arg in metadata_action.argv if arg.startswith("--extern=first=")]), ) - extern_rlib = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and arg.endswith(".rmeta")] + extern_rlib_full = [arg for arg in rlib_action.argv if arg.startswith("--extern=first=") and "libfirst" in arg and not arg.endswith("-hollow.rlib")] asserts.true( env, - len(extern_rlib) == 1, - "did not find a --extern=first=*.rlib but expected one", + len(extern_rlib_full) == 1, + "expected --extern=first=libfirst*.rlib (full rlib) for rlib action, got: " + str([arg for arg in rlib_action.argv if arg.startswith("--extern=first=")]), ) - # Check that the input to both actions is the metadata of :first + # The metadata action's input is first's hollow rlib only (no full rlib needed). input_metadata = [i for i in metadata_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input, found " + str([i.path for i in input_metadata])) - asserts.true(env, input_metadata[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_metadata[0].path) - input_rlib = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst")] - asserts.true(env, len(input_rlib) == 1, "expected only one libfirst input, found " + str([i.path for i in input_rlib])) - asserts.true(env, input_rlib[0].extension == "rmeta", "expected libfirst dependency to be rmeta, found " + input_rlib[0].path) + asserts.true(env, len(input_metadata) == 1, "expected only one libfirst input for metadata, found " + str([i.path for i in input_metadata])) + asserts.true(env, input_metadata[0].basename.endswith("-hollow.rlib"), "expected hollow rlib for metadata action, found " + input_metadata[0].path) + + # The Rustc action's inputs contain the full rlib (referenced by --extern) and the + # hollow rlib (present in the sandbox for -Ldependency=<_hollow_dir> resolution of + # transitive deps that were compiled against hollow rlibs). + input_rlib_full = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and not i.basename.endswith("-hollow.rlib")] + input_rlib_hollow = [i for i in rlib_action.inputs.to_list() if i.basename.startswith("libfirst") and i.basename.endswith("-hollow.rlib")] + asserts.true(env, len(input_rlib_full) == 1, "expected full rlib in rlib action inputs, found " + str([i.path for i in input_rlib_full])) + asserts.true(env, len(input_rlib_hollow) == 1, "expected hollow rlib in rlib action inputs (for sandbox), found " + str([i.path for i in input_rlib_hollow])) return analysistest.end(env) @@ -124,10 +152,16 @@ def _pipelined_compilation_test(): target_under_test = ":bin", target_compatible_with = _NO_WINDOWS, ) + hollow_rlib_env_test( + name = "hollow_rlib_env_test", + target_under_test = ":second", + target_compatible_with = _NO_WINDOWS, + ) return [ ":second_lib_test", ":bin_test", + ":hollow_rlib_env_test", ] def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): @@ -138,8 +172,8 @@ def _rmeta_is_propagated_through_custom_rule_test_impl(ctx): # also depend on metadata for 'wrapper'. rust_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] - metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rmeta")] - rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib")] + metadata_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith("-hollow.rlib")] + rlib_inputs = [i for i in rust_action.inputs.to_list() if i.path.endswith(".rlib") and not i.path.endswith("-hollow.rlib")] seen_wrapper_metadata = False seen_to_wrap_metadata = False @@ -176,22 +210,30 @@ def _rmeta_is_used_when_building_custom_rule_test_impl(ctx): # This is the custom rule invocation of rustc. rust_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] - # We want to check that the action depends on metadata, regardless of ctx.attr.generate_metadata seen_to_wrap_rlib = False - seen_to_wrap_rmeta = False + seen_to_wrap_hollow = False for act in rust_action.inputs.to_list(): - if "libto_wrap" in act.path and act.path.endswith(".rlib"): + if "libto_wrap" in act.path and act.path.endswith("-hollow.rlib"): + seen_to_wrap_hollow = True + elif "libto_wrap" in act.path and act.path.endswith(".rlib") and not act.path.endswith("-hollow.rlib"): seen_to_wrap_rlib = True - elif "libto_wrap" in act.path and act.path.endswith(".rmeta"): - seen_to_wrap_rmeta = True - asserts.true(env, seen_to_wrap_rmeta, "expected dependency on metadata for 'to_wrap' but not found") - asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") + if ctx.attr.generate_metadata: + # When wrapper generates its own hollow rlib, the Rustc action uses the full + # rlib of to_wrap for --extern (SVH consistency) and also has the hollow rlib + # in the sandbox for -Ldependency= resolution. + asserts.true(env, seen_to_wrap_hollow, "expected hollow rlib in inputs (for sandbox) when generate_metadata=True") + asserts.true(env, seen_to_wrap_rlib, "expected full rlib in inputs for --extern when generate_metadata=True") + else: + # When wrapper does not generate its own hollow rlib, the Rustc action uses + # hollow rlib deps via normal _depend_on_metadata logic (pipelined rlib deps). + asserts.true(env, seen_to_wrap_hollow, "expected dependency on metadata for 'to_wrap' but not found") + asserts.false(env, seen_to_wrap_rlib, "expected no dependency on object for 'to_wrap' but it was found") return analysistest.end(env) rmeta_is_propagated_through_custom_rule_test = analysistest.make(_rmeta_is_propagated_through_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) -rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, config_settings = ENABLE_PIPELINING) +rmeta_is_used_when_building_custom_rule_test = analysistest.make(_rmeta_is_used_when_building_custom_rule_test_impl, attrs = {"generate_metadata": attr.bool()}, config_settings = ENABLE_PIPELINING) def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): env = analysistest.begin(ctx) @@ -204,6 +246,33 @@ def _rmeta_not_produced_if_pipelining_disabled_test_impl(ctx): rmeta_not_produced_if_pipelining_disabled_test = analysistest.make(_rmeta_not_produced_if_pipelining_disabled_test_impl, config_settings = ENABLE_PIPELINING) +def _hollow_rlib_env_test_impl(ctx): + """Verify RUSTC_BOOTSTRAP=1 is set consistently on both Rustc and RustcMetadata actions. + + RUSTC_BOOTSTRAP=1 changes the crate hash (SVH), so it must be set on both actions + to keep the hollow rlib and full rlib SVHs consistent.""" + env = analysistest.begin(ctx) + tut = analysistest.target_under_test(env) + metadata_action = [act for act in tut.actions if act.mnemonic == "RustcMetadata"][0] + rlib_action = [act for act in tut.actions if act.mnemonic == "Rustc"][0] + + asserts.equals( + env, + "1", + metadata_action.env.get("RUSTC_BOOTSTRAP", ""), + "Metadata action should have RUSTC_BOOTSTRAP=1 for hollow rlib approach", + ) + asserts.equals( + env, + "1", + rlib_action.env.get("RUSTC_BOOTSTRAP", ""), + "Rustc action should have RUSTC_BOOTSTRAP=1 for SVH compatibility with hollow rlib", + ) + + return analysistest.end(env) + +hollow_rlib_env_test = analysistest.make(_hollow_rlib_env_test_impl, config_settings = ENABLE_PIPELINING) + def _disable_pipelining_test(): rust_library( name = "lib", @@ -249,6 +318,7 @@ def _custom_rule_test(generate_metadata, suffix): rmeta_is_used_when_building_custom_rule_test( name = "rmeta_is_used_when_building_custom_rule_test" + suffix, + generate_metadata = generate_metadata, target_under_test = ":wrapper" + suffix, target_compatible_with = _NO_WINDOWS, ) @@ -258,6 +328,59 @@ def _custom_rule_test(generate_metadata, suffix): ":rmeta_is_used_when_building_custom_rule_test" + suffix, ] +def _svh_mismatch_test(): + """Creates a rust_test demonstrating SVH mismatch with non-deterministic proc macros. + + Without pipelining (default): each library is compiled exactly once, SVH + is consistent across the dependency graph, and the test builds and passes. + + With pipelining (//rust/settings:pipelined_compilation=true): rules_rust + compiles svh_lib twice in separate rustc invocations — once for the hollow + metadata (.rmeta), once for the full .rlib. Because the proc macro uses + HashMap with OS-seeded randomness, these two invocations typically produce + different token streams and therefore different SVH values. The consumer is + compiled against the hollow .rmeta (recording SVH_1); when rustc links the + test binary against the full .rlib (SVH_2), it detects SVH_1 ≠ SVH_2 and + fails with E0460. The test is therefore expected to FAIL TO BUILD most of + the time (~99.2% with 5 HashMap entries) when pipelining is enabled. + + The test is marked flaky because the SVH mismatch is non-deterministic: + on rare occasions (~0.8%) both rustc invocations produce the same HashMap + iteration order and the build succeeds even with pipelining enabled. + """ + + rust_proc_macro( + name = "svh_nondeterministic_macro", + srcs = ["svh_mismatch/svh_mismatch_nondeterministic_macro.rs"], + crate_name = "nondeterministic_macro", + edition = "2021", + ) + + rust_library( + name = "svh_lib", + srcs = ["svh_mismatch/svh_mismatch_lib.rs"], + edition = "2021", + proc_macro_deps = [":svh_nondeterministic_macro"], + ) + + rust_library( + name = "svh_consumer", + srcs = ["svh_mismatch/svh_mismatch_consumer.rs"], + edition = "2021", + deps = [":svh_lib"], + ) + + rust_test( + name = "svh_mismatch_test", + srcs = ["svh_mismatch/svh_mismatch_test.rs"], + edition = "2021", + deps = [":svh_consumer"], + flaky = True, + target_compatible_with = _NO_WINDOWS, + ) + + return [":svh_mismatch_test"] + def pipelined_compilation_test_suite(name): """Entry-point macro called from the BUILD file. @@ -269,6 +392,7 @@ def pipelined_compilation_test_suite(name): tests.extend(_disable_pipelining_test()) tests.extend(_custom_rule_test(generate_metadata = True, suffix = "_with_metadata")) tests.extend(_custom_rule_test(generate_metadata = False, suffix = "_without_metadata")) + tests.extend(_svh_mismatch_test()) native.test_suite( name = name, diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs new file mode 100644 index 0000000000..99b0ea9bf4 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_consumer.rs @@ -0,0 +1,6 @@ +/// A library that depends on svh_lib. When compiled against a hollow `.rmeta` +/// of svh_lib, this crate's metadata records svh_lib's SVH at that point in +/// time. If the full `.rlib` of svh_lib was produced by a separate rustc +/// invocation (with a different HashMap seed), it may have a different SVH, +/// causing a mismatch when a downstream binary tries to link against both. +pub use svh_lib::Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs new file mode 100644 index 0000000000..e2f3985399 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_lib.rs @@ -0,0 +1,8 @@ +use nondeterministic_macro::NondeterministicHash; + +/// A struct whose derivation runs the non-deterministic proc macro. +/// The macro generates a public constant whose value depends on HashMap +/// iteration order, so this crate's SVH varies between separate rustc +/// invocations. +#[derive(NondeterministicHash)] +pub struct Widget; diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs new file mode 100644 index 0000000000..7ba44425b7 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_nondeterministic_macro.rs @@ -0,0 +1,37 @@ +extern crate proc_macro; +use proc_macro::TokenStream; +use std::collections::HashMap; + +/// A derive macro that produces non-deterministic output due to HashMap's +/// random iteration order. Each separate process invocation initializes +/// `HashMap` with a different OS-seeded `RandomState`, so iteration order +/// varies between invocations. This makes the generated constant—and thus +/// the crate's SVH—differ when the macro is run twice (e.g., once for a +/// hollow `.rmeta` and once for a full `.rlib` in pipelined compilation). +#[proc_macro_derive(NondeterministicHash)] +pub fn nondeterministic_hash_derive(_input: TokenStream) -> TokenStream { + // HashMap::new() uses RandomState, which seeds from OS entropy. + // Each separate process invocation gets a different seed, so iteration + // order over the map is non-deterministic across invocations. + let mut map = HashMap::new(); + map.insert("alpha", 1u64); + map.insert("beta", 2u64); + map.insert("gamma", 4u64); + map.insert("delta", 8u64); + map.insert("epsilon", 16u64); + + // Position-weighted sum: not commutative, so different iteration orders + // produce different values. With 5 entries (5! = 120 orderings), the + // probability of identical output in two separate invocations is ~0.8%. + let fingerprint: u64 = map + .iter() + .enumerate() + .map(|(pos, (_, &val))| val.wrapping_mul(pos as u64 + 1)) + .fold(0u64, u64::wrapping_add); + + // Exposing this as a public constant makes it part of the crate's + // exported API, which is included in the SVH computation. + format!("pub const NONDETERMINISTIC_HASH_FINGERPRINT: u64 = {};", fingerprint) + .parse() + .unwrap() +} diff --git a/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs new file mode 100644 index 0000000000..6ecfe83553 --- /dev/null +++ b/test/unit/pipelined_compilation/svh_mismatch/svh_mismatch_test.rs @@ -0,0 +1,28 @@ +/// Demonstrates SVH (Strict Version Hash) mismatch with pipelined compilation. +/// +/// Without pipelining this test always builds and passes: each library is +/// compiled exactly once, so the SVH embedded in every `.rmeta` and `.rlib` +/// is identical. +/// +/// With `//rust/settings:pipelined_compilation=true` rules_rust compiles +/// `svh_lib` **twice** in separate rustc processes — once to emit the hollow +/// `.rmeta` (metadata only), once to emit the full `.rlib`. Because +/// `nondeterministic_macro` uses `HashMap` with OS-seeded randomness, the two +/// rustc invocations typically produce different token streams and therefore +/// different SVH values. `svh_consumer` is compiled against the hollow `.rmeta` +/// and records SVH_1 in its own metadata; when rustc later tries to link the +/// test binary against the full `.rlib` (which carries SVH_2), it detects the +/// mismatch and fails with E0460. The test therefore **fails to build** most of +/// the time (~99.2% probability) when pipelining is enabled. +/// +/// The `flaky = True` attribute on this target acknowledges that the mismatch +/// is non-deterministic: on rare occasions (~0.8%) both rustc invocations +/// happen to produce the same HashMap iteration order, the SVHs agree, and the +/// build succeeds. +use svh_consumer::Widget; + +#[test] +fn svh_consistent() { + // If we reach here the SVH was consistent (no pipelining, or a lucky run). + let _: Widget = Widget; +} diff --git a/test/unit/pipelined_compilation/wrap.bzl b/test/unit/pipelined_compilation/wrap.bzl index f24a0e421a..e3f4ac5482 100644 --- a/test/unit/pipelined_compilation/wrap.bzl +++ b/test/unit/pipelined_compilation/wrap.bzl @@ -40,12 +40,23 @@ def _wrap_impl(ctx): lib_hash = output_hash, extension = ".rlib", ) - rust_metadata_name = "{prefix}{name}-{lib_hash}{extension}".format( - prefix = "lib", - name = crate_name, - lib_hash = output_hash, - extension = ".rmeta", - ) + + # Use -hollow.rlib extension (not .rmeta) so rustc reads it as an rlib archive + # containing optimized MIR. See rust/private/rust.bzl for the same logic. + # The hollow rlib is placed in a "_hollow/" subdirectory to avoid the full rlib + # and hollow rlib appearing in the same -Ldependency= search directory, which + # would cause E0463 "can't find crate" errors due to ambiguous crate candidates. + metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and ctx.attr.generate_metadata + if metadata_supports_pipelining: + rust_metadata_name = "_hollow/lib{name}-{lib_hash}-hollow.rlib".format( + name = crate_name, + lib_hash = output_hash, + ) + else: + rust_metadata_name = "lib{name}-{lib_hash}.rmeta".format( + name = crate_name, + lib_hash = output_hash, + ) tgt = ctx.attr.target deps = [DepVariantInfo( @@ -73,8 +84,7 @@ def _wrap_impl(ctx): aliases = {}, output = rust_lib, metadata = rust_metadata, - metadata_supports_pipelining = can_use_metadata_for_pipelining(toolchain, crate_type) and - ctx.attr.generate_metadata, + metadata_supports_pipelining = metadata_supports_pipelining, owner = ctx.label, edition = "2018", compile_data = depset([]), diff --git a/test/unit/windows_lib_name/BUILD.bazel b/test/unit/windows_lib_name/BUILD.bazel new file mode 100644 index 0000000000..e2a5113ec1 --- /dev/null +++ b/test/unit/windows_lib_name/BUILD.bazel @@ -0,0 +1,3 @@ +load(":windows_lib_name_test.bzl", "windows_lib_name_test_suite") + +windows_lib_name_test_suite(name = "windows_lib_name_test_suite") diff --git a/test/unit/windows_lib_name/windows_lib_name_test.bzl b/test/unit/windows_lib_name/windows_lib_name_test.bzl new file mode 100644 index 0000000000..1749e16700 --- /dev/null +++ b/test/unit/windows_lib_name/windows_lib_name_test.bzl @@ -0,0 +1,185 @@ +"""Analysistests for Windows-specific library naming and link flags.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") + +# buildifier: disable=bzl-visibility +load("//rust/private:rustc.bzl", "portable_link_flags_for_testing", "symlink_for_ambiguous_lib_for_testing") + +# buildifier: disable=bzl-visibility +load("//rust/private:utils.bzl", "get_lib_name_default", "get_lib_name_for_windows") + +# buildifier: disable=provider-params +LinkFlagsInfo = provider(fields = {"flags": "List[str]"}) + +# buildifier: disable=provider-params +SymlinkInfo = provider(fields = {"symlink": "File"}) + +def _portable_link_flags_probe_impl(ctx): + lib_artifact = ctx.actions.declare_file(ctx.attr.lib_basename) + ctx.actions.write(lib_artifact, "", is_executable = False) + library_to_link = struct( + static_library = lib_artifact, + pic_static_library = None, + dynamic_library = None, + interface_library = None, + alwayslink = False, + ) + + get_lib_name = get_lib_name_for_windows if ctx.attr.flavor_msvc else get_lib_name_default + flags = portable_link_flags_for_testing( + lib = library_to_link, + use_pic = False, + ambiguous_libs = {}, + get_lib_name = get_lib_name, + for_windows = True, + flavor_msvc = ctx.attr.flavor_msvc, + ) + + return [ + DefaultInfo(files = depset([])), + LinkFlagsInfo(flags = flags), + ] + +portable_link_flags_probe = rule( + implementation = _portable_link_flags_probe_impl, + attrs = { + "flavor_msvc": attr.bool(default = False), + "lib_basename": attr.string(mandatory = True), + }, +) + +def _symlink_probe_impl(ctx): + lib_artifact = ctx.actions.declare_file(ctx.attr.lib_basename) + ctx.actions.write(lib_artifact, "", is_executable = False) + crate_output = ctx.actions.declare_file("crate.rlib") + ctx.actions.write(crate_output, "", is_executable = False) + symlink = symlink_for_ambiguous_lib_for_testing( + ctx.actions, + toolchain = struct(target_abi = ctx.attr.target_abi), + crate_info = struct(output = crate_output), + lib = lib_artifact, + ) + + return [ + SymlinkInfo(symlink = symlink), + DefaultInfo(files = depset([symlink])), + ] + +symlink_probe = rule( + implementation = _symlink_probe_impl, + attrs = { + "lib_basename": attr.string(mandatory = True), + "target_abi": attr.string(mandatory = True), + }, +) + +def _portable_link_flags_windows_gnu_test_impl(ctx): + env = analysistest.begin(ctx) + flags = analysistest.target_under_test(env)[LinkFlagsInfo].flags + + asserts.equals( + env, + ["-lstatic=foo.dll", "-Clink-arg=-lfoo.dll"], + flags, + ) + return analysistest.end(env) + +portable_link_flags_windows_gnu_test = analysistest.make( + _portable_link_flags_windows_gnu_test_impl, +) + +def _portable_link_flags_windows_msvc_test_impl(ctx): + env = analysistest.begin(ctx) + flags = analysistest.target_under_test(env)[LinkFlagsInfo].flags + + asserts.equals( + env, + ["-lstatic=libfoo.dll", "-Clink-arg=libfoo.dll.lib"], + flags, + ) + return analysistest.end(env) + +portable_link_flags_windows_msvc_test = analysistest.make( + _portable_link_flags_windows_msvc_test_impl, +) + +def _symlink_name_windows_gnu_test_impl(ctx): + env = analysistest.begin(ctx) + symlink = analysistest.target_under_test(env)[SymlinkInfo].symlink + + asserts.true(env, symlink.basename.startswith("libfoo.dll-")) + asserts.true(env, symlink.basename.endswith(".a")) + asserts.false(env, symlink.basename.startswith("liblib")) + + return analysistest.end(env) + +symlink_name_windows_gnu_test = analysistest.make(_symlink_name_windows_gnu_test_impl) + +def _symlink_name_windows_msvc_test_impl(ctx): + env = analysistest.begin(ctx) + symlink = analysistest.target_under_test(env)[SymlinkInfo].symlink + + asserts.true(env, symlink.basename.startswith("native_dep-")) + asserts.true(env, symlink.basename.endswith(".lib")) + + return analysistest.end(env) + +symlink_name_windows_msvc_test = analysistest.make(_symlink_name_windows_msvc_test_impl) + +def _define_targets(): + portable_link_flags_probe( + name = "portable_link_flags_windows_gnu_probe", + flavor_msvc = False, + lib_basename = "libfoo.dll.a", + ) + portable_link_flags_probe( + name = "portable_link_flags_windows_msvc_probe", + flavor_msvc = True, + lib_basename = "libfoo.dll.lib", + ) + + symlink_probe( + name = "symlink_windows_gnu_probe", + lib_basename = "libfoo.dll.a", + target_abi = "gnu", + ) + symlink_probe( + name = "symlink_windows_msvc_probe", + lib_basename = "native_dep.lib", + target_abi = "msvc", + ) + +def windows_lib_name_test_suite(name): + """Entry-point macro for Windows library naming tests. + + Args: + name: test suite name + """ + _define_targets() + + portable_link_flags_windows_gnu_test( + name = "portable_link_flags_windows_gnu_test", + target_under_test = ":portable_link_flags_windows_gnu_probe", + ) + portable_link_flags_windows_msvc_test( + name = "portable_link_flags_windows_msvc_test", + target_under_test = ":portable_link_flags_windows_msvc_probe", + ) + symlink_name_windows_gnu_test( + name = "symlink_name_windows_gnu_test", + target_under_test = ":symlink_windows_gnu_probe", + ) + symlink_name_windows_msvc_test( + name = "symlink_name_windows_msvc_test", + target_under_test = ":symlink_windows_msvc_probe", + ) + + native.test_suite( + name = name, + tests = [ + ":portable_link_flags_windows_gnu_test", + ":portable_link_flags_windows_msvc_test", + ":symlink_name_windows_gnu_test", + ":symlink_name_windows_msvc_test", + ], + ) diff --git a/test/unit/windows_stdlib/BUILD.bazel b/test/unit/windows_stdlib/BUILD.bazel new file mode 100644 index 0000000000..91b803e6a7 --- /dev/null +++ b/test/unit/windows_stdlib/BUILD.bazel @@ -0,0 +1,3 @@ +load(":windows_stdlib_test.bzl", "windows_stdlib_test_suite") + +windows_stdlib_test_suite(name = "windows_stdlib_test_suite") diff --git a/test/unit/windows_stdlib/windows_stdlib_test.bzl b/test/unit/windows_stdlib/windows_stdlib_test.bzl new file mode 100644 index 0000000000..7008a86aa6 --- /dev/null +++ b/test/unit/windows_stdlib/windows_stdlib_test.bzl @@ -0,0 +1,122 @@ +"""Analysistests covering Windows-specific stdlib link flags.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("//rust/platform:triple.bzl", "triple") +load("//rust/platform:triple_mappings.bzl", "system_to_stdlib_linkflags") + +# buildifier: disable=bzl-visibility +load("//rust/private:repository_utils.bzl", "BUILD_for_rust_toolchain") + +def _stdlib_linkflags_windows_test_impl(ctx): + env = analysistest.begin(ctx) + analysistest.target_under_test(env) # Ensure target is configured. + + msvc_flags = system_to_stdlib_linkflags(triple("x86_64-pc-windows-msvc")) + gnu_flags = system_to_stdlib_linkflags(triple("x86_64-pc-windows-gnu")) + gnullvm_flags = system_to_stdlib_linkflags(triple("aarch64-pc-windows-gnullvm")) + + asserts.equals( + env, + ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"], + msvc_flags, + ) + asserts.equals( + env, + ["-ladvapi32", "-lws2_32", "-luserenv"], + gnu_flags, + ) + asserts.equals(env, gnu_flags, gnullvm_flags) + + return analysistest.end(env) + +stdlib_linkflags_windows_test = analysistest.make(_stdlib_linkflags_windows_test_impl) + +def _build_for_rust_toolchain_windows_flags_test_impl(ctx): + env = analysistest.begin(ctx) + analysistest.target_under_test(env) + + msvc_triple = triple("x86_64-pc-windows-msvc") + gnu_triple = triple("x86_64-pc-windows-gnu") + + rendered_msvc = BUILD_for_rust_toolchain( + name = "tc_msvc", + exec_triple = msvc_triple, + target_triple = msvc_triple, + version = "1.75.0", + allocator_library = None, + global_allocator_library = None, + default_edition = "2021", + include_rustfmt = False, + include_llvm_tools = False, + include_linker = False, + stdlib_linkflags = None, + extra_rustc_flags = None, + extra_exec_rustc_flags = None, + opt_level = None, + strip_level = None, + ) + rendered_gnu = BUILD_for_rust_toolchain( + name = "tc_gnu", + exec_triple = gnu_triple, + target_triple = gnu_triple, + version = "1.75.0", + allocator_library = None, + global_allocator_library = None, + default_edition = "2021", + include_rustfmt = False, + include_llvm_tools = False, + include_linker = False, + stdlib_linkflags = None, + extra_rustc_flags = None, + extra_exec_rustc_flags = None, + opt_level = None, + strip_level = None, + ) + + asserts.true( + env, + 'stdlib_linkflags = ["advapi32.lib", "ws2_32.lib", "userenv.lib", "Bcrypt.lib"],' in rendered_msvc, + "MSVC toolchain should render .lib stdlib linkflags:\n%s" % rendered_msvc, + ) + asserts.true( + env, + 'stdlib_linkflags = ["-ladvapi32", "-lws2_32", "-luserenv"],' in rendered_gnu, + "GNU toolchain should render -l stdlib linkflags:\n%s" % rendered_gnu, + ) + + return analysistest.end(env) + +build_for_rust_toolchain_windows_flags_test = analysistest.make( + _build_for_rust_toolchain_windows_flags_test_impl, +) + +def _define_targets(): + # Target under test is unused beyond satisfying analysistest requirements. + native.filegroup( + name = "dummy_target", + srcs = [], + ) + +def windows_stdlib_test_suite(name): + """Entry-point macro for Windows stdlib linkflag tests. + + Args: + name: test suite name""" + _define_targets() + + stdlib_linkflags_windows_test( + name = "stdlib_linkflags_windows_test", + target_under_test = ":dummy_target", + ) + build_for_rust_toolchain_windows_flags_test( + name = "build_for_rust_toolchain_windows_flags_test", + target_under_test = ":dummy_target", + ) + + native.test_suite( + name = name, + tests = [ + ":build_for_rust_toolchain_windows_flags_test", + ":stdlib_linkflags_windows_test", + ], + ) diff --git a/util/process_wrapper/BUILD.bazel b/util/process_wrapper/BUILD.bazel index af56264e4c..08dd7ecadf 100644 --- a/util/process_wrapper/BUILD.bazel +++ b/util/process_wrapper/BUILD.bazel @@ -2,7 +2,6 @@ load("@bazel_skylib//lib:selects.bzl", "selects") # buildifier: disable=bzl-visibility load("//rust/private:rust.bzl", "rust_binary_without_process_wrapper", "rust_test_without_process_wrapper_test") -load("//util/process_wrapper/private:bootstrap_process_wrapper.bzl", "bootstrap_process_wrapper") config_setting( name = "compilation_mode_opt", @@ -55,11 +54,8 @@ rust_test_without_process_wrapper_test( edition = "2018", ) -bootstrap_process_wrapper( +alias( name = "bootstrap_process_wrapper", - is_windows = select({ - "@platforms//os:windows": True, - "//conditions:default": False, - }), + actual = "//util/process_wrapper/private:bootstrap_process_wrapper", visibility = ["//visibility:public"], ) diff --git a/util/process_wrapper/main.rs b/util/process_wrapper/main.rs index 39a6d6db16..2a7cbd8565 100644 --- a/util/process_wrapper/main.rs +++ b/util/process_wrapper/main.rs @@ -19,40 +19,23 @@ mod rustc; mod util; use std::collections::HashMap; +#[cfg(windows)] +use std::collections::{HashSet, VecDeque}; use std::fmt; -use std::fs::{copy, OpenOptions}; +use std::fs::{self, copy, OpenOptions}; use std::io; -use std::process::{exit, Command, ExitStatus, Stdio}; +use std::path::PathBuf; +use std::process::{exit, Command, Stdio}; +#[cfg(windows)] +use std::time::{SystemTime, UNIX_EPOCH}; use tinyjson::JsonValue; use crate::options::options; use crate::output::{process_output, LineOutput}; use crate::rustc::ErrorFormat; - #[cfg(windows)] -fn status_code(status: ExitStatus, was_killed: bool) -> i32 { - // On windows, there's no good way to know if the process was killed by a signal. - // If we killed the process, we override the code to signal success. - if was_killed { - 0 - } else { - status.code().unwrap_or(1) - } -} - -#[cfg(not(windows))] -fn status_code(status: ExitStatus, was_killed: bool) -> i32 { - // On unix, if code is None it means that the process was killed by a signal. - // https://doc.rust-lang.org/std/process/struct.ExitStatus.html#method.success - match status.code() { - Some(code) => code, - // If we killed the process, we expect None here - None if was_killed => 0, - // Otherwise it's some unexpected signal - None => 1, - } -} +use crate::util::read_file_to_array; #[derive(Debug)] struct ProcessWrapperError(String); @@ -73,6 +56,204 @@ macro_rules! debug_log { }; } +#[cfg(windows)] +struct TemporaryDirectoryGuard { + path: Option, +} + +#[cfg(windows)] +impl TemporaryDirectoryGuard { + fn new(path: Option) -> Self { + Self { path } + } + + fn take(&mut self) -> Option { + self.path.take() + } +} + +#[cfg(windows)] +impl Drop for TemporaryDirectoryGuard { + fn drop(&mut self) { + if let Some(path) = self.path.take() { + let _ = fs::remove_dir_all(path); + } + } +} + +#[cfg(not(windows))] +struct TemporaryDirectoryGuard; + +#[cfg(not(windows))] +impl TemporaryDirectoryGuard { + fn new(_: Option) -> Self { + TemporaryDirectoryGuard + } + + fn take(&mut self) -> Option { + None + } +} + +#[cfg(windows)] +fn get_dependency_search_paths_from_args( + initial_args: &[String], +) -> Result<(Vec, Vec), ProcessWrapperError> { + let mut dependency_paths = Vec::new(); + let mut filtered_args = Vec::new(); + let mut argfile_contents: HashMap> = HashMap::new(); + + let mut queue: VecDeque<(String, Option)> = initial_args + .iter() + .map(|arg| (arg.clone(), None)) + .collect(); + + while let Some((arg, parent_argfile)) = queue.pop_front() { + let target = match &parent_argfile { + Some(p) => argfile_contents.entry(format!("{}.filtered", p)).or_default(), + None => &mut filtered_args, + }; + + if arg == "-L" { + let next_arg = queue.front().map(|(a, _)| a.as_str()); + if let Some(path) = next_arg.and_then(|n| n.strip_prefix("dependency=")) { + dependency_paths.push(PathBuf::from(path)); + queue.pop_front(); + } else { + target.push(arg); + } + } else if let Some(path) = arg.strip_prefix("-Ldependency=") { + dependency_paths.push(PathBuf::from(path)); + } else if let Some(argfile_path) = arg.strip_prefix('@') { + let lines = read_file_to_array(argfile_path).map_err(|e| { + ProcessWrapperError(format!("unable to read argfile {}: {}", argfile_path, e)) + })?; + + for line in lines { + queue.push_back((line, Some(argfile_path.to_string()))); + } + + target.push(format!("@{}.filtered", argfile_path)); + } else { + target.push(arg); + } + } + + for (path, content) in argfile_contents { + fs::write(&path, content.join("\n")).map_err(|e| { + ProcessWrapperError(format!("unable to write filtered argfile {}: {}", path, e)) + })?; + } + + Ok((dependency_paths, filtered_args)) +} + +#[cfg(windows)] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + let (dependency_paths, mut filtered_args) = get_dependency_search_paths_from_args(args)?; + + if dependency_paths.is_empty() { + return Ok((filtered_args, None)); + } + + let unique_suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + let dir_name = format!( + "rules_rust_process_wrapper_deps_{}_{}", + std::process::id(), + unique_suffix + ); + + let base_dir = std::env::current_dir().map_err(|e| { + ProcessWrapperError(format!("unable to read current working directory: {}", e)) + })?; + let unified_dir = base_dir.join(&dir_name); + fs::create_dir_all(&unified_dir).map_err(|e| { + ProcessWrapperError(format!( + "unable to create unified dependency directory {}: {}", + unified_dir.display(), + e + )) + })?; + + let mut seen = HashSet::new(); + for path in dependency_paths { + let entries = fs::read_dir(&path).map_err(|e| { + ProcessWrapperError(format!( + "unable to read dependency search path {}: {}", + path.display(), + e + )) + })?; + + for entry in entries { + let entry = entry.map_err(|e| { + ProcessWrapperError(format!( + "unable to iterate dependency search path {}: {}", + path.display(), + e + )) + })?; + let file_type = entry.file_type().map_err(|e| { + ProcessWrapperError(format!( + "unable to inspect dependency search path {}: {}", + path.display(), + e + )) + })?; + if !(file_type.is_file() || file_type.is_symlink()) { + continue; + } + + let file_name = entry.file_name(); + let file_name_lower = file_name + .to_string_lossy() + .to_ascii_lowercase(); + if !seen.insert(file_name_lower) { + continue; + } + + let dest = unified_dir.join(&file_name); + let src = entry.path(); + match fs::hard_link(&src, &dest) { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(err) => { + debug_log!( + "failed to hardlink {} to {} ({}), falling back to copy", + src.display(), + dest.display(), + err + ); + fs::copy(&src, &dest).map_err(|copy_err| { + ProcessWrapperError(format!( + "unable to copy {} into unified dependency dir {}: {}", + src.display(), + dest.display(), + copy_err + )) + })?; + } + } + } + } + + filtered_args.push(format!("-Ldependency={}", unified_dir.display())); + + Ok((filtered_args, Some(unified_dir))) +} + +#[cfg(not(windows))] +fn consolidate_dependency_search_paths( + args: &[String], +) -> Result<(Vec, Option), ProcessWrapperError> { + Ok((args.to_vec(), None)) +} + fn json_warning(line: &str) -> JsonValue { JsonValue::Object(HashMap::from([ ( @@ -93,9 +274,7 @@ fn json_warning(line: &str) -> JsonValue { fn process_line( mut line: String, - quit_on_rmeta: bool, format: ErrorFormat, - metadata_emitted: &mut bool, ) -> Result { // LLVM can emit lines that look like the following, and these will be interspersed // with the regular JSON output. Arguably, rustc should be fixed not to emit lines @@ -110,19 +289,19 @@ fn process_line( return Ok(LineOutput::Skip); } } - if quit_on_rmeta { - rustc::stop_on_rmeta_completion(line, format, metadata_emitted) - } else { - rustc::process_json(line, format) - } + rustc::process_json(line, format) } fn main() -> Result<(), ProcessWrapperError> { let opts = options().map_err(|e| ProcessWrapperError(e.to_string()))?; + let (child_arguments, dep_dir_cleanup) = + consolidate_dependency_search_paths(&opts.child_arguments)?; + let mut temp_dir_guard = TemporaryDirectoryGuard::new(dep_dir_cleanup); + let mut command = Command::new(opts.executable); command - .args(opts.child_arguments) + .args(child_arguments) .env_clear() .envs(opts.child_environment) .stdout(if let Some(stdout_file) = opts.stdout_file { @@ -172,26 +351,13 @@ fn main() -> Result<(), ProcessWrapperError> { None }; - let mut was_killed = false; let result = if let Some(format) = opts.rustc_output_format { - let quit_on_rmeta = opts.rustc_quit_on_rmeta; - // Process json rustc output and kill the subprocess when we get a signal - // that we emitted a metadata file. - let mut me = false; - let metadata_emitted = &mut me; - let result = process_output( + process_output( &mut child_stderr, stderr.as_mut(), output_file.as_mut(), - move |line| process_line(line, quit_on_rmeta, format, metadata_emitted), - ); - if me { - // If recv returns Ok(), a signal was sent in this channel so we should terminate the child process. - // We can safely ignore the Result from kill() as we don't care if the process already terminated. - let _ = child.kill(); - was_killed = true; - } - result + move |line| process_line(line, format), + ) } else { // Process output normally by forwarding stderr process_output( @@ -206,10 +372,8 @@ fn main() -> Result<(), ProcessWrapperError> { let status = child .wait() .map_err(|e| ProcessWrapperError(format!("failed to wait for child process: {}", e)))?; - // If the child process is rustc and is killed after metadata generation, that's also a success. - let code = status_code(status, was_killed); - let success = code == 0; - if success { + let code = status.code().unwrap_or(1); + if code == 0 { if let Some(tf) = opts.touch_file { OpenOptions::new() .create(true) @@ -228,6 +392,10 @@ fn main() -> Result<(), ProcessWrapperError> { } } + if let Some(path) = temp_dir_guard.take() { + let _ = fs::remove_dir_all(path); + } + exit(code) } @@ -241,7 +409,6 @@ mod test { #[test] fn test_process_line_diagnostic_json() -> Result<(), String> { - let mut metadata_emitted = false; let LineOutput::Message(msg) = process_line( r#" { @@ -250,9 +417,7 @@ mod test { } "# .to_string(), - false, ErrorFormat::Json, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -273,7 +438,6 @@ mod test { #[test] fn test_process_line_diagnostic_rendered() -> Result<(), String> { - let mut metadata_emitted = false; let LineOutput::Message(msg) = process_line( r#" { @@ -282,9 +446,7 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ false, ErrorFormat::Rendered, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -295,16 +457,13 @@ mod test { #[test] fn test_process_line_noise() -> Result<(), String> { - let mut metadata_emitted = false; for text in [ "'+zaamo' is not a recognized feature for this target (ignoring feature)", " WARN rustc_errors::emitter Invalid span...", ] { let LineOutput::Message(msg) = process_line( text.to_string(), - /*quit_on_rmeta=*/ false, ErrorFormat::Json, - &mut metadata_emitted, )? else { return Err("Expected a LineOutput::Message".to_string()); @@ -330,7 +489,6 @@ mod test { #[test] fn test_process_line_emit_link() -> Result<(), String> { - let mut metadata_emitted = false; assert!(matches!( process_line( r#" @@ -340,19 +498,15 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ true, ErrorFormat::Rendered, - &mut metadata_emitted, )?, LineOutput::Skip )); - assert!(!metadata_emitted); Ok(()) } #[test] fn test_process_line_emit_metadata() -> Result<(), String> { - let mut metadata_emitted = false; assert!(matches!( process_line( r#" @@ -362,13 +516,10 @@ mod test { } "# .to_string(), - /*quit_on_rmeta=*/ true, ErrorFormat::Rendered, - &mut metadata_emitted, )?, - LineOutput::Terminate + LineOutput::Skip )); - assert!(metadata_emitted); Ok(()) } } diff --git a/util/process_wrapper/options.rs b/util/process_wrapper/options.rs index 2f252cadc7..6dbc898a11 100644 --- a/util/process_wrapper/options.rs +++ b/util/process_wrapper/options.rs @@ -44,9 +44,6 @@ pub(crate) struct Options { // If set, also logs all unprocessed output from the rustc output to this file. // Meant to be used to get json output out of rustc for tooling usage. pub(crate) output_file: Option, - // If set, it configures rustc to emit an rmeta file and then - // quit. - pub(crate) rustc_quit_on_rmeta: bool, // This controls the output format of rustc messages. pub(crate) rustc_output_format: Option, } @@ -64,7 +61,6 @@ pub(crate) fn options() -> Result { let mut stdout_file = None; let mut stderr_file = None; let mut output_file = None; - let mut rustc_quit_on_rmeta_raw = None; let mut rustc_output_format_raw = None; let mut flags = Flags::new(); let mut require_explicit_unstable_features = None; @@ -102,17 +98,9 @@ pub(crate) fn options() -> Result { "Log all unprocessed subprocess stderr in this file.", &mut output_file, ); - flags.define_flag( - "--rustc-quit-on-rmeta", - "If enabled, this wrapper will terminate rustc after rmeta has been emitted.", - &mut rustc_quit_on_rmeta_raw, - ); flags.define_flag( "--rustc-output-format", - "Controls the rustc output format if --rustc-quit-on-rmeta is set.\n\ - 'json' will cause the json output to be output, \ - 'rendered' will extract the rendered message and print that.\n\ - Default: `rendered`", + "The expected rustc output format. Valid values: json, rendered.", &mut rustc_output_format_raw, ); flags.define_flag( @@ -179,7 +167,6 @@ pub(crate) fn options() -> Result { }) .transpose()?; - let rustc_quit_on_rmeta = rustc_quit_on_rmeta_raw.is_some_and(|s| s == "true"); let rustc_output_format = rustc_output_format_raw .map(|v| match v.as_str() { "json" => Ok(rustc::ErrorFormat::Json), @@ -227,7 +214,6 @@ pub(crate) fn options() -> Result { stdout_file, stderr_file, output_file, - rustc_quit_on_rmeta, rustc_output_format, }) } diff --git a/util/process_wrapper/output.rs b/util/process_wrapper/output.rs index 4b3604b18d..5dabad8179 100644 --- a/util/process_wrapper/output.rs +++ b/util/process_wrapper/output.rs @@ -18,15 +18,11 @@ use std::io::{self, prelude::*}; /// LineOutput tells process_output what to do when a line is processed. /// If a Message is returned, it will be written to write_end, if -/// Skip is returned nothing will be printed and execution continues, -/// if Terminate is returned, process_output returns immediately. -/// Terminate is used to stop processing when we see an emit metadata -/// message. +/// Skip is returned nothing will be printed and execution continues. #[derive(Debug)] pub(crate) enum LineOutput { Message(String), Skip, - Terminate, } #[derive(Debug)] @@ -95,7 +91,6 @@ where match process_line(line.clone()) { Ok(LineOutput::Message(to_write)) => output_writer.write_all(to_write.as_bytes())?, Ok(LineOutput::Skip) => {} - Ok(LineOutput::Terminate) => return Ok(()), Err(msg) => { failed_on = Some((line, msg)); break; diff --git a/util/process_wrapper/private/BUILD.bazel b/util/process_wrapper/private/BUILD.bazel index badd4a695d..6cbbbc07da 100644 --- a/util/process_wrapper/private/BUILD.bazel +++ b/util/process_wrapper/private/BUILD.bazel @@ -1,4 +1,7 @@ -exports_files([ - "process_wrapper.sh", - "process_wrapper.bat", -]) +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") + +cc_binary( + name = "bootstrap_process_wrapper", + srcs = ["bootstrap_process_wrapper.cc"], + visibility = ["//util/process_wrapper:__pkg__"], +) diff --git a/util/process_wrapper/private/bootstrap_process_wrapper.bzl b/util/process_wrapper/private/bootstrap_process_wrapper.bzl deleted file mode 100644 index c47a12fd7b..0000000000 --- a/util/process_wrapper/private/bootstrap_process_wrapper.bzl +++ /dev/null @@ -1,73 +0,0 @@ -"""Bootstrap rustc process wrapper""" - -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") - -def _bootstrap_process_wrapper_impl_unix(ctx): - output = ctx.actions.declare_file("{}.sh".format(ctx.label.name)) - - setting = ctx.attr._use_sh_toolchain_for_bootstrap_process_wrapper[BuildSettingInfo].value - sh_toolchain = ctx.toolchains["@bazel_tools//tools/sh:toolchain_type"] - if setting and sh_toolchain: - shebang = "#!{}".format(sh_toolchain.path) - ctx.actions.expand_template( - output = output, - template = ctx.file._bash, - substitutions = { - # Replace the shebang with one constructed from the configured - # shell toolchain. - "#!/usr/bin/env bash": shebang, - }, - ) - else: - ctx.actions.symlink( - output = output, - target_file = ctx.file._bash, - is_executable = True, - ) - - return [DefaultInfo( - files = depset([output]), - executable = output, - )] - -def _bootstrap_process_wrapper_impl_windows(ctx): - output = ctx.actions.declare_file("{}.bat".format(ctx.label.name)) - ctx.actions.symlink( - output = output, - target_file = ctx.file._batch, - is_executable = True, - ) - - return [DefaultInfo( - files = depset([output]), - executable = output, - )] - -def _bootstrap_process_wrapper_impl(ctx): - if ctx.attr.is_windows: - return _bootstrap_process_wrapper_impl_windows(ctx) - return _bootstrap_process_wrapper_impl_unix(ctx) - -bootstrap_process_wrapper = rule( - doc = "A rule which produces a bootstrapping script for the rustc process wrapper.", - implementation = _bootstrap_process_wrapper_impl, - attrs = { - "is_windows": attr.bool( - doc = "Indicate whether or not the target platform is windows.", - mandatory = True, - ), - "_bash": attr.label( - allow_single_file = True, - default = Label("//util/process_wrapper/private:process_wrapper.sh"), - ), - "_batch": attr.label( - allow_single_file = True, - default = Label("//util/process_wrapper/private:process_wrapper.bat"), - ), - "_use_sh_toolchain_for_bootstrap_process_wrapper": attr.label( - default = Label("//rust/settings:experimental_use_sh_toolchain_for_bootstrap_process_wrapper"), - ), - }, - toolchains = [config_common.toolchain_type("@bazel_tools//tools/sh:toolchain_type", mandatory = False)], - executable = True, -) diff --git a/util/process_wrapper/private/bootstrap_process_wrapper.cc b/util/process_wrapper/private/bootstrap_process_wrapper.cc new file mode 100644 index 0000000000..e6e58513e3 --- /dev/null +++ b/util/process_wrapper/private/bootstrap_process_wrapper.cc @@ -0,0 +1,90 @@ +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#include +#define getcwd _getcwd +#else +#include +#endif + +namespace { + +constexpr const char* kPwdPlaceholder = "${pwd}"; + +std::string replace_pwd_placeholder(const std::string& arg, + const std::string& pwd) { + std::string out = arg; + std::string::size_type pos = 0; + while ((pos = out.find(kPwdPlaceholder, pos)) != std::string::npos) { + out.replace(pos, std::strlen(kPwdPlaceholder), pwd); + pos += pwd.size(); + } + return out; +} + +std::vector build_exec_argv(const std::vector& args) { + std::vector exec_argv; + exec_argv.reserve(args.size() + 1); + for (const std::string& arg : args) { + exec_argv.push_back(const_cast(arg.c_str())); + } + exec_argv.push_back(nullptr); + return exec_argv; +} + +} // namespace + +int main(int argc, char** argv) { + int first_arg_index = 1; + if (argc > 1 && std::strcmp(argv[1], "--") == 0) { + first_arg_index = 2; + } + + if (first_arg_index >= argc) { + std::fprintf(stderr, "bootstrap_process_wrapper: missing command\n"); + return 1; + } + + char* pwd_raw = getcwd(nullptr, 0); + if (pwd_raw == nullptr) { + std::perror("bootstrap_process_wrapper: getcwd"); + return 1; + } + std::string pwd = pwd_raw; + std::free(pwd_raw); + + std::vector command_args; + command_args.reserve(static_cast(argc - first_arg_index)); + for (int i = first_arg_index; i < argc; ++i) { + command_args.push_back(replace_pwd_placeholder(argv[i], pwd)); + } + +#if defined(_WIN32) + for (char& c : command_args[0]) { + if (c == '/') { + c = '\\'; + } + } +#endif + + std::vector exec_argv = build_exec_argv(command_args); + +#if defined(_WIN32) + int exit_code = _spawnvp(_P_WAIT, exec_argv[0], exec_argv.data()); + if (exit_code == -1) { + std::perror("bootstrap_process_wrapper: _spawnvp"); + return 1; + } + return exit_code; +#else + execvp(exec_argv[0], exec_argv.data()); + std::perror("bootstrap_process_wrapper: execvp"); + return 1; +#endif +} diff --git a/util/process_wrapper/private/process_wrapper.bat b/util/process_wrapper/private/process_wrapper.bat deleted file mode 100755 index 36fff8699a..0000000000 --- a/util/process_wrapper/private/process_wrapper.bat +++ /dev/null @@ -1,31 +0,0 @@ -@ECHO OFF -SETLOCAL enabledelayedexpansion - -SET command=%* - -:: Resolve the `${pwd}` placeholders -SET command=!command:${pwd}=%CD%! - -:: Strip out the leading `--` argument. -SET command=!command:~3! - -:: Find the rustc.exe argument and sanitize it's path -for %%A in (%*) do ( - SET arg=%%~A - if "!arg:~-9!"=="rustc.exe" ( - SET sanitized=!arg:/=\! - - SET command=!sanitized! !command:%%~A=! - goto :break - ) -) - -:break - -%command% - -:: Capture the exit code of rustc.exe -SET exit_code=!errorlevel! - -:: Exit with the same exit code -EXIT /b %exit_code% diff --git a/util/process_wrapper/private/process_wrapper.sh b/util/process_wrapper/private/process_wrapper.sh deleted file mode 100755 index 97b3478c9f..0000000000 --- a/util/process_wrapper/private/process_wrapper.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -# Skip the first argument which is expected to be `--` -shift - -args=() - -for arg in "$@"; do - # Check if the argument contains "${PWD}" and replace it with the actual value of PWD - if [[ "${arg}" == *'${pwd}'* ]]; then - arg="${arg//\$\{pwd\}/$PWD}" - fi - args+=("${arg}") -done - -exec "${args[@]}" diff --git a/util/process_wrapper/rustc.rs b/util/process_wrapper/rustc.rs index 97ee466337..3bb4a8c2d9 100644 --- a/util/process_wrapper/rustc.rs +++ b/util/process_wrapper/rustc.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::convert::{TryFrom, TryInto}; - use tinyjson::JsonValue; use crate::output::{LineOutput, LineResult}; @@ -37,66 +35,20 @@ fn get_key(value: &JsonValue, key: &str) -> Option { } } -#[derive(Debug)] -enum RustcMessage { - Emit(String), - Message(String), -} - -impl TryFrom for RustcMessage { - type Error = (); - fn try_from(val: JsonValue) -> Result { - if let Some(emit) = get_key(&val, "emit") { - return Ok(Self::Emit(emit)); - } - if let Some(rendered) = get_key(&val, "rendered") { - return Ok(Self::Message(rendered)); - } - Err(()) - } -} - /// process_rustc_json takes an output line from rustc configured with /// --error-format=json, parses the json and returns the appropriate output /// according to the original --error-format supplied. -/// Only messages are returned, emits are ignored. +/// Only diagnostics with a rendered message are returned. /// Returns an errors if parsing json fails. pub(crate) fn process_json(line: String, error_format: ErrorFormat) -> LineResult { let parsed: JsonValue = line .parse() .map_err(|_| "error parsing rustc output as json".to_owned())?; - Ok(match parsed.try_into() { - Ok(RustcMessage::Message(rendered)) => { - output_based_on_error_format(line, rendered, error_format) - } - _ => LineOutput::Skip, - }) -} - -/// stop_on_rmeta_completion parses the json output of rustc in the same way -/// process_rustc_json does. In addition, it will signal to stop when metadata -/// is emitted so the compiler can be terminated. -/// This is used to implement pipelining in rules_rust, please see -/// https://internals.rust-lang.org/t/evaluating-pipelined-rustc-compilation/10199 -/// Returns an error if parsing json fails. -/// TODO: pass a function to handle the emit event and merge with process_json -pub(crate) fn stop_on_rmeta_completion( - line: String, - error_format: ErrorFormat, - kill: &mut bool, -) -> LineResult { - let parsed: JsonValue = line - .parse() - .map_err(|_| "error parsing rustc output as json".to_owned())?; - Ok(match parsed.try_into() { - Ok(RustcMessage::Emit(emit)) if emit == "metadata" => { - *kill = true; - LineOutput::Terminate - } - Ok(RustcMessage::Message(rendered)) => { - output_based_on_error_format(line, rendered, error_format) - } - _ => LineOutput::Skip, + Ok(if let Some(rendered) = get_key(&parsed, "rendered") { + output_based_on_error_format(line, rendered, error_format) + } else { + // Ignore non-diagnostic messages such as artifact notifications. + LineOutput::Skip }) }